lib: Cancel tpm threads instead of marking exit

This makes the threadpool use pthread_cancel instead of setting an
exit flag that threadpool managed threads check periodically. This
drastically reduces CPU consumption in the irmd when running a lot of
applications. It requires cancellation handlers in the ipcp and irmd
to be implemented to ensure safe cancellation during operation and
shutdown.

1 files changed, 21 insertions, 41 deletions

diff --git a/src/lib/tpm.c b/src/lib/tpm.c
index c883e0a8..1317758b 100644
--- a/src/lib/tpm.c
+++ b/src/lib/tpm.c
@@ -38,8 +38,8 @@
 struct pthr_el {
         struct list_head next;
 
-        bool             join;
         bool             kill;
+        bool             busy;
 
         pthread_t        thr;
 };
@@ -78,13 +78,12 @@ static void tpm_join(struct tpm * tpm)
                 if (tpm->state != TPM_RUNNING) {
                         if (!e->kill) {
                                 e->kill = true;
+                                pthread_cancel(e->thr);
                                 --tpm->cur;
                         }
-                        while (!e->join)
-                                pthread_cond_wait(&tpm->cond, &tpm->lock);
                 }
 
-                if (e->join) {
+                if (e->kill) {
                         pthread_join(e->thr, NULL);
                         list_del(&e->next);
                         free(e);
@@ -92,32 +91,15 @@ static void tpm_join(struct tpm * tpm)
         }
 }
 
-static struct pthr_el * tpm_pthr_el(struct tpm * tpm,
-                                    pthread_t    thr)
-{
-        struct list_head * p;
-        struct pthr_el *   e;
-
-        list_for_each(p, &tpm->pool) {
-                e = list_entry(p, struct pthr_el, next);
-                if (e->thr == thr)
-                        return e;
-
-        }
-
-        assert(false);
-
-        return NULL;
-}
-
 static void tpm_kill(struct tpm * tpm)
 {
         struct list_head * p;
 
         list_for_each(p, &tpm->pool) {
                 struct pthr_el * e = list_entry(p, struct pthr_el, next);
-                if (!e->kill) {
+                if (!e->busy && !e->kill) {
                         e->kill = true;
+                        pthread_cancel(e->thr);
                         --tpm->cur;
                         return;
                 }
@@ -152,8 +134,8 @@ static void * tpmgr(void * o)
                                 if (e == NULL)
                                         break;
 
-                                e->join = false;
                                 e->kill = false;
+                                e->busy = false;
 
                                 if (pthread_create(&e->thr, NULL,
                                                    tpm->func, tpm->o)) {
@@ -261,23 +243,30 @@ void tpm_destroy(struct tpm * tpm)
         free(tpm);
 }
 
-bool tpm_check(struct tpm * tpm)
+static struct pthr_el * tpm_pthr_el(struct tpm * tpm,
+                                    pthread_t    thr)
 {
-        bool ret;
+        struct list_head * p;
+        struct pthr_el *   e;
 
-        pthread_mutex_lock(&tpm->lock);
+        list_for_each(p, &tpm->pool) {
+                e = list_entry(p, struct pthr_el, next);
+                if (e->thr == thr)
+                        return e;
 
-        ret = tpm_pthr_el(tpm, pthread_self())->kill;
+        }
 
-        pthread_mutex_unlock(&tpm->lock);
+        assert(false);
 
-        return ret;
+        return NULL;
 }
 
 void tpm_inc(struct tpm * tpm)
 {
         pthread_mutex_lock(&tpm->lock);
 
+        tpm_pthr_el(tpm, pthread_self())->busy = false;
+
         --tpm->wrk;
 
         pthread_mutex_unlock(&tpm->lock);
@@ -287,18 +276,9 @@ void tpm_dec(struct tpm * tpm)
 {
         pthread_mutex_lock(&tpm->lock);
 
-        ++tpm->wrk;
-
-        pthread_cond_signal(&tpm->cond);
-
-        pthread_mutex_unlock(&tpm->lock);
-}
+        tpm_pthr_el(tpm, pthread_self())->busy = true;
 
-void tpm_exit(struct tpm * tpm)
-{
-        pthread_mutex_lock(&tpm->lock);
-
-        tpm_pthr_el(tpm, pthread_self())->join = true;
+        ++tpm->wrk;
 
         pthread_cond_signal(&tpm->cond);