ipcpd: Add condition variable to IPCP state

This adds a condition variable to the IPCP state, so that upon state
changes any listeners to state changes can be notified. It also
replaces the read/write lock with a mutex in order to be able to do
so.

3 files changed, 63 insertions, 72 deletions

diff --git a/src/ipcpd/normal/fmgr.c b/src/ipcpd/normal/fmgr.c
index f9de16c4..70afff37 100644
--- a/src/ipcpd/normal/fmgr.c
+++ b/src/ipcpd/normal/fmgr.c
@@ -80,21 +80,22 @@ static void * fmgr_listen(void * o)
         int fd;
         char * ae_name;
 
-        /* FIXME: Avoid busy wait and react to pthread_cond_t */
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
-        while (!(_ipcp->state == IPCP_ENROLLED ||
-                 _ipcp->state == IPCP_SHUTDOWN)) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
-                sched_yield();
-                pthread_rwlock_rdlock(&_ipcp->state_lock);
-        }
+        while (true) {
+                pthread_mutex_lock(&_ipcp->state_lock);
+                while (!(_ipcp->state == IPCP_ENROLLED ||
+                         _ipcp->state == IPCP_SHUTDOWN))
+                        pthread_cond_wait(&_ipcp->state_cond,
+                                          &_ipcp->state_lock);
+
+                if (_ipcp->state == IPCP_SHUTDOWN) {
+                        pthread_mutex_unlock(&_ipcp->state_lock);
+                        return 0;
+                }
+                pthread_mutex_unlock(&_ipcp->state_lock);
 
-        while (_ipcp->state != IPCP_SHUTDOWN) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
                 fd = flow_accept(&ae_name);
                 if (fd < 0) {
                         LOG_ERR("Failed to accept flow.");
-                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         continue;
                 }
 
@@ -103,14 +104,12 @@ static void * fmgr_listen(void * o)
                         if (flow_alloc_resp(fd, -1))
                                 LOG_ERR("Failed to reply to flow allocation.");
                         flow_dealloc(fd);
-                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         continue;
                 }
 
                 if (flow_alloc_resp(fd, 0)) {
                         LOG_ERR("Failed to reply to flow allocation.");
                         flow_dealloc(fd);
-                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         continue;
                 }
 
@@ -121,7 +120,6 @@ static void * fmgr_listen(void * o)
                         if (ribmgr_add_flow(fd)) {
                                 LOG_ERR("Failed to hand fd to RIB.");
                                 flow_dealloc(fd);
-                                pthread_rwlock_rdlock(&_ipcp->state_lock);
                                 continue;
                         }
                 }
@@ -130,7 +128,6 @@ static void * fmgr_listen(void * o)
                         if (frct_dt_flow(fd)) {
                                 LOG_ERR("Failed to hand fd to FRCT.");
                                 flow_dealloc(fd);
-                                pthread_rwlock_rdlock(&_ipcp->state_lock);
                                 continue;
                         }
                 }
@@ -138,11 +135,8 @@ static void * fmgr_listen(void * o)
                 if (add_n_1_fd(fd, ae_name)) {
                         LOG_ERR("Failed to add file descriptor to list.");
                         flow_dealloc(fd);
-                        pthread_rwlock_rdlock(&_ipcp->state_lock);
                         continue;
                 }
-
-                pthread_rwlock_rdlock(&_ipcp->state_lock);
         }
 
         return (void *) 0;
diff --git a/src/ipcpd/normal/main.c b/src/ipcpd/normal/main.c
index b5aad271..ea8d75e1 100644
--- a/src/ipcpd/normal/main.c
+++ b/src/ipcpd/normal/main.c
@@ -73,11 +73,11 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         LOG_DBG("Terminating by order of %d. Bye.",
                                 info->si_pid);
 
-                        pthread_rwlock_wrlock(&_ipcp->state_lock);
+                        pthread_mutex_lock(&_ipcp->state_lock);
 
-                        _ipcp->state = IPCP_SHUTDOWN;
+                        ipcp_state_change(_ipcp, IPCP_SHUTDOWN);
 
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
 
                         pthread_cancel(normal_data(_ipcp)->mainloop);
 
@@ -97,21 +97,21 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
 
 static int normal_ipcp_name_reg(char * name)
 {
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_ENROLLED) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_DBGF("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
         }
 
         if (ipcp_data_add_reg_entry(_ipcp->data, name)) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to add %s to local registry.", name);
                 return -1;
         }
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         LOG_DBG("Registered %s.", name);
 
@@ -120,59 +120,56 @@ static int normal_ipcp_name_reg(char * name)
 
 static int normal_ipcp_name_unreg(char * name)
 {
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         ipcp_data_del_reg_entry(_ipcp->data, name);
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         return 0;
 }
 
 static int normal_ipcp_enroll(char * dif_name)
 {
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_INIT) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't enroll an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         if (fmgr_mgmt_flow(dif_name)) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to establish management flow.");
                 return -1;
         }
 
-        /* FIXME: Wait passively until state changed to ENROLLED */
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
-        while (_ipcp->state != IPCP_ENROLLED) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
-                sched_yield();
-                pthread_rwlock_rdlock(&_ipcp->state_lock);
-        }
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        /* FIXME: Change into timedwait, see solution in irmd first */
+        pthread_mutex_lock(&_ipcp->state_lock);
+        while (_ipcp->state != IPCP_ENROLLED)
+                pthread_cond_wait(&_ipcp->state_cond, &_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         return 0;
 }
 
 static int normal_ipcp_bootstrap(struct dif_config * conf)
 {
-        LOG_DBGF("bootstrapping in dif %s.", conf->dif_name);
+        LOG_DBGF("Bootstrapping in DIF %s.", conf->dif_name);
 
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_INIT) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Won't bootstrap an IPCP that is not in INIT.");
                 return -1; /* -ENOTINIT */
         }
 
         if (ribmgr_bootstrap(conf)) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to bootstrap RIB manager.");
                 return -1;
         }
@@ -182,9 +179,9 @@ static int normal_ipcp_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        _ipcp->state = IPCP_ENROLLED;
+        ipcp_state_change(_ipcp, IPCP_ENROLLED);
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -239,7 +236,7 @@ void normal_ipcp_data_destroy()
         if (_ipcp == NULL)
                 return;
 
-        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         if (_ipcp->state != IPCP_SHUTDOWN)
                 LOG_WARN("Cleaning up while not in shutdown.");
@@ -249,7 +246,7 @@ void normal_ipcp_data_destroy()
         if (normal_data(_ipcp)->rb != NULL)
                 shm_ap_rbuff_close(normal_data(_ipcp)->rb);
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         free(_ipcp->data);
 }
@@ -323,7 +320,7 @@ int main(int argc, char * argv[])
                 exit(EXIT_FAILURE);
         }
 
-        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
 
         pthread_sigmask(SIG_BLOCK, &sigset, NULL);
 
@@ -332,7 +329,7 @@ int main(int argc, char * argv[])
 
         pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
 
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         if (ipcp_create_r(getpid())) {
                 LOG_ERR("Failed to notify IRMd we are initialized.");
diff --git a/src/ipcpd/normal/ribmgr.c b/src/ipcpd/normal/ribmgr.c
index 2a68877a..bbc29b64 100644
--- a/src/ipcpd/normal/ribmgr.c
+++ b/src/ipcpd/normal/ribmgr.c
@@ -220,15 +220,15 @@ int ribmgr_cdap_write(struct cdap * instance,
         static_info_msg_t * msg;
         int ret = 0;
 
-        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
         if (_ipcp->state == IPCP_PENDING_ENROLL &&
             strcmp(name, STATIC_INFO) == 0) {
                 LOG_DBG("Received static DIF information.");
 
                 msg = static_info_msg__unpack(NULL, len, data);
                 if (msg == NULL) {
-                        _ipcp->state = IPCP_INIT;
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        ipcp_state_change(_ipcp, IPCP_INIT);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         LOG_ERR("Failed to unpack static info message.");
                         return -1;
@@ -246,8 +246,8 @@ int ribmgr_cdap_write(struct cdap * instance,
                 rib->address = msg->address;
 
                 if (frct_init(&rib->dtc, rib->address)) {
-                        _ipcp->state = IPCP_INIT;
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        ipcp_state_change(_ipcp, IPCP_INIT);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         cdap_send_reply(instance, invoke_id, -1, NULL, 0);
                         static_info_msg__free_unpacked(msg, NULL);
                         LOG_ERR("Failed to init FRCT");
@@ -257,7 +257,7 @@ int ribmgr_cdap_write(struct cdap * instance,
                 static_info_msg__free_unpacked(msg, NULL);
         } else
                 ret = -1;
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
                 LOG_ERR("Failed to send reply to write request.");
@@ -298,13 +298,13 @@ int ribmgr_cdap_start(struct cdap * instance,
         size_t len = 0;
         int iid = 0;
 
-        pthread_rwlock_rdlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
         if (_ipcp->state == IPCP_ENROLLED &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("New enrollment request.");
 
                 if (cdap_send_reply(instance, invoke_id, 0, NULL, 0)) {
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to enrollment request.");
                         return -1;
                 }
@@ -323,14 +323,14 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 len = static_info_msg__get_packed_size(&stat_info);
                 if (len == 0) {
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to get size of static information.");
                         return -1;
                 }
 
                 data = malloc(len);
                 if (data == NULL) {
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to allocate memory.");
                         return -1;
                 }
@@ -344,7 +344,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_write(instance, STATIC_INFO, data, len, 0);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send static information.");
                         return -1;
@@ -352,7 +352,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_request_add(instance, WRITE, STATIC_INFO, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to add CDAP request to list.");
                         return -1;
@@ -368,7 +368,7 @@ int ribmgr_cdap_start(struct cdap * instance,
                 iid = cdap_send_stop(instance, ENROLLMENT);
                 if (iid < 0) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to send stop of enrollment.");
                         return -1;
@@ -376,7 +376,7 @@ int ribmgr_cdap_start(struct cdap * instance,
 
                 if (cdap_request_add(instance, STOP, ENROLLMENT, iid)) {
                         pthread_mutex_unlock(&rib->cdap_reqs_lock);
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         free(data);
                         LOG_ERR("Failed to add CDAP request to list.");
                         return -1;
@@ -386,12 +386,12 @@ int ribmgr_cdap_start(struct cdap * instance,
                 free(data);
         } else {
                 if (cdap_send_reply(instance, invoke_id, -1, NULL, 0)) {
-                        pthread_rwlock_unlock(&_ipcp->state_lock);
+                        pthread_mutex_unlock(&_ipcp->state_lock);
                         LOG_ERR("Failed to send reply to start request.");
                         return -1;
                 }
         }
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -402,21 +402,21 @@ int ribmgr_cdap_stop(struct cdap * instance,
 {
         int ret = 0;
 
-        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
         if (_ipcp->state == IPCP_PENDING_ENROLL &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("Stop enrollment received.");
 
-                _ipcp->state = IPCP_ENROLLED;
+                ipcp_state_change(_ipcp, IPCP_ENROLLED);
         } else
                 ret = -1;
 
         if (cdap_send_reply(instance, invoke_id, ret, NULL, 0)) {
-                pthread_rwlock_unlock(&_ipcp->state_lock);
+                pthread_mutex_unlock(&_ipcp->state_lock);
                 LOG_ERR("Failed to send reply to stop request.");
                 return -1;
         }
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         return 0;
 }
@@ -452,11 +452,11 @@ int ribmgr_add_flow(int fd)
         flow->instance = instance;
         flow->fd = fd;
 
-        pthread_rwlock_wrlock(&_ipcp->state_lock);
+        pthread_mutex_lock(&_ipcp->state_lock);
         pthread_rwlock_wrlock(&rib->flows_lock);
         if (list_empty(&rib->flows) &&
             _ipcp->state == IPCP_INIT) {
-                _ipcp->state = IPCP_PENDING_ENROLL;
+                ipcp_state_change(_ipcp, IPCP_PENDING_ENROLL);
 
                 pthread_mutex_lock(&rib->cdap_reqs_lock);
                 iid = cdap_send_start(instance,
@@ -480,7 +480,7 @@ int ribmgr_add_flow(int fd)
                 }
                 pthread_mutex_unlock(&rib->cdap_reqs_lock);
         }
-        pthread_rwlock_unlock(&_ipcp->state_lock);
+        pthread_mutex_unlock(&_ipcp->state_lock);
 
         list_add(&flow->next, &rib->flows);
         pthread_rwlock_unlock(&rib->flows_lock);