ipcpd: Split IPCP state PENDING_ENROL

This will split the IPCP state PENDING_ENROL into IPCP_CONFIG and
IPCP_BOOTING. IPCP_CONFIG is concerned only with configuring the IPCP
with the bare essence. When in IPCP_BOOTING, the IPCP will complete
its configuration by starting its policies, and thus making the IPCP
completely functioning.

8 files changed, 136 insertions, 90 deletions

diff --git a/src/ipcpd/ipcp.h b/src/ipcpd/ipcp.h
index c89fe438..58cb6309 100644
--- a/src/ipcpd/ipcp.h
+++ b/src/ipcpd/ipcp.h
@@ -34,8 +34,9 @@
 enum ipcp_state {
         IPCP_NULL = 0,
         IPCP_INIT,
-        IPCP_PENDING_ENROLL,
-        IPCP_ENROLLED,
+        IPCP_CONFIG,
+        IPCP_BOOTING,
+        IPCP_RUNNING,
         IPCP_DISCONNECTED,
         IPCP_SHUTDOWN
 };
diff --git a/src/ipcpd/local/main.c b/src/ipcpd/local/main.c
index f0c85084..d8614afb 100644
--- a/src/ipcpd/local/main.c
+++ b/src/ipcpd/local/main.c
@@ -94,7 +94,7 @@ static void * ipcp_local_sdu_loop(void * o)
 
                 pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-                if (ipcp_get_state() != IPCP_ENROLLED) {
+                if (ipcp_get_state() != IPCP_RUNNING) {
                         pthread_rwlock_unlock(&ipcpi.state_lock);
                         return (void *) 1; /* -ENOTENROLLED */
                 }
@@ -137,7 +137,7 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         if (ipcp_get_state() == IPCP_INIT)
                                 ipcp_set_state(IPCP_NULL);
 
-                        if (ipcp_get_state() == IPCP_ENROLLED)
+                        if (ipcp_get_state() == IPCP_RUNNING)
                                 ipcp_set_state(IPCP_SHUTDOWN);
 
                         pthread_rwlock_unlock(&ipcpi.state_lock);
@@ -163,7 +163,7 @@ static int ipcp_local_bootstrap(struct dif_config * conf)
         /* this IPCP doesn't need to maintain its dif_name */
         free(conf->dif_name);
 
-        ipcp_set_state(IPCP_ENROLLED);
+        ipcp_set_state(IPCP_RUNNING);
 
         pthread_create(&local_data.sduloop, NULL, ipcp_local_sdu_loop, NULL);
 
@@ -234,7 +234,7 @@ static int ipcp_local_flow_alloc(int           fd,
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
@@ -296,7 +296,7 @@ static int ipcp_local_flow_dealloc(int fd)
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
diff --git a/src/ipcpd/normal/fmgr.c b/src/ipcpd/normal/fmgr.c
index 607308c0..35217283 100644
--- a/src/ipcpd/normal/fmgr.c
+++ b/src/ipcpd/normal/fmgr.c
@@ -127,7 +127,7 @@ static void * fmgr_nm1_acceptor(void * o)
         (void) o;
 
         while (true) {
-                ipcp_wait_state(IPCP_ENROLLED, NULL);
+                ipcp_wait_state(IPCP_RUNNING, NULL);
 
                 pthread_rwlock_rdlock(&ipcpi.state_lock);
 
@@ -412,7 +412,7 @@ int fmgr_np1_alloc(int           fd,
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_ERR("IPCP is not enrolled yet.");
                 return -1; /* -ENOTINIT */
diff --git a/src/ipcpd/normal/main.c b/src/ipcpd/normal/main.c
index a5161718..810fbca5 100644
--- a/src/ipcpd/normal/main.c
+++ b/src/ipcpd/normal/main.c
@@ -100,8 +100,6 @@ static int normal_ipcp_name_query(char * name)
 {
         LOG_MISSING;
 
-        (void) name;
-
         /*
          * NOTE: For the moment we just return -1,
          * for testing purposes we may return zero here
@@ -135,18 +133,26 @@ static int normal_ipcp_enroll(char * dif_name)
                 return -1;
         }
 
-        if (ipcp_wait_state(IPCP_ENROLLED, &timeout) == -ETIMEDOUT) {
-                LOG_ERR("Enrollment timed out.");
+        if (ribmgr_enrol()) {
+                LOG_ERR("Failed to enrol IPCP.");
                 return -1;
         }
 
-        pthread_rwlock_rdlock(&ipcpi.state_lock);
+        if (ipcp_wait_state(IPCP_BOOTING, &timeout) == -ETIMEDOUT) {
+                LOG_ERR("Enrollment timed out.");
+                return -1;
+        }
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ribmgr_start_policies()) {
+                pthread_rwlock_wrlock(&ipcpi.state_lock);
+                ipcp_set_state(IPCP_INIT);
                 pthread_rwlock_unlock(&ipcpi.state_lock);
+                LOG_ERR("Failed to start policies.");
                 return -1;
         }
 
+        pthread_rwlock_wrlock(&ipcpi.state_lock);
+        ipcp_set_state(IPCP_RUNNING);
         pthread_rwlock_unlock(&ipcpi.state_lock);
 
         /* FIXME: Remove once we obtain neighbors during enrollment */
@@ -174,8 +180,20 @@ static int normal_ipcp_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        ipcp_set_state(IPCP_ENROLLED);
+        ipcp_set_state(IPCP_BOOTING);
+        pthread_rwlock_unlock(&ipcpi.state_lock);
+
+        if (ribmgr_start_policies()) {
+                pthread_rwlock_wrlock(&ipcpi.state_lock);
+                ipcp_set_state(IPCP_INIT);
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                LOG_ERR("Failed to start policies.");
+                return -1;
+        }
+
+        pthread_rwlock_wrlock(&ipcpi.state_lock);
 
+        ipcp_set_state(IPCP_RUNNING);
         ipcpi.data->dif_name = conf->dif_name;
 
         pthread_rwlock_unlock(&ipcpi.state_lock);
diff --git a/src/ipcpd/normal/ribmgr.c b/src/ipcpd/normal/ribmgr.c
index 0f76d960..3cebde0c 100644
--- a/src/ipcpd/normal/ribmgr.c
+++ b/src/ipcpd/normal/ribmgr.c
@@ -127,6 +127,7 @@ struct {
         pthread_mutex_t     cdap_reqs_lock;
 
         struct addr_auth *  addr_auth;
+        enum pol_addr_auth  addr_auth_type;
 } rib;
 
 void ribmgr_ro_created(const char * name,
@@ -136,7 +137,7 @@ void ribmgr_ro_created(const char * name,
         static_info_msg_t * stat_msg;
 
         pthread_rwlock_wrlock(&ipcpi.state_lock);
-        if (ipcp_get_state() == IPCP_PENDING_ENROLL &&
+        if (ipcp_get_state() == IPCP_CONFIG &&
             strcmp(name, RIBMGR_PREFIX STAT_INFO) == 0) {
                 LOG_DBG("Received static DIF information.");
 
@@ -156,18 +157,7 @@ void ribmgr_ro_created(const char * name,
                 rib.dtc.has_chk = stat_msg->has_chk;
                 rib.dtc.min_pdu_size = stat_msg->min_pdu_size;
                 rib.dtc.max_pdu_size = stat_msg->max_pdu_size;
-
-                rib.addr_auth = addr_auth_create(stat_msg->addr_auth_type);
-                if (rib.addr_auth == NULL) {
-                        ipcp_set_state(IPCP_INIT);
-                        pthread_rwlock_unlock(&ipcpi.state_lock);
-                        static_info_msg__free_unpacked(stat_msg, NULL);
-                        LOG_ERR("Failed to create address authority");
-                        return;
-                }
-
-                rib.address = rib.addr_auth->address();
-                LOG_DBG("IPCP has address %lu", rib.address);
+                rib.addr_auth_type = stat_msg->addr_auth_type;
 
                 if (frct_init()) {
                         ipcp_set_state(IPCP_INIT);
@@ -946,7 +936,7 @@ static int ribmgr_cdap_start(struct cdap * instance,
         int iid = 0;
 
         pthread_rwlock_wrlock(&ipcpi.state_lock);
-        if (ipcp_get_state() == IPCP_ENROLLED &&
+        if (ipcp_get_state() == IPCP_RUNNING &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("New enrollment request.");
 
@@ -1008,11 +998,11 @@ static int ribmgr_cdap_stop(struct cdap * instance,
         int ret = 0;
 
         pthread_rwlock_wrlock(&ipcpi.state_lock);
-        if (ipcp_get_state() == IPCP_PENDING_ENROLL &&
+        if (ipcp_get_state() == IPCP_CONFIG &&
             strcmp(name, ENROLLMENT) == 0) {
                 LOG_DBG("Stop enrollment received.");
 
-                ipcp_set_state(IPCP_ENROLLED);
+                ipcp_set_state(IPCP_BOOTING);
         } else
                 ret = -1;
 
@@ -1175,7 +1165,6 @@ int ribmgr_add_flow(int fd)
 {
         struct cdap * instance = NULL;
         struct mgmt_flow * flow;
-        int iid = 0;
 
         flow = malloc(sizeof(*flow));
         if (flow == NULL)
@@ -1192,42 +1181,9 @@ int ribmgr_add_flow(int fd)
         flow->instance = instance;
         flow->fd = fd;
 
-        pthread_rwlock_wrlock(&ipcpi.state_lock);
         pthread_rwlock_wrlock(&rib.flows_lock);
-        if (list_empty(&rib.flows) && ipcp_get_state() == IPCP_INIT) {
-                ipcp_set_state(IPCP_PENDING_ENROLL);
-
-                pthread_mutex_lock(&rib.cdap_reqs_lock);
-                iid = cdap_send_request(instance,
-                                        CDAP_START,
-                                        ENROLLMENT,
-                                        NULL, 0, 0);
-                if (iid < 0) {
-                        pthread_mutex_unlock(&rib.cdap_reqs_lock);
-                        pthread_rwlock_unlock(&rib.flows_lock);
-                        pthread_rwlock_unlock(&ipcpi.state_lock);
-                        LOG_ERR("Failed to start enrollment.");
-                        cdap_destroy(instance);
-                        free(flow);
-                        return -1;
-                }
-
-                if (cdap_result_wait(instance, CDAP_START,
-                                     ENROLLMENT, iid)) {
-                        pthread_mutex_unlock(&rib.cdap_reqs_lock);
-                        pthread_rwlock_unlock(&rib.flows_lock);
-                        pthread_rwlock_unlock(&ipcpi.state_lock);
-                        LOG_ERR("Failed to start enrollment.");
-                        cdap_destroy(instance);
-                        free(flow);
-                        return -1;
-                }
-                pthread_mutex_unlock(&rib.cdap_reqs_lock);
-        }
-
         list_add(&flow->next, &rib.flows);
         pthread_rwlock_unlock(&rib.flows_lock);
-        pthread_rwlock_unlock(&ipcpi.state_lock);
 
         return 0;
 }
@@ -1284,15 +1240,7 @@ int ribmgr_bootstrap(struct dif_config * conf)
         stat_info.has_chk = rib.dtc.has_chk = conf->has_chk;
         stat_info.min_pdu_size = rib.dtc.min_pdu_size = conf->min_pdu_size;
         stat_info.max_pdu_size = rib.dtc.max_pdu_size = conf->max_pdu_size;
-
-        rib.addr_auth = addr_auth_create(conf->addr_auth_type);
-        if (rib.addr_auth == NULL) {
-                LOG_ERR("Failed to create address authority.");
-                ro_delete(RIBMGR_PREFIX);
-                return -1;
-        }
-
-        stat_info.addr_auth_type = rib.addr_auth->type;
+        stat_info.addr_auth_type = rib.addr_auth_type = conf->addr_auth_type;
 
         len = static_info_msg__get_packed_size(&stat_info);
         if (len == 0) {
@@ -1321,9 +1269,6 @@ int ribmgr_bootstrap(struct dif_config * conf)
                 return -1;
         }
 
-        rib.address = rib.addr_auth->address();
-        LOG_DBG("IPCP has address %lu", rib.address);
-
         if (frct_init()) {
                 LOG_ERR("Failed to initialize FRCT.");
                 ribmgr_ro_delete(RIBMGR_PREFIX STAT_INFO);
@@ -1337,6 +1282,84 @@ int ribmgr_bootstrap(struct dif_config * conf)
         return 0;
 }
 
+int ribmgr_enrol(void)
+{
+        struct cdap * instance = NULL;
+        struct mgmt_flow * flow;
+        int iid = 0;
+
+        pthread_rwlock_wrlock(&ipcpi.state_lock);
+
+        if (ipcp_get_state() != IPCP_INIT) {
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                return -1;
+        }
+
+        ipcp_set_state(IPCP_CONFIG);
+
+        pthread_rwlock_wrlock(&rib.flows_lock);
+        if (list_empty(&rib.flows)) {
+                ipcp_set_state(IPCP_INIT);
+                pthread_rwlock_unlock(&rib.flows_lock);
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                return -1;
+        }
+
+        flow = list_entry((&rib.flows)->next, struct mgmt_flow, next);
+        instance = flow->instance;
+
+        pthread_mutex_lock(&rib.cdap_reqs_lock);
+        iid = cdap_send_request(instance,
+                                CDAP_START,
+                                ENROLLMENT,
+                                NULL, 0, 0);
+        if (iid < 0) {
+                ipcp_set_state(IPCP_INIT);
+                pthread_mutex_unlock(&rib.cdap_reqs_lock);
+                pthread_rwlock_unlock(&rib.flows_lock);
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                LOG_ERR("Failed to start enrollment.");
+                return -1;
+        }
+
+        if (cdap_result_wait(instance, CDAP_START,
+                             ENROLLMENT, iid)) {
+                ipcp_set_state(IPCP_INIT);
+                pthread_mutex_unlock(&rib.cdap_reqs_lock);
+                pthread_rwlock_unlock(&rib.flows_lock);
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                LOG_ERR("Failed to start enrollment.");
+                return -1;
+        }
+        pthread_mutex_unlock(&rib.cdap_reqs_lock);
+        pthread_rwlock_unlock(&rib.flows_lock);
+        pthread_rwlock_unlock(&ipcpi.state_lock);
+
+        return 0;
+}
+
+int ribmgr_start_policies(void)
+{
+        pthread_rwlock_rdlock(&ipcpi.state_lock);
+        if (ipcp_get_state() != IPCP_BOOTING) {
+                pthread_rwlock_unlock(&ipcpi.state_lock);
+                LOG_ERR("Cannot start policies in wrong state");
+                return -1;
+        }
+        pthread_rwlock_unlock(&ipcpi.state_lock);
+
+        rib.addr_auth = addr_auth_create(rib.addr_auth_type);
+        if (rib.addr_auth == NULL) {
+                LOG_ERR("Failed to create address authority.");
+                return -1;
+        }
+
+        rib.address = rib.addr_auth->address();
+        LOG_DBG("IPCP has address %lu", rib.address);
+
+        return 0;
+}
+
 struct dt_const * ribmgr_dt_const()
 {
         return &(rib.dtc);
diff --git a/src/ipcpd/normal/ribmgr.h b/src/ipcpd/normal/ribmgr.h
index 36f771c1..b76ff3bd 100644
--- a/src/ipcpd/normal/ribmgr.h
+++ b/src/ipcpd/normal/ribmgr.h
@@ -38,6 +38,10 @@ int               ribmgr_remove_flow(int fd);
 
 int               ribmgr_bootstrap(struct dif_config * conf);
 
+int               ribmgr_enrol(void);
+
+int               ribmgr_start_policies(void);
+
 struct dt_const * ribmgr_dt_const(void);
 
 uint64_t          ribmgr_address(void);
diff --git a/src/ipcpd/shim-eth-llc/main.c b/src/ipcpd/shim-eth-llc/main.c
index fafe8651..fe2c3336 100644
--- a/src/ipcpd/shim-eth-llc/main.c
+++ b/src/ipcpd/shim-eth-llc/main.c
@@ -629,7 +629,7 @@ static void * eth_llc_ipcp_sdu_writer(void * o)
                                &timeout)) {
                 pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-                if (ipcp_get_state() != IPCP_ENROLLED) {
+                if (ipcp_get_state() != IPCP_RUNNING) {
                         pthread_rwlock_unlock(&ipcpi.state_lock);
                         return (void *) -1; /* -ENOTENROLLED */
                 }
@@ -680,7 +680,7 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         if (ipcp_get_state() == IPCP_INIT)
                                 ipcp_set_state(IPCP_NULL);
 
-                        if (ipcp_get_state() == IPCP_ENROLLED)
+                        if (ipcp_get_state() == IPCP_RUNNING)
                                 ipcp_set_state(IPCP_SHUTDOWN);
 
                         pthread_rwlock_unlock(&ipcpi.state_lock);
@@ -845,7 +845,7 @@ static int eth_llc_ipcp_bootstrap(struct dif_config * conf)
         eth_llc_data.tx_offset = 0;
 #endif
 
-        ipcp_set_state(IPCP_ENROLLED);
+        ipcp_set_state(IPCP_RUNNING);
 
         pthread_create(&eth_llc_data.sdu_reader,
                        NULL,
@@ -951,7 +951,7 @@ static int eth_llc_ipcp_flow_alloc(int           fd,
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
@@ -1051,7 +1051,7 @@ static int eth_llc_ipcp_flow_dealloc(int fd)
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
diff --git a/src/ipcpd/shim-udp/main.c b/src/ipcpd/shim-udp/main.c
index eff0bd94..67161d90 100644
--- a/src/ipcpd/shim-udp/main.c
+++ b/src/ipcpd/shim-udp/main.c
@@ -486,7 +486,7 @@ static void * ipcp_udp_sdu_loop(void * o)
         while (flow_event_wait(udp_data.np1_flows, udp_data.fq, &timeout)) {
                 pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-                if (ipcp_get_state() != IPCP_ENROLLED) {
+                if (ipcp_get_state() != IPCP_RUNNING) {
                         pthread_rwlock_unlock(&ipcpi.state_lock);
                         return (void *) -1; /* -ENOTENROLLED */
                 }
@@ -531,7 +531,7 @@ void ipcp_sig_handler(int sig, siginfo_t * info, void * c)
                         if (ipcp_get_state() == IPCP_INIT)
                                 ipcp_set_state(IPCP_NULL);
 
-                        if (ipcp_get_state() == IPCP_ENROLLED)
+                        if (ipcp_get_state() == IPCP_RUNNING)
                                 ipcp_set_state(IPCP_SHUTDOWN);
 
                         pthread_rwlock_unlock(&ipcpi.state_lock);
@@ -616,7 +616,7 @@ static int ipcp_udp_bootstrap(struct dif_config * conf)
 
         FD_CLR(udp_data.s_fd, &udp_data.flow_fd_s);
 
-        ipcp_set_state(IPCP_ENROLLED);
+        ipcp_set_state(IPCP_RUNNING);
 
         pthread_create(&udp_data.handler,
                        NULL,
@@ -890,7 +890,7 @@ static int ipcp_udp_name_query(char * name)
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't query a name on a non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */
@@ -915,7 +915,7 @@ static int ipcp_udp_name_query(char * name)
 
                 pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-                if (ipcp_get_state() != IPCP_ENROLLED) {
+                if (ipcp_get_state() != IPCP_RUNNING) {
                         pthread_rwlock_unlock(&ipcpi.state_lock);
                         LOG_DBG("Won't add name to the directory.");
                         return -1; /* -ENOTENROLLED */
@@ -991,7 +991,7 @@ static int ipcp_udp_flow_alloc(int           fd,
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't allocate flow with non-enrolled IPCP.");
                 close(skfd);
@@ -1113,7 +1113,7 @@ static int ipcp_udp_flow_dealloc(int fd)
 
         pthread_rwlock_rdlock(&ipcpi.state_lock);
 
-        if (ipcp_get_state() != IPCP_ENROLLED) {
+        if (ipcp_get_state() != IPCP_RUNNING) {
                 pthread_rwlock_unlock(&ipcpi.state_lock);
                 LOG_DBG("Won't register with non-enrolled IPCP.");
                 return -1; /* -ENOTENROLLED */