ipcpd: Fix request handling at shutdown

The IPCP states were not entirely correct causing some operations to
be serviced during shutdown. This caused some use-after-free in the
pff. States in the IPCP are now correctly set. IRMd states updated to
the same strategy. The IRMd registry tracks if the IPCP was ENROLLED
or BOOTSTRAPPED, the IPCP just goes to OPERATIONAL.

IPCP state diagram::

NULL -> init() -> INIT -> start() -> BOOT ->
     bootstrap/enroll() -> OPERATIONAL -> shutdown()
     -> SHUTDOWN -> stop_components() -> BOOT ->
stop() -> INIT -> fini() -> NULL

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>

3 files changed, 24 insertions, 20 deletions

diff --git a/src/ipcpd/ipcp.c b/src/ipcpd/ipcp.c
index 7fe3e7eb..a47b6226 100644
--- a/src/ipcpd/ipcp.c
+++ b/src/ipcpd/ipcp.c
@@ -302,7 +302,7 @@ static void * acceptloop(void * o)
         (void) o;
 
         while (ipcp_get_state() != IPCP_SHUTDOWN &&
-               ipcp_get_state() != IPCP_NULL) {
+               ipcp_get_state() != IPCP_INIT) {
                 struct cmd * cmd;
 
                 csockfd = accept(ipcpd.sockfd, 0, 0);
@@ -444,11 +444,11 @@ static void do_bootstrap(ipcp_config_msg_t * conf_msg,
                 return;
         }
 
-        if (ipcp_get_state() != IPCP_INIT) {
+        if (ipcp_get_state() != IPCP_BOOT) {
 
                 log_err("Failed to bootstrap: IPCP in state <%s>, need <%s>.",
                         ipcp_state_str[ipcp_get_state()],
-                        ipcp_state_str[IPCP_INIT]);
+                        ipcp_state_str[IPCP_BOOT]);
                 ret_msg->result = -EIPCPSTATE;
                 return;
         }
@@ -476,14 +476,13 @@ static void do_bootstrap(ipcp_config_msg_t * conf_msg,
         strcpy(ipcpd.layer_name, info->name);
         ipcpd.dir_hash_algo = (enum hash_algo) info->dir_hash_algo;
         ret_msg->layer_info = layer_info_s_to_msg(info);
+        ipcp_set_state(IPCP_OPERATIONAL);
 
         log_info("Finished bootstrapping in %s.", info->name);
         log_info("  type: %s", ipcp_type_str[ipcpd.type]);
         log_info("  hash: %s [%zd bytes]",
                 dir_hash_str[ipcpd.dir_hash_algo],
                 ipcp_dir_hash_len());
-
-        ipcp_set_state(IPCP_OPERATIONAL);
 }
 
 static void do_enroll(const char * dst,
@@ -499,10 +498,10 @@ static void do_enroll(const char * dst,
                 return;
         }
 
-        if (ipcp_get_state() != IPCP_INIT) {
+        if (ipcp_get_state() != IPCP_BOOT) {
                 log_err("Failed to enroll: IPCP in state <%s>, need <%s>.",
                         ipcp_state_str[ipcp_get_state()],
-                        ipcp_state_str[IPCP_INIT]);
+                        ipcp_state_str[IPCP_BOOT]);
                 ret_msg->result = -EIPCPSTATE;
                 return;
         }
@@ -949,7 +948,6 @@ int ipcp_init(int               argc,
 
         log_init(log);
 
-        ipcpd.state = IPCP_NULL;
         ipcpd.type  = type;
 
 #if defined (__linux__)
@@ -1031,6 +1029,9 @@ int ipcp_init(int               argc,
 
         ipcp_set_state(IPCP_INIT);
 
+        log_info("IPCP %s %d initialized.", ipcp_type_str[ipcpd.type],
+                 getpid());
+
         return 0;
 
  fail_tpm_create:
@@ -1075,7 +1076,9 @@ int ipcp_start(void)
         info.pid  = getpid();
         info.type = ipcpd.type;
         strcpy(info.name, ipcpd.name);
-        info.state = IPCP_OPERATIONAL;
+        info.state = IPCP_BOOT;
+
+        ipcp_set_state(IPCP_BOOT);
 
         if (tpm_start(ipcpd.tpm)) {
                 log_err("Failed to start threadpool manager.");
@@ -1087,8 +1090,6 @@ int ipcp_start(void)
                 goto fail_acceptor;
         }
 
-        info.state = IPCP_OPERATIONAL;
-
         if (ipcp_create_r(&info)) {
                 log_err("Failed to notify IRMd we are initialized.");
                 goto fail_create_r;
@@ -1103,8 +1104,7 @@ int ipcp_start(void)
         tpm_stop(ipcpd.tpm);
  fail_tpm_start:
         tpm_destroy(ipcpd.tpm);
-        ipcp_set_state(IPCP_NULL);
-        info.state = IPCP_NULL;
+        ipcp_set_state(IPCP_INIT);
         ipcp_create_r(&info);
         return -1;
 }
@@ -1124,7 +1124,7 @@ void ipcp_sigwait(void)
         sigaddset(&sigset, SIGTERM);
         sigaddset(&sigset, SIGPIPE);
 
-        while(ipcp_get_state() != IPCP_NULL &&
+        while(ipcp_get_state() != IPCP_INIT &&
               ipcp_get_state() != IPCP_SHUTDOWN) {
 #ifdef __APPLE__
                 if (sigwait(&sigset, &sig) < 0) {
@@ -1149,8 +1149,8 @@ void ipcp_sigwait(void)
                         /* FALLTHRU */
                 case SIGQUIT:
                         if (info.si_pid == ipcpd.irmd_pid) {
-                                if (ipcp_get_state() == IPCP_INIT)
-                                        ipcp_set_state(IPCP_NULL);
+                                if (ipcp_get_state() == IPCP_BOOT)
+                                        ipcp_set_state(IPCP_INIT);
 
                                 if (ipcp_get_state() == IPCP_OPERATIONAL)
                                         ipcp_set_state(IPCP_SHUTDOWN);
@@ -1173,6 +1173,8 @@ void ipcp_stop(void)
         pthread_join(ipcpd.acceptor, NULL);
 
         tpm_stop(ipcpd.tpm);
+
+        ipcp_set_state(IPCP_INIT);
 }
 
 void ipcp_fini(void)
@@ -1200,6 +1202,8 @@ void ipcp_fini(void)
         log_info("IPCP %d out.", getpid());
 
         log_fini();
+
+        ipcpd.state = IPCP_NULL;
 }
 
 void ipcp_set_state(enum ipcp_state state)
diff --git a/src/ipcpd/unicast/dir/dht.c b/src/ipcpd/unicast/dir/dht.c
index f7de7bb7..d0523eee 100644
--- a/src/ipcpd/unicast/dir/dht.c
+++ b/src/ipcpd/unicast/dir/dht.c
@@ -39,7 +39,6 @@
 #include <ouroboros/errno.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/list.h>
-#include <ouroboros/notifier.h>
 #include <ouroboros/random.h>
 #include <ouroboros/rib.h>
 #include <ouroboros/time.h>
diff --git a/src/ipcpd/unicast/main.c b/src/ipcpd/unicast/main.c
index c2348242..cd0977d4 100644
--- a/src/ipcpd/unicast/main.c
+++ b/src/ipcpd/unicast/main.c
@@ -172,7 +172,7 @@ static void stop_components(void)
 
         dt_stop();
 
-        ipcp_set_state(IPCP_INIT);
+        ipcp_set_state(IPCP_BOOT);
 }
 
 static int unicast_ipcp_enroll(const char *        dst,
@@ -308,11 +308,12 @@ int main(int    argc,
 
         if (ipcp_get_state() == IPCP_SHUTDOWN) {
                 stop_components();
+                ipcp_stop();
                 finalize_components();
+        } else {
+                ipcp_stop();
         }
 
-        ipcp_stop();
-
         enroll_fini();
 
         connmgr_fini();