irmd: Initial Flow Allocation Protocol Headerbe

This adds the initial version for the flow allocation protocol header
between IRMd instances. This is a step towards flow authentication.

The header supports secure and authenticated flow allocation,
supporting certificate-based authentication and ephemeral key
exchange for end-to-end encryption.

 id:          128-bit identifier for the entity.
 timestamp:   64-bit timestamp (replay protection).
 certificate: Certificate for authentication.
 public key:  ECDHE public key for key exchange.
 data:        Application data.
 signature:   Signature for integrity/authenticity.

Authentication and encryption require OpenSSL to be installed.

The IRMd compares the allocation request delay with the MPL of the
Layer over which the flow allocation was sent. MPL is now reported by
the Layer in ms instead of seconds.

Time functions revised for consistency and adds some tests.

The TPM can now print thread running times in Debug builds
(TPM_DEBUG_REPORT_INTERVAL) and abort processes with hung threads
(TPM_DEBUG_ABORT_TIMEOUT). Long running threads waiting for input
should call tpm_wait_work() to avoid trigger a process abort.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

40 files changed, 1499 insertions, 304 deletions

diff --git a/include/ouroboros/crypt.h b/include/ouroboros/crypt.h
index b510a3b2..a1339330 100644
--- a/include/ouroboros/crypt.h
+++ b/include/ouroboros/crypt.h
@@ -61,6 +61,9 @@ int                crypt_load_crt_file(const char * path,
 int                crypt_load_crt_str(const char * str,
                                       void **      crt);
 
+int                crypt_load_crt_der(buffer_t buf,
+                                      void **  crt);
+
 int                crypt_get_pubkey_crt(void *  crt,
                                         void ** pk);
 
@@ -80,8 +83,11 @@ int                crypt_cmp_key(const void * key1,
 
 void               crypt_free_key(void * key);
 
-int                crypt_crt_str(void * crt,
-                                 char * buf);
+int                crypt_crt_str(const void * crt,
+                                 char *       buf);
+
+int                crypt_crt_der(const void * crt,
+                                 buffer_t *   buf);
 
 int                crypt_check_crt_name(void *       crt,
                                         const char * name);
diff --git a/include/ouroboros/endian.h b/include/ouroboros/endian.h
index addb2ed3..6c3493d9 100644
--- a/include/ouroboros/endian.h
+++ b/include/ouroboros/endian.h
@@ -66,8 +66,8 @@
 #endif
 
 #define hton64(x) htobe64(x)
-#define hton32(x) htobe32(x)
 #define ntoh64(x) betoh64(x)
+#define hton32(x) htobe32(x)
 #define ntoh32(x) betoh32(x)
 #define hton16(x) htobe16(x)
 #define ntoh16(x) betoh16(x)
diff --git a/include/ouroboros/hash.h b/include/ouroboros/hash.h
index 6b0087ce..3d5734f5 100644
--- a/include/ouroboros/hash.h
+++ b/include/ouroboros/hash.h
@@ -52,6 +52,10 @@ enum hash_algo {
 #define HASH_VAL128(hash128)                              \
         HASH_VAL64(hash128), HASH_VAL64(hash128 + 8)
 
+#define HASH_FMT192 HASH_FMT128 HASH_FMT64
+#define HASH_VAL192(hash192)                              \
+        HASH_VAL128(hash192), HASH_VAL64(hash192 + 16)
+
 #define HASH_FMT224 HASH_FMT128 HASH_FMT64 HASH_FMT32
 #define HASH_VAL224(hash224)                              \
         HASH_VAL128(hash224), HASH_VAL64(hash224 + 16),   \
diff --git a/include/ouroboros/time.h b/include/ouroboros/time.h
index b274c35b..470c99a0 100644
--- a/include/ouroboros/time.h
+++ b/include/ouroboros/time.h
@@ -36,29 +36,33 @@
 
 #include <time.h>
 #include <sys/time.h>
+#include <sys/types.h>
 
 #define TIMESPEC_INIT_S(s)   {(s), 0}
 #define TIMESPEC_INIT_MS(ms) {(ms) / 1000, ((ms) % 1000) * MILLION}
 #define TIMESPEC_INIT_US(us) {(us) / MILLION, ((us) % MILLION) * 1000}
 #define TIMESPEC_INIT_NS(ns) {(ns) / BILLION, ((ns) % BILLION)}
 
+#define TS_TO_UINT64(ts)                                                \
+        ((uint64_t)(ts).tv_sec * BILLION + (uint64_t)(ts).tv_nsec)
+
 #define TIMEVAL_INIT_S(s)   {(s), 0}
 #define TIMEVAL_INIT_MS(ms) {(ms) / 1000, ((ms) % 1000) * 1000}
 #define TIMEVAL_INIT_US(us) {(us) / MILLION, ((us) % MILLION)}
 
 /* functions for timespecs */
-#define ts_diff_ns(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * BILLION     \
+#define ts_diff_ns(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * BILLION     \
                             + ((tx)->tv_nsec - (t0)->tv_nsec))
-#define ts_diff_us(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
+#define ts_diff_us(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
                             + ((tx)->tv_nsec - (t0)->tv_nsec) / 1000L)
-#define ts_diff_ms(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
+#define ts_diff_ms(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
                             + ((tx)->tv_nsec - (t0)->tv_nsec) / MILLION)
 
 /* functions for timevals are the same */
-#define tv_diff_us(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
+#define tv_diff_us(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
+                            + ((tx)->tv_usec - (t0)->tv_usec))
+#define tv_diff_ms(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
                             + ((tx)->tv_usec - (t0)->tv_usec) / 1000L)
-#define tv_diff_ms(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
-                            + ((tx)->tv_usec - (t0)->tv_usec) / MILLION)
 
 /* functions for timespecs */
 
diff --git a/include/ouroboros/tpm.h b/include/ouroboros/tpm.h
index 7188dc91..3fb49b88 100644
--- a/include/ouroboros/tpm.h
+++ b/include/ouroboros/tpm.h
@@ -40,6 +40,8 @@ void         tpm_stop(struct tpm * tpm);
 
 void         tpm_begin_work(struct tpm * tpm);
 
+void         tpm_wait_work(struct tpm * tpm);
+
 void         tpm_end_work(struct tpm * tpm);
 
 #endif /* OUROBOROS_LIB_TPM_H */
diff --git a/include/ouroboros/utils.h b/include/ouroboros/utils.h
index 7f625c90..9c48b039 100644
--- a/include/ouroboros/utils.h
+++ b/include/ouroboros/utils.h
@@ -33,6 +33,7 @@
 #define ABS(a)   ((a) > 0 ? (a) : -(a))
 #define clrbuf(buf) do { memset(&(buf), 0, sizeof(buf)); } while (0);
 #define freebuf(buf) do { free((buf).data); clrbuf(buf); } while (0);
+#define BUF_INIT { NULL, 0 }
 
 typedef struct {
         uint8_t * data;
diff --git a/src/ipcpd/eth/CMakeLists.txt b/src/ipcpd/eth/CMakeLists.txt
index d57e1848..17ae74fc 100644
--- a/src/ipcpd/eth/CMakeLists.txt
+++ b/src/ipcpd/eth/CMakeLists.txt
@@ -85,8 +85,8 @@ if (HAVE_ETH)
     "Bypass the Qdisc in the kernel when using raw sockets")
   set(IPCP_ETH_LO_MTU 1500 CACHE STRING
     "Restrict Ethernet MTU over loopback interfaces")
-  set(IPCP_ETH_MPL 5 CACHE STRING
-    "Default maximum packet lifetime for the Ethernet IPCPs, in seconds")
+  set(IPCP_ETH_MPL 100 CACHE STRING
+    "Default maximum packet lifetime for the Ethernet IPCPs, in ms")
 
   set(ETH_LLC_SOURCES
     # Add source files here
diff --git a/src/ipcpd/local/CMakeLists.txt b/src/ipcpd/local/CMakeLists.txt
index 10fd0120..08abff57 100644
--- a/src/ipcpd/local/CMakeLists.txt
+++ b/src/ipcpd/local/CMakeLists.txt
@@ -13,8 +13,8 @@ include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_BINARY_DIR}/include)
 
 set(IPCP_LOCAL_TARGET ipcpd-local CACHE INTERNAL "")
-set(IPCP_LOCAL_MPL 2 CACHE STRING
-    "Default maximum packet lifetime for the Ethernet IPCPs, in seconds")
+set(IPCP_LOCAL_MPL 100 CACHE STRING
+    "Default maximum packet lifetime for the Ethernet IPCPs, in ms")
 
 set(LOCAL_SOURCES
   # Add source files here
diff --git a/src/ipcpd/udp/CMakeLists.txt b/src/ipcpd/udp/CMakeLists.txt
index 8ae5518e..5abf5a00 100644
--- a/src/ipcpd/udp/CMakeLists.txt
+++ b/src/ipcpd/udp/CMakeLists.txt
@@ -58,8 +58,8 @@ set(IPCP_UDP_RD_THR 3 CACHE STRING
   "Number of reader threads in UDP IPCP")
 set(IPCP_UDP_WR_THR 3 CACHE STRING
   "Number of writer threads in UDP IPCP")
-set(IPCP_UDP_MPL 60 CACHE STRING
-    "Default maximum packet lifetime for the UDP IPCP, in seconds")
+set(IPCP_UDP_MPL 5000 CACHE STRING
+    "Default maximum packet lifetime for the UDP IPCP, in ms")
 
 include(AddCompileFlags)
 if (CMAKE_BUILD_TYPE MATCHES "Debug*")
diff --git a/src/ipcpd/unicast/CMakeLists.txt b/src/ipcpd/unicast/CMakeLists.txt
index ca742871..b0dd3acc 100644
--- a/src/ipcpd/unicast/CMakeLists.txt
+++ b/src/ipcpd/unicast/CMakeLists.txt
@@ -13,8 +13,8 @@ include_directories(${CMAKE_SOURCE_DIR}/include)
 include_directories(${CMAKE_BINARY_DIR}/include)
 
 set(IPCP_UNICAST_TARGET ipcpd-unicast CACHE INTERNAL "")
-set(IPCP_UNICAST_MPL 60 CACHE STRING
-    "Default maximum packet lifetime for the unicast IPCP, in seconds")
+set(IPCP_UNICAST_MPL 10000 CACHE STRING
+    "Default maximum packet lifetime for the unicast IPCP, in ms")
 
 protobuf_generate_c(DHT_PROTO_SRCS DHT_PROTO_HDRS dir/dht.proto)
 
diff --git a/src/ipcpd/unicast/dir/dht.c b/src/ipcpd/unicast/dir/dht.c
index 95c5f19a..483570e8 100644
--- a/src/ipcpd/unicast/dir/dht.c
+++ b/src/ipcpd/unicast/dir/dht.c
@@ -439,6 +439,12 @@ static void cancel_req_destroy(void * o)
 
 static void kad_req_destroy(struct kad_req * req)
 {
+        struct timespec t;
+        struct timespec intv = TIMESPEC_INIT_S(20);
+
+        clock_gettime(PTHREAD_COND_CLOCK, &t);
+        ts_add(&t, &intv, &t);
+
         assert(req);
 
         pthread_mutex_lock(&req->lock);
@@ -464,7 +470,7 @@ static void kad_req_destroy(struct kad_req * req)
         pthread_cleanup_push(cancel_req_destroy, req);
 
         while (req->state != REQ_NULL && req->state != REQ_DONE)
-                pthread_cond_wait(&req->cond, &req->lock);
+                pthread_cond_timedwait(&req->cond, &req->lock, &t);
 
         pthread_cleanup_pop(true);
 }
@@ -497,7 +503,7 @@ static int kad_req_wait(struct kad_req * req,
         case REQ_DESTROY:
                 ret = -1;
                 req->state = REQ_NULL;
-                pthread_cond_signal(&req->cond);
+                pthread_cond_broadcast(&req->cond);
                 break;
         case REQ_PENDING: /* ETIMEDOUT */
         case REQ_RESPONSE:
@@ -518,7 +524,7 @@ static void kad_req_respond(struct kad_req * req)
         pthread_mutex_lock(&req->lock);
 
         req->state = REQ_RESPONSE;
-        pthread_cond_signal(&req->cond);
+        pthread_cond_broadcast(&req->cond);
 
         pthread_mutex_unlock(&req->lock);
 }
@@ -886,10 +892,9 @@ static void lookup_update(struct dht *    dht,
                         struct contact * d;
                         d = list_last_entry(&lu->contacts,
                                             struct contact, next);
+                        list_add_tail(&c->next, p);
                         list_del(&d->next);
-                        assert(lu->contacts.prv != &d->next);
                         contact_destroy(d);
-                        list_add_tail(&c->next, p);
                         mod = true;
                 }
         }
diff --git a/src/irmd/CMakeLists.txt b/src/irmd/CMakeLists.txt
index c7e4cbd9..fce89bef 100644
--- a/src/irmd/CMakeLists.txt
+++ b/src/irmd/CMakeLists.txt
@@ -71,6 +71,7 @@ set(SOURCE_FILES
   ipcp.c
   configfile.c
   main.c
+  oap.c
   reg/flow.c
   reg/ipcp.c
   reg/proc.c
@@ -95,6 +96,5 @@ endif ()
 
 install(TARGETS irmd RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR})
 
-# Enable once irmd has tests
-#add_subdirectory(tests)
 add_subdirectory(reg)
+add_subdirectory(tests)
diff --git a/src/irmd/config.h.in b/src/irmd/config.h.in
index fa1156b9..aa37b0ac 100644
--- a/src/irmd/config.h.in
+++ b/src/irmd/config.h.in
@@ -66,6 +66,7 @@
 
 #cmakedefine IRMD_KILL_ALL_PROCESSES
 #cmakedefine HAVE_LIBGCRYPT
+#cmakedefine HAVE_OPENSSL
 
 #define O7S_ASCII_ART                                                          \
 "\n"                                                                       \
diff --git a/src/irmd/ipcp.c b/src/irmd/ipcp.c
index 28f870a7..8a42be42 100644
--- a/src/irmd/ipcp.c
+++ b/src/irmd/ipcp.c
@@ -170,7 +170,7 @@ ipcp_msg_t * send_recv_ipcp_msg(pid_t        pid,
                 recv_msg = ipcp_msg__unpack(NULL, len, buf);
         else {
                 if (errno == EAGAIN && !dealloc) {
-                        int diff = ts_diff_ms(&tic, &toc);
+                        int diff = ts_diff_ms(&toc, &tic);
                         log_warn("IPCP %s timed out after %d ms.",
                                  str_ipcp_cmd(msg->code), diff);
                 }
diff --git a/src/irmd/main.c b/src/irmd/main.c
index cf079698..5cda9559 100644
--- a/src/irmd/main.c
+++ b/src/irmd/main.c
@@ -40,6 +40,7 @@
 #include <ouroboros/lockfile.h>
 #include <ouroboros/logs.h>
 #include <ouroboros/pthread.h>
+#include <ouroboros/random.h>
 #include <ouroboros/rib.h>
 #include <ouroboros/shm_rdrbuff.h>
 #include <ouroboros/sockets.h>
@@ -50,6 +51,7 @@
 
 #include "irmd.h"
 #include "ipcp.h"
+#include "oap.h"
 #include "reg/reg.h"
 #include "configfile.h"
 
@@ -760,66 +762,46 @@ static void __cleanup_flow(void * flow)
 }
 
 static int flow_accept(struct flow_info * flow,
+                       buffer_t *         symmkey,
                        buffer_t *         data,
                        struct timespec *  abstime)
 {
-        uint8_t   buf[MSGBUFSZ];
-        buffer_t  lpk; /* local public key           */
-        buffer_t  rpk; /* remote public key          */
-        void *    pkp; /* my public/private key pair */
-        ssize_t   key_len;
-        uint8_t * s;
-        int       err;
+        struct oap_hdr  oap_hdr;        /* incoming request           */
+        struct oap_hdr  r_oap_hdr;      /* outgoing response          */
+        uint8_t         buf[MSGBUFSZ];  /* buffer for local ephkey    */
+        buffer_t        lpk = BUF_INIT; /* local ephemeral pubkey     */
+        int             err;
+        struct timespec now;
 
         /* piggyback of user data not yet implemented */
         assert(data != NULL && data->len == 0 && data->data == NULL);
+        assert(symmkey != NULL && symmkey->len == 0 && symmkey->data == NULL);
 
         if (!reg_has_proc(flow->n_pid)) {
                 log_err("Unknown process %d calling accept.", flow->n_pid);
                 err = -EINVAL;
-                goto fail;
-        }
-
-        s = malloc(SYMMKEYSZ);
-        if (s == NULL) {
-                log_err("Failed to malloc symmkey.");
-                err = -ENOMEM;
-                goto fail;
-        }
-
-        key_len = crypt_dh_pkp_create(&pkp, buf);
-        if (key_len < 0) {
-                log_err("Failed to generate key pair.");
-                err = -ECRYPT;
-                goto fail_pkp;
+                goto fail_flow;
         }
 
-        lpk.data = buf;
-        lpk.len  = (size_t) key_len;
-
-        log_dbg("Generated ephemeral keys for %d.", flow->n_pid);
-
         if (reg_create_flow(flow) < 0) {
                 log_err("Failed to create flow.");
                 err = -EBADF;
                 goto fail_flow;
         }
 
-        if (reg_prepare_flow_accept(flow, &lpk) < 0) {
+        if (reg_prepare_flow_accept(flow) < 0) {
                 log_err("Failed to prepare accept.");
                 err = -EBADF;
                 goto fail_wait;
         }
 
+        log_dbg("Waiting for flow accept %d.", flow->id);
+
         pthread_cleanup_push(__cleanup_flow, flow);
-        pthread_cleanup_push(__cleanup_pkp, pkp);
-        pthread_cleanup_push(free, s);
 
-        err = reg_wait_flow_accepted(flow, &rpk, abstime);
+        err = reg_wait_flow_accepted(flow, &oap_hdr.hdr, abstime);
 
         pthread_cleanup_pop(false);
-        pthread_cleanup_pop(false);
-        pthread_cleanup_pop(false);
 
         if (err == -ETIMEDOUT) {
                 log_err("Flow accept timed out.");
@@ -834,45 +816,96 @@ static int flow_accept(struct flow_info * flow,
 
         assert(err == 0);
 
-        if (flow->qs.cypher_s != 0) { /* crypto requested */
-                if (crypt_dh_derive(pkp, rpk, s) < 0) {
+        if (oap_hdr_decode(oap_hdr.hdr, &oap_hdr) < 0) {
+                log_err("Failed to decode OAP header.");
+                err = -EIPCP;
+                goto fail_oap_hdr;
+        }
+
+        clock_gettime(CLOCK_REALTIME, &now);
+
+        if (now.tv_sec - (time_t) (oap_hdr.timestamp / MILLION) > flow->mpl)
+                log_warn("Flow alloc time exceeds MPL by %zu ms.",
+                        now.tv_sec - oap_hdr.timestamp / MILLION);
+
+        if (flow->qs.cypher_s != 0) {     /* crypto requested           */
+                uint8_t * s;              /* symmetric encryption key   */
+                ssize_t   key_len;        /* length of local pubkey     */
+                void *    pkp = NULL;     /* ephemeral private key pair */
+
+                s = malloc(SYMMKEYSZ);
+                if (s == NULL) {
+                        log_err("Failed to malloc symmkey.");
+                        err = -ENOMEM;
+                        goto fail_keys;
+                }
+
+                key_len = crypt_dh_pkp_create(&pkp, buf);
+                if (key_len < 0) {
+                        free(s);
+                        log_err("Failed to generate key pair.");
+                        err = -ECRYPT;
+                        goto fail_keys;
+                }
+
+                lpk.data = buf;
+                lpk.len  = (size_t) key_len;
+
+                log_dbg("Generated ephemeral keys for %d.", flow->n_pid);
+
+                if (crypt_dh_derive(pkp, oap_hdr.eph, s) < 0) {
                         log_err("Failed to derive secret for %d.", flow->id);
+                        crypt_dh_pkp_destroy(pkp);
+                        free(s);
                         err = -ECRYPT;
                         goto fail_derive;
                 }
-                freebuf(rpk);
-                data->data = s;
-                data->len  = SYMMKEYSZ;
-                s= NULL;
-        } else {
-                clrbuf(lpk);
+
+                symmkey->data = s;
+                symmkey->len  = SYMMKEYSZ;
+
+                crypt_dh_pkp_destroy(pkp);
+        }
+
+        if (oap_hdr_init(oap_hdr.id, NULL, NULL, lpk, *data, &r_oap_hdr) < 0) {
+                log_err("Failed to create OAP header.");
+                err = -ENOMEM;
+                goto fail_r_oap_hdr;
         }
 
-        if (ipcp_flow_alloc_resp(flow, 0, lpk) < 0) {
+        if (ipcp_flow_alloc_resp(flow, 0, r_oap_hdr.hdr) < 0) {
                 log_err("Failed to respond to flow allocation.");
-                err = -EIPCP;
-                goto fail_alloc_resp;
+                goto fail_resp;
         }
 
-        crypt_dh_pkp_destroy(pkp);
-        free(s);
+        oap_hdr_fini(&oap_hdr);
+        oap_hdr_fini(&r_oap_hdr);
 
         return 0;
 
+ fail_r_oap_hdr:
+        freebuf(*symmkey);
  fail_derive:
-        freebuf(rpk);
         clrbuf(lpk);
+ fail_keys:
+        oap_hdr_fini(&oap_hdr);
+ fail_oap_hdr:
+        assert(lpk.data == NULL && lpk.len == 0);
         ipcp_flow_alloc_resp(flow, err, lpk);
- fail_alloc_resp:
-        flow->state = FLOW_NULL;
  fail_wait:
         reg_destroy_flow(flow->id);
  fail_flow:
-        crypt_dh_pkp_destroy(pkp);
- fail_pkp:
-        free(s);
- fail:
         return err;
+
+ fail_resp:
+        flow->state = FLOW_NULL;
+        oap_hdr_fini(&r_oap_hdr);
+        freebuf(*symmkey);
+        clrbuf(lpk);
+        oap_hdr_fini(&oap_hdr);
+        assert(lpk.data == NULL && lpk.len == 0);
+        reg_destroy_flow(flow->id);
+        return -EIPCP;
 }
 
 static int flow_join(struct flow_info * flow,
@@ -910,6 +943,7 @@ static int flow_join(struct flow_info * flow,
 
         reg_prepare_flow_alloc(flow);
 
+
         if (ipcp_flow_join(flow, hash)) {
                 log_err("Flow join with layer %s failed.", dst);
                 err = -ENOTALLOC;
@@ -935,6 +969,7 @@ static int flow_join(struct flow_info * flow,
                 goto fail_alloc;
         }
 
+        assert(pbuf.data == NULL && pbuf.len == 0);
         assert(err == 0);
 
         freebuf(hash);
@@ -1008,20 +1043,33 @@ static int get_ipcp_by_dst(const char *     dst,
 
 static int flow_alloc(struct flow_info * flow,
                       const char *       dst,
+                      buffer_t *         symmkey,
                       buffer_t *         data,
                       struct timespec *  abstime)
 {
-        uint8_t   buf[MSGBUFSZ];
-        buffer_t  lpk ={NULL, 0}; /* local public key           */
-        buffer_t  rpk;            /* remote public key          */
-        void *    pkp = NULL;     /* my public/private key pair */
-        uint8_t * s = NULL;
-        buffer_t  hash;
-        int       err;
+        struct oap_hdr oap_hdr;        /* outgoing request           */
+        struct oap_hdr r_oap_hdr;      /* incoming response          */
+        uint8_t        buf[MSGBUFSZ];  /* buffer for local ephkey    */
+        buffer_t       lpk = BUF_INIT; /* local ephemeral pubkey     */
+        void *         pkp = NULL;     /* ephemeral private key pair */
+        uint8_t *      s = NULL;       /* symmetric key              */
+        buffer_t       hash;
+        uint8_t        idbuf[OAP_ID_SIZE];
+        buffer_t       id;
+        int            err;
+
         /* piggyback of user data not yet implemented */
         assert(data != NULL && data->len == 0 && data->data == NULL);
+        assert(symmkey != NULL && symmkey->len == 0 && symmkey->data == NULL);
 
-        log_info("Allocating flow for %d to %s.", flow->n_pid, dst);
+        if (random_buffer(idbuf, OAP_ID_SIZE) < 0) {
+                log_err("Failed to generate ID.");
+                err = -EIRMD;
+                goto fail_id;
+        }
+
+        id.data = idbuf;
+        id.len  = OAP_ID_SIZE;
 
         if (flow->qs.cypher_s > 0) {
                 ssize_t key_len;
@@ -1046,6 +1094,14 @@ static int flow_alloc(struct flow_info * flow,
                 log_dbg("Generated ephemeral keys for %d.", flow->n_pid);
         }
 
+        if (oap_hdr_init(id, NULL, NULL, lpk, *data, &oap_hdr) < 0) {
+                log_err("Failed to create OAP header.");
+                err = -ENOMEM;
+                goto fail_oap_hdr;
+        }
+
+        log_info("Allocating flow for %d to %s.", flow->n_pid, dst);
+
         if (reg_create_flow(flow) < 0) {
                 log_err("Failed to create flow.");
                 err = -EBADF;
@@ -1060,7 +1116,7 @@ static int flow_alloc(struct flow_info * flow,
 
         reg_prepare_flow_alloc(flow);
 
-        if (ipcp_flow_alloc(flow, hash, lpk)) {
+        if (ipcp_flow_alloc(flow, hash, oap_hdr.hdr)) {
                 log_err("Flow allocation %d failed.", flow->id);
                 err = -ENOTALLOC;
                 goto fail_alloc;
@@ -1071,7 +1127,7 @@ static int flow_alloc(struct flow_info * flow,
         pthread_cleanup_push(free, hash.data);
         pthread_cleanup_push(free, s);
 
-        err = reg_wait_flow_allocated(flow, &rpk, abstime);
+        err = reg_wait_flow_allocated(flow, &r_oap_hdr.hdr, abstime);
 
         pthread_cleanup_pop(false);
         pthread_cleanup_pop(false);
@@ -1091,37 +1147,57 @@ static int flow_alloc(struct flow_info * flow,
 
         assert(err == 0);
 
+        if (oap_hdr_decode(r_oap_hdr.hdr, &r_oap_hdr) < 0) {
+                log_err("Failed to decode OAP header.");
+                err = -EIPCP;
+                goto fail_r_oap_hdr;
+        }
+
+        if (memcmp(r_oap_hdr.id.data, oap_hdr.id.data, r_oap_hdr.id.len) != 0) {
+                log_err("OAP ID mismatch in flow allocation.");
+                err = -EIPCP;
+                goto fail_r_oap_hdr;
+        }
+
         if (flow->qs.cypher_s != 0) { /* crypto requested */
-                if (crypt_dh_derive(pkp, rpk, s) < 0) {
+                if (crypt_dh_derive(pkp, r_oap_hdr.eph, s) < 0) {
                         log_err("Failed to derive secret for %d.", flow->id);
                         err = -ECRYPT;
-                        goto fail_derive;
+                        goto fail_r_oap_hdr;
                 }
                 crypt_dh_pkp_destroy(pkp);
-                freebuf(rpk);
-                data->data = s;
-                data->len  = SYMMKEYSZ;
+
+                symmkey->data = s;
+                symmkey->len  = SYMMKEYSZ;
                 s = NULL;
         }
 
+        oap_hdr_fini(&r_oap_hdr);
+        oap_hdr_fini(&oap_hdr);
+
+        /* TODO: piggyback user data if needed */
+
         freebuf(hash);
         free(s);
 
         return 0;
 
- fail_derive:
-        freebuf(rpk);
+ fail_r_oap_hdr:
         flow->state = FLOW_DEALLOCATED;
+        oap_hdr_fini(&r_oap_hdr);
  fail_alloc:
         freebuf(hash);
  fail_ipcp:
         reg_destroy_flow(flow->id);
  fail_flow:
-        if (flow->qs.cypher_s > 0)
-               crypt_dh_pkp_destroy(pkp);
+        oap_hdr_fini(&oap_hdr);
+ fail_oap_hdr:
+        crypt_dh_pkp_destroy(pkp);
  fail_pkp:
         free(s);
  fail_malloc:
+        clrbuf(id);
+ fail_id:
         return err;
 }
 
@@ -1325,6 +1401,7 @@ static irm_msg_t * do_command_msg(irm_msg_t * msg)
         int                res;
         irm_msg_t *        ret_msg;
         buffer_t           data;
+        buffer_t           symmkey = BUF_INIT;;
 
         memset(&flow, 0, sizeof(flow));
 
@@ -1421,17 +1498,21 @@ static irm_msg_t * do_command_msg(irm_msg_t * msg)
                 res = name_unreg(msg->name, msg->pid);
                 break;
         case IRM_MSG_CODE__IRM_FLOW_ACCEPT:
+                tpm_wait_work(irmd.tpm);
                 data.len  = msg->pk.len;
                 data.data = msg->pk.data;
                 msg->has_pk = false;
                 assert(data.len > 0 ? data.data != NULL : data.data == NULL);
                 flow = flow_info_msg_to_s(msg->flow_info);
-                res = flow_accept(&flow, &data, abstime);
+                res = flow_accept(&flow, &symmkey, &data, abstime);
                 if (res == 0) {
                         ret_msg->flow_info    = flow_info_s_to_msg(&flow);
-                        ret_msg->has_symmkey  = data.len != 0;
-                        ret_msg->symmkey.data = data.data;
-                        ret_msg->symmkey.len  = data.len;
+                        ret_msg->has_symmkey  = symmkey.len != 0;
+                        ret_msg->symmkey.data = symmkey.data;
+                        ret_msg->symmkey.len  = symmkey.len;
+                        ret_msg->has_pk       = data.len != 0;
+                        ret_msg->pk.data      = data.data;
+                        ret_msg->pk.len       = data.len;
                 }
                 break;
         case IRM_MSG_CODE__IRM_FLOW_ALLOC:
@@ -1441,12 +1522,15 @@ static irm_msg_t * do_command_msg(irm_msg_t * msg)
                 assert(data.len > 0 ? data.data != NULL : data.data == NULL);
                 flow = flow_info_msg_to_s(msg->flow_info);
                 abstime = abstime == NULL ? &max : abstime;
-                res = flow_alloc(&flow, msg->dst, &data, abstime);
+                res = flow_alloc(&flow, msg->dst, &symmkey, &data, abstime);
                 if (res == 0) {
                         ret_msg->flow_info    = flow_info_s_to_msg(&flow);
-                        ret_msg->has_symmkey  = data.len != 0;
-                        ret_msg->symmkey.data = data.data;
-                        ret_msg->symmkey.len  = data.len;
+                        ret_msg->has_symmkey  = symmkey.len != 0;
+                        ret_msg->symmkey.data = symmkey.data;
+                        ret_msg->symmkey.len  = symmkey.len;
+                        ret_msg->has_pk       = data.len != 0;
+                        ret_msg->pk.data      = data.data;
+                        ret_msg->pk.len       = data.len;
                 }
                 break;
         case IRM_MSG_CODE__IRM_FLOW_JOIN:
diff --git a/src/irmd/oap.c b/src/irmd/oap.c
new file mode 100644
index 00000000..d5e5b7cc
--- /dev/null
+++ b/src/irmd/oap.c
@@ -0,0 +1,220 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2024
+ *
+ * Ouroboros flow allocation protocol header
+ *
+ *    Dimitri Staessens <dimitri@ouroboros.rocks>
+ *    Sander Vrijders   <sander@ouroboros.rocks>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#if defined(__linux__) || defined(__CYGWIN__)
+ #define _DEFAULT_SOURCE
+#else
+ #define _POSIX_C_SOURCE 200809L
+#endif
+
+#include <ouroboros/crypt.h>
+#include <ouroboros/endian.h>
+#include <ouroboros/time.h>
+
+#include "oap.h"
+
+#include <assert.h>
+
+int oap_hdr_init(buffer_t         id,
+                 void *           pkp,
+                 void *           pubcrt,
+                 buffer_t         ephkey,
+                 buffer_t         data,
+                 struct oap_hdr * oap_hdr)
+{
+        struct timespec now;
+        uint64_t        stamp;
+        buffer_t        hdr;
+        buffer_t        der = BUF_INIT;
+        buffer_t        sig = BUF_INIT;
+        buffer_t        sign;
+        uint16_t        len;
+        off_t           offset;
+
+        assert(id.data != NULL && id.len == OAP_ID_SIZE);
+        assert(oap_hdr != NULL);
+        memset(oap_hdr, 0, sizeof(*oap_hdr));
+
+        clock_gettime(CLOCK_REALTIME, &now);
+        stamp = hton64(TS_TO_UINT64(now));
+
+        if (pubcrt != NULL && crypt_crt_der(pubcrt, &der) < 0)
+                goto fail_der;
+
+        hdr.len = id.len +
+                sizeof(stamp) +
+                sizeof(len) + der.len +
+                sizeof(len) + ephkey.len +
+                sizeof(len) + data.len +
+                sizeof(len); /* sig len */
+
+        hdr.data = malloc(hdr.len);
+        if (hdr.data == NULL)
+                goto fail_hdr;
+
+        offset = 0;
+
+        memcpy(hdr.data, id.data, id.len);
+        offset += id.len;
+
+        memcpy(hdr.data + offset, &stamp, sizeof(stamp));
+        offset += sizeof(stamp);
+
+        /* pubcrt */
+        len = hton16((uint16_t) der.len);
+        memcpy(hdr.data + offset, &len, sizeof(len));
+        offset += sizeof(len);
+        if (der.len != 0)
+                memcpy(hdr.data + offset, der.data, der.len);
+        offset += der.len;
+
+        /* ephkey */
+        len = hton16((uint16_t) ephkey.len);
+        memcpy(hdr.data + offset, &len, sizeof(len));
+        offset += sizeof(len);
+        if (ephkey.len != 0)
+                memcpy(hdr.data + offset, ephkey.data, ephkey.len);
+        offset += ephkey.len;
+
+        /* data */
+        len = hton16((uint16_t) data.len);
+        memcpy(hdr.data + offset, &len, sizeof(len));
+        offset += sizeof(len);
+        if (data.len != 0)
+                memcpy(hdr.data + offset, data.data, data.len);
+        offset += data.len;
+
+        sign.data = hdr.data;
+        sign.len  = hdr.len - sizeof(len);
+
+        if (pkp != NULL && auth_sign(pkp, sign, &sig) < 0)
+                goto fail_sig;
+
+        len = hton16((uint16_t) sig.len);
+        memcpy(hdr.data + offset, &len, sizeof(len));
+        offset += sizeof(len);
+
+        oap_hdr->hdr = hdr;
+
+        assert((size_t) offset == hdr.len);
+
+        if (sig.len > 0) {
+                oap_hdr->hdr.len += sig.len;
+                oap_hdr->hdr.data = realloc(hdr.data, oap_hdr->hdr.len);
+                if (oap_hdr->hdr.data == NULL)
+                        goto fail_oap_hdr;
+
+                memcpy(oap_hdr->hdr.data + offset, sig.data, sig.len);
+                clrbuf(hdr);
+        }
+
+        if (oap_hdr_decode(oap_hdr->hdr, oap_hdr) < 0)
+                goto fail_decode;
+
+        freebuf(der);
+        freebuf(sig);
+
+        return 0;
+
+ fail_decode:
+        oap_hdr_fini(oap_hdr);
+ fail_oap_hdr:
+        freebuf(sig);
+ fail_sig:
+        freebuf(hdr);
+ fail_hdr:
+        freebuf(der);
+ fail_der:
+        memset(oap_hdr, 0, sizeof(*oap_hdr));
+        return -1;
+}
+
+void oap_hdr_fini(struct oap_hdr * oap_hdr)
+{
+        assert(oap_hdr != NULL);
+
+        freebuf(oap_hdr->hdr);
+        memset(oap_hdr, 0, sizeof(*oap_hdr));
+}
+
+int oap_hdr_decode(buffer_t             hdr,
+                   struct oap_hdr * oap_hdr)
+{
+        off_t offset;
+
+        assert(oap_hdr != NULL);
+        memset(oap_hdr, 0, sizeof(*oap_hdr));
+
+        if (hdr.len < OAP_HDR_MIN_SIZE)
+                goto fail_decode;
+
+        oap_hdr->id.data = hdr.data;
+        oap_hdr->id.len  = OAP_ID_SIZE;
+
+        offset = OAP_ID_SIZE;
+
+        oap_hdr->timestamp = ntoh64(*(uint64_t *)(hdr.data + offset));
+
+        offset += sizeof(uint64_t);
+
+        oap_hdr->crt.len = (size_t) ntoh16(*(uint16_t *)(hdr.data + offset));
+        oap_hdr->crt.data = hdr.data + offset + sizeof(uint16_t);
+
+        offset += sizeof(uint16_t) + oap_hdr->crt.len;
+
+        if ((size_t) offset + sizeof(uint16_t) >= hdr.len)
+                goto fail_decode;
+
+        oap_hdr->eph.len = (size_t) ntoh16(*(uint16_t *)(hdr.data + offset));
+        oap_hdr->eph.data = hdr.data + offset + sizeof(uint16_t);
+
+        offset += sizeof(uint16_t) + oap_hdr->eph.len;
+
+        if ((size_t) offset + sizeof(uint16_t) >= hdr.len)
+                goto fail_decode;
+
+        oap_hdr->data.len = (size_t) ntoh16(*(uint16_t *)(hdr.data + offset));
+        oap_hdr->data.data = hdr.data + offset + sizeof(uint16_t);
+
+        offset += sizeof(uint16_t) + oap_hdr->data.len;
+
+        if ((size_t) offset + sizeof(uint16_t) > hdr.len)
+                goto fail_decode;
+
+        oap_hdr->sig.len = (size_t) ntoh16(*(uint16_t *)(hdr.data + offset));
+        oap_hdr->sig.data = hdr.data + offset + sizeof(uint16_t);
+
+        offset += sizeof(uint16_t) + oap_hdr->sig.len;
+
+        if ((size_t) offset != hdr.len)
+                goto fail_decode;
+
+        oap_hdr->hdr = hdr;
+
+        return 0;
+
+ fail_decode:
+        memset(oap_hdr, 0, sizeof(*oap_hdr));
+        return -1;
+}
+
+
diff --git a/src/irmd/oap.h b/src/irmd/oap.h
new file mode 100644
index 00000000..460a89de
--- /dev/null
+++ b/src/irmd/oap.h
@@ -0,0 +1,88 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2024
+ *
+ * Ouroboros flow allocation protocol header
+ *
+ *    Dimitri Staessens <dimitri@ouroboros.rocks>
+ *    Sander Vrijders   <sander@ouroboros.rocks>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#ifndef OUROBOROS_IRMD_OAP_H
+#define OUROBOROS_IRMD_OAP_H
+
+#include <ouroboros/utils.h>
+
+#define OAP_ID_SIZE      (16)
+#define OAP_HDR_MIN_SIZE (OAP_ID_SIZE + sizeof(uint64_t) + 4 * sizeof(uint16_t))
+
+
+/*
+ *  0                   1                   2                   3
+ *  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ * +---------------------------------------------------------------+
+ * |                                                               |
+ * |                        id (128 bits)                          |
+ * |                                                               |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ * |                      timestamp (64 bits)                      |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ * |      crt_len  (16 bits)     |                                 |
+ * +-----------+-----------------+                                 |
+ * |                        certificate                            |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ * |      eph_len  (16 bits)     |                                 |
+ * +-----------+-----------------+                                 |
+ * |                   public key for ECDHE                        |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ * |     data_len (16 bits)      |                                 |
+ * +-----------+-----------------+                                 |
+ * |               piggy backed application data                   |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ * |     sig_len  (16 bits)      |                                 |
+ * +-----------+-----------------+                                 |
+ * |                         signature                             |
+ * |                                                               |
+ * +---------------------------------------------------------------+
+ */
+
+struct oap_hdr {
+        uint64_t timestamp;
+        buffer_t id;
+        buffer_t crt;
+        buffer_t eph;
+        buffer_t data;
+        buffer_t sig;
+        buffer_t hdr;
+};
+
+int  oap_hdr_init(buffer_t         id,
+                  void *           pkp,
+                  void *           pubcrt,
+                  buffer_t         ephkey,
+                  buffer_t         data,
+                  struct oap_hdr * oap_hdr);
+
+void oap_hdr_fini(struct oap_hdr * oap_hdr);
+
+int  oap_hdr_decode(buffer_t         hdr,
+                    struct oap_hdr * oap_hdr);
+
+#endif /* OUROBOROS_IRMD_OAP_H */
diff --git a/src/irmd/reg/reg.c b/src/irmd/reg/reg.c
index d95a4722..339e7fa0 100644
--- a/src/irmd/reg/reg.c
+++ b/src/irmd/reg/reg.c
@@ -1771,8 +1771,7 @@ int reg_respond_alloc(struct flow_info * info,
         return -1;
 }
 
-int reg_prepare_flow_accept(struct flow_info * info,
-                            buffer_t *         pbuf)
+int reg_prepare_flow_accept(struct flow_info * info)
 {
         struct reg_flow * flow;
         int               ret;
@@ -1790,8 +1789,6 @@ int reg_prepare_flow_accept(struct flow_info * info,
 
         ret = reg_flow_update(flow, info);
 
-        reg_flow_set_data(flow, pbuf);
-
         pthread_mutex_unlock(&reg.mtx);
 
         return ret;
@@ -1915,7 +1912,6 @@ int reg_respond_accept(struct flow_info * info,
                        buffer_t *         pbuf)
 {
         struct reg_flow * flow;
-        buffer_t          temp;
 
         assert(info != NULL);
         assert(info->state == FLOW_ALLOCATED);
@@ -1933,11 +1929,8 @@ int reg_respond_accept(struct flow_info * info,
 
         info->n_pid = flow->info.n_pid;
 
-        if (info->qs.cypher_s > 0) {
-                reg_flow_get_data(flow, &temp);
-                reg_flow_set_data(flow, pbuf);
-                *pbuf = temp;
-        }
+        reg_flow_set_data(flow, pbuf);
+        clrbuf(pbuf);
 
         if (reg_flow_update(flow, info) < 0) {
                 log_err("Failed to create flow structs.");
diff --git a/src/irmd/reg/reg.h b/src/irmd/reg/reg.h
index 17dfcc32..f1899d65 100644
--- a/src/irmd/reg/reg.h
+++ b/src/irmd/reg/reg.h
@@ -119,8 +119,7 @@ int   reg_wait_flow_allocated(struct flow_info *      info,
 int   reg_respond_alloc(struct flow_info * info,
                         buffer_t *         pbuf);
 
-int   reg_prepare_flow_accept(struct flow_info * info,
-                              buffer_t *         pbuf);
+int   reg_prepare_flow_accept(struct flow_info * info);
 
 int   reg_wait_flow_accepted(struct flow_info *      info,
                              buffer_t *              pbuf,
diff --git a/src/irmd/reg/tests/reg_test.c b/src/irmd/reg/tests/reg_test.c
index 80fc64d6..b69cf476 100644
--- a/src/irmd/reg/tests/reg_test.c
+++ b/src/irmd/reg/tests/reg_test.c
@@ -115,7 +115,6 @@ static int test_reg_allocate_flow_timeout(void)
 {
         struct timespec abstime;
         struct timespec timeo = TIMESPEC_INIT_MS(1);
-        buffer_t        pbuf;
         buffer_t        rbuf = {NULL, 0};
 
         struct flow_info info = {
@@ -125,14 +124,6 @@ static int test_reg_allocate_flow_timeout(void)
 
         TEST_START();
 
-        pbuf.data = (uint8_t *) strdup(TEST_DATA);;
-        if (pbuf.data == NULL) {
-                printf("Failed to strdup data.\n");
-                goto fail;
-        }
-
-        pbuf.len  = strlen((char *) pbuf.data) + 1;
-
         clock_gettime(PTHREAD_COND_CLOCK, &abstime);
 
         ts_add(&abstime, &timeo, &abstime);
@@ -147,7 +138,7 @@ static int test_reg_allocate_flow_timeout(void)
                 goto fail;
         }
 
-        if (reg_prepare_flow_accept(&info, &pbuf) < 0) {
+        if (reg_prepare_flow_accept(&info) < 0) {
                 printf("Failed to prepare flow for accept.\n");
                 goto fail;
         }
@@ -162,12 +153,6 @@ static int test_reg_allocate_flow_timeout(void)
                 goto fail;
         }
 
-        if (pbuf.data == NULL) {
-                printf("Flow data was updated on timeout.");
-                goto fail;
-        }
-
-        freebuf(pbuf);
         reg_destroy_flow(info.id);
 
         if (reg.n_flows != 0) {
@@ -220,13 +205,6 @@ static void * test_flow_respond_accept(void * o)
 
         reg_respond_accept(info, &pbuf);
 
-        if (info->qs.cypher_s == 0) {
-                freebuf(pbuf);
-        } else if (strcmp((char *) pbuf.data, TEST_DATA) != 0) {
-                printf("Data was not passed correctly.\n");
-                goto fail;
-        }
-
         return (void *) 0;
  fail:
         return (void *) -1;
@@ -237,7 +215,6 @@ static int test_reg_accept_flow_success(void)
         pthread_t       thr;
         struct timespec abstime;
         struct timespec timeo = TIMESPEC_INIT_S(1);
-        buffer_t        pbuf = {(uint8_t *) TEST_DATA, strlen(TEST_DATA)};
         buffer_t        rbuf  = {NULL, 0};
 
         struct flow_info info = {
@@ -267,7 +244,7 @@ static int test_reg_accept_flow_success(void)
                 goto fail;
         }
 
-        if (reg_prepare_flow_accept(&info, &pbuf) < 0) {
+        if (reg_prepare_flow_accept(&info) < 0) {
                 printf("Failed to prepare flow for accept.\n");
                 goto fail;
         }
@@ -332,7 +309,6 @@ static int test_reg_accept_flow_success_no_crypt(void)
         pthread_t       thr;
         struct timespec abstime;
         struct timespec timeo = TIMESPEC_INIT_S(1);
-        buffer_t        pbuf = {(uint8_t *) TEST_DATA, strlen(TEST_DATA)};
         buffer_t        rbuf  = {NULL, 0};
 
         struct flow_info info = {
@@ -362,7 +338,7 @@ static int test_reg_accept_flow_success_no_crypt(void)
                 goto fail;
         }
 
-        if (reg_prepare_flow_accept(&info, &pbuf) < 0) {
+        if (reg_prepare_flow_accept(&info) < 0) {
                 printf("Failed to prepare flow for accept.\n");
                 goto fail;
         }
@@ -389,10 +365,7 @@ static int test_reg_accept_flow_success_no_crypt(void)
                 goto fail;
         }
 
-        if (strcmp((char *) rbuf.data, TEST_DATA) != 0) {
-                printf("Data was updated.\n");
-                goto fail;
-        }
+        freebuf(rbuf);
 
         n_1_info.state = FLOW_DEALLOCATED;
 
@@ -1177,7 +1150,7 @@ static void * test_call_flow_accept(void * o)
         clock_gettime(PTHREAD_COND_CLOCK, &abstime);
         ts_add(&abstime, &timeo, &abstime);
 
-        reg_prepare_flow_accept(&info, &pbuf);
+        reg_prepare_flow_accept(&info);
 
         if (reg_wait_flow_accepted(&info, &pbuf, &abstime) != -ETIMEDOUT) {
                 printf("Wait allocated did not timeout.\n");
diff --git a/src/irmd/tests/CMakeLists.txt b/src/irmd/tests/CMakeLists.txt
index 99743052..e860acce 100644
--- a/src/irmd/tests/CMakeLists.txt
+++ b/src/irmd/tests/CMakeLists.txt
@@ -3,6 +3,8 @@ get_filename_component(src_folder "${tmp}" NAME)
 
 create_test_sourcelist(${src_folder}_tests test_suite.c
   # Add new tests here
+  irm_test.c
+  oap_test.c
 )
 
 add_executable(${src_folder}_test EXCLUDE_FROM_ALL ${${src_folder}_tests})
@@ -21,3 +23,5 @@ foreach(test ${tests_to_run})
   get_filename_component(test_name ${test} NAME_WE)
   add_test(irmd/${test_name} ${C_TEST_PATH}/${src_folder}_test ${test_name})
 endforeach(test)
+
+set_property(TEST irmd/oap_test PROPERTY SKIP_RETURN_CODE 1)
diff --git a/src/irmd/tests/irm_test.c b/src/irmd/tests/irm_test.c
new file mode 100644
index 00000000..d440289c
--- /dev/null
+++ b/src/irmd/tests/irm_test.c
@@ -0,0 +1,33 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2024
+ *
+ * Unit tests of IRMd functions
+ *    Dimitri Staessens <dimitri@ouroboros.rocks>
+ *    Sander Vrijders   <sander@ouroboros.rocks>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+
+
+int irm_test(int argc,
+             char **argv)
+{
+        int ret = 0;
+
+        (void) argc;
+        (void) argv;
+
+        return ret;
+}
diff --git a/src/irmd/tests/oap_test.c b/src/irmd/tests/oap_test.c
new file mode 100644
index 00000000..e9f67505
--- /dev/null
+++ b/src/irmd/tests/oap_test.c
@@ -0,0 +1,273 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2024
+ *
+ * Unit tests of Ouroboros flow allocation protocol
+ *    Dimitri Staessens <dimitri@ouroboros.rocks>
+ *    Sander Vrijders   <sander@ouroboros.rocks>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#include "config.h"
+
+#include "oap.c"
+
+#include <ouroboros/random.h>
+#include <ouroboros/test.h>
+
+static const char * pkp_str = \
+"-----BEGIN EC PRIVATE KEY-----\n"
+"MHcCAQEEIC13y+5jdKe80HBJD7WITpQamcn3rrkTX1r0v+JwSk4NoAoGCCqGSM49\n"
+"AwEHoUQDQgAEcC0yLAfUtufH8cdLybrdWPc6U+xRuhDhqqrEcBO5+eob2xyqEaNk\n"
+"nIV/86724zPptGRahWz0rzW2PvNppJdNBg==\n"
+"-----END EC PRIVATE KEY-----\n";
+
+/* Valid signed server certificate for server-2.unittest.o7s */
+static const char * crt_str = \
+"-----BEGIN CERTIFICATE-----\n"
+"MIIDgjCCAyigAwIBAgICEAIwCgYIKoZIzj0EAwIwWzELMAkGA1UEBhMCQkUxDDAK\n"
+"BgNVBAgMA09WTDEMMAoGA1UECgwDbzdzMRUwEwYDVQQLDAx1bml0dGVzdC5vN3Mx\n"
+"GTAXBgNVBAMMEGltMi51bml0dGVzdC5vN3MwHhcNMjUwNzA0MTMxODI5WhcNMzUw\n"
+"NzAyMTMxODI5WjBwMQswCQYDVQQGEwJCRTEMMAoGA1UECAwDT1ZMMQ4wDAYDVQQH\n"
+"DAVHaGVudDEMMAoGA1UECgwDbzdzMRUwEwYDVQQLDAx1bml0dGVzdC5vN3MxHjAc\n"
+"BgNVBAMMFXNlcnZlci0yLnVuaXR0ZXN0Lm83czBZMBMGByqGSM49AgEGCCqGSM49\n"
+"AwEHA0IABHAtMiwH1Lbnx/HHS8m63Vj3OlPsUboQ4aqqxHATufnqG9scqhGjZJyF\n"
+"f/Ou9uMz6bRkWoVs9K81tj7zaaSXTQajggHFMIIBwTAJBgNVHRMEAjAAMBEGCWCG\n"
+"SAGG+EIBAQQEAwIGQDA6BglghkgBhvhCAQ0ELRYrR3JpbGxlZCBDaGVlc2UgR2Vu\n"
+"ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUTt3xHTwE9amoglxh\n"
+"cEMqWv+PpDMwgb8GA1UdIwSBtzCBtIAUFfeZRx8QWWKQr7Aw8zjDu2shvcShgZek\n"
+"gZQwgZExCzAJBgNVBAYTAkJFMQwwCgYDVQQIDANPVkwxDjAMBgNVBAcMBUdoZW50\n"
+"MQwwCgYDVQQKDANvN3MxFTATBgNVBAsMDHVuaXR0ZXN0Lm83czEZMBcGA1UEAwwQ\n"
+"Y2EyLnVuaXR0ZXN0Lm83czEkMCIGCSqGSIb3DQEJARYVZHVtbXlAb3Vyb2Jvcm9z\n"
+"LnJvY2tzggIQAjAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEw\n"
+"EQYDVR0fBAowCDAGoASgAoYAMCoGCCsGAQUFBwEBBB4wHDAMBggrBgEFBQcwAoYA\n"
+"MAwGCCsGAQUFBzABhgAwIAYDVR0RBBkwF4IVc2VydmVyLTEudW5pdHRlc3Qubzdz\n"
+"MAoGCCqGSM49BAMCA0gAMEUCIQDHuDb62w/Uah4nKwUFoJVkr4rgdNGh2Rn3SWaK\n"
+"0FV/gAIgOLKorTwSgrTFdyOUkuPOhRs8BEMpah+dp8UTO8AnLvY=\n"
+"-----END CERTIFICATE-----\n";
+
+static int test_oap_hdr_init_fini(void)
+{
+        struct oap_hdr  oap_hdr;
+        struct timespec now;
+        uint64_t        stamp;
+        buffer_t        ephkey = BUF_INIT;
+        buffer_t        data   = BUF_INIT;
+        uint8_t         buf[OAP_ID_SIZE];
+        buffer_t        id;
+        void *          pkp    = NULL;
+        void *          pubcrt = NULL;
+
+        TEST_START();
+
+        random_buffer(buf, OAP_ID_SIZE);
+        id.data = buf;
+        id.len  = OAP_ID_SIZE;
+
+        clock_gettime(CLOCK_REALTIME, &now);
+        stamp = TS_TO_UINT64(now);
+
+        if (oap_hdr_init(id, pkp, pubcrt, ephkey, data, &oap_hdr) < 0) {
+                printf("Failed to init OAP request header.\n");
+                goto fail_req_hdr;
+        }
+
+        if (oap_hdr.hdr.len != OAP_HDR_MIN_SIZE) {
+                printf("OAP request header wrong: %zu < %zu.\n",
+                       oap_hdr.hdr.len, OAP_HDR_MIN_SIZE);
+                goto fail_req_hdr_chk;
+        }
+
+        if (oap_hdr.id.len != OAP_ID_SIZE) {
+                printf("OAP request header ID wrong size: %zu != %zu.\n",
+                       oap_hdr.id.len, (size_t) OAP_ID_SIZE);
+                goto fail_req_hdr_chk;
+        }
+
+        if (memcmp(oap_hdr.id.data, id.data, OAP_ID_SIZE) != 0) {
+                printf("OAP request header ID mismatch.\n");
+                goto fail_req_hdr_chk;
+        }
+
+        if (oap_hdr.timestamp < stamp) {
+                printf("OAP request header timestamp is too old.\n");
+                goto fail_req_hdr_chk;
+        }
+
+        if (oap_hdr.timestamp > stamp + 1 * BILLION) {
+                printf("OAP request header timestamp is too new.\n");
+                goto fail_req_hdr_chk;
+        }
+
+        oap_hdr_fini(&oap_hdr);
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+
+ fail_req_hdr_chk:
+        oap_hdr_fini(&oap_hdr);
+ fail_req_hdr:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
+}
+
+static int test_oap_hdr_init_fini_data(void)
+
+{
+        struct oap_hdr oap_hdr;
+        buffer_t       data;
+        buffer_t       ephkey = BUF_INIT;
+        uint8_t         buf[OAP_ID_SIZE];
+        buffer_t        id;
+        void *         pkp    = NULL;
+        void *         pubcrt = NULL;
+
+        TEST_START();
+
+        random_buffer(buf, OAP_ID_SIZE);
+        id.data = buf;
+        id.len  = OAP_ID_SIZE;
+
+        data.len = 100;
+        data.data = malloc(data.len);
+        if (data.data == NULL) {
+                printf("Failed to allocate data buffer.\n");
+                goto fail_data;
+        }
+
+        random_buffer(data.data, data.len);
+
+        if (oap_hdr_init(id, pkp, pubcrt, ephkey, data, &oap_hdr) < 0) {
+                printf("Failed to create OAP request header.\n");
+                goto fail_req_hdr;
+        }
+
+        if (oap_hdr.hdr.len != OAP_HDR_MIN_SIZE + data.len) {
+                printf("OAP request header wrong: %zu < %zu.\n",
+                       oap_hdr.hdr.len, OAP_HDR_MIN_SIZE + data.len);
+                goto fail_req_hdr_sz;
+        }
+
+        oap_hdr_fini(&oap_hdr);
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+
+ fail_req_hdr_sz:
+        oap_hdr_fini(&oap_hdr);
+ fail_req_hdr:
+        freebuf(data);
+ fail_data:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
+}
+
+static int test_oap_hdr_init_fini_signed(void)
+{
+        struct oap_hdr oap_hdr;
+        buffer_t       ephkey = BUF_INIT;
+        buffer_t       data   = BUF_INIT;
+        buffer_t       sign;
+        buffer_t       id;
+        uint8_t        buf[OAP_ID_SIZE];
+        void *         pkp;
+        void *         pk;
+        void *         pubcrt;
+        void *         pubcrt2;
+
+        TEST_START();
+
+        random_buffer(buf, OAP_ID_SIZE);
+        id.data = buf;
+        id.len  = OAP_ID_SIZE;
+
+        if (crypt_load_privkey_str(pkp_str, &pkp) < 0) {
+                printf("Failed to load private key.\n");
+                goto fail_pkp;
+        }
+
+        if (crypt_load_crt_str(crt_str, &pubcrt) < 0) {
+                printf("Failed to load public certificate.\n");
+                goto fail_pubcrt;
+        }
+
+        if (oap_hdr_init(id, pkp, pubcrt, ephkey, data, &oap_hdr) < 0) {
+                printf("Failed to create OAP request header.\n");
+                goto fail_req_hdr;
+        }
+
+        if (crypt_load_crt_der(oap_hdr.crt, &pubcrt2) < 0) {
+                printf("Failed to load public certificate from DER.\n");
+                goto fail_crt_der;
+        }
+
+        if (crypt_get_pubkey_crt(pubcrt2, &pk) < 0) {
+                printf("Failed to get public key from certificate.\n");
+                goto fail_crt_pk;
+        }
+
+        sign = oap_hdr.hdr;
+        sign.len -= (oap_hdr.sig.len + sizeof(uint16_t));
+
+        if (auth_verify_sig(pk, sign, oap_hdr.sig) < 0) {
+                printf("Failed to verify OAP request header signature.\n");
+                goto fail_check_sig;
+        }
+
+        oap_hdr_fini(&oap_hdr);
+
+        crypt_free_crt(pubcrt2);
+        crypt_free_crt(pubcrt);
+        crypt_free_key(pk);
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+
+ fail_check_sig:
+        crypt_free_key(pk);
+ fail_crt_pk:
+        crypt_free_crt(pubcrt2);
+ fail_crt_der:
+        oap_hdr_fini(&oap_hdr);
+ fail_req_hdr:
+        crypt_free_crt(pubcrt);
+ fail_pubcrt:
+        crypt_free_key(pkp);
+ fail_pkp:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
+}
+
+int oap_test(int    argc,
+             char **argv)
+{
+        int ret = 0;
+
+        (void) argc;
+        (void) argv;
+
+        ret |= test_oap_hdr_init_fini();
+        ret |= test_oap_hdr_init_fini_data();
+#ifdef HAVE_OPENSSL
+        ret |= test_oap_hdr_init_fini_signed();
+#else
+        (void) test_oap_hdr_init_fini_signed;
+
+        ret = TEST_RC_SKIP;
+#endif
+        return ret;
+}
diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt
index cc73a9fc..04a8f089 100644
--- a/src/lib/CMakeLists.txt
+++ b/src/lib/CMakeLists.txt
@@ -115,7 +115,7 @@ if (OPENSSL_FOUND)
     set(DISABLE_OPENSSL FALSE CACHE BOOL "Disable OpenSSL support")
     if (NOT DISABLE_OPENSSL)
       message(STATUS "OpenSSL support enabled")
-      set(HAVE_OPENSSL TRUE)
+      set(HAVE_OPENSSL TRUE CACHE INTERNAL "")
     else()
       message(STATUS "OpenSSL support disabled")
       unset(HAVE_OPENSSL)
@@ -234,6 +234,10 @@ set(ACK_WHEEL_SLOTS 256 CACHE STRING
   "Number of slots in the acknowledgment wheel, must be a power of 2")
 set(ACK_WHEEL_RESOLUTION 18 CACHE STRING
   "Minimum acknowledgment delay (ns), as a power to 2")
+set(TPM_DEBUG_REPORT_INTERVAL 0 CACHE STRING
+  "Interval at wich the TPM will report long running threads (s), 0 disables")
+set(TPM_DEBUG_ABORT_TIMEOUT 0 CACHE STRING
+  "TPM abort process after a thread reaches this timeout (s), 0 disables")
 
 if (HAVE_FUSE)
   set(PROC_FLOW_STATS TRUE CACHE BOOL
diff --git a/src/lib/config.h.in b/src/lib/config.h.in
index 21a30493..3fcac7b8 100644
--- a/src/lib/config.h.in
+++ b/src/lib/config.h.in
@@ -29,21 +29,24 @@
 #define HAVE_ENCRYPTION
 #endif
 
-#define SYS_MAX_FLOWS       @SYS_MAX_FLOWS@
-
-#cmakedefine                SHM_RBUFF_LOCKLESS
-#cmakedefine                SHM_RDRB_MULTI_BLOCK
-#cmakedefine                QOS_DISABLE_CRC
-#cmakedefine                HAVE_OPENSSL_RNG
-
-#define SHM_RBUFF_PREFIX    "@SHM_RBUFF_PREFIX@"
-#define SHM_LOCKFILE_NAME   "@SHM_LOCKFILE_NAME@"
-#define SHM_FLOW_SET_PREFIX "@SHM_FLOW_SET_PREFIX@"
-#define SHM_RDRB_NAME       "@SHM_RDRB_NAME@"
-#define SHM_RDRB_BLOCK_SIZE @SHM_RDRB_BLOCK_SIZE@
-#define SHM_BUFFER_SIZE     @SHM_BUFFER_SIZE@
-#define SHM_RBUFF_SIZE      @SHM_RBUFF_SIZE@
-#define FLOW_ALLOC_TIMEOUT  @FLOW_ALLOC_TIMEOUT@
+#define SYS_MAX_FLOWS            @SYS_MAX_FLOWS@
+
+#cmakedefine                     SHM_RBUFF_LOCKLESS
+#cmakedefine                     SHM_RDRB_MULTI_BLOCK
+#cmakedefine                     QOS_DISABLE_CRC
+#cmakedefine                     HAVE_OPENSSL_RNG
+
+#define SHM_RBUFF_PREFIX         "@SHM_RBUFF_PREFIX@"
+#define SHM_LOCKFILE_NAME        "@SHM_LOCKFILE_NAME@"
+#define SHM_FLOW_SET_PREFIX      "@SHM_FLOW_SET_PREFIX@"
+#define SHM_RDRB_NAME            "@SHM_RDRB_NAME@"
+#define SHM_RDRB_BLOCK_SIZE       @SHM_RDRB_BLOCK_SIZE@
+#define SHM_BUFFER_SIZE           @SHM_BUFFER_SIZE@
+#define SHM_RBUFF_SIZE            @SHM_RBUFF_SIZE@
+#define FLOW_ALLOC_TIMEOUT        @FLOW_ALLOC_TIMEOUT@
+
+#define TPM_DEBUG_REPORT_INTERVAL @TPM_DEBUG_REPORT_INTERVAL@
+#define TPM_DEBUG_ABORT_TIMEOUT   @TPM_DEBUG_ABORT_TIMEOUT@
 
 #if defined(__linux__) || (defined(__MACH__) && !defined(__APPLE__))
 /* Avoid a bug in robust mutex implementation of glibc 2.25 */
diff --git a/src/lib/crypt.c b/src/lib/crypt.c
index 756fcccc..e8c4d5ab 100644
--- a/src/lib/crypt.c
+++ b/src/lib/crypt.c
@@ -60,10 +60,13 @@ int crypt_dh_pkp_create(void **   pkp,
 
 void crypt_dh_pkp_destroy(void * pkp)
 {
+        if (pkp == NULL)
+                return;
 #ifdef HAVE_OPENSSL
         openssl_ecdh_pkp_destroy(pkp);
 #else
         (void) pkp;
+
         return;
 #endif
 }
@@ -179,7 +182,7 @@ int crypt_load_privkey_file(const char * path,
 }
 
 int crypt_load_privkey_str(const char * str,
-                       void **      key)
+                           void **      key)
 {
         *key = NULL;
 
@@ -232,6 +235,8 @@ void crypt_free_key(void * key)
 int crypt_load_crt_file(const char * path,
                         void **      crt)
 {
+        assert(crt != NULL);
+
         *crt = NULL;
 
 #ifdef HAVE_OPENSSL
@@ -246,6 +251,8 @@ int crypt_load_crt_file(const char * path,
 int crypt_load_crt_str(const char * str,
                        void **      crt)
 {
+        assert(crt != NULL);
+
         *crt = NULL;
 
 #ifdef HAVE_OPENSSL
@@ -257,6 +264,21 @@ int crypt_load_crt_str(const char * str,
 #endif
 }
 
+int crypt_load_crt_der(const buffer_t buf,
+                       void **        crt)
+{
+        assert(crt != NULL);
+#ifdef HAVE_OPENSSL
+        return openssl_load_crt_der(buf, crt);
+#else
+        *crt = NULL;
+
+        (void) buf;
+
+        return 0;
+#endif
+}
+
 int crypt_get_pubkey_crt(void *  crt,
                          void ** pk)
 {
@@ -283,8 +305,8 @@ void crypt_free_crt(void * crt)
 #endif
 }
 
-int crypt_crt_str(void * crt,
-                  char * buf)
+int crypt_crt_str(const void * crt,
+                  char *       buf)
 {
 #ifdef HAVE_OPENSSL
         return openssl_crt_str(crt, buf);
@@ -296,6 +318,23 @@ int crypt_crt_str(void * crt,
 #endif
 }
 
+int crypt_crt_der(const void * crt,
+                  buffer_t *   buf)
+{
+        assert(crt != NULL);
+        assert(buf != NULL);
+
+#ifdef HAVE_OPENSSL
+        return openssl_crt_der(crt, buf);
+#else
+        (void) crt;
+
+        clrbuf(*buf);
+
+        return 0;
+#endif
+}
+
 int crypt_check_crt_name(void *       crt,
                          const char * name)
 {
diff --git a/src/lib/crypt/openssl.c b/src/lib/crypt/openssl.c
index 6e7a5dab..1824d879 100644
--- a/src/lib/crypt/openssl.c
+++ b/src/lib/crypt/openssl.c
@@ -159,7 +159,7 @@ ssize_t openssl_ecdh_pkp_create(void **   pkp,
         if (__openssl_ecdh_gen_key(pkp) < 0)
                 goto fail_key;
 
-        pos = pk; /* i2d_PUBKEY increments the pointer, don't use buf! */
+        pos = pk; /* i2d_PUBKEY increments the pointer, don't use pk! */
         len = i2d_PUBKEY(*pkp, &pos);
         if (len < 0)
                 goto fail_pubkey;
@@ -388,6 +388,28 @@ int openssl_load_crt_str(const char * str,
         return -1;
 }
 
+int openssl_load_crt_der(buffer_t buf,
+                         void **  crt)
+{
+        const uint8_t * p;
+        X509 *          xcrt;
+
+        assert(crt != NULL);
+
+        p = buf.data;
+
+        xcrt = d2i_X509(NULL, &p, buf.len);
+        if (xcrt == NULL)
+                goto fail_crt;
+
+        *crt = (void *) xcrt;
+
+        return 0;
+ fail_crt:
+        *crt = NULL;
+        return -1;
+}
+
 int openssl_get_pubkey_crt(void *  crt,
                            void ** key)
 {
@@ -578,8 +600,8 @@ int openssl_check_crt_name(void *       crt,
         return -1;
 }
 
-int openssl_crt_str(void * crt,
-                    char * str)
+int openssl_crt_str(const void * crt,
+                    char *       str)
 {
         BIO *  bio;
         X509 * xcrt;
@@ -608,6 +630,28 @@ int openssl_crt_str(void * crt,
         return -1;
 }
 
+int openssl_crt_der(const void * crt,
+                    buffer_t *   buf)
+{
+        int len;
+
+        assert(crt != NULL);
+        assert(buf != NULL);
+
+        len = i2d_X509((X509 *) crt, &buf->data);
+        if (len < 0)
+                goto fail_der;
+
+        buf->len = (size_t) len;
+
+        return 0;
+
+ fail_der:
+        clrbuf(*buf);
+        return -1;
+}
+
+
 void * openssl_auth_create_store(void)
 {
         return X509_STORE_new();
diff --git a/src/lib/crypt/openssl.h b/src/lib/crypt/openssl.h
index 5d6f50dd..d4ee73b9 100644
--- a/src/lib/crypt/openssl.h
+++ b/src/lib/crypt/openssl.h
@@ -57,6 +57,9 @@ int     openssl_load_crt_file(const char * path,
 int     openssl_load_crt_str(const char * str,
                              void **      crt);
 
+int     openssl_load_crt_der(buffer_t buf,
+                             void **  crt);
+
 int     openssl_get_pubkey_crt(void *  crt,
                                void ** pk);
 
@@ -82,8 +85,11 @@ void    openssl_free_key(void * key);
 int     openssl_check_crt_name(void *       crt,
                                const char * name);
 
-int     openssl_crt_str(void * crt,
-                        char * str);
+int     openssl_crt_str(const void * crt,
+                        char *       str);
+
+int     openssl_crt_der(const void * crt,
+                        buffer_t *   buf);
 
 void *  openssl_auth_create_store(void);
 
diff --git a/src/lib/dev.c b/src/lib/dev.c
index cb35f3f9..c22b4f06 100644
--- a/src/lib/dev.c
+++ b/src/lib/dev.c
@@ -377,13 +377,13 @@ static void _flow_keepalive(struct flow * flow)
 
         clock_gettime(PTHREAD_COND_CLOCK, &now);
 
-        if (ts_diff_ns(&r_act, &now) > (int64_t) timeo * MILLION) {
+        if (ts_diff_ns(&now, &r_act) > (int64_t) timeo * MILLION) {
                 shm_rbuff_set_acl(flow->rx_rb, ACL_FLOWPEER);
                 shm_flow_set_notify(ai.fqset, flow_id, FLOW_PEER);
                 return;
         }
 
-        if (ts_diff_ns(&s_act, &now) > (int64_t) timeo * (MILLION >> 2)) {
+        if (ts_diff_ns(&now, &s_act) > (int64_t) timeo * (MILLION >> 2)) {
                 pthread_rwlock_unlock(&ai.lock);
 
                 flow_send_keepalive(flow, now);
diff --git a/src/lib/frct.c b/src/lib/frct.c
index d51cf006..4d871efe 100644
--- a/src/lib/frct.c
+++ b/src/lib/frct.c
@@ -159,11 +159,11 @@ static int frct_rib_read(const char * path,
                 frcti->rto,
                 frcti->snd_cr.lwe,
                 frcti->snd_cr.rwe,
-                ts_diff_ns(&frcti->snd_cr.act, &now),
+                ts_diff_ns(&now, &frcti->snd_cr.act),
                 frcti->snd_cr.seqno,
                 frcti->rcv_cr.lwe,
                 frcti->rcv_cr.rwe,
-                ts_diff_ns(&frcti->rcv_cr.act, &now),
+                ts_diff_ns(&now, &frcti->rcv_cr.act),
                 frcti->rcv_cr.seqno,
                 frcti->n_rtx,
                 frcti->n_prb,
@@ -303,13 +303,13 @@ static void send_frct_pkt(struct frcti * frcti)
         ackno = frcti->rcv_cr.lwe;
         rwe   = frcti->rcv_cr.rwe;
 
-        diff = ts_diff_ns(&frcti->rcv_cr.act, &now);
+        diff = ts_diff_ns(&now, &frcti->rcv_cr.act);
         if (diff > frcti->a) {
                 pthread_rwlock_unlock(&frcti->lock);
                 return;
         }
 
-        diff = ts_diff_ns(&frcti->snd_cr.act, &now);
+        diff = ts_diff_ns(&now, &frcti->snd_cr.act);
         if (diff < TICTIME) {
                 pthread_rwlock_unlock(&frcti->lock);
                 return;
@@ -339,7 +339,7 @@ static struct frcti * frcti_create(int    fd,
 #ifdef PROC_FLOW_STATS
         char                frctstr[FRCT_NAME_STRLEN + 1];
 #endif
-        mpl *= BILLION;
+        mpl *= MILLION;
         a   *= BILLION;
         r   *= BILLION;
 
@@ -517,14 +517,14 @@ static bool __frcti_is_window_open(struct frcti * frcti)
                         frcti->t_rdvs = now;
                 } else {
                         time_t diff;
-                        diff = ts_diff_ns(&frcti->t_wnd, &now);
+                        diff = ts_diff_ns(&now, &frcti->t_wnd);
                         if (diff > MAX_RDV) {
                                 pthread_mutex_unlock(&frcti->mtx);
                                 pthread_rwlock_unlock(&frcti->lock);
                                 return false;
                         }
 
-                        diff = ts_diff_ns(&frcti->t_rdvs, &now);
+                        diff = ts_diff_ns(&now, &frcti->t_rdvs);
                         if  (diff > frcti->rdv) {
                                 frcti->t_rdvs = now;
                                 __send_rdv(frcti->fd);
@@ -580,13 +580,13 @@ static int __frcti_window_wait(struct frcti *    frcti,
 
                         clock_gettime(PTHREAD_COND_CLOCK, &now);
 
-                        diff = ts_diff_ns(&frcti->t_wnd, &now);
+                        diff = ts_diff_ns(&now, &frcti->t_wnd);
                         if (diff > MAX_RDV) {
                                 pthread_mutex_unlock(&frcti->mtx);
                                 return -ECONNRESET; /* write fails! */
                         }
 
-                        diff = ts_diff_ns(&frcti->t_rdvs, &now);
+                        diff = ts_diff_ns(&now, &frcti->t_rdvs);
                         if  (diff > frcti->rdv) {
                                 frcti->t_rdvs = now;
                                 __send_rdv(frcti->fd);
@@ -855,7 +855,7 @@ static void __frcti_rcv(struct frcti *       frcti,
                         if (!(pci->flags & FRCT_DATA))
                                 frcti->n_dak++;
 #endif
-                        rtt_estimator(frcti, ts_diff_ns(&frcti->t_probe, &now));
+                        rtt_estimator(frcti, ts_diff_ns(&now, &frcti->t_probe));
                         frcti->probe = false;
                 }
         }
diff --git a/src/lib/tests/auth_test.c b/src/lib/tests/auth_test.c
index c3d42b8f..271fdabf 100644
--- a/src/lib/tests/auth_test.c
+++ b/src/lib/tests/auth_test.c
@@ -36,7 +36,7 @@
 *   https://community.f5.com/kb/technicalarticles/
 */
 
-/* Root certificate for CA ca.unittest.o7s */
+/* Root certificate for CA ca2.unittest.o7s */
 static const char * root_ca_crt = \
 "-----BEGIN CERTIFICATE-----\n"
 "MIICiTCCAi+gAwIBAgIUe4iFIymeUTgutBrdvcxFihOVHnowCgYIKoZIzj0EAwIw\n"
@@ -55,7 +55,7 @@ static const char * root_ca_crt = \
 "JSTSWB29kFFiM9ZdMV7M/tiZH9nSz1M8XhsTIGk=\n"
 "-----END CERTIFICATE-----\n";
 
-/* Certificate for intermediary im.unittest.o7s used for signing */
+/* Certificate for intermediary im2.unittest.o7s used for signing */
 static const char * intermediate_ca_crt = \
 "-----BEGIN CERTIFICATE-----\n"
 "MIIChTCCAiqgAwIBAgICEAIwCgYIKoZIzj0EAwIwgZExCzAJBgNVBAYTAkJFMQww\n"
@@ -74,7 +74,7 @@ static const char * intermediate_ca_crt = \
 "NnkLn+73oMj8w4pXGLNKAkX0z7yPJ4QhwA==\n"
 "-----END CERTIFICATE-----\n";
 
-/* Server server-1.unittest.o7s private-public key pair */
+/* Server server-2.unittest.o7s private-public key pair */
 static const char * server_ec_pkp = \
 "-----BEGIN EC PRIVATE KEY-----\n"
 "MHcCAQEEIC13y+5jdKe80HBJD7WITpQamcn3rrkTX1r0v+JwSk4NoAoGCCqGSM49\n"
@@ -89,7 +89,7 @@ static const char * server_ec_pk = \
 "uhDhqqrEcBO5+eob2xyqEaNknIV/86724zPptGRahWz0rzW2PvNppJdNBg==\n"
 "-----END PUBLIC KEY-----\n";
 
-/* Valid signed server certificate for server-1.unittest.o7s, SHA2 */
+/* Valid signed server certificate for server-2.unittest.o7s */
 static const char * signed_server_crt = \
 "-----BEGIN CERTIFICATE-----\n"
 "MIIDgjCCAyigAwIBAgICEAIwCgYIKoZIzj0EAwIwWzELMAkGA1UEBhMCQkUxDDAK\n"
diff --git a/src/lib/tests/time_test.c b/src/lib/tests/time_test.c
index 77fecdac..2b75b873 100644
--- a/src/lib/tests/time_test.c
+++ b/src/lib/tests/time_test.c
@@ -22,145 +22,508 @@
 
 #define _POSIX_C_SOURCE 200809L
 
+#include <ouroboros/test.h>
 #include <ouroboros/time.h>
 
 #include <stdio.h>
 
-static void ts_print(struct timespec * s)
+static int ts_check(struct timespec * s,
+                    time_t            sec,
+                    time_t            nsec)
 {
-        printf("timespec is %zd:%ld.\n",
-               (ssize_t) s->tv_sec, s->tv_nsec);
+        return s->tv_sec == sec && s->tv_nsec == nsec;
 }
 
-static void tv_print(struct timeval * v)
+static int tv_check(struct timeval * v,
+                    time_t           sec,
+                    time_t           usec)
 {
-        printf("timeval is %zd:%zu.\n",
-               (ssize_t) v->tv_sec, (size_t) v->tv_usec);
+        return v->tv_sec == sec && v->tv_usec == usec;
 }
 
-static void ts_init(struct timespec * s,
-                    time_t            sec,
-                    time_t            nsec)
+
+static int test_time_ts_init(void)
 {
-        s->tv_sec  = sec;
-        s->tv_nsec = nsec;
+        struct timespec s  = TIMESPEC_INIT_S (100);
+        struct timespec ms = TIMESPEC_INIT_MS(100);
+        struct timespec us = TIMESPEC_INIT_US(100);
+        struct timespec ns = TIMESPEC_INIT_NS(100);
+
+        TEST_START();
+
+        if (!ts_check(&s, 100, 0)) {
+                printf("timespec_init_s failed.\n");
+                goto fail;
+        }
+
+        if (!ts_check(&ms, 0, 100 * MILLION)) {
+                printf("timespec_init_ms failed.\n");
+                goto fail;
+        }
+
+        if (!ts_check(&us, 0, 100* 1000L)) {
+                printf("timespec_init_us failed.\n");
+                goto fail;
+        }
+
+        if (!ts_check(&ns, 0, 100)) {
+                printf("timespec_init_ns failed.\n");
+                goto fail;
+        }
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
 }
 
-static void tv_init(struct timeval * v,
-                    time_t           sec,
-                    time_t           usec)
+static int test_time_tv_init(void)
 {
-        v->tv_sec  = sec;
-        v->tv_usec = usec;
+        struct timeval s  = TIMEVAL_INIT_S (100);
+        struct timeval ms = TIMEVAL_INIT_MS(100);
+        struct timeval us = TIMEVAL_INIT_US(100);
+
+        TEST_START();
+
+        if (!tv_check(&s, 100, 0)) {
+                printf("timeval_init_s failed.\n");
+                goto fail;
+        }
+
+        if (!tv_check(&ms, 0, 100 * 1000L)) {
+                printf("timeval_init_ms failed.\n");
+                goto fail;
+        }
+
+        if (!tv_check(&us, 0, 100)) {
+                printf("timeval_init_us failed.\n");
+                goto fail;
+        }
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
 }
 
-static int ts_check(struct timespec * s,
-                    time_t            sec,
-                    time_t            nsec)
+static int test_ts_diff(void)
 {
-        return s->tv_sec == sec && s->tv_nsec == nsec;
+        struct timespec s0  = TIMESPEC_INIT_S (100);
+        struct timespec s1  = TIMESPEC_INIT_S (200);
+        struct timespec ms0 = TIMESPEC_INIT_MS(100);
+        struct timespec ms1 = TIMESPEC_INIT_MS(200);
+        struct timespec us0 = TIMESPEC_INIT_US(100);
+        struct timespec us1 = TIMESPEC_INIT_US(200);
+        struct timespec ns0 = TIMESPEC_INIT_NS(100);
+        struct timespec ns1 = TIMESPEC_INIT_NS(200);
+        struct timespec res;
+
+        TEST_START();
+
+        ts_diff(&s0, &s1, &res);
+        if (!ts_check(&res, -100, 0)) {
+                printf("timespec_diff failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        ts_diff(&s1, &s0, &res);
+        if (!ts_check(&res, 100, 0)) {
+                printf("timespec_diff failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        ts_diff(&ms0, &ms1, &res);
+        if (!ts_check(&res, -1, 900 * MILLION)) {
+                printf("timespec_diff failed at ms0 - ms1.\n");
+                goto fail;
+        }
+
+        ts_diff(&ms1, &ms0, &res);
+        if (!ts_check(&res, 0, 100 * MILLION)) {
+                printf("timespec_diff failed at ms1 - ms0.\n");
+                goto fail;
+        }
+
+        ts_diff(&us0, &us1, &res);
+        if (!ts_check(&res, -1, 999900 * 1000L)) {
+                printf("timespec_diff failed at us0 - us1.\n");
+                goto fail;
+        }
+
+        ts_diff(&us1, &us0, &res);
+        if (!ts_check(&res, 0, 100 * 1000L)) {
+                printf("timespec_diff failed at us1 - us0.\n");
+                goto fail;
+        }
+
+        ts_diff(&ns0, &ns1, &res);
+        if (!ts_check(&res, -1, 999999900)) {
+                printf("timespec_diff failed at ns0 - ns1.\n");
+                goto fail;
+        }
+
+        ts_diff(&ns1, &ns0, &res);
+        if (!ts_check(&res, 0, 100)) {
+                printf("timespec_diff failed at ns1 - ns0.\n");
+                goto fail;
+        }
+
+        ts_diff(&s0, &ms0, &res);
+        if (!ts_check(&res, 99, 900 * MILLION)) {
+                printf("timespec_diff failed at s0 - ms0.\n");
+                goto fail;
+        }
+
+        ts_diff(&s0, &us0, &res);
+        if (!ts_check(&res, 99, 999900 * 1000L)) {
+                printf("timespec_diff failed at s0 - us0.\n");
+                goto fail;
+        }
+
+        ts_diff(&s0, &ns0, &res);
+        if (!ts_check(&res, 99, 999999900)) {
+                printf("timespec_diff failed at s0 - ns0.\n");
+                goto fail;
+        }
+
+        ts_diff(&ms0, &us0, &res);
+        if (!ts_check(&res, 0, 99900 * 1000L)) {
+                printf("timespec_diff failed at ms0 - us0.\n");
+                goto fail;
+        }
+
+        ts_diff(&ms0, &ns0, &res);
+        if (!ts_check(&res, 0, 99999900)) {
+                printf("timespec_diff failed at ms0 - ns0.\n");
+                goto fail;
+        }
+
+        ts_diff(&us0, &ns0, &res);
+        if (!ts_check(&res, 0, 99900)) {
+                printf("timespec_diff failed at us0 - ns0.\n");
+                goto fail;
+        }
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
 }
 
-static int tv_check(struct timeval * v,
-                    time_t           sec,
-                    time_t           usec)
+static int test_tv_diff(void)
 {
-        return v->tv_sec == sec && v->tv_usec == usec;
+        struct timeval s0  = TIMEVAL_INIT_S (100);
+        struct timeval s1  = TIMEVAL_INIT_S (200);
+        struct timeval ms0 = TIMEVAL_INIT_MS(100);
+        struct timeval ms1 = TIMEVAL_INIT_MS(200);
+        struct timeval us0 = TIMEVAL_INIT_US(100);
+        struct timeval us1 = TIMEVAL_INIT_US(200);
+        struct timeval res;
+
+        TEST_START();
+
+        tv_diff(&s0, &s1, &res);
+        if (!tv_check(&res, -100, 0)) {
+                printf("timeval_diff failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        tv_diff(&s1, &s0, &res);
+        if (!tv_check(&res, 100, 0)) {
+                printf("timeval_diff failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        tv_diff(&ms0, &ms1, &res);
+        if (!tv_check(&res, -1, 900 * 1000L)) {
+                printf("timeval_diff failed at ms0 - ms1.\n");
+                goto fail;
+        }
+
+        tv_diff(&ms1, &ms0, &res);
+        if (!tv_check(&res, 0, 100 * 1000L)) {
+                printf("timeval_diff failed at ms1 - ms0.\n");
+                goto fail;
+        }
+
+        tv_diff(&us0, &us1, &res);
+        if (!tv_check(&res, -1, 999900)) {
+                printf("timeval_diff failed at us0 - us1.\n");
+                goto fail;
+        }
+
+        tv_diff(&us1, &us0, &res);
+        if (!tv_check(&res, 0, 100)) {
+                printf("timeval_diff failed at us1 - us0.\n");
+                goto fail;
+        }
+
+        tv_diff(&s0, &ms0, &res);
+        if (!tv_check(&res, 99, 900 * 1000L)) {
+                printf("timeval_diff failed at s0 - ms0.\n");
+                goto fail;
+        }
+
+        tv_diff(&s0, &us0, &res);
+        if (!tv_check(&res, 99, 999900)) {
+                printf("timeval_diff failed at s0 - us0.\n");
+                goto fail;
+        }
+
+        tv_diff(&ms0, &us0, &res);
+        if (!tv_check(&res, 0, 99900)) {
+                printf("timeval_diff failed at ms0 - us0.\n");
+                goto fail;
+        }
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
 }
 
-int time_test(int     argc,
-              char ** argv)
+static int test_ts_diff_time(void)
 {
-        struct timespec s0;
-        struct timespec s1;
-        struct timespec s2;
+        struct timespec s0  = TIMESPEC_INIT_S (100);
+        struct timespec s1  = TIMESPEC_INIT_S (200);
+        struct timespec ms0 = TIMESPEC_INIT_MS(100);
+        struct timespec ms1 = TIMESPEC_INIT_MS(200);
+        struct timespec us0 = TIMESPEC_INIT_US(100);
+        struct timespec us1 = TIMESPEC_INIT_US(200);
+        struct timespec ns0 = TIMESPEC_INIT_NS(100);
+        struct timespec ns1 = TIMESPEC_INIT_NS(200);
 
-        struct timeval v0;
-        struct timeval v1;
-        struct timeval v2;
+        TEST_START();
 
-        (void) argc;
-        (void) argv;
+        if (ts_diff_ms(&s0, &s1) != -100 * 1000L) {
+                printf("timespec_diff_ms failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        if (ts_diff_ms(&s1, &s0) != 100 * 1000L) {
+                printf("timespec_diff_ms failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        if (ts_diff_us(&s0, &s1) != -100 * MILLION) {
+                printf("timespec_diff_us failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        if (ts_diff_us(&s1, &s0) != 100 * MILLION) {
+                printf("timespec_diff_us failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        if (ts_diff_ns(&s0, &s1) != -100 * BILLION) {
+                printf("timespec_diff_ns failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        if (ts_diff_ns(&s1, &s0) != 100 * BILLION) {
+                printf("timespec_diff_ns failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        if (ts_diff_ms(&ms0, &ms1) != -100) {
+                printf("timespec_diff_ms failed at ms0 - ms1.\n");
+                goto fail;
+        }
+
+        if (ts_diff_ms(&ms1, &ms0) != 100) {
+                printf("timespec_diff_ms failed at ms1 - ms0.\n");
+                goto fail;
+        }
 
-        ts_init(&s0, 0, 0);
-        ts_init(&s1, 5, 0);
+        if (ts_diff_us(&ms0, &ms1) != -100 * 1000L) {
+                printf("timespec_diff_us failed at ms0 - ms1.\n");
+                goto fail;
+        }
 
-        ts_add(&s0, &s1, &s2);
-        if (!ts_check(&s2, 5, 0)) {
-                printf("ts_add failed.\n");
-                ts_print(&s2);
-                return -1;
+        if (ts_diff_us(&ms1, &ms0) != 100 * 1000L) {
+                printf("timespec_diff_us failed at ms1 - ms0.\n");
+                goto fail;
         }
 
-        tv_init(&v0, 0, 0);
-        tv_init(&v1, 5, 0);
+        if (ts_diff_ns(&ms0, &ms1) != -100 * MILLION) {
+                printf("timespec_diff_ns failed at ms0 - ms1.\n");
+                goto fail;
+        }
 
-        tv_add(&v0, &v1, &v2);
-        if (!tv_check(&v2, 5, 0)) {
-                printf("tv_add failed.\n");
-                tv_print(&v2);
-                return -1;
+        if (ts_diff_ns(&ms1, &ms0) != 100 * MILLION) {
+                printf("timespec_diff_ns failed at ms1 - ms0.\n");
+                goto fail;
         }
 
-        ts_init(&s0, 0, 500 * MILLION);
-        ts_init(&s1, 0, 600 * MILLION);
+        if (ts_diff_ms(&us0, &us1) != 0) {
+                printf("timespec_diff_ms failed at us0 - us1.\n");
+                goto fail;
+        }
 
-        ts_add(&s0, &s1, &s2);
-        if (!ts_check(&s2, 1, 100 * MILLION)) {
-                printf("ts_add with nano overflow failed.\n");
-                ts_print(&s2);
-                return -1;
+        if (ts_diff_ms(&us1, &us0) != 0) {
+                printf("timespec_diff_ms failed at us1 - us0.\n");
+                goto fail;
         }
 
-        tv_init(&v0, 0, 500 * 1000);
-        tv_init(&v1, 0, 600 * 1000);
+        if (ts_diff_us(&us0, &us1) != -100) {
+                printf("timespec_diff_us failed at us0 - us1.\n");
+                goto fail;
+        }
 
-        tv_add(&v0, &v1, &v2);
-        if (!tv_check(&v2, 1, 100 * 1000)) {
-                printf("tv_add with nano overflow failed.\n");
-                tv_print(&v2);
-                return -1;
+        if (ts_diff_us(&us1, &us0) != 100) {
+                printf("timespec_diff_us failed at us1 - us0.\n");
+                goto fail;
         }
 
-        ts_init(&s0, 0, 0);
-        ts_init(&s1, 5, 0);
+        if (ts_diff_ns(&us0, &us1) != -100 * 1000L) {
+                printf("timespec_diff_ns failed at us0 - us1.\n");
+                goto fail;
+        }
 
-        ts_diff(&s0, &s1, &s2);
-        if (!ts_check(&s2, -5, 0)) {
-                printf("ts_diff failed.\n");
-                ts_print(&s2);
-                return -1;
+        if (ts_diff_ns(&us1, &us0) != 100 * 1000L) {
+                printf("timespec_diff_ns failed at us1 - us0.\n");
+                goto fail;
         }
 
-        tv_init(&v0, 0, 0);
-        tv_init(&v1, 5, 0);
+        if (ts_diff_ms(&ns0, &ns1) != 0) {
+                printf("timespec_diff_ms failed at ns0 - ns1.\n");
+                goto fail;
+        }
 
-        tv_diff(&v0, &v1, &v2);
-        if (!tv_check(&v2, -5, 0)) {
-                printf("tv_diff failed.\n");
-                tv_print(&v2);
-                return -1;
+        if (ts_diff_ms(&ns1, &ns0) != 0) {
+                printf("timespec_diff_ms failed at ns1 - ns0.\n");
+                goto fail;
         }
 
-        ts_init(&s0, 0, 500 * MILLION);
-        ts_init(&s1, 0, 600 * MILLION);
+        if (ts_diff_us(&ns0, &ns1) != 0) {
+                printf("timespec_diff_us failed at ns0 - ns1.\n");
+                goto fail;
+        }
 
-        ts_diff(&s0, &s1, &s2);
-        if (!ts_check(&s2, -1, 900 * MILLION)) {
-                printf("ts_diff with nano underflow failed.\n");
-                ts_print(&s2);
-                return -1;
+        if (ts_diff_us(&ns1, &ns0) != 0) {
+                printf("timespec_diff_us failed at ns1 - ns0.\n");
+                goto fail;
         }
 
-        tv_init(&v0, 0, 500 * 1000);
-        tv_init(&v1, 0, 600 * 1000);
+        if (ts_diff_ns(&ns0, &ns1) != -100) {
+                printf("timespec_diff_ns failed at ns0 - ns1.\n");
+                goto fail;
+        }
 
-        tv_diff(&v0, &v1, &v2);
-        if (!tv_check(&v2, -1, 900 * 1000)) {
-                printf("tv_diff with nano underflow failed.\n");
-                tv_print(&v2);
-                return -1;
+        if (ts_diff_ns(&ns1, &ns0) != 100) {
+                printf("timespec_diff_ns failed at ns1 - ns0.\n");
+                goto fail;
         }
 
-        return 0;
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
+}
+
+static int test_tv_diff_time(void)
+{
+        struct timeval s0  = TIMEVAL_INIT_S (100);
+        struct timeval s1  = TIMEVAL_INIT_S (200);
+        struct timeval ms0 = TIMEVAL_INIT_MS(100);
+        struct timeval ms1 = TIMEVAL_INIT_MS(200);
+        struct timeval us0 = TIMEVAL_INIT_US(100);
+        struct timeval us1 = TIMEVAL_INIT_US(200);
+
+        TEST_START();
+
+        if (tv_diff_ms(&s0, &s1) != -100 * 1000L) {
+                printf("timeval_diff_ms failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_ms(&s1, &s0) != 100 * 1000L) {
+                printf("timeval_diff_ms failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&s0, &s1) != -100 * MILLION) {
+                printf("timeval_diff_us failed at s0 - s1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&s1, &s0) != 100 * MILLION) {
+                printf("timeval_diff_us failed at s1 - s0.\n");
+                goto fail;
+        }
+
+        if (tv_diff_ms(&ms0, &ms1) != -100) {
+                printf("timeval_diff_ms failed at ms0 - ms1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_ms(&ms1, &ms0) != 100) {
+                printf("timeval_diff_ms failed at ms1 - ms0.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&ms0, &ms1) != -100 * 1000L) {
+                printf("timeval_diff_us failed at ms0 - ms1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&ms1, &ms0) != 100 * 1000L) {
+                printf("timeval_diff_us failed at ms1 - ms0.\n");
+                goto fail;
+        }
+
+        if (tv_diff_ms(&us0, &us1) != 0) {
+                printf("timeval_diff_ms failed at us0 - us1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_ms(&us1, &us0) != 0) {
+                printf("timeval_diff_ms failed at us1 - us0.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&us0, &us1) != -100) {
+                printf("timeval_diff_us failed at us0 - us1.\n");
+                goto fail;
+        }
+
+        if (tv_diff_us(&us1, &us0) != 100) {
+                printf("timeval_diff_us failed at us1 - us0.\n");
+                goto fail;
+        }
+
+        TEST_SUCCESS();
+
+        return TEST_RC_SUCCESS;
+ fail:
+        TEST_FAIL();
+        return TEST_RC_FAIL;
+}
+
+int time_test(int argc,
+              char ** argv)
+{
+        int ret = 0;
+
+        (void) argc;
+        (void) argv;
+
+        ret |= test_time_ts_init();
+        ret |= test_time_tv_init();
+        ret |= test_ts_diff();
+        ret |= test_tv_diff();
+        ret |= test_ts_diff_time();
+        ret |= test_tv_diff_time();
+
+        return ret;
 }
diff --git a/src/lib/tpm.c b/src/lib/tpm.c
index 64777815..a7391bd7 100644
--- a/src/lib/tpm.c
+++ b/src/lib/tpm.c
@@ -31,16 +31,23 @@
 
 #include <assert.h>
 #include <pthread.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
 
-#define TPM_TIMEOUT 1000
+#define TPM_TIMEOUT      1000
 
 struct pthr_el {
         struct list_head next;
 
         bool             kill;
         bool             busy;
-
+        bool             wait;
+#ifdef CONFIG_OUROBOROS_DEBUG
+        struct timespec  start;
+        struct timespec  last;
+#endif
         pthread_t        thr;
 };
 
@@ -72,6 +79,10 @@ static void tpm_join(struct tpm * tpm)
 {
         struct list_head * p;
         struct list_head * h;
+#ifdef CONFIG_OUROBOROS_DEBUG
+        struct timespec now;
+        clock_gettime(CLOCK_REALTIME, &now);
+#endif
 
         list_for_each_safe(p, h, &tpm->pool) {
                 struct pthr_el * e = list_entry(p, struct pthr_el, next);
@@ -86,6 +97,21 @@ static void tpm_join(struct tpm * tpm)
 
         list_for_each_safe(p, h, &tpm->pool) {
                 struct pthr_el * e = list_entry(p, struct pthr_el, next);
+#ifdef CONFIG_OUROBOROS_DEBUG
+                time_t diff = ts_diff_ms(&now, &e->start) / 1000;
+                bool hung;
+                if (TPM_DEBUG_REPORT_INTERVAL > 0) {
+                        time_t ldiff = ts_diff_ms(&now, &e->last) / 1000;
+                        if(e->busy && ldiff > TPM_DEBUG_REPORT_INTERVAL) {
+                                e->last = now;
+                                printf("Thread %d:%lx running for %ld s.\n",
+                                       getpid(),e->thr, diff);
+                        }
+                }
+                hung = e->busy && !e->wait && diff > TPM_DEBUG_ABORT_TIMEOUT;
+                if (TPM_DEBUG_ABORT_TIMEOUT > 0 && hung)
+                        assert(false); /* coredump */
+#endif
                 if (e->kill) {
                         pthread_t thr = e->thr;
                         list_del(&e->next);
@@ -139,11 +165,9 @@ static int __tpm(struct tpm * tpm)
                         if (e == NULL)
                                 break;
 
-                        e->kill = false;
-                        e->busy = false;
+                        memset(e, 0, sizeof(*e));
 
-                        if (pthread_create(&e->thr, NULL,
-                                                tpm->func, tpm->o)) {
+                        if (pthread_create(&e->thr, NULL, tpm->func, tpm->o)) {
                                 free(e);
                                 break;
                         }
@@ -280,12 +304,21 @@ void tpm_begin_work(struct tpm * tpm)
 {
         struct pthr_el * e;
 
+#ifdef CONFIG_OUROBOROS_DEBUG
+        struct timespec now;
+        clock_gettime(CLOCK_REALTIME, &now);
+#endif
+
         pthread_mutex_lock(&tpm->mtx);
 
         e = tpm_pthr_el(tpm, pthread_self());
         if (e != NULL) {
                 e->busy = true;
                 ++tpm->wrk;
+#ifdef CONFIG_OUROBOROS_DEBUG
+                e->start = now;
+                e->last  = now;
+#endif
         }
 
         pthread_cond_signal(&tpm->cond);
@@ -293,6 +326,19 @@ void tpm_begin_work(struct tpm * tpm)
         pthread_mutex_unlock(&tpm->mtx);
 }
 
+void tpm_wait_work(struct tpm * tpm)
+{
+        struct pthr_el * e;
+
+        pthread_mutex_lock(&tpm->mtx);
+
+        e = tpm_pthr_el(tpm, pthread_self());
+        if (e != NULL)
+                e->wait = true;
+
+        pthread_mutex_unlock(&tpm->mtx);
+}
+
 void tpm_end_work(struct tpm * tpm)
 {
         struct pthr_el * e;
diff --git a/src/tools/ocbr/ocbr_client.c b/src/tools/ocbr/ocbr_client.c
index ba7b41f4..eada6e60 100644
--- a/src/tools/ocbr/ocbr_client.c
+++ b/src/tools/ocbr/ocbr_client.c
@@ -129,7 +129,7 @@ int client_main(char * server,
 
                         ++seqnr;
 
-                        if (ts_diff_us(&start, &end) / MILLION >= duration)
+                        if (ts_diff_us(&end, &start) / MILLION >= duration)
                                 stop = true;
                 }
         } else { /* flood */
@@ -142,7 +142,7 @@ int client_main(char * server,
 
                         ++seqnr;
 
-                        if (ts_diff_us(&start, &end) / MILLION
+                        if (ts_diff_us(&end, &start) / MILLION
                             >= (long) duration)
                                 stop = true;
                 }
@@ -151,7 +151,7 @@ int client_main(char * server,
 
         clock_gettime(CLOCK_REALTIME, &end);
 
-        ms = ts_diff_ms(&start, &end);
+        ms = ts_diff_ms(&end, &start);
 
         printf("sent statistics: "
                "%9ld packets, %12ld bytes in %9d ms, %4.4f Mb/s\n",
diff --git a/src/tools/ocbr/ocbr_server.c b/src/tools/ocbr/ocbr_server.c
index a4bbadd4..34c4fa94 100644
--- a/src/tools/ocbr/ocbr_server.c
+++ b/src/tools/ocbr/ocbr_server.c
@@ -114,14 +114,14 @@ static void handle_flow(int fd)
                         bytes_read += count;
                 }
 
-                if (ts_diff_us(&alive, &now)
+                if (ts_diff_us(&now, &alive)
                     > server_settings.timeout * MILLION) {
                         printf("Test on flow %d timed out\n", fd);
                         stop = true;
                 }
 
-                if (stop || ts_diff_ms(&now, &iv_end) < 0) {
-                        long us = ts_diff_us(&iv_start, &now);
+                if (stop || ts_diff_ms(&now, &iv_end) > 0) {
+                        long us = ts_diff_us(&now, &iv_start);
                         printf("Flow %4d: %9ld packets (%12ld bytes) in %9ld ms"
                                " => %9.4f pps, %9.4f Mbps\n",
                                fd,
diff --git a/src/tools/operf/operf_client.c b/src/tools/operf/operf_client.c
index 63f98299..7060ce5b 100644
--- a/src/tools/operf/operf_client.c
+++ b/src/tools/operf/operf_client.c
@@ -133,7 +133,7 @@ void * writer(void * o)
         clock_gettime(CLOCK_REALTIME, &start);
         clock_gettime(CLOCK_REALTIME, &now);
 
-        while (!stop && ts_diff_ms(&start, &now) < client.duration) {
+        while (!stop && ts_diff_ms(&now, &start) > client.duration) {
                 if (!client.flood) {
                         clock_gettime(CLOCK_REALTIME, &now);
                         ts_add(&now, &intv, &end);
@@ -230,10 +230,10 @@ int client_main(void)
                 printf("%ld received, ", client.rcvd);
                 printf("%ld%% packet loss, ", client.sent == 0 ? 0 :
                        100 - ((100 * client.rcvd) / client.sent));
-                printf("time: %.3f ms, ", ts_diff_us(&tic, &toc) / 1000.0);
+                printf("time: %.3f ms, ", ts_diff_us(&toc, &tic) / 1000.0);
                 printf("bandwidth: %.3lf Mb/s.\n",
                        (client.rcvd * client.size * 8)
-                       / (double) ts_diff_us(&tic, &toc));
+                       / (double) ts_diff_us(&toc, &tic));
         }
 
         flow_dealloc(fd);
diff --git a/src/tools/operf/operf_server.c b/src/tools/operf/operf_server.c
index d11f3486..a611f79c 100644
--- a/src/tools/operf/operf_server.c
+++ b/src/tools/operf/operf_server.c
@@ -66,7 +66,7 @@ void * cleaner_thread(void * o)
                 pthread_mutex_lock(&server.lock);
                 for (i = 0; i < OPERF_MAX_FLOWS; ++i)
                         if (fset_has(server.flows, i) &&
-                            ts_diff_ms(&server.times[i], &now)
+                            ts_diff_ms(&now, &server.times[i])
                             > server.timeout) {
                                 printf("Flow %d timed out.\n", i);
                                 fset_del(server.flows, i);
diff --git a/src/tools/oping/oping_client.c b/src/tools/oping/oping_client.c
index 7b03c83d..5a9e03dc 100644
--- a/src/tools/oping/oping_client.c
+++ b/src/tools/oping/oping_client.c
@@ -100,7 +100,7 @@ void * reader(void * o)
                 sent.tv_sec = msg->tv_sec;
                 sent.tv_nsec = msg->tv_nsec;
 
-                ms = ts_diff_us(&sent, &now) / 1000.0;
+                ms = ts_diff_us(&now, &sent) / 1000.0;
 
                 if (id < exp_id)
                         ++client.ooo;
@@ -256,7 +256,7 @@ static int client_main(void)
         printf("%zd out-of-order, ", client.ooo);
         printf("%.0lf%% packet loss, ", client.sent == 0 ? 0 :
                ceil(100 - (100 * (client.rcvd / (float) client.sent))));
-        printf("time: %.3f ms\n", ts_diff_us(&tic, &toc) / 1000.0);
+        printf("time: %.3f ms\n", ts_diff_us(&toc, &tic) / 1000.0);
 
         if (client.rcvd > 0) {
                 printf("rtt min/avg/max/mdev = %.3f/%.3f/%.3f/",
diff --git a/src/tools/oping/oping_server.c b/src/tools/oping/oping_server.c
index 11e10a77..3adce244 100644
--- a/src/tools/oping/oping_server.c
+++ b/src/tools/oping/oping_server.c
@@ -69,7 +69,7 @@ void * cleaner_thread(void * o)
                                 time_t diff;
 
                                 pthread_mutex_lock(&server.lock);
-                                diff = ts_diff_ms(&server.times[i], &now);
+                                diff = ts_diff_ms(&now, &server.times[i]);
                                 pthread_mutex_unlock(&server.lock);
 
                                 if (diff > deadline_ms) {
diff --git a/src/tools/time_utils.h b/src/tools/time_utils.h
index c17282dc..a4117f44 100644
--- a/src/tools/time_utils.h
+++ b/src/tools/time_utils.h
@@ -53,17 +53,17 @@
 #include <sys/time.h>
 
 /* functions for timespecs */
-#define ts_diff_ns(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * BILLION     \
+#define ts_diff_ns(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * BILLION     \
                             + ((tx)->tv_nsec - (t0)->tv_nsec))
-#define ts_diff_us(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
+#define ts_diff_us(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
                             + ((tx)->tv_nsec - (t0)->tv_nsec) / 1000L)
-#define ts_diff_ms(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
+#define ts_diff_ms(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
                             + ((tx)->tv_nsec - (t0)->tv_nsec) / MILLION)
 
 /* functions for timevals are the same */
-#define tv_diff_us(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
+#define tv_diff_us(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * MILLION     \
                             + ((tx)->tv_usec - (t0)->tv_usec) / 1000L)
-#define tv_diff_ms(t0, tx) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
+#define tv_diff_ms(tx, t0) (((tx)->tv_sec - (t0)->tv_sec) * 1000L       \
                             + ((tx)->tv_usec - (t0)->tv_usec) / MILLION)
 
 /* functions for timespecs */