lib: Revise du buff API towards a memory allocator

This changes the API to the rdrbuff to treat it as a pool memory
allocator. The head and tailspace to allocate in a buffer is now set
system-wide instead of being passed as a parameter.

Signed-off-by: Dimitri Staessens <dimitri.staessens@ugent.be>
Signed-off-by: Sander Vrijders <sander.vrijders@ugent.be>

4 files changed, 41 insertions, 55 deletions

diff --git a/include/ouroboros/shm_rdrbuff.h b/include/ouroboros/shm_rdrbuff.h
index 49551497..dc8418fd 100644
--- a/include/ouroboros/shm_rdrbuff.h
+++ b/include/ouroboros/shm_rdrbuff.h
@@ -42,18 +42,16 @@ void                 shm_rdrbuff_destroy(struct shm_rdrbuff * rdrb);
 
 void                 shm_rdrbuff_purge(void);
 
-/* returns the index of the buffer in the DU map */
-ssize_t              shm_rdrbuff_write(struct shm_rdrbuff * rdrb,
-                                       size_t               headspace,
-                                       size_t               tailspace,
-                                       const uint8_t *      data,
-                                       size_t               data_len);
+/* Returns block index, a ptr and du_buff.  */
+ssize_t              shm_rdrbuff_alloc(struct shm_rdrbuff *  rdrb,
+                                       size_t                count,
+                                       uint8_t **            ptr,
+                                       struct shm_du_buff ** sdb);
 
-ssize_t              shm_rdrbuff_write_b(struct shm_rdrbuff *    rdrb,
-                                         size_t                  headspace,
-                                         size_t                  tailspace,
-                                         const uint8_t *         data,
-                                         size_t                  data_len,
+ssize_t              shm_rdrbuff_alloc_b(struct shm_rdrbuff *    rdrb,
+                                         size_t                  count,
+                                         uint8_t **              ptr,
+                                         struct shm_du_buff **   sdb,
                                          const struct timespec * abstime);
 
 ssize_t              shm_rdrbuff_read(uint8_t **           dst,
diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt
index 42164fac..c18860a4 100644
--- a/src/lib/CMakeLists.txt
+++ b/src/lib/CMakeLists.txt
@@ -147,9 +147,9 @@ set(PROG_RES_FDS 64 CACHE STRING
   "Number of reserved flow descriptors per application")
 set(PROG_MAX_FQUEUES 32 CACHE STRING
   "Maximum number of flow sets per application")
-set(DU_BUFF_HEADSPACE 128 CACHE STRING
+set(DU_BUFF_HEADSPACE 256 CACHE STRING
   "Bytes of headspace to reserve for future headers")
-set(DU_BUFF_TAILSPACE 16 CACHE STRING
+set(DU_BUFF_TAILSPACE 32 CACHE STRING
   "Bytes of tailspace to reserve for future tails")
 if (NOT APPLE)
   set(PTHREAD_COND_CLOCK "CLOCK_MONOTONIC" CACHE STRING
diff --git a/src/lib/dev.c b/src/lib/dev.c
index 3c1494ee..d596758e 100644
--- a/src/lib/dev.c
+++ b/src/lib/dev.c
@@ -855,6 +855,7 @@ ssize_t flow_write(int          fd,
         struct timespec      abs;
         struct timespec *    abstime = NULL;
         struct shm_du_buff * sdb;
+        uint8_t *            ptr;
 
         if (buf == NULL)
                 return 0;
@@ -886,22 +887,20 @@ ssize_t flow_write(int          fd,
                 return -EPERM;
 
         if (flags & FLOWFWNOBLOCK)
-                idx = shm_rdrbuff_write(ai.rdrb,
-                                        DU_BUFF_HEADSPACE,
-                                        DU_BUFF_TAILSPACE,
-                                        buf,
-                                        count);
+                idx = shm_rdrbuff_alloc(ai.rdrb,
+                                        count,
+                                        &ptr,
+                                        &sdb);
         else  /* Blocking. */
-                idx = shm_rdrbuff_write_b(ai.rdrb,
-                                          DU_BUFF_HEADSPACE,
-                                          DU_BUFF_TAILSPACE,
-                                          buf,
+                idx = shm_rdrbuff_alloc_b(ai.rdrb,
                                           count,
+                                          &ptr,
+                                          &sdb,
                                           abstime);
         if (idx < 0)
                 return idx;
 
-        sdb = shm_rdrbuff_get(ai.rdrb, idx);
+        memcpy(ptr, buf, count);
 
         if (frcti_snd(flow->frcti, sdb) < 0) {
                 shm_rdrbuff_remove(ai.rdrb, idx);
@@ -1442,21 +1441,7 @@ int ipcp_flow_write(int                  fd,
 int ipcp_sdb_reserve(struct shm_du_buff ** sdb,
                      size_t                len)
 {
-        ssize_t idx;
-
-        idx = shm_rdrbuff_write_b(ai.rdrb,
-                                  DU_BUFF_HEADSPACE,
-                                  DU_BUFF_TAILSPACE,
-                                  NULL,
-                                  len,
-                                  NULL);
-
-        if (idx < 0)
-                return -1;
-
-        *sdb = shm_rdrbuff_get(ai.rdrb, idx);
-
-        return 0;
+        return shm_rdrbuff_alloc_b(ai.rdrb, len, NULL, sdb, NULL) < 0 ? -1 : 0;
 }
 
 void ipcp_sdb_release(struct shm_du_buff * sdb)
diff --git a/src/lib/shm_rdrbuff.c b/src/lib/shm_rdrbuff.c
index e9ef9222..0d3faf17 100644
--- a/src/lib/shm_rdrbuff.c
+++ b/src/lib/shm_rdrbuff.c
@@ -45,6 +45,7 @@
 #define SHM_FILE_SIZE (SHM_BLOCKS_SIZE + 2 * sizeof(size_t)                    \
                        + sizeof(pthread_mutex_t) + 2 * sizeof(pthread_cond_t)  \
                        + sizeof(pid_t))
+#define DU_BUFF_OVERHEAD (DU_BUFF_HEADSPACE + DU_BUFF_TAILSPACE)
 
 #define get_head_ptr(rdrb)                                                     \
         idx_to_du_buff_ptr(rdrb, *rdrb->head)
@@ -277,14 +278,13 @@ void shm_rdrbuff_purge(void)
         free(shm_rdrb_fn);
 }
 
-ssize_t shm_rdrbuff_write(struct shm_rdrbuff * rdrb,
-                          size_t               headspace,
-                          size_t               tailspace,
-                          const uint8_t *      data,
-                          size_t               len)
+ssize_t shm_rdrbuff_alloc(struct shm_rdrbuff *  rdrb,
+                          size_t                len,
+                          uint8_t **            ptr,
+                          struct shm_du_buff ** psdb)
 {
         struct shm_du_buff * sdb;
-        size_t               size = headspace + len + tailspace;
+        size_t               size = DU_BUFF_OVERHEAD + len;
 #ifdef SHM_RDRB_MULTI_BLOCK
         size_t               blocks = 0;
         size_t               padblocks = 0;
@@ -292,6 +292,7 @@ ssize_t shm_rdrbuff_write(struct shm_rdrbuff * rdrb,
         ssize_t              sz = size + sizeof(*sdb);
 
         assert(rdrb);
+        assert(psdb);
 
 #ifndef SHM_RDRB_MULTI_BLOCK
         if (sz > SHM_RDRB_BLOCK_SIZE)
@@ -346,24 +347,24 @@ ssize_t shm_rdrbuff_write(struct shm_rdrbuff * rdrb,
         pthread_mutex_unlock(rdrb->lock);
 
         sdb->size    = size;
-        sdb->du_head = headspace;
+        sdb->du_head = DU_BUFF_HEADSPACE;
         sdb->du_tail = sdb->du_head + len;
 
-        if (data != NULL)
-                memcpy(((uint8_t *) (sdb + 1)) + headspace, data, len);
+        *psdb = sdb;
+        if (ptr != NULL)
+                *ptr = (uint8_t *) (sdb + 1) + sdb->du_head;
 
         return sdb->idx;
 }
 
-ssize_t shm_rdrbuff_write_b(struct shm_rdrbuff *    rdrb,
-                            size_t                  headspace,
-                            size_t                  tailspace,
-                            const uint8_t *         data,
+ssize_t shm_rdrbuff_alloc_b(struct shm_rdrbuff *    rdrb,
                             size_t                  len,
+                            uint8_t **              ptr,
+                            struct shm_du_buff **   psdb,
                             const struct timespec * abstime)
 {
         struct shm_du_buff * sdb;
-        size_t               size      = headspace + len + tailspace;
+        size_t               size      = DU_BUFF_OVERHEAD + len;
 #ifdef SHM_RDRB_MULTI_BLOCK
         size_t               blocks    = 0;
         size_t               padblocks = 0;
@@ -372,6 +373,7 @@ ssize_t shm_rdrbuff_write_b(struct shm_rdrbuff *    rdrb,
         int                  ret       = 0;
 
         assert(rdrb);
+        assert(psdb);
 
 #ifndef SHM_RDRB_MULTI_BLOCK
         if (sz > SHM_RDRB_BLOCK_SIZE)
@@ -444,11 +446,12 @@ ssize_t shm_rdrbuff_write_b(struct shm_rdrbuff *    rdrb,
                 return -ETIMEDOUT;
 
         sdb->size    = size;
-        sdb->du_head = headspace;
+        sdb->du_head = DU_BUFF_HEADSPACE;
         sdb->du_tail = sdb->du_head + len;
 
-        if (data != NULL)
-                memcpy(((uint8_t *) (sdb + 1)) + headspace, data, len);
+        *psdb = sdb;
+        if (ptr != NULL)
+                *ptr = (uint8_t *) (sdb + 1) + sdb->du_head;
 
         return sdb->idx;
 }