lib: Fix some issues in lockless rbuff

3 files changed, 38 insertions, 9 deletions

diff --git a/include/ouroboros/config.h.in b/include/ouroboros/config.h.in
index 46f784d4..30bab68f 100644
--- a/include/ouroboros/config.h.in
+++ b/include/ouroboros/config.h.in
@@ -44,7 +44,7 @@
 #define SHM_BUFFER_SIZE        1 << 14
 #define DU_BUFF_HEADSPACE      128
 #define DU_BUFF_TAILSPACE      0
-#define SHM_RBUFF_LOCKLESS
+#define SHM_RBUFF_LOCKLESS     0
 #define SHM_RBUFF_PREFIX       "/ouroboros.rbuff."
 #define SHM_FLOW_SET_PREFIX    "/ouroboros.sets."
 #define IRMD_MAX_FLOWS         4096
diff --git a/src/lib/shm_rbuff.c b/src/lib/shm_rbuff.c
index dc1d9f0a..dc3f83b3 100644
--- a/src/lib/shm_rbuff.c
+++ b/src/lib/shm_rbuff.c
@@ -1,4 +1,28 @@
-#if (defined (SHM_RBUFF_LOCKLESS) && \
+/*
+ * Ouroboros - Copyright (C) 2016 - 2017
+ *
+ * Ring buffer implementations for incoming SDUs
+ *
+ *    Dimitri Staessens <dimitri.staessens@ugent.be>
+ *    Sander Vrijders   <sander.vrijders@ugent.be>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
+#include <ouroboros/config.h>
+
+#if ((SHM_RBUFF_LOCKLESS > 0) &&                        \
      (defined(__GNUC__) || defined (__clang__)))
 #include "shm_rbuff_ll.c"
 #else
diff --git a/src/lib/shm_rbuff_ll.c b/src/lib/shm_rbuff_ll.c
index b5b8bed2..d777de8b 100644
--- a/src/lib/shm_rbuff_ll.c
+++ b/src/lib/shm_rbuff_ll.c
@@ -228,6 +228,8 @@ int shm_rbuff_write(struct shm_rbuff * rb,
         size_t ohead;
         size_t nhead;
 
+        bool was_empty = false;
+
         assert(rb);
         assert(idx < SHM_BUFFER_SIZE);
 
@@ -238,11 +240,9 @@ int shm_rbuff_write(struct shm_rbuff * rb,
                 return -EAGAIN;
 
         if (shm_rbuff_empty(rb))
-                pthread_cond_broadcast(rb->add);
+                was_empty = true;
 
-        ohead = RB_HEAD;
-
-        *head_el_ptr(rb) = (ssize_t) idx;
+        nhead = RB_HEAD;
 
         do {
                 ohead = nhead;
@@ -250,6 +250,11 @@ int shm_rbuff_write(struct shm_rbuff * rb,
                 nhead = __sync_val_compare_and_swap(rb->head, ohead, nhead);
         } while (nhead != ohead);
 
+        *(rb->shm_base + nhead) = (ssize_t) idx;
+
+        if (was_empty)
+                pthread_cond_broadcast(rb->add);
+
         return 0;
 }
 
@@ -263,7 +268,7 @@ ssize_t shm_rbuff_read(struct shm_rbuff * rb)
         if (shm_rbuff_empty(rb))
                 return -EAGAIN;
 
-        otail = RB_TAIL;
+        ntail = RB_TAIL;
 
         do {
                 otail = ntail;
@@ -273,7 +278,7 @@ ssize_t shm_rbuff_read(struct shm_rbuff * rb)
 
         pthread_cond_broadcast(rb->del);
 
-        return *tail_el_ptr(rb);
+        return *(rb->shm_base + ntail);
 }
 
 ssize_t shm_rbuff_read_b(struct shm_rbuff *      rb,
@@ -348,7 +353,7 @@ void shm_rbuff_fini(struct shm_rbuff * rb)
 
         assert(__sync_fetch_and_add(rb->acl, 0) == RB_CLOSED);
 
-        if (shm_rbuff_empty)
+        if (shm_rbuff_empty(rb))
                 return;
 
 #ifdef __APPLE__