Add barriers

Use smp_rmb() is busy-loops to make sure we re-read the variable.
Kick the reader threads we are waiting for after a few loops.
Documents memory ordering in rcu_read_lock().

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

diff --git a/urcu.c b/urcu.c
index 018e09b8601b041621fc6f72afdaeb57839e895c..0e42207e2bcd59d82127dc308def83b85c424798 100644 (file)
--- a/urcu.c
+++ b/urcu.c
@@ -77,11 +77,32 @@ static void switch_next_urcu_qparity(void)
 }
 
 #ifdef DEBUG_FULL_MB
+static void force_mb_single_thread(pthread_t tid)
+{
+       smp_mb();
+}
+
 static void force_mb_all_threads(void)
 {
        smp_mb();
 }
 #else
+
+static void force_mb_single_thread(pthread_t tid)
+{
+       assert(reader_data);
+       sig_done = 0;
+       smp_mb();       /* write sig_done before sending the signals */
+       pthread_kill(tid, SIGURCU);
+       /*
+        * Wait for sighandler (and thus mb()) to execute on every thread.
+        * BUSY-LOOP.
+        */
+       while (sig_done < 1)
+               smp_rmb();      /* ensure we re-read sig-done */
+       smp_mb();       /* read sig_done before ending the barrier */
+}
+
 static void force_mb_all_threads(void)
 {
        struct reader_data *index;
@@ -100,7 +121,7 @@ static void force_mb_all_threads(void)
         * BUSY-LOOP.
         */
        while (sig_done < num_readers)
-               barrier();
+               smp_rmb();      /* ensure we re-read sig-done */
        smp_mb();       /* read sig_done before ending the barrier */
 }
 #endif
@@ -111,14 +132,21 @@ void wait_for_quiescent_state(void)
 
        if (!reader_data)
                return;
-       /* Wait for each thread urcu_active_readers count to become 0.
+       /*
+        * Wait for each thread urcu_active_readers count to become 0.
         */
        for (index = reader_data; index < reader_data + num_readers; index++) {
+               int wait_loops = 0;
                /*
-                * BUSY-LOOP.
+                * BUSY-LOOP. Force the reader thread to commit its
+                * urcu_active_readers update to memory if we wait for too long.
                 */
-               while (rcu_old_gp_ongoing(index->urcu_active_readers))
-                       barrier();
+               while (rcu_old_gp_ongoing(index->urcu_active_readers)) {
+                       if (wait_loops++ == KICK_READER_LOOPS) {
+                               force_mb_single_thread(index->tid);
+                               wait_loops = 0;
+                       }
+               }
        }
 }
 
@@ -243,6 +271,11 @@ void urcu_unregister_thread(void)
 #ifndef DEBUG_FULL_MB
 void sigurcu_handler(int signo, siginfo_t *siginfo, void *context)
 {
+       /*
+        * Executing this smp_mb() is the only purpose of this signal handler.
+        * It punctually promotes barrier() into smp_mb() on every thread it is
+        * executed on.
+        */
        smp_mb();
        atomic_inc(&sig_done);
 }

diff --git a/urcu.h b/urcu.h
index 1b663c7ff1492e3beeab7832a592e6c2b7d90925..3eca5ea3c44ccc7135635229c41141d5050df1e2 100644 (file)
--- a/urcu.h
+++ b/urcu.h
@@ -130,6 +130,13 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
 
 #define SIGURCU SIGUSR1
 
+/*
+ * If a reader is really non-cooperative and refuses to commit its
+ * urcu_active_readers count to memory (there is no barrier in the reader
+ * per-se), kick it after a few loops waiting for it.
+ */
+#define KICK_READER_LOOPS 10000
+
 #ifdef DEBUG_YIELD
 #include <sched.h>
 #include <time.h>
@@ -234,6 +241,8 @@ static inline void rcu_read_lock(void)
 
        tmp = urcu_active_readers;
        /* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
+       /* The data dependency "read urcu_gp_ctr, write urcu_active_readers",
+        * serializes those two memory operations. */
        if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
                urcu_active_readers = urcu_gp_ctr;
        else