From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Date: Fri, 13 Feb 2009 15:37:58 +0000 (-0500)
Subject: Add _STORE_SHARED() and _LOAD_SHARED()
X-Git-Tag: v0.1~281
X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=b0d5e7908b8049b9f9e790d69e0f41179b5e415d;p=urcu.git

Add _STORE_SHARED() and _LOAD_SHARED()

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
---

diff --git a/urcu.c b/urcu.c
index 8cb32a1..5e2d2af 100644
--- a/urcu.c
+++ b/urcu.c
@@ -23,9 +23,14 @@ pthread_mutex_t urcu_mutex = PTHREAD_MUTEX_INITIALIZER;
  * Global grace period counter.
  * Contains the current RCU_GP_CTR_BIT.
  * Also has a RCU_GP_CTR_BIT of 1, to accelerate the reader fast path.
+ * Written to only by writer with mutex taken. Read by both writer and readers.
  */
 long urcu_gp_ctr = RCU_GP_COUNT;
 
+/*
+ * Written to only by each individual reader. Read by both the reader and the
+ * writers.
+ */
 long __thread urcu_active_readers;
 
 /* Thread IDs of registered readers */
@@ -73,7 +78,7 @@ void internal_urcu_unlock(void)
  */
 static void switch_next_urcu_qparity(void)
 {
-	urcu_gp_ctr ^= RCU_GP_CTR_BIT;
+	STORE_SHARED(urcu_gp_ctr, urcu_gp_ctr ^ RCU_GP_CTR_BIT);
 }
 
 #ifdef DEBUG_FULL_MB
@@ -183,8 +188,8 @@ void synchronize_rcu(void)
 	 * 0 quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data (no
 	 * progress).
+	 * Ensured by STORE_SHARED and LOAD_SHARED.
 	 */
-	smp_mc();
 
 	/*
 	 * Wait for previous parity to be empty of readers.
@@ -196,8 +201,8 @@ void synchronize_rcu(void)
 	 * committing qparity update to memory. Failure to do so could result in
 	 * the writer waiting forever while new readers are always accessing
 	 * data (no progress).
+	 * Ensured by STORE_SHARED and LOAD_SHARED.
 	 */
-	smp_mc();
 
 	switch_next_urcu_qparity();	/* 1 -> 0 */
 
@@ -206,8 +211,8 @@ void synchronize_rcu(void)
 	 * 1 quiescent state. Failure to do so could result in the writer
 	 * waiting forever while new readers are always accessing data (no
 	 * progress).
+	 * Ensured by STORE_SHARED and LOAD_SHARED.
 	 */
-	smp_mc();
 
 	/*
 	 * Wait for previous parity to be empty of readers.
diff --git a/urcu.h b/urcu.h
index df27032..539b1d5 100644
--- a/urcu.h
+++ b/urcu.h
@@ -179,14 +179,28 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  */
 #define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x))
 
+/*
+ * Identify a shared load. A smp_rmc() or smp_mc() should come before the load.
+ */
+#define _LOAD_SHARED(p)	       ACCESS_ONCE(p)
+
 /*
  * Load a data from shared memory, doing a cache flush if required.
  */
-#define LOAD_SHARED(p)	       ({ \
-				smp_rmc(); \
-				typeof(p) _________p1 = ACCESS_ONCE(p); \
-				(_________p1); \
-				})
+#define LOAD_SHARED(p) \
+	({ \
+		smp_rmc(); \
+		_LOAD_SHARED(p); \
+	})
+
+
+/*
+ * Identify a shared store. A smp_wmc() or smp_mc() should follow the store.
+ */
+#define _STORE_SHARED(x, v) \
+	do { \
+		(x) = (v); \
+	} while (0)
 
 /*
  * Store v into x, where x is located in shared memory. Performs the required
@@ -194,8 +208,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
  */
 #define STORE_SHARED(x, v) \
 	do { \
-		(x) = (v); \
-		smp_wmc; \
+		_STORE_SHARED(x, v); \
+		smp_wmc(); \
 	} while (0)
 
 /**
@@ -214,8 +228,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr,
 				(_________p1); \
 				})
 
-
-
 #define SIGURCU SIGUSR1
 
 /*
@@ -331,15 +343,15 @@ static inline void rcu_read_lock(void)
 	/* urcu_gp_ctr = RCU_GP_COUNT | (~RCU_GP_CTR_BIT or RCU_GP_CTR_BIT) */
 	/*
 	 * The data dependency "read urcu_gp_ctr, write urcu_active_readers",
-	 * serializes those two memory operations. We are not using STORE_SHARED
-	 * and LOAD_SHARED here (although we should) because the writer will
-	 * wake us up with a signal which does a flush in its handler to perform
-	 * urcu_gp_ctr re-read and urcu_active_readers commit to main memory.
+	 * serializes those two memory operations. The memory barrier in the
+	 * signal handler ensures we receive the proper memory commit barriers
+	 * required by _STORE_SHARED and _LOAD_SHARED whenever communication
+	 * with the writer is needed.
 	 */
 	if (likely(!(tmp & RCU_GP_CTR_NEST_MASK)))
-		urcu_active_readers = ACCESS_ONCE(urcu_gp_ctr);
+		_STORE_SHARED(urcu_active_readers, _LOAD_SHARED(urcu_gp_ctr));
 	else
-		urcu_active_readers = tmp + RCU_GP_COUNT;
+		_STORE_SHARED(urcu_active_readers, tmp + RCU_GP_COUNT);
 	/*
 	 * Increment active readers count before accessing the pointer.
 	 * See force_mb_all_threads().
@@ -354,7 +366,7 @@ static inline void rcu_read_unlock(void)
 	 * Finish using rcu before decrementing the pointer.
 	 * See force_mb_all_threads().
 	 */
-	urcu_active_readers -= RCU_GP_COUNT;
+	_STORE_SHARED(urcu_active_readers, urcu_active_readers - RCU_GP_COUNT);
 }
 
 /**
@@ -375,8 +387,7 @@ static inline void rcu_read_unlock(void)
 		if (!__builtin_constant_p(v) || \
 		    ((v) != NULL)) \
 			wmb(); \
-		(p) = (v); \
-		smp_wmc(); \
+		STORE_SHARED(p, v); \
 	})
 
 #define rcu_xchg_pointer(p, v) \