From: Mathieu Desnoyers Date: Fri, 10 Jun 2011 19:32:31 +0000 (-0400) Subject: Headers: move *-static.h headers to urcu/static/ X-Git-Tag: v0.6.0~14 X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=af7c2dbeac32c663b64ad05e4eca70e18784463b;p=userspace-rcu.git Headers: move *-static.h headers to urcu/static/ Signed-off-by: Mathieu Desnoyers --- diff --git a/Makefile.am b/Makefile.am index 9fb1568..5723b21 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,9 +10,7 @@ nobase_dist_include_HEADERS = urcu/compiler.h urcu/hlist.h urcu/list.h \ urcu/rculist.h urcu/rcuhlist.h urcu/system.h urcu/urcu-futex.h \ urcu/uatomic_generic.h urcu/arch_generic.h urcu/wfstack.h \ urcu/wfqueue.h urcu/rculfstack.h urcu/rculfqueue.h \ - urcu/wfqueue-static.h urcu/wfstack-static.h \ - urcu/rculfqueue-static.h urcu/rculfstack-static.h \ - urcu/urcu_ref.h urcu/map/*.h + urcu/urcu_ref.h urcu/map/*.h urcu/static/*.h nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic_arch.h urcu/config.h EXTRA_DIST = $(top_srcdir)/urcu/arch_*.h $(top_srcdir)/urcu/uatomic_arch_*.h \ diff --git a/rculfqueue.c b/rculfqueue.c index c844525..0daee5d 100644 --- a/rculfqueue.c +++ b/rculfqueue.c @@ -27,7 +27,7 @@ #undef _LGPL_SOURCE /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu/rculfqueue.h" -#include "urcu/rculfqueue-static.h" +#include "urcu/static/rculfqueue.h" /* * library wrappers to be used by non-LGPL compatible source code. diff --git a/rculfstack.c b/rculfstack.c index ca50232..4a3041d 100644 --- a/rculfstack.c +++ b/rculfstack.c @@ -27,7 +27,7 @@ #undef _LGPL_SOURCE /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu/rculfstack.h" -#include "urcu/rculfstack-static.h" +#include "urcu/static/rculfstack.h" /* * library wrappers to be used by non-LGPL compatible source code. diff --git a/urcu-bp-static.h b/urcu-bp-static.h deleted file mode 100644 index 14c6cfe..0000000 --- a/urcu-bp-static.h +++ /dev/null @@ -1,210 +0,0 @@ -#ifndef _URCU_BP_STATIC_H -#define _URCU_BP_STATIC_H - -/* - * urcu-bp-static.h - * - * Userspace RCU header. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking - * dynamically with the userspace rcu library. - * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * IBM's contributions to this file may be relicensed under LGPLv2 or later. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -/* - * This code section can only be included in LGPL 2.1 compatible source code. - * See below for the function call wrappers which can be used in code meant to - * be only linked with the Userspace RCU library. This comes with a small - * performance degradation on the read-side due to the added function calls. - * This is required to permit relinking with newer versions of the library. - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Active attempts to check for reader Q.S. before calling sleep(). - */ -#define RCU_QS_ACTIVE_ATTEMPTS 100 - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define YIELD_READ (1 << 0) -#define YIELD_WRITE (1 << 1) - -/* - * Updates without RCU_MB are much slower. Account this in - * the delay. - */ -/* maximum sleep delay, in us */ -#define MAX_SLEEP 50 - -extern unsigned int yield_active; -extern unsigned int __thread rand_yield; - -static inline void debug_yield_read(void) -{ - if (yield_active & YIELD_READ) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_write(void) -{ - if (yield_active & YIELD_WRITE) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_init(void) -{ - rand_yield = time(NULL) ^ pthread_self(); -} -#else -static inline void debug_yield_read(void) -{ -} - -static inline void debug_yield_write(void) -{ -} - -static inline void debug_yield_init(void) -{ - -} -#endif - -/* - * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a - * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. - */ -#define RCU_GP_COUNT (1UL << 0) -/* Use the amount of bits equal to half of the architecture long size */ -#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2)) -#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) - -/* - * Used internally by _rcu_read_lock. - */ -extern void rcu_bp_register(void); - -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern long rcu_gp_ctr; - -struct rcu_reader { - /* Data used by both reader and synchronize_rcu() */ - long ctr; - /* Data used for registry */ - struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); - pthread_t tid; - int alloc; /* registry entry allocated */ -}; - -/* - * Bulletproof version keeps a pointer to a registry not part of the TLS. - * Adds a pointer dereference on the read-side, but won't require to unregister - * the reader thread. - */ -extern struct rcu_reader __thread *rcu_reader; - -static inline int rcu_old_gp_ongoing(long *value) -{ - long v; - - if (value == NULL) - return 0; - /* - * Make sure both tests below are done on the same version of *value - * to insure consistency. - */ - v = CMM_LOAD_SHARED(*value); - return (v & RCU_GP_CTR_NEST_MASK) && - ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); -} - -static inline void _rcu_read_lock(void) -{ - long tmp; - - /* Check if registered */ - if (unlikely(!rcu_reader)) - rcu_bp_register(); - - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ - tmp = rcu_reader->ctr; - /* - * rcu_gp_ctr is - * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) - */ - if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { - _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); - /* - * Set active readers count for outermost nesting level before - * accessing the pointer. - */ - cmm_smp_mb(); - } else { - _CMM_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT); - } -} - -static inline void _rcu_read_unlock(void) -{ - /* - * Finish using rcu before decrementing the pointer. - */ - cmm_smp_mb(); - _CMM_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT); - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_BP_STATIC_H */ diff --git a/urcu-bp.c b/urcu-bp.c index 915debe..203642c 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -37,7 +37,7 @@ #include "urcu/map/urcu-bp.h" -#include "urcu-bp-static.h" +#include "urcu/static/urcu-bp.h" /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu-bp.h" diff --git a/urcu-bp.h b/urcu-bp.h index b2b382f..763eb05 100644 --- a/urcu-bp.h +++ b/urcu-bp.h @@ -38,7 +38,8 @@ #include /* - * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers. + * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer + * publication headers. */ #include @@ -58,7 +59,7 @@ extern "C" { #ifdef _LGPL_SOURCE -#include +#include /* * Mappings for static use of the userspace RCU library. @@ -78,7 +79,7 @@ extern "C" { /* * library wrappers to be used by non-LGPL compatible source code. - * See LGPL-only urcu-pointer-static.h for documentation. + * See LGPL-only urcu/static/urcu-pointer.h for documentation. */ extern void rcu_read_lock(void); diff --git a/urcu-pointer-static.h b/urcu-pointer-static.h deleted file mode 100644 index b644486..0000000 --- a/urcu-pointer-static.h +++ /dev/null @@ -1,132 +0,0 @@ -#ifndef _URCU_POINTER_STATIC_H -#define _URCU_POINTER_STATIC_H - -/* - * urcu-pointer-static.h - * - * Userspace RCU header. Operations on pointers. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-pointer.h for - * linking dynamically with the userspace rcu library. - * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * IBM's contributions to this file may be relicensed under LGPLv2 or later. - */ - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable - * into a RCU read-side critical section. The pointer can later be safely - * dereferenced within the critical section. - * - * This ensures that the pointer copy is invariant thorough the whole critical - * section. - * - * Inserts memory barriers on architectures that require them (currently only - * Alpha) and documents which pointers are protected by RCU. - * - * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative - * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the - * data read before the pointer read by speculating the value of the pointer. - * Correct ordering is ensured because the pointer is read as a volatile access. - * This acts as a global side-effect operation, which forbids reordering of - * dependent memory operations. Note that such concern about dependency-breaking - * optimizations will eventually be taken care of by the "memory_order_consume" - * addition to forthcoming C++ standard. - * - * Should match rcu_assign_pointer() or rcu_xchg_pointer(). - */ - -#define _rcu_dereference(p) ({ \ - typeof(p) _________p1 = CMM_LOAD_SHARED(p); \ - cmm_smp_read_barrier_depends(); \ - (_________p1); \ - }) - -/** - * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer - * is as expected by "old". If succeeds, returns the previous pointer to the - * data structure, which can be safely freed after waiting for a quiescent state - * using synchronize_rcu(). If fails (unexpected value), returns old (which - * should not be freed !). - */ - -#define _rcu_cmpxchg_pointer(p, old, _new) \ - ({ \ - typeof(*p) _________pold = (old); \ - typeof(*p) _________pnew = (_new); \ - if (!__builtin_constant_p(_new) || \ - ((_new) != NULL)) \ - cmm_wmb(); \ - uatomic_cmpxchg(p, _________pold, _________pnew); \ - }) - -/** - * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous - * pointer to the data structure, which can be safely freed after waiting for a - * quiescent state using synchronize_rcu(). - */ - -#define _rcu_xchg_pointer(p, v) \ - ({ \ - typeof(*p) _________pv = (v); \ - if (!__builtin_constant_p(v) || \ - ((v) != NULL)) \ - cmm_wmb(); \ - uatomic_xchg(p, _________pv); \ - }) - - -#define _rcu_set_pointer(p, v) \ - ({ \ - typeof(*p) _________pv = (v); \ - if (!__builtin_constant_p(v) || \ - ((v) != NULL)) \ - cmm_wmb(); \ - uatomic_set(p, _________pv); \ - }) - -/** - * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure - * meant to be read by RCU read-side critical sections. Returns the assigned - * value. - * - * Documents which pointers will be dereferenced by RCU read-side critical - * sections and adds the required memory barriers on architectures requiring - * them. It also makes sure the compiler does not reorder code initializing the - * data structure before its publication. - * - * Should match rcu_dereference_pointer(). - */ - -#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v) - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_POINTER_STATIC_H */ diff --git a/urcu-pointer.c b/urcu-pointer.c index 180c834..7dfb53a 100644 --- a/urcu-pointer.c +++ b/urcu-pointer.c @@ -26,7 +26,7 @@ #include -#include "urcu-pointer-static.h" +#include "urcu/static/urcu-pointer.h" /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu-pointer.h" diff --git a/urcu-pointer.h b/urcu-pointer.h index 7b240a7..359a99f 100644 --- a/urcu-pointer.h +++ b/urcu-pointer.h @@ -36,7 +36,7 @@ extern "C" { #ifdef _LGPL_SOURCE -#include +#include /* * rcu_dereference(ptr) diff --git a/urcu-qsbr-static.h b/urcu-qsbr-static.h deleted file mode 100644 index e0b12be..0000000 --- a/urcu-qsbr-static.h +++ /dev/null @@ -1,204 +0,0 @@ -#ifndef _URCU_QSBR_STATIC_H -#define _URCU_QSBR_STATIC_H - -/* - * urcu-qsbr-static.h - * - * Userspace RCU QSBR header. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-qsbr.h for linking - * dynamically with the userspace rcu QSBR library. - * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * IBM's contributions to this file may be relicensed under LGPLv2 or later. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * This code section can only be included in LGPL 2.1 compatible source code. - * See below for the function call wrappers which can be used in code meant to - * be only linked with the Userspace RCU library. This comes with a small - * performance degradation on the read-side due to the added function calls. - * This is required to permit relinking with newer versions of the library. - */ - -/* - * If a reader is really non-cooperative and refuses to commit its - * rcu_reader.ctr count to memory (there is no barrier in the reader - * per-se), kick it after a few loops waiting for it. - */ -#define KICK_READER_LOOPS 10000 - -/* - * Active attempts to check for reader Q.S. before calling futex(). - */ -#define RCU_QS_ACTIVE_ATTEMPTS 100 - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define YIELD_READ (1 << 0) -#define YIELD_WRITE (1 << 1) - -/* maximum sleep delay, in us */ -#define MAX_SLEEP 50 - -extern unsigned int yield_active; -extern unsigned int __thread rand_yield; - -static inline void debug_yield_read(void) -{ - if (yield_active & YIELD_READ) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_write(void) -{ - if (yield_active & YIELD_WRITE) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_init(void) -{ - rand_yield = time(NULL) ^ pthread_self(); -} -#else -static inline void debug_yield_read(void) -{ -} - -static inline void debug_yield_write(void) -{ -} - -static inline void debug_yield_init(void) -{ - -} -#endif - -#define RCU_GP_ONLINE (1UL << 0) -#define RCU_GP_CTR (1UL << 1) - -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern unsigned long rcu_gp_ctr; - -struct rcu_reader { - /* Data used by both reader and synchronize_rcu() */ - unsigned long ctr; - /* Data used for registry */ - struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); - pthread_t tid; -}; - -extern struct rcu_reader __thread rcu_reader; - -extern int gp_futex; - -/* - * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. - */ -static inline void wake_up_gp(void) -{ - if (unlikely(uatomic_read(&gp_futex) == -1)) { - uatomic_set(&gp_futex, 0); - futex_noasync(&gp_futex, FUTEX_WAKE, 1, - NULL, NULL, 0); - } -} - -static inline int rcu_gp_ongoing(unsigned long *ctr) -{ - unsigned long v; - - v = CMM_LOAD_SHARED(*ctr); - return v && (v != rcu_gp_ctr); -} - -static inline void _rcu_read_lock(void) -{ - rcu_assert(rcu_reader.ctr); -} - -static inline void _rcu_read_unlock(void) -{ -} - -static inline void _rcu_quiescent_state(void) -{ - cmm_smp_mb(); - _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); - cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ - wake_up_gp(); - cmm_smp_mb(); -} - -static inline void _rcu_thread_offline(void) -{ - cmm_smp_mb(); - CMM_STORE_SHARED(rcu_reader.ctr, 0); - cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ - wake_up_gp(); - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ -} - -static inline void _rcu_thread_online(void) -{ - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ - _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr)); - cmm_smp_mb(); -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_QSBR_STATIC_H */ diff --git a/urcu-qsbr.c b/urcu-qsbr.c index bb10bb8..5eda71d 100644 --- a/urcu-qsbr.c +++ b/urcu-qsbr.c @@ -36,7 +36,7 @@ #include "urcu/map/urcu-qsbr.h" #define BUILD_QSBR_LIB -#include "urcu-qsbr-static.h" +#include "urcu/static/urcu-qsbr.h" /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu-qsbr.h" diff --git a/urcu-qsbr.h b/urcu-qsbr.h index 1d5c568..20dbf47 100644 --- a/urcu-qsbr.h +++ b/urcu-qsbr.h @@ -32,7 +32,8 @@ #include /* - * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers. + * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer + * publication headers. */ #include @@ -52,7 +53,7 @@ extern "C" { #ifdef _LGPL_SOURCE -#include +#include /* * Mappings for static use of the userspace RCU library. diff --git a/urcu-static.h b/urcu-static.h deleted file mode 100644 index 18e4826..0000000 --- a/urcu-static.h +++ /dev/null @@ -1,305 +0,0 @@ -#ifndef _URCU_STATIC_H -#define _URCU_STATIC_H - -/* - * urcu-static.h - * - * Userspace RCU header. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking - * dynamically with the userspace rcu library. - * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * IBM's contributions to this file may be relicensed under LGPLv2 or later. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* Default is RCU_MEMBARRIER */ -#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL) -#define RCU_MEMBARRIER -#endif - -#ifdef RCU_MEMBARRIER -#include -#include - -/* If the headers do not support SYS_membarrier, statically use RCU_MB */ -#ifdef SYS_membarrier -# define MEMBARRIER_EXPEDITED (1 << 0) -# define MEMBARRIER_DELAYED (1 << 1) -# define MEMBARRIER_QUERY (1 << 16) -# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) -#else -# undef RCU_MEMBARRIER -# define RCU_MB -#endif -#endif - -/* - * This code section can only be included in LGPL 2.1 compatible source code. - * See below for the function call wrappers which can be used in code meant to - * be only linked with the Userspace RCU library. This comes with a small - * performance degradation on the read-side due to the added function calls. - * This is required to permit relinking with newer versions of the library. - */ - -/* - * The signal number used by the RCU library can be overridden with - * -DSIGRCU= when compiling the library. - * Provide backward compatibility for liburcu 0.3.x SIGURCU. - */ -#ifdef SIGURCU -#define SIGRCU SIGURCU -#endif - -#ifndef SIGRCU -#define SIGRCU SIGUSR1 -#endif - -/* - * If a reader is really non-cooperative and refuses to commit its - * rcu_active_readers count to memory (there is no barrier in the reader - * per-se), kick it after a few loops waiting for it. - */ -#define KICK_READER_LOOPS 10000 - -/* - * Active attempts to check for reader Q.S. before calling futex(). - */ -#define RCU_QS_ACTIVE_ATTEMPTS 100 - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define YIELD_READ (1 << 0) -#define YIELD_WRITE (1 << 1) - -/* - * Updates with RCU_SIGNAL are much slower. Account this in the delay. - */ -#ifdef RCU_SIGNAL -/* maximum sleep delay, in us */ -#define MAX_SLEEP 30000 -#else -#define MAX_SLEEP 50 -#endif - -extern unsigned int yield_active; -extern unsigned int __thread rand_yield; - -static inline void debug_yield_read(void) -{ - if (yield_active & YIELD_READ) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_write(void) -{ - if (yield_active & YIELD_WRITE) - if (rand_r(&rand_yield) & 0x1) - usleep(rand_r(&rand_yield) % MAX_SLEEP); -} - -static inline void debug_yield_init(void) -{ - rand_yield = time(NULL) ^ pthread_self(); -} -#else -static inline void debug_yield_read(void) -{ -} - -static inline void debug_yield_write(void) -{ -} - -static inline void debug_yield_init(void) -{ - -} -#endif - -/* - * RCU memory barrier broadcast group. Currently, only broadcast to all process - * threads is supported (group 0). - * - * Slave barriers are only guaranteed to be ordered wrt master barriers. - * - * The pair ordering is detailed as (O: ordered, X: not ordered) : - * slave master - * slave X O - * master O O - */ - -#define MB_GROUP_ALL 0 -#define RCU_MB_GROUP MB_GROUP_ALL - -#ifdef RCU_MEMBARRIER -extern int has_sys_membarrier; - -static inline void smp_mb_slave(int group) -{ - if (likely(has_sys_membarrier)) - cmm_barrier(); - else - cmm_smp_mb(); -} -#endif - -#ifdef RCU_MB -static inline void smp_mb_slave(int group) -{ - cmm_smp_mb(); -} -#endif - -#ifdef RCU_SIGNAL -static inline void smp_mb_slave(int group) -{ - cmm_barrier(); -} -#endif - -/* - * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use - * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. - */ -#define RCU_GP_COUNT (1UL << 0) -/* Use the amount of bits equal to half of the architecture long size */ -#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2)) -#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) - -/* - * Global quiescent period counter with low-order bits unused. - * Using a int rather than a char to eliminate false register dependencies - * causing stalls on some architectures. - */ -extern unsigned long rcu_gp_ctr; - -struct rcu_reader { - /* Data used by both reader and synchronize_rcu() */ - unsigned long ctr; - char need_mb; - /* Data used for registry */ - struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); - pthread_t tid; -}; - -extern struct rcu_reader __thread rcu_reader; - -extern int gp_futex; - -/* - * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. - */ -static inline void wake_up_gp(void) -{ - if (unlikely(uatomic_read(&gp_futex) == -1)) { - uatomic_set(&gp_futex, 0); - futex_async(&gp_futex, FUTEX_WAKE, 1, - NULL, NULL, 0); - } -} - -static inline int rcu_gp_ongoing(unsigned long *ctr) -{ - unsigned long v; - - /* - * Make sure both tests below are done on the same version of *value - * to insure consistency. - */ - v = CMM_LOAD_SHARED(*ctr); - return (v & RCU_GP_CTR_NEST_MASK) && - ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); -} - -static inline void _rcu_read_lock(void) -{ - unsigned long tmp; - - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ - tmp = rcu_reader.ctr; - /* - * rcu_gp_ctr is - * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) - */ - if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { - _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); - /* - * Set active readers count for outermost nesting level before - * accessing the pointer. See smp_mb_master(). - */ - smp_mb_slave(RCU_MB_GROUP); - } else { - _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT); - } -} - -static inline void _rcu_read_unlock(void) -{ - unsigned long tmp; - - tmp = rcu_reader.ctr; - /* - * Finish using rcu before decrementing the pointer. - * See smp_mb_master(). - */ - if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { - smp_mb_slave(RCU_MB_GROUP); - _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); - /* write rcu_reader.ctr before read futex */ - smp_mb_slave(RCU_MB_GROUP); - wake_up_gp(); - } else { - _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); - } - cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_STATIC_H */ diff --git a/urcu.c b/urcu.c index 98c5fbc..ccaf3e9 100644 --- a/urcu.c +++ b/urcu.c @@ -36,7 +36,7 @@ #include "urcu/map/urcu.h" -#include "urcu-static.h" +#include "urcu/static/urcu.h" /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu.h" diff --git a/urcu.h b/urcu.h index 0fa2fea..d4c8753 100644 --- a/urcu.h +++ b/urcu.h @@ -35,7 +35,8 @@ #include /* - * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers. + * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer + * publication headers. */ #include @@ -55,7 +56,7 @@ extern "C" { #ifdef _LGPL_SOURCE -#include +#include /* * Mappings for static use of the userspace RCU library. @@ -85,7 +86,7 @@ extern "C" { /* * library wrappers to be used by non-LGPL compatible source code. - * See LGPL-only urcu-pointer-static.h for documentation. + * See LGPL-only urcu/static/urcu-pointer.h for documentation. */ extern void rcu_read_lock(void); diff --git a/urcu/rculfqueue-static.h b/urcu/rculfqueue-static.h deleted file mode 100644 index 410a4cf..0000000 --- a/urcu/rculfqueue-static.h +++ /dev/null @@ -1,144 +0,0 @@ -#ifndef _URCU_RCULFQUEUE_STATIC_H -#define _URCU_RCULFQUEUE_STATIC_H - -/* - * rculfqueue-static.h - * - * Userspace RCU library - Lock-Free RCU Queue - * - * Copyright 2010 - Mathieu Desnoyers - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking - * dynamically with the userspace rcu library. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include -#include -#include -/* A urcu implementation header should be already included. */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Lock-free RCU queue using reference counting. Enqueue and dequeue operations - * hold a RCU read lock to deal with cmpxchg ABA problem. This implementation - * keeps a dummy head node to ensure we can always update the queue locklessly. - * Given that this is a queue, the dummy head node must always advance as we - * dequeue entries. Therefore, we keep a reference count on each entry we are - * dequeueing, so they can be kept as dummy head node until the next dequeue, at - * which point their reference count will be decremented. - */ - -#define URCU_LFQ_PERMANENT_REF 128 - -void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node) -{ - node->next = NULL; - urcu_ref_init(&node->ref); -} - -void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q, - void (*release)(struct urcu_ref *ref)) -{ - _cds_lfq_node_init_rcu(&q->init); - /* Make sure the initial node is never freed. */ - urcu_ref_set(&q->init.ref, URCU_LFQ_PERMANENT_REF); - q->head = q->tail = &q->init; - q->release = release; -} - -/* - * Should be called under rcu read lock critical section. - */ -void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q, - struct cds_lfq_node_rcu *node) -{ - urcu_ref_get(&node->ref); - node->queue = q; - - /* - * uatomic_cmpxchg() implicit memory barrier orders earlier stores to - * node before publication. - */ - - for (;;) { - struct cds_lfq_node_rcu *tail, *next; - - tail = rcu_dereference(q->tail); - /* - * Typically expect tail->next to be NULL. - */ - next = uatomic_cmpxchg(&tail->next, NULL, node); - if (next == NULL) { - /* - * Tail was at the end of queue, we successfully - * appended to it. - * Now move tail (another enqueue might beat - * us to it, that's fine). - */ - (void) uatomic_cmpxchg(&q->tail, tail, node); - return; - } else { - /* - * Failure to append to current tail. Help moving tail - * further and retry. - */ - (void) uatomic_cmpxchg(&q->tail, tail, next); - continue; - } - } -} - -/* - * Should be called under rcu read lock critical section. - * - * The entry returned by dequeue must be taken care of by doing a - * sequence of urcu_ref_put which release handler should do a call_rcu. - * - * In other words, the entry lfq node returned by dequeue must not be - * modified/re-used/freed until the reference count reaches zero and a grace - * period has elapsed. - */ -struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q) -{ - for (;;) { - struct cds_lfq_node_rcu *head, *next; - - head = rcu_dereference(q->head); - next = rcu_dereference(head->next); - if (next) { - if (uatomic_cmpxchg(&q->head, head, next) == head) { - urcu_ref_put(&head->ref, q->release); - return next; - } else { - /* Concurrently pushed, retry */ - continue; - } - } else { - /* Empty */ - return NULL; - } - } -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_RCULFQUEUE_STATIC_H */ diff --git a/urcu/rculfqueue.h b/urcu/rculfqueue.h index fa54ca5..c3084b7 100644 --- a/urcu/rculfqueue.h +++ b/urcu/rculfqueue.h @@ -57,7 +57,7 @@ struct cds_lfq_queue_rcu { #ifdef _LGPL_SOURCE -#include +#include #define cds_lfq_node_init_rcu _cds_lfq_node_init_rcu #define cds_lfq_init_rcu _cds_lfq_init_rcu diff --git a/urcu/rculfstack-static.h b/urcu/rculfstack-static.h deleted file mode 100644 index 7caf3c8..0000000 --- a/urcu/rculfstack-static.h +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef _URCU_RCULFSTACK_STATIC_H -#define _URCU_RCULFSTACK_STATIC_H - -/* - * rculfstack-static.h - * - * Userspace RCU library - Lock-Free RCU Stack - * - * Copyright 2010 - Mathieu Desnoyers - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking - * dynamically with the userspace rcu library. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include -/* A urcu implementation header should be already included. */ - -#ifdef __cplusplus -extern "C" { -#endif - -void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node) -{ -} - -void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s) -{ - s->head = NULL; -} - -void _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s, struct cds_lfs_node_rcu *node) -{ - struct cds_lfs_node_rcu *head = NULL; - - for (;;) { - struct cds_lfs_node_rcu *old_head = head; - - node->next = head; - /* - * uatomic_cmpxchg() implicit memory barrier orders earlier - * stores to node before publication. - */ - head = uatomic_cmpxchg(&s->head, old_head, node); - if (old_head == head) - break; - } -} - -/* - * Should be called under rcu read-side lock. - * - * The caller must wait for a grace period to pass before freeing the returned - * node or modifying the cds_lfs_node_rcu structure. - * Returns NULL if stack is empty. - */ -struct cds_lfs_node_rcu * -_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s) -{ - for (;;) { - struct cds_lfs_node_rcu *head; - - head = rcu_dereference(s->head); - if (head) { - struct cds_lfs_node_rcu *next = rcu_dereference(head->next); - - if (uatomic_cmpxchg(&s->head, head, next) == head) { - return head; - } else { - /* Concurrent modification. Retry. */ - continue; - } - } else { - /* Empty stack */ - return NULL; - } - } -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_RCULFSTACK_STATIC_H */ diff --git a/urcu/rculfstack.h b/urcu/rculfstack.h index d759854..6a240fd 100644 --- a/urcu/rculfstack.h +++ b/urcu/rculfstack.h @@ -37,7 +37,7 @@ struct cds_lfs_stack_rcu { #ifdef _LGPL_SOURCE -#include +#include #define cds_lfs_node_init_rcu _cds_lfs_node_init_rcu #define cds_lfs_init_rcu _cds_lfs_init_rcu diff --git a/urcu/static/rculfqueue.h b/urcu/static/rculfqueue.h new file mode 100644 index 0000000..410a4cf --- /dev/null +++ b/urcu/static/rculfqueue.h @@ -0,0 +1,144 @@ +#ifndef _URCU_RCULFQUEUE_STATIC_H +#define _URCU_RCULFQUEUE_STATIC_H + +/* + * rculfqueue-static.h + * + * Userspace RCU library - Lock-Free RCU Queue + * + * Copyright 2010 - Mathieu Desnoyers + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking + * dynamically with the userspace rcu library. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +/* A urcu implementation header should be already included. */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Lock-free RCU queue using reference counting. Enqueue and dequeue operations + * hold a RCU read lock to deal with cmpxchg ABA problem. This implementation + * keeps a dummy head node to ensure we can always update the queue locklessly. + * Given that this is a queue, the dummy head node must always advance as we + * dequeue entries. Therefore, we keep a reference count on each entry we are + * dequeueing, so they can be kept as dummy head node until the next dequeue, at + * which point their reference count will be decremented. + */ + +#define URCU_LFQ_PERMANENT_REF 128 + +void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node) +{ + node->next = NULL; + urcu_ref_init(&node->ref); +} + +void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q, + void (*release)(struct urcu_ref *ref)) +{ + _cds_lfq_node_init_rcu(&q->init); + /* Make sure the initial node is never freed. */ + urcu_ref_set(&q->init.ref, URCU_LFQ_PERMANENT_REF); + q->head = q->tail = &q->init; + q->release = release; +} + +/* + * Should be called under rcu read lock critical section. + */ +void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q, + struct cds_lfq_node_rcu *node) +{ + urcu_ref_get(&node->ref); + node->queue = q; + + /* + * uatomic_cmpxchg() implicit memory barrier orders earlier stores to + * node before publication. + */ + + for (;;) { + struct cds_lfq_node_rcu *tail, *next; + + tail = rcu_dereference(q->tail); + /* + * Typically expect tail->next to be NULL. + */ + next = uatomic_cmpxchg(&tail->next, NULL, node); + if (next == NULL) { + /* + * Tail was at the end of queue, we successfully + * appended to it. + * Now move tail (another enqueue might beat + * us to it, that's fine). + */ + (void) uatomic_cmpxchg(&q->tail, tail, node); + return; + } else { + /* + * Failure to append to current tail. Help moving tail + * further and retry. + */ + (void) uatomic_cmpxchg(&q->tail, tail, next); + continue; + } + } +} + +/* + * Should be called under rcu read lock critical section. + * + * The entry returned by dequeue must be taken care of by doing a + * sequence of urcu_ref_put which release handler should do a call_rcu. + * + * In other words, the entry lfq node returned by dequeue must not be + * modified/re-used/freed until the reference count reaches zero and a grace + * period has elapsed. + */ +struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q) +{ + for (;;) { + struct cds_lfq_node_rcu *head, *next; + + head = rcu_dereference(q->head); + next = rcu_dereference(head->next); + if (next) { + if (uatomic_cmpxchg(&q->head, head, next) == head) { + urcu_ref_put(&head->ref, q->release); + return next; + } else { + /* Concurrently pushed, retry */ + continue; + } + } else { + /* Empty */ + return NULL; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_RCULFQUEUE_STATIC_H */ diff --git a/urcu/static/rculfstack.h b/urcu/static/rculfstack.h new file mode 100644 index 0000000..7caf3c8 --- /dev/null +++ b/urcu/static/rculfstack.h @@ -0,0 +1,97 @@ +#ifndef _URCU_RCULFSTACK_STATIC_H +#define _URCU_RCULFSTACK_STATIC_H + +/* + * rculfstack-static.h + * + * Userspace RCU library - Lock-Free RCU Stack + * + * Copyright 2010 - Mathieu Desnoyers + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking + * dynamically with the userspace rcu library. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +/* A urcu implementation header should be already included. */ + +#ifdef __cplusplus +extern "C" { +#endif + +void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node) +{ +} + +void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s) +{ + s->head = NULL; +} + +void _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s, struct cds_lfs_node_rcu *node) +{ + struct cds_lfs_node_rcu *head = NULL; + + for (;;) { + struct cds_lfs_node_rcu *old_head = head; + + node->next = head; + /* + * uatomic_cmpxchg() implicit memory barrier orders earlier + * stores to node before publication. + */ + head = uatomic_cmpxchg(&s->head, old_head, node); + if (old_head == head) + break; + } +} + +/* + * Should be called under rcu read-side lock. + * + * The caller must wait for a grace period to pass before freeing the returned + * node or modifying the cds_lfs_node_rcu structure. + * Returns NULL if stack is empty. + */ +struct cds_lfs_node_rcu * +_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s) +{ + for (;;) { + struct cds_lfs_node_rcu *head; + + head = rcu_dereference(s->head); + if (head) { + struct cds_lfs_node_rcu *next = rcu_dereference(head->next); + + if (uatomic_cmpxchg(&s->head, head, next) == head) { + return head; + } else { + /* Concurrent modification. Retry. */ + continue; + } + } else { + /* Empty stack */ + return NULL; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_RCULFSTACK_STATIC_H */ diff --git a/urcu/static/urcu-bp.h b/urcu/static/urcu-bp.h new file mode 100644 index 0000000..14c6cfe --- /dev/null +++ b/urcu/static/urcu-bp.h @@ -0,0 +1,210 @@ +#ifndef _URCU_BP_STATIC_H +#define _URCU_BP_STATIC_H + +/* + * urcu-bp-static.h + * + * Userspace RCU header. + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking + * dynamically with the userspace rcu library. + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* + * This code section can only be included in LGPL 2.1 compatible source code. + * See below for the function call wrappers which can be used in code meant to + * be only linked with the Userspace RCU library. This comes with a small + * performance degradation on the read-side due to the added function calls. + * This is required to permit relinking with newer versions of the library. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Active attempts to check for reader Q.S. before calling sleep(). + */ +#define RCU_QS_ACTIVE_ATTEMPTS 100 + +#ifdef DEBUG_RCU +#define rcu_assert(args...) assert(args) +#else +#define rcu_assert(args...) +#endif + +#ifdef DEBUG_YIELD +#include +#include +#include +#include + +#define YIELD_READ (1 << 0) +#define YIELD_WRITE (1 << 1) + +/* + * Updates without RCU_MB are much slower. Account this in + * the delay. + */ +/* maximum sleep delay, in us */ +#define MAX_SLEEP 50 + +extern unsigned int yield_active; +extern unsigned int __thread rand_yield; + +static inline void debug_yield_read(void) +{ + if (yield_active & YIELD_READ) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_write(void) +{ + if (yield_active & YIELD_WRITE) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_init(void) +{ + rand_yield = time(NULL) ^ pthread_self(); +} +#else +static inline void debug_yield_read(void) +{ +} + +static inline void debug_yield_write(void) +{ +} + +static inline void debug_yield_init(void) +{ + +} +#endif + +/* + * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a + * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. + */ +#define RCU_GP_COUNT (1UL << 0) +/* Use the amount of bits equal to half of the architecture long size */ +#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2)) +#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) + +/* + * Used internally by _rcu_read_lock. + */ +extern void rcu_bp_register(void); + +/* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register dependencies + * causing stalls on some architectures. + */ +extern long rcu_gp_ctr; + +struct rcu_reader { + /* Data used by both reader and synchronize_rcu() */ + long ctr; + /* Data used for registry */ + struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + pthread_t tid; + int alloc; /* registry entry allocated */ +}; + +/* + * Bulletproof version keeps a pointer to a registry not part of the TLS. + * Adds a pointer dereference on the read-side, but won't require to unregister + * the reader thread. + */ +extern struct rcu_reader __thread *rcu_reader; + +static inline int rcu_old_gp_ongoing(long *value) +{ + long v; + + if (value == NULL) + return 0; + /* + * Make sure both tests below are done on the same version of *value + * to insure consistency. + */ + v = CMM_LOAD_SHARED(*value); + return (v & RCU_GP_CTR_NEST_MASK) && + ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); +} + +static inline void _rcu_read_lock(void) +{ + long tmp; + + /* Check if registered */ + if (unlikely(!rcu_reader)) + rcu_bp_register(); + + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ + tmp = rcu_reader->ctr; + /* + * rcu_gp_ctr is + * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) + */ + if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { + _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); + /* + * Set active readers count for outermost nesting level before + * accessing the pointer. + */ + cmm_smp_mb(); + } else { + _CMM_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT); + } +} + +static inline void _rcu_read_unlock(void) +{ + /* + * Finish using rcu before decrementing the pointer. + */ + cmm_smp_mb(); + _CMM_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT); + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_BP_STATIC_H */ diff --git a/urcu/static/urcu-pointer.h b/urcu/static/urcu-pointer.h new file mode 100644 index 0000000..b644486 --- /dev/null +++ b/urcu/static/urcu-pointer.h @@ -0,0 +1,132 @@ +#ifndef _URCU_POINTER_STATIC_H +#define _URCU_POINTER_STATIC_H + +/* + * urcu-pointer-static.h + * + * Userspace RCU header. Operations on pointers. + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-pointer.h for + * linking dynamically with the userspace rcu library. + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable + * into a RCU read-side critical section. The pointer can later be safely + * dereferenced within the critical section. + * + * This ensures that the pointer copy is invariant thorough the whole critical + * section. + * + * Inserts memory barriers on architectures that require them (currently only + * Alpha) and documents which pointers are protected by RCU. + * + * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative + * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the + * data read before the pointer read by speculating the value of the pointer. + * Correct ordering is ensured because the pointer is read as a volatile access. + * This acts as a global side-effect operation, which forbids reordering of + * dependent memory operations. Note that such concern about dependency-breaking + * optimizations will eventually be taken care of by the "memory_order_consume" + * addition to forthcoming C++ standard. + * + * Should match rcu_assign_pointer() or rcu_xchg_pointer(). + */ + +#define _rcu_dereference(p) ({ \ + typeof(p) _________p1 = CMM_LOAD_SHARED(p); \ + cmm_smp_read_barrier_depends(); \ + (_________p1); \ + }) + +/** + * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer + * is as expected by "old". If succeeds, returns the previous pointer to the + * data structure, which can be safely freed after waiting for a quiescent state + * using synchronize_rcu(). If fails (unexpected value), returns old (which + * should not be freed !). + */ + +#define _rcu_cmpxchg_pointer(p, old, _new) \ + ({ \ + typeof(*p) _________pold = (old); \ + typeof(*p) _________pnew = (_new); \ + if (!__builtin_constant_p(_new) || \ + ((_new) != NULL)) \ + cmm_wmb(); \ + uatomic_cmpxchg(p, _________pold, _________pnew); \ + }) + +/** + * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous + * pointer to the data structure, which can be safely freed after waiting for a + * quiescent state using synchronize_rcu(). + */ + +#define _rcu_xchg_pointer(p, v) \ + ({ \ + typeof(*p) _________pv = (v); \ + if (!__builtin_constant_p(v) || \ + ((v) != NULL)) \ + cmm_wmb(); \ + uatomic_xchg(p, _________pv); \ + }) + + +#define _rcu_set_pointer(p, v) \ + ({ \ + typeof(*p) _________pv = (v); \ + if (!__builtin_constant_p(v) || \ + ((v) != NULL)) \ + cmm_wmb(); \ + uatomic_set(p, _________pv); \ + }) + +/** + * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure + * meant to be read by RCU read-side critical sections. Returns the assigned + * value. + * + * Documents which pointers will be dereferenced by RCU read-side critical + * sections and adds the required memory barriers on architectures requiring + * them. It also makes sure the compiler does not reorder code initializing the + * data structure before its publication. + * + * Should match rcu_dereference_pointer(). + */ + +#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v) + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_POINTER_STATIC_H */ diff --git a/urcu/static/urcu-qsbr.h b/urcu/static/urcu-qsbr.h new file mode 100644 index 0000000..e0b12be --- /dev/null +++ b/urcu/static/urcu-qsbr.h @@ -0,0 +1,204 @@ +#ifndef _URCU_QSBR_STATIC_H +#define _URCU_QSBR_STATIC_H + +/* + * urcu-qsbr-static.h + * + * Userspace RCU QSBR header. + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-qsbr.h for linking + * dynamically with the userspace rcu QSBR library. + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This code section can only be included in LGPL 2.1 compatible source code. + * See below for the function call wrappers which can be used in code meant to + * be only linked with the Userspace RCU library. This comes with a small + * performance degradation on the read-side due to the added function calls. + * This is required to permit relinking with newer versions of the library. + */ + +/* + * If a reader is really non-cooperative and refuses to commit its + * rcu_reader.ctr count to memory (there is no barrier in the reader + * per-se), kick it after a few loops waiting for it. + */ +#define KICK_READER_LOOPS 10000 + +/* + * Active attempts to check for reader Q.S. before calling futex(). + */ +#define RCU_QS_ACTIVE_ATTEMPTS 100 + +#ifdef DEBUG_RCU +#define rcu_assert(args...) assert(args) +#else +#define rcu_assert(args...) +#endif + +#ifdef DEBUG_YIELD +#include +#include +#include +#include + +#define YIELD_READ (1 << 0) +#define YIELD_WRITE (1 << 1) + +/* maximum sleep delay, in us */ +#define MAX_SLEEP 50 + +extern unsigned int yield_active; +extern unsigned int __thread rand_yield; + +static inline void debug_yield_read(void) +{ + if (yield_active & YIELD_READ) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_write(void) +{ + if (yield_active & YIELD_WRITE) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_init(void) +{ + rand_yield = time(NULL) ^ pthread_self(); +} +#else +static inline void debug_yield_read(void) +{ +} + +static inline void debug_yield_write(void) +{ +} + +static inline void debug_yield_init(void) +{ + +} +#endif + +#define RCU_GP_ONLINE (1UL << 0) +#define RCU_GP_CTR (1UL << 1) + +/* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register dependencies + * causing stalls on some architectures. + */ +extern unsigned long rcu_gp_ctr; + +struct rcu_reader { + /* Data used by both reader and synchronize_rcu() */ + unsigned long ctr; + /* Data used for registry */ + struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + pthread_t tid; +}; + +extern struct rcu_reader __thread rcu_reader; + +extern int gp_futex; + +/* + * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. + */ +static inline void wake_up_gp(void) +{ + if (unlikely(uatomic_read(&gp_futex) == -1)) { + uatomic_set(&gp_futex, 0); + futex_noasync(&gp_futex, FUTEX_WAKE, 1, + NULL, NULL, 0); + } +} + +static inline int rcu_gp_ongoing(unsigned long *ctr) +{ + unsigned long v; + + v = CMM_LOAD_SHARED(*ctr); + return v && (v != rcu_gp_ctr); +} + +static inline void _rcu_read_lock(void) +{ + rcu_assert(rcu_reader.ctr); +} + +static inline void _rcu_read_unlock(void) +{ +} + +static inline void _rcu_quiescent_state(void) +{ + cmm_smp_mb(); + _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); + cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ + wake_up_gp(); + cmm_smp_mb(); +} + +static inline void _rcu_thread_offline(void) +{ + cmm_smp_mb(); + CMM_STORE_SHARED(rcu_reader.ctr, 0); + cmm_smp_mb(); /* write rcu_reader.ctr before read futex */ + wake_up_gp(); + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ +} + +static inline void _rcu_thread_online(void) +{ + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ + _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr)); + cmm_smp_mb(); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_QSBR_STATIC_H */ diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h new file mode 100644 index 0000000..18e4826 --- /dev/null +++ b/urcu/static/urcu.h @@ -0,0 +1,305 @@ +#ifndef _URCU_STATIC_H +#define _URCU_STATIC_H + +/* + * urcu-static.h + * + * Userspace RCU header. + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking + * dynamically with the userspace rcu library. + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Default is RCU_MEMBARRIER */ +#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL) +#define RCU_MEMBARRIER +#endif + +#ifdef RCU_MEMBARRIER +#include +#include + +/* If the headers do not support SYS_membarrier, statically use RCU_MB */ +#ifdef SYS_membarrier +# define MEMBARRIER_EXPEDITED (1 << 0) +# define MEMBARRIER_DELAYED (1 << 1) +# define MEMBARRIER_QUERY (1 << 16) +# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__) +#else +# undef RCU_MEMBARRIER +# define RCU_MB +#endif +#endif + +/* + * This code section can only be included in LGPL 2.1 compatible source code. + * See below for the function call wrappers which can be used in code meant to + * be only linked with the Userspace RCU library. This comes with a small + * performance degradation on the read-side due to the added function calls. + * This is required to permit relinking with newer versions of the library. + */ + +/* + * The signal number used by the RCU library can be overridden with + * -DSIGRCU= when compiling the library. + * Provide backward compatibility for liburcu 0.3.x SIGURCU. + */ +#ifdef SIGURCU +#define SIGRCU SIGURCU +#endif + +#ifndef SIGRCU +#define SIGRCU SIGUSR1 +#endif + +/* + * If a reader is really non-cooperative and refuses to commit its + * rcu_active_readers count to memory (there is no barrier in the reader + * per-se), kick it after a few loops waiting for it. + */ +#define KICK_READER_LOOPS 10000 + +/* + * Active attempts to check for reader Q.S. before calling futex(). + */ +#define RCU_QS_ACTIVE_ATTEMPTS 100 + +#ifdef DEBUG_RCU +#define rcu_assert(args...) assert(args) +#else +#define rcu_assert(args...) +#endif + +#ifdef DEBUG_YIELD +#include +#include +#include +#include + +#define YIELD_READ (1 << 0) +#define YIELD_WRITE (1 << 1) + +/* + * Updates with RCU_SIGNAL are much slower. Account this in the delay. + */ +#ifdef RCU_SIGNAL +/* maximum sleep delay, in us */ +#define MAX_SLEEP 30000 +#else +#define MAX_SLEEP 50 +#endif + +extern unsigned int yield_active; +extern unsigned int __thread rand_yield; + +static inline void debug_yield_read(void) +{ + if (yield_active & YIELD_READ) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_write(void) +{ + if (yield_active & YIELD_WRITE) + if (rand_r(&rand_yield) & 0x1) + usleep(rand_r(&rand_yield) % MAX_SLEEP); +} + +static inline void debug_yield_init(void) +{ + rand_yield = time(NULL) ^ pthread_self(); +} +#else +static inline void debug_yield_read(void) +{ +} + +static inline void debug_yield_write(void) +{ +} + +static inline void debug_yield_init(void) +{ + +} +#endif + +/* + * RCU memory barrier broadcast group. Currently, only broadcast to all process + * threads is supported (group 0). + * + * Slave barriers are only guaranteed to be ordered wrt master barriers. + * + * The pair ordering is detailed as (O: ordered, X: not ordered) : + * slave master + * slave X O + * master O O + */ + +#define MB_GROUP_ALL 0 +#define RCU_MB_GROUP MB_GROUP_ALL + +#ifdef RCU_MEMBARRIER +extern int has_sys_membarrier; + +static inline void smp_mb_slave(int group) +{ + if (likely(has_sys_membarrier)) + cmm_barrier(); + else + cmm_smp_mb(); +} +#endif + +#ifdef RCU_MB +static inline void smp_mb_slave(int group) +{ + cmm_smp_mb(); +} +#endif + +#ifdef RCU_SIGNAL +static inline void smp_mb_slave(int group) +{ + cmm_barrier(); +} +#endif + +/* + * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use + * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. + */ +#define RCU_GP_COUNT (1UL << 0) +/* Use the amount of bits equal to half of the architecture long size */ +#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2)) +#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1) + +/* + * Global quiescent period counter with low-order bits unused. + * Using a int rather than a char to eliminate false register dependencies + * causing stalls on some architectures. + */ +extern unsigned long rcu_gp_ctr; + +struct rcu_reader { + /* Data used by both reader and synchronize_rcu() */ + unsigned long ctr; + char need_mb; + /* Data used for registry */ + struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE))); + pthread_t tid; +}; + +extern struct rcu_reader __thread rcu_reader; + +extern int gp_futex; + +/* + * Wake-up waiting synchronize_rcu(). Called from many concurrent threads. + */ +static inline void wake_up_gp(void) +{ + if (unlikely(uatomic_read(&gp_futex) == -1)) { + uatomic_set(&gp_futex, 0); + futex_async(&gp_futex, FUTEX_WAKE, 1, + NULL, NULL, 0); + } +} + +static inline int rcu_gp_ongoing(unsigned long *ctr) +{ + unsigned long v; + + /* + * Make sure both tests below are done on the same version of *value + * to insure consistency. + */ + v = CMM_LOAD_SHARED(*ctr); + return (v & RCU_GP_CTR_NEST_MASK) && + ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE); +} + +static inline void _rcu_read_lock(void) +{ + unsigned long tmp; + + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ + tmp = rcu_reader.ctr; + /* + * rcu_gp_ctr is + * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) + */ + if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { + _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); + /* + * Set active readers count for outermost nesting level before + * accessing the pointer. See smp_mb_master(). + */ + smp_mb_slave(RCU_MB_GROUP); + } else { + _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT); + } +} + +static inline void _rcu_read_unlock(void) +{ + unsigned long tmp; + + tmp = rcu_reader.ctr; + /* + * Finish using rcu before decrementing the pointer. + * See smp_mb_master(). + */ + if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { + smp_mb_slave(RCU_MB_GROUP); + _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); + /* write rcu_reader.ctr before read futex */ + smp_mb_slave(RCU_MB_GROUP); + wake_up_gp(); + } else { + _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); + } + cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_STATIC_H */ diff --git a/urcu/static/wfqueue.h b/urcu/static/wfqueue.h new file mode 100644 index 0000000..790931b --- /dev/null +++ b/urcu/static/wfqueue.h @@ -0,0 +1,151 @@ +#ifndef _URCU_WFQUEUE_STATIC_H +#define _URCU_WFQUEUE_STATIC_H + +/* + * wfqueue-static.h + * + * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking + * dynamically with the userspace rcu library. + * + * Copyright 2010 - Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Queue with wait-free enqueue/blocking dequeue. + * This implementation adds a dummy head node when the queue is empty to ensure + * we can always update the queue locklessly. + * + * Inspired from half-wait-free/half-blocking queue implementation done by + * Paul E. McKenney. + */ + +#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */ +#define WFQ_WAIT 10 /* Wait 10 ms if being set */ + +static inline void _cds_wfq_node_init(struct cds_wfq_node *node) +{ + node->next = NULL; +} + +static inline void _cds_wfq_init(struct cds_wfq_queue *q) +{ + int ret; + + _cds_wfq_node_init(&q->dummy); + /* Set queue head and tail */ + q->head = &q->dummy; + q->tail = &q->dummy.next; + ret = pthread_mutex_init(&q->lock, NULL); + assert(!ret); +} + +static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q, + struct cds_wfq_node *node) +{ + struct cds_wfq_node **old_tail; + + /* + * uatomic_xchg() implicit memory barrier orders earlier stores to data + * structure containing node and setting node->next to NULL before + * publication. + */ + old_tail = uatomic_xchg(&q->tail, node); + /* + * At this point, dequeuers see a NULL old_tail->next, which indicates + * that the queue is being appended to. The following store will append + * "node" to the queue from a dequeuer perspective. + */ + CMM_STORE_SHARED(*old_tail, node); +} + +/* + * It is valid to reuse and free a dequeued node immediately. + * + * No need to go on a waitqueue here, as there is no possible state in which the + * list could cause dequeue to busy-loop needlessly while waiting for another + * thread to be scheduled. The queue appears empty until tail->next is set by + * enqueue. + */ +static inline struct cds_wfq_node * +___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q) +{ + struct cds_wfq_node *node, *next; + int attempt = 0; + + /* + * Queue is empty if it only contains the dummy node. + */ + if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next) + return NULL; + node = q->head; + + /* + * Adaptative busy-looping waiting for enqueuer to complete enqueue. + */ + while ((next = CMM_LOAD_SHARED(node->next)) == NULL) { + if (++attempt >= WFQ_ADAPT_ATTEMPTS) { + poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */ + attempt = 0; + } else + caa_cpu_relax(); + } + /* + * Move queue head forward. + */ + q->head = next; + /* + * Requeue dummy node if we just dequeued it. + */ + if (node == &q->dummy) { + _cds_wfq_node_init(node); + _cds_wfq_enqueue(q, node); + return ___cds_wfq_dequeue_blocking(q); + } + return node; +} + +static inline struct cds_wfq_node * +_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q) +{ + struct cds_wfq_node *retnode; + int ret; + + ret = pthread_mutex_lock(&q->lock); + assert(!ret); + retnode = ___cds_wfq_dequeue_blocking(q); + ret = pthread_mutex_unlock(&q->lock); + assert(!ret); + return retnode; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_WFQUEUE_STATIC_H */ diff --git a/urcu/static/wfstack.h b/urcu/static/wfstack.h new file mode 100644 index 0000000..ff18c4a --- /dev/null +++ b/urcu/static/wfstack.h @@ -0,0 +1,121 @@ +#ifndef _URCU_WFSTACK_STATIC_H +#define _URCU_WFSTACK_STATIC_H + +/* + * wfstack-static.h + * + * Userspace RCU library - Stack with Wait-Free push, Blocking pop. + * + * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfstack.h for linking + * dynamically with the userspace rcu library. + * + * Copyright 2010 - Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define CDS_WF_STACK_END ((void *)0x1UL) +#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */ +#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */ + +void _cds_wfs_node_init(struct cds_wfs_node *node) +{ + node->next = NULL; +} + +void _cds_wfs_init(struct cds_wfs_stack *s) +{ + int ret; + + s->head = CDS_WF_STACK_END; + ret = pthread_mutex_init(&s->lock, NULL); + assert(!ret); +} + +void _cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node) +{ + struct cds_wfs_node *old_head; + + assert(node->next == NULL); + /* + * uatomic_xchg() implicit memory barrier orders earlier stores to node + * (setting it to NULL) before publication. + */ + old_head = uatomic_xchg(&s->head, node); + /* + * At this point, dequeuers see a NULL node->next, they should busy-wait + * until node->next is set to old_head. + */ + CMM_STORE_SHARED(node->next, old_head); +} + +/* + * Returns NULL if stack is empty. + */ +struct cds_wfs_node * +___cds_wfs_pop_blocking(struct cds_wfs_stack *s) +{ + struct cds_wfs_node *head, *next; + int attempt = 0; + +retry: + head = CMM_LOAD_SHARED(s->head); + if (head == CDS_WF_STACK_END) + return NULL; + /* + * Adaptative busy-looping waiting for push to complete. + */ + while ((next = CMM_LOAD_SHARED(head->next)) == NULL) { + if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) { + poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */ + attempt = 0; + } else + caa_cpu_relax(); + } + if (uatomic_cmpxchg(&s->head, head, next) == head) + return head; + else + goto retry; /* Concurrent modification. Retry. */ +} + +struct cds_wfs_node * +_cds_wfs_pop_blocking(struct cds_wfs_stack *s) +{ + struct cds_wfs_node *retnode; + int ret; + + ret = pthread_mutex_lock(&s->lock); + assert(!ret); + retnode = ___cds_wfs_pop_blocking(s); + ret = pthread_mutex_unlock(&s->lock); + assert(!ret); + return retnode; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _URCU_WFSTACK_STATIC_H */ diff --git a/urcu/wfqueue-static.h b/urcu/wfqueue-static.h deleted file mode 100644 index 790931b..0000000 --- a/urcu/wfqueue-static.h +++ /dev/null @@ -1,151 +0,0 @@ -#ifndef _URCU_WFQUEUE_STATIC_H -#define _URCU_WFQUEUE_STATIC_H - -/* - * wfqueue-static.h - * - * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking - * dynamically with the userspace rcu library. - * - * Copyright 2010 - Mathieu Desnoyers - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Queue with wait-free enqueue/blocking dequeue. - * This implementation adds a dummy head node when the queue is empty to ensure - * we can always update the queue locklessly. - * - * Inspired from half-wait-free/half-blocking queue implementation done by - * Paul E. McKenney. - */ - -#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */ -#define WFQ_WAIT 10 /* Wait 10 ms if being set */ - -static inline void _cds_wfq_node_init(struct cds_wfq_node *node) -{ - node->next = NULL; -} - -static inline void _cds_wfq_init(struct cds_wfq_queue *q) -{ - int ret; - - _cds_wfq_node_init(&q->dummy); - /* Set queue head and tail */ - q->head = &q->dummy; - q->tail = &q->dummy.next; - ret = pthread_mutex_init(&q->lock, NULL); - assert(!ret); -} - -static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q, - struct cds_wfq_node *node) -{ - struct cds_wfq_node **old_tail; - - /* - * uatomic_xchg() implicit memory barrier orders earlier stores to data - * structure containing node and setting node->next to NULL before - * publication. - */ - old_tail = uatomic_xchg(&q->tail, node); - /* - * At this point, dequeuers see a NULL old_tail->next, which indicates - * that the queue is being appended to. The following store will append - * "node" to the queue from a dequeuer perspective. - */ - CMM_STORE_SHARED(*old_tail, node); -} - -/* - * It is valid to reuse and free a dequeued node immediately. - * - * No need to go on a waitqueue here, as there is no possible state in which the - * list could cause dequeue to busy-loop needlessly while waiting for another - * thread to be scheduled. The queue appears empty until tail->next is set by - * enqueue. - */ -static inline struct cds_wfq_node * -___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q) -{ - struct cds_wfq_node *node, *next; - int attempt = 0; - - /* - * Queue is empty if it only contains the dummy node. - */ - if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next) - return NULL; - node = q->head; - - /* - * Adaptative busy-looping waiting for enqueuer to complete enqueue. - */ - while ((next = CMM_LOAD_SHARED(node->next)) == NULL) { - if (++attempt >= WFQ_ADAPT_ATTEMPTS) { - poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */ - attempt = 0; - } else - caa_cpu_relax(); - } - /* - * Move queue head forward. - */ - q->head = next; - /* - * Requeue dummy node if we just dequeued it. - */ - if (node == &q->dummy) { - _cds_wfq_node_init(node); - _cds_wfq_enqueue(q, node); - return ___cds_wfq_dequeue_blocking(q); - } - return node; -} - -static inline struct cds_wfq_node * -_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q) -{ - struct cds_wfq_node *retnode; - int ret; - - ret = pthread_mutex_lock(&q->lock); - assert(!ret); - retnode = ___cds_wfq_dequeue_blocking(q); - ret = pthread_mutex_unlock(&q->lock); - assert(!ret); - return retnode; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_WFQUEUE_STATIC_H */ diff --git a/urcu/wfqueue.h b/urcu/wfqueue.h index f063900..03a73f1 100644 --- a/urcu/wfqueue.h +++ b/urcu/wfqueue.h @@ -52,7 +52,7 @@ struct cds_wfq_queue { #ifdef _LGPL_SOURCE -#include +#include #define cds_wfq_node_init _cds_wfq_node_init #define cds_wfq_init _cds_wfq_init diff --git a/urcu/wfstack-static.h b/urcu/wfstack-static.h deleted file mode 100644 index ff18c4a..0000000 --- a/urcu/wfstack-static.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef _URCU_WFSTACK_STATIC_H -#define _URCU_WFSTACK_STATIC_H - -/* - * wfstack-static.h - * - * Userspace RCU library - Stack with Wait-Free push, Blocking pop. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfstack.h for linking - * dynamically with the userspace rcu library. - * - * Copyright 2010 - Mathieu Desnoyers - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define CDS_WF_STACK_END ((void *)0x1UL) -#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */ -#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */ - -void _cds_wfs_node_init(struct cds_wfs_node *node) -{ - node->next = NULL; -} - -void _cds_wfs_init(struct cds_wfs_stack *s) -{ - int ret; - - s->head = CDS_WF_STACK_END; - ret = pthread_mutex_init(&s->lock, NULL); - assert(!ret); -} - -void _cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node) -{ - struct cds_wfs_node *old_head; - - assert(node->next == NULL); - /* - * uatomic_xchg() implicit memory barrier orders earlier stores to node - * (setting it to NULL) before publication. - */ - old_head = uatomic_xchg(&s->head, node); - /* - * At this point, dequeuers see a NULL node->next, they should busy-wait - * until node->next is set to old_head. - */ - CMM_STORE_SHARED(node->next, old_head); -} - -/* - * Returns NULL if stack is empty. - */ -struct cds_wfs_node * -___cds_wfs_pop_blocking(struct cds_wfs_stack *s) -{ - struct cds_wfs_node *head, *next; - int attempt = 0; - -retry: - head = CMM_LOAD_SHARED(s->head); - if (head == CDS_WF_STACK_END) - return NULL; - /* - * Adaptative busy-looping waiting for push to complete. - */ - while ((next = CMM_LOAD_SHARED(head->next)) == NULL) { - if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) { - poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */ - attempt = 0; - } else - caa_cpu_relax(); - } - if (uatomic_cmpxchg(&s->head, head, next) == head) - return head; - else - goto retry; /* Concurrent modification. Retry. */ -} - -struct cds_wfs_node * -_cds_wfs_pop_blocking(struct cds_wfs_stack *s) -{ - struct cds_wfs_node *retnode; - int ret; - - ret = pthread_mutex_lock(&s->lock); - assert(!ret); - retnode = ___cds_wfs_pop_blocking(s); - ret = pthread_mutex_unlock(&s->lock); - assert(!ret); - return retnode; -} - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_WFSTACK_STATIC_H */ diff --git a/urcu/wfstack.h b/urcu/wfstack.h index c13ba64..354646d 100644 --- a/urcu/wfstack.h +++ b/urcu/wfstack.h @@ -42,7 +42,7 @@ struct cds_wfs_stack { #ifdef _LGPL_SOURCE -#include +#include #define cds_wfs_node_init _cds_wfs_node_init #define cds_wfs_init _cds_wfs_init diff --git a/wfqueue.c b/wfqueue.c index dc7bdfc..3337171 100644 --- a/wfqueue.c +++ b/wfqueue.c @@ -22,7 +22,7 @@ /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu/wfqueue.h" -#include "urcu/wfqueue-static.h" +#include "urcu/static/wfqueue.h" /* * library wrappers to be used by non-LGPL compatible source code. diff --git a/wfstack.c b/wfstack.c index db29558..d999a5b 100644 --- a/wfstack.c +++ b/wfstack.c @@ -22,7 +22,7 @@ /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu/wfstack.h" -#include "urcu/wfstack-static.h" +#include "urcu/static/wfstack.h" /* * library wrappers to be used by non-LGPL compatible source code.