urcu/rculist.h urcu/rcuhlist.h urcu/system.h urcu/urcu-futex.h \
urcu/uatomic_generic.h urcu/arch_generic.h urcu/wfstack.h \
urcu/wfqueue.h urcu/rculfstack.h urcu/rculfqueue.h \
- urcu/wfqueue-static.h urcu/wfstack-static.h \
- urcu/rculfqueue-static.h urcu/rculfstack-static.h \
- urcu/urcu_ref.h urcu/map/*.h
+ urcu/urcu_ref.h urcu/map/*.h urcu/static/*.h
nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic_arch.h urcu/config.h
EXTRA_DIST = $(top_srcdir)/urcu/arch_*.h $(top_srcdir)/urcu/uatomic_arch_*.h \
#undef _LGPL_SOURCE
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu/rculfqueue.h"
-#include "urcu/rculfqueue-static.h"
+#include "urcu/static/rculfqueue.h"
/*
* library wrappers to be used by non-LGPL compatible source code.
#undef _LGPL_SOURCE
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu/rculfstack.h"
-#include "urcu/rculfstack-static.h"
+#include "urcu/static/rculfstack.h"
/*
* library wrappers to be used by non-LGPL compatible source code.
+++ /dev/null
-#ifndef _URCU_BP_STATIC_H
-#define _URCU_BP_STATIC_H
-
-/*
- * urcu-bp-static.h
- *
- * Userspace RCU header.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking
- * dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <syscall.h>
-#include <unistd.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic_arch.h>
-#include <urcu/list.h>
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Active attempts to check for reader Q.S. before calling sleep().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-#ifdef DEBUG_RCU
-#define rcu_assert(args...) assert(args)
-#else
-#define rcu_assert(args...)
-#endif
-
-#ifdef DEBUG_YIELD
-#include <sched.h>
-#include <time.h>
-#include <pthread.h>
-#include <unistd.h>
-
-#define YIELD_READ (1 << 0)
-#define YIELD_WRITE (1 << 1)
-
-/*
- * Updates without RCU_MB are much slower. Account this in
- * the delay.
- */
-/* maximum sleep delay, in us */
-#define MAX_SLEEP 50
-
-extern unsigned int yield_active;
-extern unsigned int __thread rand_yield;
-
-static inline void debug_yield_read(void)
-{
- if (yield_active & YIELD_READ)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_write(void)
-{
- if (yield_active & YIELD_WRITE)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_init(void)
-{
- rand_yield = time(NULL) ^ pthread_self();
-}
-#else
-static inline void debug_yield_read(void)
-{
-}
-
-static inline void debug_yield_write(void)
-{
-}
-
-static inline void debug_yield_init(void)
-{
-
-}
-#endif
-
-/*
- * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a
- * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
- */
-#define RCU_GP_COUNT (1UL << 0)
-/* Use the amount of bits equal to half of the architecture long size */
-#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2))
-#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
-
-/*
- * Used internally by _rcu_read_lock.
- */
-extern void rcu_bp_register(void);
-
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern long rcu_gp_ctr;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- long ctr;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- pthread_t tid;
- int alloc; /* registry entry allocated */
-};
-
-/*
- * Bulletproof version keeps a pointer to a registry not part of the TLS.
- * Adds a pointer dereference on the read-side, but won't require to unregister
- * the reader thread.
- */
-extern struct rcu_reader __thread *rcu_reader;
-
-static inline int rcu_old_gp_ongoing(long *value)
-{
- long v;
-
- if (value == NULL)
- return 0;
- /*
- * Make sure both tests below are done on the same version of *value
- * to insure consistency.
- */
- v = CMM_LOAD_SHARED(*value);
- return (v & RCU_GP_CTR_NEST_MASK) &&
- ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
-}
-
-static inline void _rcu_read_lock(void)
-{
- long tmp;
-
- /* Check if registered */
- if (unlikely(!rcu_reader))
- rcu_bp_register();
-
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
- tmp = rcu_reader->ctr;
- /*
- * rcu_gp_ctr is
- * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
- */
- if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
- _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
- /*
- * Set active readers count for outermost nesting level before
- * accessing the pointer.
- */
- cmm_smp_mb();
- } else {
- _CMM_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT);
- }
-}
-
-static inline void _rcu_read_unlock(void)
-{
- /*
- * Finish using rcu before decrementing the pointer.
- */
- cmm_smp_mb();
- _CMM_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT);
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_BP_STATIC_H */
#include "urcu/map/urcu-bp.h"
-#include "urcu-bp-static.h"
+#include "urcu/static/urcu-bp.h"
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu-bp.h"
#include <pthread.h>
/*
- * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers.
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
*/
#include <urcu-pointer.h>
#ifdef _LGPL_SOURCE
-#include <urcu-bp-static.h>
+#include <urcu/static/urcu-bp.h>
/*
* Mappings for static use of the userspace RCU library.
/*
* library wrappers to be used by non-LGPL compatible source code.
- * See LGPL-only urcu-pointer-static.h for documentation.
+ * See LGPL-only urcu/static/urcu-pointer.h for documentation.
*/
extern void rcu_read_lock(void);
+++ /dev/null
-#ifndef _URCU_POINTER_STATIC_H
-#define _URCU_POINTER_STATIC_H
-
-/*
- * urcu-pointer-static.h
- *
- * Userspace RCU header. Operations on pointers.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-pointer.h for
- * linking dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic_arch.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable
- * into a RCU read-side critical section. The pointer can later be safely
- * dereferenced within the critical section.
- *
- * This ensures that the pointer copy is invariant thorough the whole critical
- * section.
- *
- * Inserts memory barriers on architectures that require them (currently only
- * Alpha) and documents which pointers are protected by RCU.
- *
- * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative
- * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
- * data read before the pointer read by speculating the value of the pointer.
- * Correct ordering is ensured because the pointer is read as a volatile access.
- * This acts as a global side-effect operation, which forbids reordering of
- * dependent memory operations. Note that such concern about dependency-breaking
- * optimizations will eventually be taken care of by the "memory_order_consume"
- * addition to forthcoming C++ standard.
- *
- * Should match rcu_assign_pointer() or rcu_xchg_pointer().
- */
-
-#define _rcu_dereference(p) ({ \
- typeof(p) _________p1 = CMM_LOAD_SHARED(p); \
- cmm_smp_read_barrier_depends(); \
- (_________p1); \
- })
-
-/**
- * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
- * is as expected by "old". If succeeds, returns the previous pointer to the
- * data structure, which can be safely freed after waiting for a quiescent state
- * using synchronize_rcu(). If fails (unexpected value), returns old (which
- * should not be freed !).
- */
-
-#define _rcu_cmpxchg_pointer(p, old, _new) \
- ({ \
- typeof(*p) _________pold = (old); \
- typeof(*p) _________pnew = (_new); \
- if (!__builtin_constant_p(_new) || \
- ((_new) != NULL)) \
- cmm_wmb(); \
- uatomic_cmpxchg(p, _________pold, _________pnew); \
- })
-
-/**
- * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
- * pointer to the data structure, which can be safely freed after waiting for a
- * quiescent state using synchronize_rcu().
- */
-
-#define _rcu_xchg_pointer(p, v) \
- ({ \
- typeof(*p) _________pv = (v); \
- if (!__builtin_constant_p(v) || \
- ((v) != NULL)) \
- cmm_wmb(); \
- uatomic_xchg(p, _________pv); \
- })
-
-
-#define _rcu_set_pointer(p, v) \
- ({ \
- typeof(*p) _________pv = (v); \
- if (!__builtin_constant_p(v) || \
- ((v) != NULL)) \
- cmm_wmb(); \
- uatomic_set(p, _________pv); \
- })
-
-/**
- * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure
- * meant to be read by RCU read-side critical sections. Returns the assigned
- * value.
- *
- * Documents which pointers will be dereferenced by RCU read-side critical
- * sections and adds the required memory barriers on architectures requiring
- * them. It also makes sure the compiler does not reorder code initializing the
- * data structure before its publication.
- *
- * Should match rcu_dereference_pointer().
- */
-
-#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_POINTER_STATIC_H */
#include <urcu/uatomic_arch.h>
-#include "urcu-pointer-static.h"
+#include "urcu/static/urcu-pointer.h"
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu-pointer.h"
#ifdef _LGPL_SOURCE
-#include <urcu-pointer-static.h>
+#include <urcu/static/urcu-pointer.h>
/*
* rcu_dereference(ptr)
+++ /dev/null
-#ifndef _URCU_QSBR_STATIC_H
-#define _URCU_QSBR_STATIC_H
-
-/*
- * urcu-qsbr-static.h
- *
- * Userspace RCU QSBR header.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-qsbr.h for linking
- * dynamically with the userspace rcu QSBR library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <assert.h>
-#include <limits.h>
-#include <syscall.h>
-#include <unistd.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic_arch.h>
-#include <urcu/list.h>
-#include <urcu/urcu-futex.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-/*
- * If a reader is really non-cooperative and refuses to commit its
- * rcu_reader.ctr count to memory (there is no barrier in the reader
- * per-se), kick it after a few loops waiting for it.
- */
-#define KICK_READER_LOOPS 10000
-
-/*
- * Active attempts to check for reader Q.S. before calling futex().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-#ifdef DEBUG_RCU
-#define rcu_assert(args...) assert(args)
-#else
-#define rcu_assert(args...)
-#endif
-
-#ifdef DEBUG_YIELD
-#include <sched.h>
-#include <time.h>
-#include <pthread.h>
-#include <unistd.h>
-
-#define YIELD_READ (1 << 0)
-#define YIELD_WRITE (1 << 1)
-
-/* maximum sleep delay, in us */
-#define MAX_SLEEP 50
-
-extern unsigned int yield_active;
-extern unsigned int __thread rand_yield;
-
-static inline void debug_yield_read(void)
-{
- if (yield_active & YIELD_READ)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_write(void)
-{
- if (yield_active & YIELD_WRITE)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_init(void)
-{
- rand_yield = time(NULL) ^ pthread_self();
-}
-#else
-static inline void debug_yield_read(void)
-{
-}
-
-static inline void debug_yield_write(void)
-{
-}
-
-static inline void debug_yield_init(void)
-{
-
-}
-#endif
-
-#define RCU_GP_ONLINE (1UL << 0)
-#define RCU_GP_CTR (1UL << 1)
-
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- unsigned long ctr;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- pthread_t tid;
-};
-
-extern struct rcu_reader __thread rcu_reader;
-
-extern int gp_futex;
-
-/*
- * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
- */
-static inline void wake_up_gp(void)
-{
- if (unlikely(uatomic_read(&gp_futex) == -1)) {
- uatomic_set(&gp_futex, 0);
- futex_noasync(&gp_futex, FUTEX_WAKE, 1,
- NULL, NULL, 0);
- }
-}
-
-static inline int rcu_gp_ongoing(unsigned long *ctr)
-{
- unsigned long v;
-
- v = CMM_LOAD_SHARED(*ctr);
- return v && (v != rcu_gp_ctr);
-}
-
-static inline void _rcu_read_lock(void)
-{
- rcu_assert(rcu_reader.ctr);
-}
-
-static inline void _rcu_read_unlock(void)
-{
-}
-
-static inline void _rcu_quiescent_state(void)
-{
- cmm_smp_mb();
- _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
- cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
- wake_up_gp();
- cmm_smp_mb();
-}
-
-static inline void _rcu_thread_offline(void)
-{
- cmm_smp_mb();
- CMM_STORE_SHARED(rcu_reader.ctr, 0);
- cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
- wake_up_gp();
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-static inline void _rcu_thread_online(void)
-{
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
- _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
- cmm_smp_mb();
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_QSBR_STATIC_H */
#include "urcu/map/urcu-qsbr.h"
#define BUILD_QSBR_LIB
-#include "urcu-qsbr-static.h"
+#include "urcu/static/urcu-qsbr.h"
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu-qsbr.h"
#include <pthread.h>
/*
- * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers.
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
*/
#include <urcu-pointer.h>
#ifdef _LGPL_SOURCE
-#include <urcu-qsbr-static.h>
+#include <urcu/static/urcu-qsbr.h>
/*
* Mappings for static use of the userspace RCU library.
+++ /dev/null
-#ifndef _URCU_STATIC_H
-#define _URCU_STATIC_H
-
-/*
- * urcu-static.h
- *
- * Userspace RCU header.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking
- * dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <syscall.h>
-#include <unistd.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic_arch.h>
-#include <urcu/list.h>
-#include <urcu/urcu-futex.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Default is RCU_MEMBARRIER */
-#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL)
-#define RCU_MEMBARRIER
-#endif
-
-#ifdef RCU_MEMBARRIER
-#include <unistd.h>
-#include <sys/syscall.h>
-
-/* If the headers do not support SYS_membarrier, statically use RCU_MB */
-#ifdef SYS_membarrier
-# define MEMBARRIER_EXPEDITED (1 << 0)
-# define MEMBARRIER_DELAYED (1 << 1)
-# define MEMBARRIER_QUERY (1 << 16)
-# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__)
-#else
-# undef RCU_MEMBARRIER
-# define RCU_MB
-#endif
-#endif
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-/*
- * The signal number used by the RCU library can be overridden with
- * -DSIGRCU= when compiling the library.
- * Provide backward compatibility for liburcu 0.3.x SIGURCU.
- */
-#ifdef SIGURCU
-#define SIGRCU SIGURCU
-#endif
-
-#ifndef SIGRCU
-#define SIGRCU SIGUSR1
-#endif
-
-/*
- * If a reader is really non-cooperative and refuses to commit its
- * rcu_active_readers count to memory (there is no barrier in the reader
- * per-se), kick it after a few loops waiting for it.
- */
-#define KICK_READER_LOOPS 10000
-
-/*
- * Active attempts to check for reader Q.S. before calling futex().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-#ifdef DEBUG_RCU
-#define rcu_assert(args...) assert(args)
-#else
-#define rcu_assert(args...)
-#endif
-
-#ifdef DEBUG_YIELD
-#include <sched.h>
-#include <time.h>
-#include <pthread.h>
-#include <unistd.h>
-
-#define YIELD_READ (1 << 0)
-#define YIELD_WRITE (1 << 1)
-
-/*
- * Updates with RCU_SIGNAL are much slower. Account this in the delay.
- */
-#ifdef RCU_SIGNAL
-/* maximum sleep delay, in us */
-#define MAX_SLEEP 30000
-#else
-#define MAX_SLEEP 50
-#endif
-
-extern unsigned int yield_active;
-extern unsigned int __thread rand_yield;
-
-static inline void debug_yield_read(void)
-{
- if (yield_active & YIELD_READ)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_write(void)
-{
- if (yield_active & YIELD_WRITE)
- if (rand_r(&rand_yield) & 0x1)
- usleep(rand_r(&rand_yield) % MAX_SLEEP);
-}
-
-static inline void debug_yield_init(void)
-{
- rand_yield = time(NULL) ^ pthread_self();
-}
-#else
-static inline void debug_yield_read(void)
-{
-}
-
-static inline void debug_yield_write(void)
-{
-}
-
-static inline void debug_yield_init(void)
-{
-
-}
-#endif
-
-/*
- * RCU memory barrier broadcast group. Currently, only broadcast to all process
- * threads is supported (group 0).
- *
- * Slave barriers are only guaranteed to be ordered wrt master barriers.
- *
- * The pair ordering is detailed as (O: ordered, X: not ordered) :
- * slave master
- * slave X O
- * master O O
- */
-
-#define MB_GROUP_ALL 0
-#define RCU_MB_GROUP MB_GROUP_ALL
-
-#ifdef RCU_MEMBARRIER
-extern int has_sys_membarrier;
-
-static inline void smp_mb_slave(int group)
-{
- if (likely(has_sys_membarrier))
- cmm_barrier();
- else
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_MB
-static inline void smp_mb_slave(int group)
-{
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_SIGNAL
-static inline void smp_mb_slave(int group)
-{
- cmm_barrier();
-}
-#endif
-
-/*
- * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use
- * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
- */
-#define RCU_GP_COUNT (1UL << 0)
-/* Use the amount of bits equal to half of the architecture long size */
-#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))
-#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
-
-/*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register dependencies
- * causing stalls on some architectures.
- */
-extern unsigned long rcu_gp_ctr;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- unsigned long ctr;
- char need_mb;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- pthread_t tid;
-};
-
-extern struct rcu_reader __thread rcu_reader;
-
-extern int gp_futex;
-
-/*
- * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
- */
-static inline void wake_up_gp(void)
-{
- if (unlikely(uatomic_read(&gp_futex) == -1)) {
- uatomic_set(&gp_futex, 0);
- futex_async(&gp_futex, FUTEX_WAKE, 1,
- NULL, NULL, 0);
- }
-}
-
-static inline int rcu_gp_ongoing(unsigned long *ctr)
-{
- unsigned long v;
-
- /*
- * Make sure both tests below are done on the same version of *value
- * to insure consistency.
- */
- v = CMM_LOAD_SHARED(*ctr);
- return (v & RCU_GP_CTR_NEST_MASK) &&
- ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
-}
-
-static inline void _rcu_read_lock(void)
-{
- unsigned long tmp;
-
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
- tmp = rcu_reader.ctr;
- /*
- * rcu_gp_ctr is
- * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
- */
- if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
- _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
- /*
- * Set active readers count for outermost nesting level before
- * accessing the pointer. See smp_mb_master().
- */
- smp_mb_slave(RCU_MB_GROUP);
- } else {
- _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT);
- }
-}
-
-static inline void _rcu_read_unlock(void)
-{
- unsigned long tmp;
-
- tmp = rcu_reader.ctr;
- /*
- * Finish using rcu before decrementing the pointer.
- * See smp_mb_master().
- */
- if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) {
- smp_mb_slave(RCU_MB_GROUP);
- _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT);
- /* write rcu_reader.ctr before read futex */
- smp_mb_slave(RCU_MB_GROUP);
- wake_up_gp();
- } else {
- _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT);
- }
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_STATIC_H */
#include "urcu/map/urcu.h"
-#include "urcu-static.h"
+#include "urcu/static/urcu.h"
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu.h"
#include <pthread.h>
/*
- * See urcu-pointer.h and urcu-pointer-static.h for pointer publication headers.
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
*/
#include <urcu-pointer.h>
#ifdef _LGPL_SOURCE
-#include <urcu-static.h>
+#include <urcu/static/urcu.h>
/*
* Mappings for static use of the userspace RCU library.
/*
* library wrappers to be used by non-LGPL compatible source code.
- * See LGPL-only urcu-pointer-static.h for documentation.
+ * See LGPL-only urcu/static/urcu-pointer.h for documentation.
*/
extern void rcu_read_lock(void);
+++ /dev/null
-#ifndef _URCU_RCULFQUEUE_STATIC_H
-#define _URCU_RCULFQUEUE_STATIC_H
-
-/*
- * rculfqueue-static.h
- *
- * Userspace RCU library - Lock-Free RCU Queue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking
- * dynamically with the userspace rcu library.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/urcu_ref.h>
-#include <urcu/uatomic_arch.h>
-#include <assert.h>
-/* A urcu implementation header should be already included. */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Lock-free RCU queue using reference counting. Enqueue and dequeue operations
- * hold a RCU read lock to deal with cmpxchg ABA problem. This implementation
- * keeps a dummy head node to ensure we can always update the queue locklessly.
- * Given that this is a queue, the dummy head node must always advance as we
- * dequeue entries. Therefore, we keep a reference count on each entry we are
- * dequeueing, so they can be kept as dummy head node until the next dequeue, at
- * which point their reference count will be decremented.
- */
-
-#define URCU_LFQ_PERMANENT_REF 128
-
-void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
-{
- node->next = NULL;
- urcu_ref_init(&node->ref);
-}
-
-void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
- void (*release)(struct urcu_ref *ref))
-{
- _cds_lfq_node_init_rcu(&q->init);
- /* Make sure the initial node is never freed. */
- urcu_ref_set(&q->init.ref, URCU_LFQ_PERMANENT_REF);
- q->head = q->tail = &q->init;
- q->release = release;
-}
-
-/*
- * Should be called under rcu read lock critical section.
- */
-void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
- struct cds_lfq_node_rcu *node)
-{
- urcu_ref_get(&node->ref);
- node->queue = q;
-
- /*
- * uatomic_cmpxchg() implicit memory barrier orders earlier stores to
- * node before publication.
- */
-
- for (;;) {
- struct cds_lfq_node_rcu *tail, *next;
-
- tail = rcu_dereference(q->tail);
- /*
- * Typically expect tail->next to be NULL.
- */
- next = uatomic_cmpxchg(&tail->next, NULL, node);
- if (next == NULL) {
- /*
- * Tail was at the end of queue, we successfully
- * appended to it.
- * Now move tail (another enqueue might beat
- * us to it, that's fine).
- */
- (void) uatomic_cmpxchg(&q->tail, tail, node);
- return;
- } else {
- /*
- * Failure to append to current tail. Help moving tail
- * further and retry.
- */
- (void) uatomic_cmpxchg(&q->tail, tail, next);
- continue;
- }
- }
-}
-
-/*
- * Should be called under rcu read lock critical section.
- *
- * The entry returned by dequeue must be taken care of by doing a
- * sequence of urcu_ref_put which release handler should do a call_rcu.
- *
- * In other words, the entry lfq node returned by dequeue must not be
- * modified/re-used/freed until the reference count reaches zero and a grace
- * period has elapsed.
- */
-struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
-{
- for (;;) {
- struct cds_lfq_node_rcu *head, *next;
-
- head = rcu_dereference(q->head);
- next = rcu_dereference(head->next);
- if (next) {
- if (uatomic_cmpxchg(&q->head, head, next) == head) {
- urcu_ref_put(&head->ref, q->release);
- return next;
- } else {
- /* Concurrently pushed, retry */
- continue;
- }
- } else {
- /* Empty */
- return NULL;
- }
- }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFQUEUE_STATIC_H */
#ifdef _LGPL_SOURCE
-#include <urcu/rculfqueue-static.h>
+#include <urcu/static/rculfqueue.h>
#define cds_lfq_node_init_rcu _cds_lfq_node_init_rcu
#define cds_lfq_init_rcu _cds_lfq_init_rcu
+++ /dev/null
-#ifndef _URCU_RCULFSTACK_STATIC_H
-#define _URCU_RCULFSTACK_STATIC_H
-
-/*
- * rculfstack-static.h
- *
- * Userspace RCU library - Lock-Free RCU Stack
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking
- * dynamically with the userspace rcu library.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/uatomic_arch.h>
-/* A urcu implementation header should be already included. */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
-{
-}
-
-void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
-{
- s->head = NULL;
-}
-
-void _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s, struct cds_lfs_node_rcu *node)
-{
- struct cds_lfs_node_rcu *head = NULL;
-
- for (;;) {
- struct cds_lfs_node_rcu *old_head = head;
-
- node->next = head;
- /*
- * uatomic_cmpxchg() implicit memory barrier orders earlier
- * stores to node before publication.
- */
- head = uatomic_cmpxchg(&s->head, old_head, node);
- if (old_head == head)
- break;
- }
-}
-
-/*
- * Should be called under rcu read-side lock.
- *
- * The caller must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_lfs_node_rcu structure.
- * Returns NULL if stack is empty.
- */
-struct cds_lfs_node_rcu *
-_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
-{
- for (;;) {
- struct cds_lfs_node_rcu *head;
-
- head = rcu_dereference(s->head);
- if (head) {
- struct cds_lfs_node_rcu *next = rcu_dereference(head->next);
-
- if (uatomic_cmpxchg(&s->head, head, next) == head) {
- return head;
- } else {
- /* Concurrent modification. Retry. */
- continue;
- }
- } else {
- /* Empty stack */
- return NULL;
- }
- }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFSTACK_STATIC_H */
#ifdef _LGPL_SOURCE
-#include <urcu/rculfstack-static.h>
+#include <urcu/static/rculfstack.h>
#define cds_lfs_node_init_rcu _cds_lfs_node_init_rcu
#define cds_lfs_init_rcu _cds_lfs_init_rcu
--- /dev/null
+#ifndef _URCU_RCULFQUEUE_STATIC_H
+#define _URCU_RCULFQUEUE_STATIC_H
+
+/*
+ * rculfqueue-static.h
+ *
+ * Userspace RCU library - Lock-Free RCU Queue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/urcu_ref.h>
+#include <urcu/uatomic_arch.h>
+#include <assert.h>
+/* A urcu implementation header should be already included. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Lock-free RCU queue using reference counting. Enqueue and dequeue operations
+ * hold a RCU read lock to deal with cmpxchg ABA problem. This implementation
+ * keeps a dummy head node to ensure we can always update the queue locklessly.
+ * Given that this is a queue, the dummy head node must always advance as we
+ * dequeue entries. Therefore, we keep a reference count on each entry we are
+ * dequeueing, so they can be kept as dummy head node until the next dequeue, at
+ * which point their reference count will be decremented.
+ */
+
+#define URCU_LFQ_PERMANENT_REF 128
+
+void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
+{
+ node->next = NULL;
+ urcu_ref_init(&node->ref);
+}
+
+void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
+ void (*release)(struct urcu_ref *ref))
+{
+ _cds_lfq_node_init_rcu(&q->init);
+ /* Make sure the initial node is never freed. */
+ urcu_ref_set(&q->init.ref, URCU_LFQ_PERMANENT_REF);
+ q->head = q->tail = &q->init;
+ q->release = release;
+}
+
+/*
+ * Should be called under rcu read lock critical section.
+ */
+void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
+ struct cds_lfq_node_rcu *node)
+{
+ urcu_ref_get(&node->ref);
+ node->queue = q;
+
+ /*
+ * uatomic_cmpxchg() implicit memory barrier orders earlier stores to
+ * node before publication.
+ */
+
+ for (;;) {
+ struct cds_lfq_node_rcu *tail, *next;
+
+ tail = rcu_dereference(q->tail);
+ /*
+ * Typically expect tail->next to be NULL.
+ */
+ next = uatomic_cmpxchg(&tail->next, NULL, node);
+ if (next == NULL) {
+ /*
+ * Tail was at the end of queue, we successfully
+ * appended to it.
+ * Now move tail (another enqueue might beat
+ * us to it, that's fine).
+ */
+ (void) uatomic_cmpxchg(&q->tail, tail, node);
+ return;
+ } else {
+ /*
+ * Failure to append to current tail. Help moving tail
+ * further and retry.
+ */
+ (void) uatomic_cmpxchg(&q->tail, tail, next);
+ continue;
+ }
+ }
+}
+
+/*
+ * Should be called under rcu read lock critical section.
+ *
+ * The entry returned by dequeue must be taken care of by doing a
+ * sequence of urcu_ref_put which release handler should do a call_rcu.
+ *
+ * In other words, the entry lfq node returned by dequeue must not be
+ * modified/re-used/freed until the reference count reaches zero and a grace
+ * period has elapsed.
+ */
+struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
+{
+ for (;;) {
+ struct cds_lfq_node_rcu *head, *next;
+
+ head = rcu_dereference(q->head);
+ next = rcu_dereference(head->next);
+ if (next) {
+ if (uatomic_cmpxchg(&q->head, head, next) == head) {
+ urcu_ref_put(&head->ref, q->release);
+ return next;
+ } else {
+ /* Concurrently pushed, retry */
+ continue;
+ }
+ } else {
+ /* Empty */
+ return NULL;
+ }
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFQUEUE_STATIC_H */
--- /dev/null
+#ifndef _URCU_RCULFSTACK_STATIC_H
+#define _URCU_RCULFSTACK_STATIC_H
+
+/*
+ * rculfstack-static.h
+ *
+ * Userspace RCU library - Lock-Free RCU Stack
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/uatomic_arch.h>
+/* A urcu implementation header should be already included. */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
+{
+}
+
+void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
+{
+ s->head = NULL;
+}
+
+void _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s, struct cds_lfs_node_rcu *node)
+{
+ struct cds_lfs_node_rcu *head = NULL;
+
+ for (;;) {
+ struct cds_lfs_node_rcu *old_head = head;
+
+ node->next = head;
+ /*
+ * uatomic_cmpxchg() implicit memory barrier orders earlier
+ * stores to node before publication.
+ */
+ head = uatomic_cmpxchg(&s->head, old_head, node);
+ if (old_head == head)
+ break;
+ }
+}
+
+/*
+ * Should be called under rcu read-side lock.
+ *
+ * The caller must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_lfs_node_rcu structure.
+ * Returns NULL if stack is empty.
+ */
+struct cds_lfs_node_rcu *
+_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
+{
+ for (;;) {
+ struct cds_lfs_node_rcu *head;
+
+ head = rcu_dereference(s->head);
+ if (head) {
+ struct cds_lfs_node_rcu *next = rcu_dereference(head->next);
+
+ if (uatomic_cmpxchg(&s->head, head, next) == head) {
+ return head;
+ } else {
+ /* Concurrent modification. Retry. */
+ continue;
+ }
+ } else {
+ /* Empty stack */
+ return NULL;
+ }
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFSTACK_STATIC_H */
--- /dev/null
+#ifndef _URCU_BP_STATIC_H
+#define _URCU_BP_STATIC_H
+
+/*
+ * urcu-bp-static.h
+ *
+ * Userspace RCU header.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <syscall.h>
+#include <unistd.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic_arch.h>
+#include <urcu/list.h>
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Active attempts to check for reader Q.S. before calling sleep().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+#ifdef DEBUG_RCU
+#define rcu_assert(args...) assert(args)
+#else
+#define rcu_assert(args...)
+#endif
+
+#ifdef DEBUG_YIELD
+#include <sched.h>
+#include <time.h>
+#include <pthread.h>
+#include <unistd.h>
+
+#define YIELD_READ (1 << 0)
+#define YIELD_WRITE (1 << 1)
+
+/*
+ * Updates without RCU_MB are much slower. Account this in
+ * the delay.
+ */
+/* maximum sleep delay, in us */
+#define MAX_SLEEP 50
+
+extern unsigned int yield_active;
+extern unsigned int __thread rand_yield;
+
+static inline void debug_yield_read(void)
+{
+ if (yield_active & YIELD_READ)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_write(void)
+{
+ if (yield_active & YIELD_WRITE)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_init(void)
+{
+ rand_yield = time(NULL) ^ pthread_self();
+}
+#else
+static inline void debug_yield_read(void)
+{
+}
+
+static inline void debug_yield_write(void)
+{
+}
+
+static inline void debug_yield_init(void)
+{
+
+}
+#endif
+
+/*
+ * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a
+ * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
+ */
+#define RCU_GP_COUNT (1UL << 0)
+/* Use the amount of bits equal to half of the architecture long size */
+#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2))
+#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
+
+/*
+ * Used internally by _rcu_read_lock.
+ */
+extern void rcu_bp_register(void);
+
+/*
+ * Global quiescent period counter with low-order bits unused.
+ * Using a int rather than a char to eliminate false register dependencies
+ * causing stalls on some architectures.
+ */
+extern long rcu_gp_ctr;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ long ctr;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ pthread_t tid;
+ int alloc; /* registry entry allocated */
+};
+
+/*
+ * Bulletproof version keeps a pointer to a registry not part of the TLS.
+ * Adds a pointer dereference on the read-side, but won't require to unregister
+ * the reader thread.
+ */
+extern struct rcu_reader __thread *rcu_reader;
+
+static inline int rcu_old_gp_ongoing(long *value)
+{
+ long v;
+
+ if (value == NULL)
+ return 0;
+ /*
+ * Make sure both tests below are done on the same version of *value
+ * to insure consistency.
+ */
+ v = CMM_LOAD_SHARED(*value);
+ return (v & RCU_GP_CTR_NEST_MASK) &&
+ ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
+}
+
+static inline void _rcu_read_lock(void)
+{
+ long tmp;
+
+ /* Check if registered */
+ if (unlikely(!rcu_reader))
+ rcu_bp_register();
+
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+ tmp = rcu_reader->ctr;
+ /*
+ * rcu_gp_ctr is
+ * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
+ */
+ if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
+ _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+ /*
+ * Set active readers count for outermost nesting level before
+ * accessing the pointer.
+ */
+ cmm_smp_mb();
+ } else {
+ _CMM_STORE_SHARED(rcu_reader->ctr, tmp + RCU_GP_COUNT);
+ }
+}
+
+static inline void _rcu_read_unlock(void)
+{
+ /*
+ * Finish using rcu before decrementing the pointer.
+ */
+ cmm_smp_mb();
+ _CMM_STORE_SHARED(rcu_reader->ctr, rcu_reader->ctr - RCU_GP_COUNT);
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_BP_STATIC_H */
--- /dev/null
+#ifndef _URCU_POINTER_STATIC_H
+#define _URCU_POINTER_STATIC_H
+
+/*
+ * urcu-pointer-static.h
+ *
+ * Userspace RCU header. Operations on pointers.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-pointer.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic_arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable
+ * into a RCU read-side critical section. The pointer can later be safely
+ * dereferenced within the critical section.
+ *
+ * This ensures that the pointer copy is invariant thorough the whole critical
+ * section.
+ *
+ * Inserts memory barriers on architectures that require them (currently only
+ * Alpha) and documents which pointers are protected by RCU.
+ *
+ * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative
+ * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
+ * data read before the pointer read by speculating the value of the pointer.
+ * Correct ordering is ensured because the pointer is read as a volatile access.
+ * This acts as a global side-effect operation, which forbids reordering of
+ * dependent memory operations. Note that such concern about dependency-breaking
+ * optimizations will eventually be taken care of by the "memory_order_consume"
+ * addition to forthcoming C++ standard.
+ *
+ * Should match rcu_assign_pointer() or rcu_xchg_pointer().
+ */
+
+#define _rcu_dereference(p) ({ \
+ typeof(p) _________p1 = CMM_LOAD_SHARED(p); \
+ cmm_smp_read_barrier_depends(); \
+ (_________p1); \
+ })
+
+/**
+ * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
+ * is as expected by "old". If succeeds, returns the previous pointer to the
+ * data structure, which can be safely freed after waiting for a quiescent state
+ * using synchronize_rcu(). If fails (unexpected value), returns old (which
+ * should not be freed !).
+ */
+
+#define _rcu_cmpxchg_pointer(p, old, _new) \
+ ({ \
+ typeof(*p) _________pold = (old); \
+ typeof(*p) _________pnew = (_new); \
+ if (!__builtin_constant_p(_new) || \
+ ((_new) != NULL)) \
+ cmm_wmb(); \
+ uatomic_cmpxchg(p, _________pold, _________pnew); \
+ })
+
+/**
+ * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
+ * pointer to the data structure, which can be safely freed after waiting for a
+ * quiescent state using synchronize_rcu().
+ */
+
+#define _rcu_xchg_pointer(p, v) \
+ ({ \
+ typeof(*p) _________pv = (v); \
+ if (!__builtin_constant_p(v) || \
+ ((v) != NULL)) \
+ cmm_wmb(); \
+ uatomic_xchg(p, _________pv); \
+ })
+
+
+#define _rcu_set_pointer(p, v) \
+ ({ \
+ typeof(*p) _________pv = (v); \
+ if (!__builtin_constant_p(v) || \
+ ((v) != NULL)) \
+ cmm_wmb(); \
+ uatomic_set(p, _________pv); \
+ })
+
+/**
+ * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure
+ * meant to be read by RCU read-side critical sections. Returns the assigned
+ * value.
+ *
+ * Documents which pointers will be dereferenced by RCU read-side critical
+ * sections and adds the required memory barriers on architectures requiring
+ * them. It also makes sure the compiler does not reorder code initializing the
+ * data structure before its publication.
+ *
+ * Should match rcu_dereference_pointer().
+ */
+
+#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_POINTER_STATIC_H */
--- /dev/null
+#ifndef _URCU_QSBR_STATIC_H
+#define _URCU_QSBR_STATIC_H
+
+/*
+ * urcu-qsbr-static.h
+ *
+ * Userspace RCU QSBR header.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-qsbr.h for linking
+ * dynamically with the userspace rcu QSBR library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <assert.h>
+#include <limits.h>
+#include <syscall.h>
+#include <unistd.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic_arch.h>
+#include <urcu/list.h>
+#include <urcu/urcu-futex.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+/*
+ * If a reader is really non-cooperative and refuses to commit its
+ * rcu_reader.ctr count to memory (there is no barrier in the reader
+ * per-se), kick it after a few loops waiting for it.
+ */
+#define KICK_READER_LOOPS 10000
+
+/*
+ * Active attempts to check for reader Q.S. before calling futex().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+#ifdef DEBUG_RCU
+#define rcu_assert(args...) assert(args)
+#else
+#define rcu_assert(args...)
+#endif
+
+#ifdef DEBUG_YIELD
+#include <sched.h>
+#include <time.h>
+#include <pthread.h>
+#include <unistd.h>
+
+#define YIELD_READ (1 << 0)
+#define YIELD_WRITE (1 << 1)
+
+/* maximum sleep delay, in us */
+#define MAX_SLEEP 50
+
+extern unsigned int yield_active;
+extern unsigned int __thread rand_yield;
+
+static inline void debug_yield_read(void)
+{
+ if (yield_active & YIELD_READ)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_write(void)
+{
+ if (yield_active & YIELD_WRITE)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_init(void)
+{
+ rand_yield = time(NULL) ^ pthread_self();
+}
+#else
+static inline void debug_yield_read(void)
+{
+}
+
+static inline void debug_yield_write(void)
+{
+}
+
+static inline void debug_yield_init(void)
+{
+
+}
+#endif
+
+#define RCU_GP_ONLINE (1UL << 0)
+#define RCU_GP_CTR (1UL << 1)
+
+/*
+ * Global quiescent period counter with low-order bits unused.
+ * Using a int rather than a char to eliminate false register dependencies
+ * causing stalls on some architectures.
+ */
+extern unsigned long rcu_gp_ctr;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ unsigned long ctr;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ pthread_t tid;
+};
+
+extern struct rcu_reader __thread rcu_reader;
+
+extern int gp_futex;
+
+/*
+ * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
+ */
+static inline void wake_up_gp(void)
+{
+ if (unlikely(uatomic_read(&gp_futex) == -1)) {
+ uatomic_set(&gp_futex, 0);
+ futex_noasync(&gp_futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0);
+ }
+}
+
+static inline int rcu_gp_ongoing(unsigned long *ctr)
+{
+ unsigned long v;
+
+ v = CMM_LOAD_SHARED(*ctr);
+ return v && (v != rcu_gp_ctr);
+}
+
+static inline void _rcu_read_lock(void)
+{
+ rcu_assert(rcu_reader.ctr);
+}
+
+static inline void _rcu_read_unlock(void)
+{
+}
+
+static inline void _rcu_quiescent_state(void)
+{
+ cmm_smp_mb();
+ _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+ cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
+ wake_up_gp();
+ cmm_smp_mb();
+}
+
+static inline void _rcu_thread_offline(void)
+{
+ cmm_smp_mb();
+ CMM_STORE_SHARED(rcu_reader.ctr, 0);
+ cmm_smp_mb(); /* write rcu_reader.ctr before read futex */
+ wake_up_gp();
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+static inline void _rcu_thread_online(void)
+{
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+ _CMM_STORE_SHARED(rcu_reader.ctr, CMM_LOAD_SHARED(rcu_gp_ctr));
+ cmm_smp_mb();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_QSBR_STATIC_H */
--- /dev/null
+#ifndef _URCU_STATIC_H
+#define _URCU_STATIC_H
+
+/*
+ * urcu-static.h
+ *
+ * Userspace RCU header.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <syscall.h>
+#include <unistd.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic_arch.h>
+#include <urcu/list.h>
+#include <urcu/urcu-futex.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Default is RCU_MEMBARRIER */
+#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL)
+#define RCU_MEMBARRIER
+#endif
+
+#ifdef RCU_MEMBARRIER
+#include <unistd.h>
+#include <sys/syscall.h>
+
+/* If the headers do not support SYS_membarrier, statically use RCU_MB */
+#ifdef SYS_membarrier
+# define MEMBARRIER_EXPEDITED (1 << 0)
+# define MEMBARRIER_DELAYED (1 << 1)
+# define MEMBARRIER_QUERY (1 << 16)
+# define membarrier(...) syscall(SYS_membarrier, __VA_ARGS__)
+#else
+# undef RCU_MEMBARRIER
+# define RCU_MB
+#endif
+#endif
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+/*
+ * The signal number used by the RCU library can be overridden with
+ * -DSIGRCU= when compiling the library.
+ * Provide backward compatibility for liburcu 0.3.x SIGURCU.
+ */
+#ifdef SIGURCU
+#define SIGRCU SIGURCU
+#endif
+
+#ifndef SIGRCU
+#define SIGRCU SIGUSR1
+#endif
+
+/*
+ * If a reader is really non-cooperative and refuses to commit its
+ * rcu_active_readers count to memory (there is no barrier in the reader
+ * per-se), kick it after a few loops waiting for it.
+ */
+#define KICK_READER_LOOPS 10000
+
+/*
+ * Active attempts to check for reader Q.S. before calling futex().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+#ifdef DEBUG_RCU
+#define rcu_assert(args...) assert(args)
+#else
+#define rcu_assert(args...)
+#endif
+
+#ifdef DEBUG_YIELD
+#include <sched.h>
+#include <time.h>
+#include <pthread.h>
+#include <unistd.h>
+
+#define YIELD_READ (1 << 0)
+#define YIELD_WRITE (1 << 1)
+
+/*
+ * Updates with RCU_SIGNAL are much slower. Account this in the delay.
+ */
+#ifdef RCU_SIGNAL
+/* maximum sleep delay, in us */
+#define MAX_SLEEP 30000
+#else
+#define MAX_SLEEP 50
+#endif
+
+extern unsigned int yield_active;
+extern unsigned int __thread rand_yield;
+
+static inline void debug_yield_read(void)
+{
+ if (yield_active & YIELD_READ)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_write(void)
+{
+ if (yield_active & YIELD_WRITE)
+ if (rand_r(&rand_yield) & 0x1)
+ usleep(rand_r(&rand_yield) % MAX_SLEEP);
+}
+
+static inline void debug_yield_init(void)
+{
+ rand_yield = time(NULL) ^ pthread_self();
+}
+#else
+static inline void debug_yield_read(void)
+{
+}
+
+static inline void debug_yield_write(void)
+{
+}
+
+static inline void debug_yield_init(void)
+{
+
+}
+#endif
+
+/*
+ * RCU memory barrier broadcast group. Currently, only broadcast to all process
+ * threads is supported (group 0).
+ *
+ * Slave barriers are only guaranteed to be ordered wrt master barriers.
+ *
+ * The pair ordering is detailed as (O: ordered, X: not ordered) :
+ * slave master
+ * slave X O
+ * master O O
+ */
+
+#define MB_GROUP_ALL 0
+#define RCU_MB_GROUP MB_GROUP_ALL
+
+#ifdef RCU_MEMBARRIER
+extern int has_sys_membarrier;
+
+static inline void smp_mb_slave(int group)
+{
+ if (likely(has_sys_membarrier))
+ cmm_barrier();
+ else
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_MB
+static inline void smp_mb_slave(int group)
+{
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_SIGNAL
+static inline void smp_mb_slave(int group)
+{
+ cmm_barrier();
+}
+#endif
+
+/*
+ * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use
+ * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
+ */
+#define RCU_GP_COUNT (1UL << 0)
+/* Use the amount of bits equal to half of the architecture long size */
+#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))
+#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
+
+/*
+ * Global quiescent period counter with low-order bits unused.
+ * Using a int rather than a char to eliminate false register dependencies
+ * causing stalls on some architectures.
+ */
+extern unsigned long rcu_gp_ctr;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ unsigned long ctr;
+ char need_mb;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ pthread_t tid;
+};
+
+extern struct rcu_reader __thread rcu_reader;
+
+extern int gp_futex;
+
+/*
+ * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
+ */
+static inline void wake_up_gp(void)
+{
+ if (unlikely(uatomic_read(&gp_futex) == -1)) {
+ uatomic_set(&gp_futex, 0);
+ futex_async(&gp_futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0);
+ }
+}
+
+static inline int rcu_gp_ongoing(unsigned long *ctr)
+{
+ unsigned long v;
+
+ /*
+ * Make sure both tests below are done on the same version of *value
+ * to insure consistency.
+ */
+ v = CMM_LOAD_SHARED(*ctr);
+ return (v & RCU_GP_CTR_NEST_MASK) &&
+ ((v ^ rcu_gp_ctr) & RCU_GP_CTR_PHASE);
+}
+
+static inline void _rcu_read_lock(void)
+{
+ unsigned long tmp;
+
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+ tmp = rcu_reader.ctr;
+ /*
+ * rcu_gp_ctr is
+ * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE)
+ */
+ if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
+ _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr));
+ /*
+ * Set active readers count for outermost nesting level before
+ * accessing the pointer. See smp_mb_master().
+ */
+ smp_mb_slave(RCU_MB_GROUP);
+ } else {
+ _CMM_STORE_SHARED(rcu_reader.ctr, tmp + RCU_GP_COUNT);
+ }
+}
+
+static inline void _rcu_read_unlock(void)
+{
+ unsigned long tmp;
+
+ tmp = rcu_reader.ctr;
+ /*
+ * Finish using rcu before decrementing the pointer.
+ * See smp_mb_master().
+ */
+ if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) {
+ smp_mb_slave(RCU_MB_GROUP);
+ _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT);
+ /* write rcu_reader.ctr before read futex */
+ smp_mb_slave(RCU_MB_GROUP);
+ wake_up_gp();
+ } else {
+ _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT);
+ }
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_H */
--- /dev/null
+#ifndef _URCU_WFQUEUE_STATIC_H
+#define _URCU_WFQUEUE_STATIC_H
+
+/*
+ * wfqueue-static.h
+ *
+ * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic_arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Queue with wait-free enqueue/blocking dequeue.
+ * This implementation adds a dummy head node when the queue is empty to ensure
+ * we can always update the queue locklessly.
+ *
+ * Inspired from half-wait-free/half-blocking queue implementation done by
+ * Paul E. McKenney.
+ */
+
+#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
+#define WFQ_WAIT 10 /* Wait 10 ms if being set */
+
+static inline void _cds_wfq_node_init(struct cds_wfq_node *node)
+{
+ node->next = NULL;
+}
+
+static inline void _cds_wfq_init(struct cds_wfq_queue *q)
+{
+ int ret;
+
+ _cds_wfq_node_init(&q->dummy);
+ /* Set queue head and tail */
+ q->head = &q->dummy;
+ q->tail = &q->dummy.next;
+ ret = pthread_mutex_init(&q->lock, NULL);
+ assert(!ret);
+}
+
+static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q,
+ struct cds_wfq_node *node)
+{
+ struct cds_wfq_node **old_tail;
+
+ /*
+ * uatomic_xchg() implicit memory barrier orders earlier stores to data
+ * structure containing node and setting node->next to NULL before
+ * publication.
+ */
+ old_tail = uatomic_xchg(&q->tail, node);
+ /*
+ * At this point, dequeuers see a NULL old_tail->next, which indicates
+ * that the queue is being appended to. The following store will append
+ * "node" to the queue from a dequeuer perspective.
+ */
+ CMM_STORE_SHARED(*old_tail, node);
+}
+
+/*
+ * It is valid to reuse and free a dequeued node immediately.
+ *
+ * No need to go on a waitqueue here, as there is no possible state in which the
+ * list could cause dequeue to busy-loop needlessly while waiting for another
+ * thread to be scheduled. The queue appears empty until tail->next is set by
+ * enqueue.
+ */
+static inline struct cds_wfq_node *
+___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ struct cds_wfq_node *node, *next;
+ int attempt = 0;
+
+ /*
+ * Queue is empty if it only contains the dummy node.
+ */
+ if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next)
+ return NULL;
+ node = q->head;
+
+ /*
+ * Adaptative busy-looping waiting for enqueuer to complete enqueue.
+ */
+ while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ if (++attempt >= WFQ_ADAPT_ATTEMPTS) {
+ poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */
+ attempt = 0;
+ } else
+ caa_cpu_relax();
+ }
+ /*
+ * Move queue head forward.
+ */
+ q->head = next;
+ /*
+ * Requeue dummy node if we just dequeued it.
+ */
+ if (node == &q->dummy) {
+ _cds_wfq_node_init(node);
+ _cds_wfq_enqueue(q, node);
+ return ___cds_wfq_dequeue_blocking(q);
+ }
+ return node;
+}
+
+static inline struct cds_wfq_node *
+_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ struct cds_wfq_node *retnode;
+ int ret;
+
+ ret = pthread_mutex_lock(&q->lock);
+ assert(!ret);
+ retnode = ___cds_wfq_dequeue_blocking(q);
+ ret = pthread_mutex_unlock(&q->lock);
+ assert(!ret);
+ return retnode;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFQUEUE_STATIC_H */
--- /dev/null
+#ifndef _URCU_WFSTACK_STATIC_H
+#define _URCU_WFSTACK_STATIC_H
+
+/*
+ * wfstack-static.h
+ *
+ * Userspace RCU library - Stack with Wait-Free push, Blocking pop.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfstack.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic_arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CDS_WF_STACK_END ((void *)0x1UL)
+#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */
+#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */
+
+void _cds_wfs_node_init(struct cds_wfs_node *node)
+{
+ node->next = NULL;
+}
+
+void _cds_wfs_init(struct cds_wfs_stack *s)
+{
+ int ret;
+
+ s->head = CDS_WF_STACK_END;
+ ret = pthread_mutex_init(&s->lock, NULL);
+ assert(!ret);
+}
+
+void _cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node)
+{
+ struct cds_wfs_node *old_head;
+
+ assert(node->next == NULL);
+ /*
+ * uatomic_xchg() implicit memory barrier orders earlier stores to node
+ * (setting it to NULL) before publication.
+ */
+ old_head = uatomic_xchg(&s->head, node);
+ /*
+ * At this point, dequeuers see a NULL node->next, they should busy-wait
+ * until node->next is set to old_head.
+ */
+ CMM_STORE_SHARED(node->next, old_head);
+}
+
+/*
+ * Returns NULL if stack is empty.
+ */
+struct cds_wfs_node *
+___cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+ struct cds_wfs_node *head, *next;
+ int attempt = 0;
+
+retry:
+ head = CMM_LOAD_SHARED(s->head);
+ if (head == CDS_WF_STACK_END)
+ return NULL;
+ /*
+ * Adaptative busy-looping waiting for push to complete.
+ */
+ while ((next = CMM_LOAD_SHARED(head->next)) == NULL) {
+ if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
+ poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */
+ attempt = 0;
+ } else
+ caa_cpu_relax();
+ }
+ if (uatomic_cmpxchg(&s->head, head, next) == head)
+ return head;
+ else
+ goto retry; /* Concurrent modification. Retry. */
+}
+
+struct cds_wfs_node *
+_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+ struct cds_wfs_node *retnode;
+ int ret;
+
+ ret = pthread_mutex_lock(&s->lock);
+ assert(!ret);
+ retnode = ___cds_wfs_pop_blocking(s);
+ ret = pthread_mutex_unlock(&s->lock);
+ assert(!ret);
+ return retnode;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFSTACK_STATIC_H */
+++ /dev/null
-#ifndef _URCU_WFQUEUE_STATIC_H
-#define _URCU_WFQUEUE_STATIC_H
-
-/*
- * wfqueue-static.h
- *
- * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking
- * dynamically with the userspace rcu library.
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <poll.h>
-#include <urcu/compiler.h>
-#include <urcu/uatomic_arch.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Queue with wait-free enqueue/blocking dequeue.
- * This implementation adds a dummy head node when the queue is empty to ensure
- * we can always update the queue locklessly.
- *
- * Inspired from half-wait-free/half-blocking queue implementation done by
- * Paul E. McKenney.
- */
-
-#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
-#define WFQ_WAIT 10 /* Wait 10 ms if being set */
-
-static inline void _cds_wfq_node_init(struct cds_wfq_node *node)
-{
- node->next = NULL;
-}
-
-static inline void _cds_wfq_init(struct cds_wfq_queue *q)
-{
- int ret;
-
- _cds_wfq_node_init(&q->dummy);
- /* Set queue head and tail */
- q->head = &q->dummy;
- q->tail = &q->dummy.next;
- ret = pthread_mutex_init(&q->lock, NULL);
- assert(!ret);
-}
-
-static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q,
- struct cds_wfq_node *node)
-{
- struct cds_wfq_node **old_tail;
-
- /*
- * uatomic_xchg() implicit memory barrier orders earlier stores to data
- * structure containing node and setting node->next to NULL before
- * publication.
- */
- old_tail = uatomic_xchg(&q->tail, node);
- /*
- * At this point, dequeuers see a NULL old_tail->next, which indicates
- * that the queue is being appended to. The following store will append
- * "node" to the queue from a dequeuer perspective.
- */
- CMM_STORE_SHARED(*old_tail, node);
-}
-
-/*
- * It is valid to reuse and free a dequeued node immediately.
- *
- * No need to go on a waitqueue here, as there is no possible state in which the
- * list could cause dequeue to busy-loop needlessly while waiting for another
- * thread to be scheduled. The queue appears empty until tail->next is set by
- * enqueue.
- */
-static inline struct cds_wfq_node *
-___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- struct cds_wfq_node *node, *next;
- int attempt = 0;
-
- /*
- * Queue is empty if it only contains the dummy node.
- */
- if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next)
- return NULL;
- node = q->head;
-
- /*
- * Adaptative busy-looping waiting for enqueuer to complete enqueue.
- */
- while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- if (++attempt >= WFQ_ADAPT_ATTEMPTS) {
- poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */
- attempt = 0;
- } else
- caa_cpu_relax();
- }
- /*
- * Move queue head forward.
- */
- q->head = next;
- /*
- * Requeue dummy node if we just dequeued it.
- */
- if (node == &q->dummy) {
- _cds_wfq_node_init(node);
- _cds_wfq_enqueue(q, node);
- return ___cds_wfq_dequeue_blocking(q);
- }
- return node;
-}
-
-static inline struct cds_wfq_node *
-_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- struct cds_wfq_node *retnode;
- int ret;
-
- ret = pthread_mutex_lock(&q->lock);
- assert(!ret);
- retnode = ___cds_wfq_dequeue_blocking(q);
- ret = pthread_mutex_unlock(&q->lock);
- assert(!ret);
- return retnode;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFQUEUE_STATIC_H */
#ifdef _LGPL_SOURCE
-#include <urcu/wfqueue-static.h>
+#include <urcu/static/wfqueue.h>
#define cds_wfq_node_init _cds_wfq_node_init
#define cds_wfq_init _cds_wfq_init
+++ /dev/null
-#ifndef _URCU_WFSTACK_STATIC_H
-#define _URCU_WFSTACK_STATIC_H
-
-/*
- * wfstack-static.h
- *
- * Userspace RCU library - Stack with Wait-Free push, Blocking pop.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfstack.h for linking
- * dynamically with the userspace rcu library.
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <poll.h>
-#include <urcu/compiler.h>
-#include <urcu/uatomic_arch.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define CDS_WF_STACK_END ((void *)0x1UL)
-#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */
-#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */
-
-void _cds_wfs_node_init(struct cds_wfs_node *node)
-{
- node->next = NULL;
-}
-
-void _cds_wfs_init(struct cds_wfs_stack *s)
-{
- int ret;
-
- s->head = CDS_WF_STACK_END;
- ret = pthread_mutex_init(&s->lock, NULL);
- assert(!ret);
-}
-
-void _cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node)
-{
- struct cds_wfs_node *old_head;
-
- assert(node->next == NULL);
- /*
- * uatomic_xchg() implicit memory barrier orders earlier stores to node
- * (setting it to NULL) before publication.
- */
- old_head = uatomic_xchg(&s->head, node);
- /*
- * At this point, dequeuers see a NULL node->next, they should busy-wait
- * until node->next is set to old_head.
- */
- CMM_STORE_SHARED(node->next, old_head);
-}
-
-/*
- * Returns NULL if stack is empty.
- */
-struct cds_wfs_node *
-___cds_wfs_pop_blocking(struct cds_wfs_stack *s)
-{
- struct cds_wfs_node *head, *next;
- int attempt = 0;
-
-retry:
- head = CMM_LOAD_SHARED(s->head);
- if (head == CDS_WF_STACK_END)
- return NULL;
- /*
- * Adaptative busy-looping waiting for push to complete.
- */
- while ((next = CMM_LOAD_SHARED(head->next)) == NULL) {
- if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
- poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */
- attempt = 0;
- } else
- caa_cpu_relax();
- }
- if (uatomic_cmpxchg(&s->head, head, next) == head)
- return head;
- else
- goto retry; /* Concurrent modification. Retry. */
-}
-
-struct cds_wfs_node *
-_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
-{
- struct cds_wfs_node *retnode;
- int ret;
-
- ret = pthread_mutex_lock(&s->lock);
- assert(!ret);
- retnode = ___cds_wfs_pop_blocking(s);
- ret = pthread_mutex_unlock(&s->lock);
- assert(!ret);
- return retnode;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFSTACK_STATIC_H */
#ifdef _LGPL_SOURCE
-#include <urcu/wfstack-static.h>
+#include <urcu/static/wfstack.h>
#define cds_wfs_node_init _cds_wfs_node_init
#define cds_wfs_init _cds_wfs_init
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu/wfqueue.h"
-#include "urcu/wfqueue-static.h"
+#include "urcu/static/wfqueue.h"
/*
* library wrappers to be used by non-LGPL compatible source code.
/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#include "urcu/wfstack.h"
-#include "urcu/wfstack-static.h"
+#include "urcu/static/wfstack.h"
/*
* library wrappers to be used by non-LGPL compatible source code.
projects / userspace-rcu.git / commitdiff
