-urcu/arch.h
-urcu/uatomic.h
+include/urcu/arch.h
+include/urcu/uatomic.h
tests/api.h
doc/examples/rculfhash/cds_lfht_for_each_entry_duplicate
#automake
-/config.h
+/include/config.h
.deps/
.libs/
Makefile.in
/m4/ltsugar.m4
/m4/ltversion.m4
/libtool
-/stamp-h1
-/config.h.in
+/include/stamp-h1
+/include/config.h.in
/config.status
/autom4te.cache/
config.h
ACLOCAL_AMFLAGS=-I m4
-AM_CPPFLAGS = -I$(top_builddir)/urcu
+SUBDIRS = include src doc tests
-#Add the -version-info directly here since we are only building
-# library that use the version-info
-AM_LDFLAGS=-version-info $(URCU_LIBRARY_VERSION)
-if USE_CYGWIN
-AM_LDFLAGS+=-no-undefined
-endif
-AM_CFLAGS=-Wall
-
-SUBDIRS = . doc tests
-
-include_HEADERS = urcu.h urcu-bp.h urcu-call-rcu.h urcu-defer.h \
- urcu-pointer.h urcu-qsbr.h urcu-flavor.h
-nobase_dist_include_HEADERS = urcu/compiler.h urcu/hlist.h urcu/list.h \
- urcu/rculist.h urcu/rcuhlist.h urcu/system.h urcu/futex.h \
- urcu/uatomic/generic.h urcu/arch/generic.h urcu/wfstack.h \
- urcu/wfqueue.h urcu/rculfstack.h urcu/rculfqueue.h \
- urcu/ref.h urcu/cds.h urcu/urcu_ref.h urcu/urcu-futex.h \
- urcu/uatomic_arch.h urcu/rculfhash.h urcu/wfcqueue.h \
- urcu/lfstack.h urcu/syscall-compat.h \
- $(top_srcdir)/urcu/map/*.h \
- $(top_srcdir)/urcu/static/*.h \
- urcu/rand-compat.h \
- urcu/tls-compat.h urcu/debug.h
-nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic.h urcu/config.h
-
-dist_noinst_HEADERS = urcu-die.h urcu-wait.h compat-getcpu.h
-
-EXTRA_DIST = $(top_srcdir)/urcu/arch/*.h $(top_srcdir)/urcu/uatomic/*.h \
- gpl-2.0.txt lgpl-2.1.txt lgpl-relicensing.txt \
- LICENSE compat_arch_x86.c \
- urcu-call-rcu-impl.h urcu-defer-impl.h \
- rculfhash-internal.h
-
-if COMPAT_ARCH
-COMPAT=compat_arch_@ARCHTYPE@.c
-else
-COMPAT=
-endif
-
-COMPAT+=compat_futex.c
-
-RCULFHASH = rculfhash.c rculfhash-mm-order.c rculfhash-mm-chunk.c \
- rculfhash-mm-mmap.c
-
-lib_LTLIBRARIES = liburcu-common.la \
- liburcu.la liburcu-qsbr.la \
- liburcu-mb.la liburcu-signal.la liburcu-bp.la \
- liburcu-cds.la
-
-#
-# liburcu-common contains wait-free queues (needed by call_rcu) as well
-# as futex fallbacks.
-#
-liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
-
-liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
-liburcu_la_LIBADD = liburcu-common.la
-
-liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT)
-liburcu_qsbr_la_LIBADD = liburcu-common.la
-
-liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
-liburcu_mb_la_CFLAGS = -DRCU_MB
-liburcu_mb_la_LIBADD = liburcu-common.la
-
-liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
-liburcu_signal_la_CFLAGS = -DRCU_SIGNAL
-liburcu_signal_la_LIBADD = liburcu-common.la
-
-liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT)
-liburcu_bp_la_LIBADD = liburcu-common.la
-
-liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \
- $(RCULFHASH) $(COMPAT)
-liburcu_cds_la_LIBADD = liburcu-common.la
-
-pkgconfigdir = $(libdir)/pkgconfig
-pkgconfig_DATA = liburcu-cds.pc liburcu.pc liburcu-bp.pc liburcu-qsbr.pc \
- liburcu-signal.pc liburcu-mb.pc
-
-dist_doc_DATA = README.md ChangeLog
+dist_doc_DATA = LICENSE \
+ README.md
dist_noinst_DATA = CodingStyle
+EXTRA_DIST = gpl-2.0.txt \
+ lgpl-2.1.txt \
+ lgpl-relicensing.txt
+
.PHONY: short_bench long_bench regtest
short_bench:
cd tests && $(MAKE) $(AM_MAKEFLAGS) short_bench
+++ /dev/null
-#ifndef _COMPAT_GETCPU_H
-#define _COMPAT_GETCPU_H
-
-/*
- * compat-getcpu.h
- *
- * Copyright (c) 2015 Michael Jeanson <mjeanson@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <config.h>
-
-#if defined(HAVE_SCHED_GETCPU)
-#include <sched.h>
-
-static inline
-int urcu_sched_getcpu(void)
-{
- return sched_getcpu();
-}
-#elif defined(HAVE_GETCPUID)
-#include <sys/processor.h>
-
-static inline
-int urcu_sched_getcpu(void)
-{
- return (int) getcpuid();
-}
-#else
-
-static inline
-int urcu_sched_getcpu(void)
-{
- return -1;
-}
-#endif
-
-#endif /* _COMPAT_GETCPU_H */
+++ /dev/null
-/*
- * compat_arch_x86.c
- *
- * Userspace RCU library - x86 compatibility checks
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <urcu/uatomic.h>
-
-/*
- * Using attribute "weak" for __rcu_cas_avail and
- * __urcu_x86_compat_mutex. Those are globally visible by the entire
- * program, even though many shared objects may have their own version.
- * The first version that gets loaded will be used by the entire
- * program (executable and all shared objects).
- */
-
-/*
- * It does not really matter if the constructor is called before using
- * the library, as long as the caller checks if __rcu_cas_avail < 0 and calls
- * compat_arch_init() explicitely if needed.
- */
-int __attribute__((constructor)) __rcu_cas_init(void);
-
-/*
- * -1: unknown
- * 1: available
- * 0: unavailable
- */
-__attribute__((weak))
-int __rcu_cas_avail = -1;
-
-__attribute__((weak))
-pthread_mutex_t __urcu_x86_compat_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-/*
- * get_eflags/set_eflags/compare_and_swap_is_available imported from glibc
- * 2.3.5. linuxthreads/sysdeps/i386/pt-machine.h.
- */
-
-static int get_eflags (void)
-{
- int res;
- __asm__ __volatile__ ("pushfl; popl %0" : "=r" (res) : );
- return res;
-}
-
-static void set_eflags (int newflags)
-{
- __asm__ __volatile__ ("pushl %0; popfl" : : "r" (newflags) : "cc");
-}
-
-static int compare_and_swap_is_available (void)
-{
- int oldflags = get_eflags ();
- int changed;
- /* Flip AC bit in EFLAGS. */
- set_eflags (oldflags ^ 0x40000);
- /* See if bit changed. */
- changed = (get_eflags () ^ oldflags) & 0x40000;
- /* Restore EFLAGS. */
- set_eflags (oldflags);
- /* If the AC flag did not change, it's a 386 and it lacks cmpxchg.
- Otherwise, it's a 486 or above and it has cmpxchg. */
- return changed != 0;
-}
-
-static void mutex_lock_signal_save(pthread_mutex_t *mutex, sigset_t *oldmask)
-{
- sigset_t newmask;
- int ret;
-
- /* Disable signals */
- ret = sigfillset(&newmask);
- assert(!ret);
- ret = pthread_sigmask(SIG_BLOCK, &newmask, oldmask);
- assert(!ret);
- ret = pthread_mutex_lock(&__urcu_x86_compat_mutex);
- assert(!ret);
-}
-
-static void mutex_lock_signal_restore(pthread_mutex_t *mutex, sigset_t *oldmask)
-{
- int ret;
-
- ret = pthread_mutex_unlock(&__urcu_x86_compat_mutex);
- assert(!ret);
- ret = pthread_sigmask(SIG_SETMASK, oldmask, NULL);
- assert(!ret);
-}
-
-unsigned long _compat_uatomic_set(void *addr, unsigned long _new, int len)
-{
- sigset_t mask;
- unsigned long result;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- *(unsigned char *)addr = (unsigned char)_new;
- result = *(unsigned char *)addr;
- break;
- case 2:
- *(unsigned short *)addr = (unsigned short)_new;
- result = *(unsigned short *)addr;
- break;
- case 4:
- *(unsigned int *)addr = (unsigned int)_new;
- result = *(unsigned int *)addr;
- break;
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- result = 0;
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
- return result;
-}
-
-unsigned long _compat_uatomic_xchg(void *addr, unsigned long _new, int len)
-{
- sigset_t mask;
- unsigned long retval;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- retval = *(unsigned char *)addr;
- *(unsigned char *)addr = (unsigned char)_new;
- break;
- case 2:
- retval = *(unsigned short *)addr;
- *(unsigned short *)addr = (unsigned short)_new;
- break;
- case 4:
- retval = *(unsigned int *)addr;
- *(unsigned int *)addr = (unsigned int)_new;
- break;
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- retval = 0; /* silence gcc warnings */
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
- return retval;
-}
-
-unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- unsigned long retval;
- sigset_t mask;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- {
- unsigned char result = *(unsigned char *)addr;
- if (result == (unsigned char)old)
- *(unsigned char *)addr = (unsigned char)_new;
- retval = result;
- break;
- }
- case 2:
- {
- unsigned short result = *(unsigned short *)addr;
- if (result == (unsigned short)old)
- *(unsigned short *)addr = (unsigned short)_new;
- retval = result;
- break;
- }
- case 4:
- {
- unsigned int result = *(unsigned int *)addr;
- if (result == (unsigned int)old)
- *(unsigned int *)addr = (unsigned int)_new;
- retval = result;
- break;
- }
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- retval = 0; /* silence gcc warnings */
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
- return retval;
-}
-
-void _compat_uatomic_or(void *addr, unsigned long v, int len)
-{
- sigset_t mask;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- *(unsigned char *)addr |= (unsigned char)v;
- break;
- case 2:
- *(unsigned short *)addr |= (unsigned short)v;
- break;
- case 4:
- *(unsigned int *)addr |= (unsigned int)v;
- break;
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
-}
-
-void _compat_uatomic_and(void *addr, unsigned long v, int len)
-{
- sigset_t mask;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- *(unsigned char *)addr &= (unsigned char)v;
- break;
- case 2:
- *(unsigned short *)addr &= (unsigned short)v;
- break;
- case 4:
- *(unsigned int *)addr &= (unsigned int)v;
- break;
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
-}
-
-unsigned long _compat_uatomic_add_return(void *addr, unsigned long v, int len)
-{
- sigset_t mask;
- unsigned long result;
-
- mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
- switch (len) {
- case 1:
- *(unsigned char *)addr += (unsigned char)v;
- result = *(unsigned char *)addr;
- break;
- case 2:
- *(unsigned short *)addr += (unsigned short)v;
- result = *(unsigned short *)addr;
- break;
- case 4:
- *(unsigned int *)addr += (unsigned int)v;
- result = *(unsigned int *)addr;
- break;
- default:
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- result = 0; /* silence gcc warnings */
- __asm__ __volatile__("ud2");
- }
- mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
- return result;
-}
-
-int __rcu_cas_init(void)
-{
- if (__rcu_cas_avail < 0)
- __rcu_cas_avail = compare_and_swap_is_available();
- return __rcu_cas_avail;
-}
+++ /dev/null
-/*
- * compat_futex.c
- *
- * Userspace RCU library - sys_futex compatibility code
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <errno.h>
-#include <poll.h>
-#include <stdint.h>
-
-#include <urcu/arch.h>
-#include <urcu/futex.h>
-#include <urcu/system.h>
-
-/*
- * Using attribute "weak" for __urcu_compat_futex_lock and
- * __urcu_compat_futex_cond. Those are globally visible by the entire
- * program, even though many shared objects may have their own version.
- * The first version that gets loaded will be used by the entire program
- * (executable and all shared objects).
- */
-
-__attribute__((weak))
-pthread_mutex_t __urcu_compat_futex_lock = PTHREAD_MUTEX_INITIALIZER;
-__attribute__((weak))
-pthread_cond_t __urcu_compat_futex_cond = PTHREAD_COND_INITIALIZER;
-
-/*
- * _NOT SIGNAL-SAFE_. pthread_cond is not signal-safe anyway. Though.
- * For now, timeout, uaddr2 and val3 are unused.
- * Waiter will relinquish the CPU until woken up.
- */
-
-int compat_futex_noasync(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- int ret = 0, lockret;
-
- /*
- * Check if NULL. Don't let users expect that they are taken into
- * account.
- */
- assert(!timeout);
- assert(!uaddr2);
- assert(!val3);
-
- /*
- * memory barriers to serialize with the previous uaddr modification.
- */
- cmm_smp_mb();
-
- lockret = pthread_mutex_lock(&__urcu_compat_futex_lock);
- if (lockret) {
- errno = lockret;
- ret = -1;
- goto end;
- }
- switch (op) {
- case FUTEX_WAIT:
- /*
- * Wait until *uaddr is changed to something else than "val".
- * Comparing *uaddr content against val figures out which
- * thread has been awakened.
- */
- while (CMM_LOAD_SHARED(*uaddr) == val)
- pthread_cond_wait(&__urcu_compat_futex_cond,
- &__urcu_compat_futex_lock);
- break;
- case FUTEX_WAKE:
- /*
- * Each wake is sending a broadcast, thus attempting wakeup of
- * all awaiting threads, independently of their respective
- * uaddr.
- */
- pthread_cond_broadcast(&__urcu_compat_futex_cond);
- break;
- default:
- errno = EINVAL;
- ret = -1;
- }
- lockret = pthread_mutex_unlock(&__urcu_compat_futex_lock);
- if (lockret) {
- errno = lockret;
- ret = -1;
- }
-end:
- return ret;
-}
-
-/*
- * _ASYNC SIGNAL-SAFE_.
- * For now, timeout, uaddr2 and val3 are unused.
- * Waiter will busy-loop trying to read the condition.
- * It is OK to use compat_futex_async() on a futex address on which
- * futex() WAKE operations are also performed.
- */
-
-int compat_futex_async(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- int ret = 0;
-
- /*
- * Check if NULL. Don't let users expect that they are taken into
- * account.
- */
- assert(!timeout);
- assert(!uaddr2);
- assert(!val3);
-
- /*
- * Ensure previous memory operations on uaddr have completed.
- */
- cmm_smp_mb();
-
- switch (op) {
- case FUTEX_WAIT:
- while (CMM_LOAD_SHARED(*uaddr) == val) {
- if (poll(NULL, 0, 10) < 0) {
- ret = -1;
- /* Keep poll errno. Caller handles EINTR. */
- goto end;
- }
- }
- break;
- case FUTEX_WAKE:
- break;
- default:
- errno = EINVAL;
- ret = -1;
- }
-end:
- return ret;
-}
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
AC_SUBST([URCU_LIBRARY_VERSION], [5:0:1])
-AC_CONFIG_SRCDIR([urcu.h])
-AC_CONFIG_HEADERS([config.h urcu/config.h])
+AC_CONFIG_HEADERS([include/config.h include/urcu/config.h])
AC_CONFIG_AUX_DIR([config])
AC_CONFIG_MACRO_DIR([m4])
AC_CANONICAL_TARGET
AC_CANONICAL_HOST
-AM_INIT_AUTOMAKE([foreign dist-bzip2 no-dist-gzip])
+AM_INIT_AUTOMAKE([foreign dist-bzip2 no-dist-gzip nostdinc])
AM_MAINTAINER_MODE([enable])
# Enable silent rules if available (Introduced in AM 1.11)
AC_SUBST(ARCHTYPE)
AC_SUBST(SUBARCHTYPE)
-UATOMICSRC=urcu/uatomic/$ARCHTYPE.h
-ARCHSRC=urcu/arch/$ARCHTYPE.h
+UATOMICSRC=include/urcu/uatomic/$ARCHTYPE.h
+ARCHSRC=include/urcu/arch/$ARCHTYPE.h
AS_IF([test "x$SUBARCHTYPE" = xx86compat],[
AC_DEFINE([CONFIG_RCU_COMPAT_ARCH], [1])
# int sched_setaffinity (pid_t __pid, const cpu_set_t *__mask);
# ~~~~
-# Since we define _GNU_SOURCE in the sources, must do so too in the
-# autoconf tests, as defining _GNU_SOURCE or not exposes
-# sched_setaffinity bits differently.
-saved_CFLAGS=$CFLAGS
-CFLAGS="$CFLAGS -D_GNU_SOURCE"
-
AC_CHECK_TYPES([cpu_set_t],
[have_cpu_set_t="yes"],
[have_cpu_set_t="no"],
AC_MSG_CHECKING([whether CPU_ZERO works])
AH_TEMPLATE([HAVE_CPU_ZERO], [Defined to 1 if we have CPU_ZERO and it works])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
- #define _GNU_SOURCE
#include <sched.h>
int main()
{
AC_MSG_CHECKING([whether CPU_SET works])
AH_TEMPLATE([HAVE_CPU_SET], [Defined to 1 if we have CPU_SET and it works])
AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
- #define _GNU_SOURCE
#include <sched.h>
int main()
{
])
])
-CFLAGS=$saved_CFLAGS
+DEFAULT_INCLUDES="-include config.h"
+AC_SUBST(DEFAULT_INCLUDES)
AC_CONFIG_LINKS([
- urcu/arch.h:$ARCHSRC
- urcu/uatomic.h:$UATOMICSRC
+ include/urcu/arch.h:$ARCHSRC
+ include/urcu/uatomic.h:$UATOMICSRC
])
AC_CONFIG_FILES([
Makefile
doc/Makefile
doc/examples/Makefile
+ include/Makefile
+ src/Makefile
tests/Makefile
tests/common/Makefile
tests/unit/Makefile
tests/regression/Makefile
tests/regression/regression_tests
tests/utils/Makefile
- liburcu.pc
- liburcu-bp.pc
- liburcu-cds.pc
- liburcu-qsbr.pc
- liburcu-mb.pc
- liburcu-signal.pc
+ src/liburcu.pc
+ src/liburcu-bp.pc
+ src/liburcu-cds.pc
+ src/liburcu-qsbr.pc
+ src/liburcu-mb.pc
+ src/liburcu-signal.pc
])
AC_OUTPUT
echo "Examples: relative top_builddir path $(top_builddir)"; \
rel_build_subdir="../"; \
fi; \
- $(MAKE) -f dist-files/Makefile AM_CC="$(CC)" AM_CPPFLAGS="$(CPPFLAGS) -I$$rel_src_subdir/$(top_srcdir)/ -I$$rel_build_subdir$(top_builddir)/" AM_CFLAGS='$(CFLAGS)' AM_LDFLAGS='$(LDFLAGS) -L../../../.libs/ -Wl,-rpath "$(PWD)/../../.libs/"' $(AM_MAKEFLAGS) all;
+ $(MAKE) -f dist-files/Makefile AM_CC="$(CC)" AM_CPPFLAGS="$(CPPFLAGS) -I$$rel_src_subdir/$(top_srcdir)/include/ -I$$rel_src_subdir/$(top_srcdir)/src/ -I$$rel_build_subdir$(top_builddir)/include/ -I$$rel_build_subdir$(top_builddir)/include/src/" AM_CFLAGS='$(CFLAGS)' AM_LDFLAGS='$(LDFLAGS) -L../../../src/.libs/ -Wl,-rpath "$(PWD)/../../src/.libs/"' $(AM_MAKEFLAGS) all;
clean-local:
@$(MAKE) -f dist-files/Makefile $(AM_MAKEFLAGS) clean; \
--- /dev/null
+nobase_dist_include_HEADERS = urcu/compiler.h urcu/hlist.h urcu/list.h \
+ urcu/rculist.h urcu/rcuhlist.h urcu/system.h urcu/futex.h \
+ urcu/uatomic/generic.h urcu/arch/generic.h urcu/wfstack.h \
+ urcu/wfqueue.h urcu/rculfstack.h urcu/rculfqueue.h \
+ urcu/ref.h urcu/cds.h urcu/urcu_ref.h urcu/urcu-futex.h \
+ urcu/uatomic_arch.h urcu/rculfhash.h urcu/wfcqueue.h \
+ urcu/lfstack.h urcu/syscall-compat.h \
+ $(top_srcdir)/include/urcu/map/*.h \
+ $(top_srcdir)/include/urcu/static/*.h \
+ urcu/rand-compat.h \
+ urcu/tls-compat.h urcu/debug.h
+
+# Don't distribute generated headers
+nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic.h urcu/config.h
+
+EXTRA_DIST = $(top_srcdir)/include/urcu/arch/*.h \
+ $(top_srcdir)/include/urcu/uatomic/*.h
--- /dev/null
+#ifndef _URCU_ARCH_AARCH64_H
+#define _URCU_ARCH_AARCH64_H
+
+/*
+ * arch/aarch64.h: definitions for aarch64 architecture
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers. aarch64 implements asm-generic/unistd.h system call
+ * numbers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 283
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_AARCH64_H */
--- /dev/null
+#ifndef _URCU_ARCH_ALPHA_H
+#define _URCU_ARCH_ALPHA_H
+
+/*
+ * arch_alpha.h: trivial definitions for the Alpha architecture.
+ *
+ * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define cmm_mb() __asm__ __volatile__ ("mb":::"memory")
+#define cmm_wmb() __asm__ __volatile__ ("wmb":::"memory")
+#define cmm_read_barrier_depends() __asm__ __volatile__ ("mb":::"memory")
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_ALPHA_H */
--- /dev/null
+#ifndef _URCU_ARCH_ARM_H
+#define _URCU_ARCH_ARM_H
+
+/*
+ * arch_arm.h: trivial definitions for the ARM architecture.
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef CONFIG_RCU_ARM_HAVE_DMB
+#define cmm_mb() __asm__ __volatile__ ("dmb":::"memory")
+#define cmm_rmb() __asm__ __volatile__ ("dmb":::"memory")
+#define cmm_wmb() __asm__ __volatile__ ("dmb":::"memory")
+#endif /* CONFIG_RCU_ARM_HAVE_DMB */
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 389
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_ARM_H */
--- /dev/null
+#ifndef _URCU_ARCH_GCC_H
+#define _URCU_ARCH_GCC_H
+
+/*
+ * arch_gcc.h: trivial definitions for architectures using gcc __sync_
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_GCC_H */
--- /dev/null
+#ifndef _URCU_ARCH_GENERIC_H
+#define _URCU_ARCH_GENERIC_H
+
+/*
+ * arch_generic.h: common definitions for multiple architectures.
+ *
+ * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CAA_CACHE_LINE_SIZE
+#define CAA_CACHE_LINE_SIZE 64
+#endif
+
+#if !defined(cmm_mc) && !defined(cmm_rmc) && !defined(cmm_wmc)
+#define CONFIG_HAVE_MEM_COHERENCY
+/*
+ * Architectures with cache coherency must _not_ define cmm_mc/cmm_rmc/cmm_wmc.
+ *
+ * For them, cmm_mc/cmm_rmc/cmm_wmc are implemented with a simple
+ * compiler barrier; in addition, we provide defaults for cmm_mb (using
+ * GCC builtins) as well as cmm_rmb and cmm_wmb (defaulting to cmm_mb).
+ */
+
+#ifndef cmm_mb
+#define cmm_mb() __sync_synchronize()
+#endif
+
+#ifndef cmm_rmb
+#define cmm_rmb() cmm_mb()
+#endif
+
+#ifndef cmm_wmb
+#define cmm_wmb() cmm_mb()
+#endif
+
+#define cmm_mc() cmm_barrier()
+#define cmm_rmc() cmm_barrier()
+#define cmm_wmc() cmm_barrier()
+#else
+/*
+ * Architectures without cache coherency need something like the following:
+ *
+ * #define cmm_mc() arch_cache_flush()
+ * #define cmm_rmc() arch_cache_flush_read()
+ * #define cmm_wmc() arch_cache_flush_write()
+ *
+ * Of these, only cmm_mc is mandatory. cmm_rmc and cmm_wmc default to
+ * cmm_mc. cmm_mb/cmm_rmb/cmm_wmb use these definitions by default:
+ *
+ * #define cmm_mb() cmm_mc()
+ * #define cmm_rmb() cmm_rmc()
+ * #define cmm_wmb() cmm_wmc()
+ */
+
+#ifndef cmm_mb
+#define cmm_mb() cmm_mc()
+#endif
+
+#ifndef cmm_rmb
+#define cmm_rmb() cmm_rmc()
+#endif
+
+#ifndef cmm_wmb
+#define cmm_wmb() cmm_wmc()
+#endif
+
+#ifndef cmm_rmc
+#define cmm_rmc() cmm_mc()
+#endif
+
+#ifndef cmm_wmc
+#define cmm_wmc() cmm_mc()
+#endif
+#endif
+
+/* Nop everywhere except on alpha. */
+#ifndef cmm_read_barrier_depends
+#define cmm_read_barrier_depends()
+#endif
+
+#ifdef CONFIG_RCU_SMP
+#ifndef cmm_smp_mb
+#define cmm_smp_mb() cmm_mb()
+#endif
+#ifndef cmm_smp_rmb
+#define cmm_smp_rmb() cmm_rmb()
+#endif
+#ifndef cmm_smp_wmb
+#define cmm_smp_wmb() cmm_wmb()
+#endif
+#ifndef cmm_smp_mc
+#define cmm_smp_mc() cmm_mc()
+#endif
+#ifndef cmm_smp_rmc
+#define cmm_smp_rmc() cmm_rmc()
+#endif
+#ifndef cmm_smp_wmc
+#define cmm_smp_wmc() cmm_wmc()
+#endif
+#ifndef cmm_smp_read_barrier_depends
+#define cmm_smp_read_barrier_depends() cmm_read_barrier_depends()
+#endif
+#else
+#ifndef cmm_smp_mb
+#define cmm_smp_mb() cmm_barrier()
+#endif
+#ifndef cmm_smp_rmb
+#define cmm_smp_rmb() cmm_barrier()
+#endif
+#ifndef cmm_smp_wmb
+#define cmm_smp_wmb() cmm_barrier()
+#endif
+#ifndef cmm_smp_mc
+#define cmm_smp_mc() cmm_barrier()
+#endif
+#ifndef cmm_smp_rmc
+#define cmm_smp_rmc() cmm_barrier()
+#endif
+#ifndef cmm_smp_wmc
+#define cmm_smp_wmc() cmm_barrier()
+#endif
+#ifndef cmm_smp_read_barrier_depends
+#define cmm_smp_read_barrier_depends()
+#endif
+#endif
+
+#ifndef caa_cpu_relax
+#define caa_cpu_relax() cmm_barrier()
+#endif
+
+#ifndef HAS_CAA_GET_CYCLES
+#define HAS_CAA_GET_CYCLES
+
+#ifdef CONFIG_RCU_HAVE_CLOCK_GETTIME
+
+#include <time.h>
+#include <stdint.h>
+
+typedef uint64_t caa_cycles_t;
+
+static inline caa_cycles_t caa_get_cycles (void)
+{
+ struct timespec ts;
+
+ if (caa_unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
+ return -1ULL;
+ return ((uint64_t) ts.tv_sec * 1000000000ULL) + ts.tv_nsec;
+}
+
+#elif defined(__APPLE__)
+
+#include <mach/mach.h>
+#include <mach/clock.h>
+#include <mach/mach_time.h>
+#include <time.h>
+#include <stdint.h>
+
+typedef uint64_t caa_cycles_t;
+
+static inline caa_cycles_t caa_get_cycles (void)
+{
+ mach_timespec_t ts = { 0, 0 };
+ static clock_serv_t clock_service;
+
+ if (caa_unlikely(!clock_service)) {
+ if (host_get_clock_service(mach_host_self(),
+ SYSTEM_CLOCK, &clock_service))
+ return -1ULL;
+ }
+ if (caa_unlikely(clock_get_time(clock_service, &ts)))
+ return -1ULL;
+ return ((uint64_t) ts.tv_sec * 1000000000ULL) + ts.tv_nsec;
+}
+
+#else
+
+#error caa_get_cycles() not implemented for this platform.
+
+#endif
+
+#endif /* HAS_CAA_GET_CYCLES */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_ARCH_GENERIC_H */
--- /dev/null
+#ifndef _URCU_ARCH_HPPA_H
+#define _URCU_ARCH_HPPA_H
+
+/*
+ * arch/hppa.h: definitions for hppa architecture
+ *
+ * Copyright (c) 2014 Helge Deller <deller@gmx.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 343
+#endif
+
+#define HAS_CAA_GET_CYCLES
+typedef unsigned long caa_cycles_t;
+
+static inline caa_cycles_t caa_get_cycles(void)
+{
+ caa_cycles_t cycles;
+
+ asm volatile("mfctl 16, %0" : "=r" (cycles));
+ return cycles;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_HPPA_H */
--- /dev/null
+#ifndef _URCU_ARCH_IA64_H
+#define _URCU_ARCH_IA64_H
+
+/*
+ * arch/ia64.h: definitions for ia64 architecture
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 1344
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_IA64_H */
--- /dev/null
+#ifndef _URCU_ARCH_MIPS_H
+#define _URCU_ARCH_MIPS_H
+
+/*
+ * arch_mips.h: trivial definitions for the MIPS architecture.
+ *
+ * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
+ * Copyright (c) 2012 Ralf Baechle <ralf@linux-mips.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define cmm_mb() __asm__ __volatile__ ( \
+ " .set mips2 \n" \
+ " sync \n" \
+ " .set mips0 \n" \
+ :::"memory")
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_MIPS_H */
--- /dev/null
+#ifndef _URCU_ARCH_NIOS2_H
+#define _URCU_ARCH_NIOS2_H
+
+/*
+ * arch_nios2.h: trivial definitions for the NIOS2 architecture.
+ *
+ * Copyright (c) 2016 Marek Vasut <marex@denx.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define cmm_mb() cmm_barrier()
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_NIOS2_H */
--- /dev/null
+#ifndef _URCU_ARCH_PPC_H
+#define _URCU_ARCH_PPC_H
+
+/*
+ * arch_ppc.h: trivial definitions for the powerpc architecture.
+ *
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Include size of POWER5+ L3 cache lines: 256 bytes */
+#define CAA_CACHE_LINE_SIZE 256
+
+#ifdef __NO_LWSYNC__
+#define LWSYNC_OPCODE "sync\n"
+#else
+#define LWSYNC_OPCODE "lwsync\n"
+#endif
+
+/*
+ * Use sync for all cmm_mb/rmb/wmb barriers because lwsync does not
+ * preserve ordering of cacheable vs. non-cacheable accesses, so it
+ * should not be used to order with respect to MMIO operations. An
+ * eieio+lwsync pair is also not enough for cmm_rmb, because it will
+ * order cacheable and non-cacheable memory operations separately---i.e.
+ * not the latter against the former.
+ */
+#define cmm_mb() __asm__ __volatile__ ("sync":::"memory")
+
+/*
+ * lwsync orders loads in cacheable memory with respect to other loads,
+ * and stores in cacheable memory with respect to other stores.
+ * Therefore, use it for barriers ordering accesses to cacheable memory
+ * only.
+ */
+#define cmm_smp_rmb() __asm__ __volatile__ (LWSYNC_OPCODE:::"memory")
+#define cmm_smp_wmb() __asm__ __volatile__ (LWSYNC_OPCODE:::"memory")
+
+#define mftbl() \
+ __extension__ \
+ ({ \
+ unsigned long rval; \
+ __asm__ __volatile__ ("mftbl %0" : "=r" (rval)); \
+ rval; \
+ })
+
+#define mftbu() \
+ __extension__ \
+ ({ \
+ unsigned long rval; \
+ __asm__ __volatile__ ("mftbu %0" : "=r" (rval)); \
+ rval; \
+ })
+
+#define mftb() \
+ __extension__ \
+ ({ \
+ unsigned long long rval; \
+ __asm__ __volatile__ ("mftb %0" : "=r" (rval)); \
+ rval; \
+ })
+
+#define HAS_CAA_GET_CYCLES
+
+typedef uint64_t caa_cycles_t;
+
+#ifdef __powerpc64__
+static inline caa_cycles_t caa_get_cycles(void)
+{
+ return (caa_cycles_t) mftb();
+}
+#else
+static inline caa_cycles_t caa_get_cycles(void)
+{
+ unsigned long h, l;
+
+ for (;;) {
+ h = mftbu();
+ cmm_barrier();
+ l = mftbl();
+ cmm_barrier();
+ if (mftbu() == h)
+ return (((caa_cycles_t) h) << 32) + l;
+ }
+}
+#endif
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 365
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_PPC_H */
--- /dev/null
+#ifndef _URCU_ARCH_S390_H
+#define _URCU_ARCH_S390_H
+
+/*
+ * Trivial definitions for the S390 architecture based on information from the
+ * Principles of Operation "CPU Serialization" (5-91), "BRANCH ON CONDITION"
+ * (7-25) and "STORE CLOCK" (7-169).
+ *
+ * Copyright (c) 2009 Novell, Inc.
+ * Author: Jan Blunck <jblunck@suse.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CAA_CACHE_LINE_SIZE 128
+
+#define cmm_mb() __asm__ __volatile__("bcr 15,0" : : : "memory")
+
+#define HAS_CAA_GET_CYCLES
+
+typedef uint64_t caa_cycles_t;
+
+static inline caa_cycles_t caa_get_cycles (void)
+{
+ caa_cycles_t cycles;
+
+ __asm__ __volatile__("stck %0" : "=m" (cycles) : : "cc", "memory" );
+
+ return cycles;
+}
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 356
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_S390_H */
--- /dev/null
+#ifndef _URCU_ARCH_SPARC64_H
+#define _URCU_ARCH_SPARC64_H
+
+/*
+ * arch_sparc64.h: trivial definitions for the Sparc64 architecture.
+ *
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 351
+#endif
+
+#define CAA_CACHE_LINE_SIZE 256
+
+/*
+ * Inspired from the Linux kernel. Workaround Spitfire bug #51.
+ */
+#define membar_safe(type) \
+__asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \
+ "membar " type "\n" \
+ "1:\n" \
+ : : : "memory")
+
+#define cmm_mb() membar_safe("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
+#define cmm_rmb() membar_safe("#LoadLoad")
+#define cmm_wmb() membar_safe("#StoreStore")
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_SPARC64_H */
--- /dev/null
+#ifndef _URCU_ARCH_TILE_H
+#define _URCU_ARCH_TILE_H
+
+/*
+ * arch/tile.h: definitions for tile architecture
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <sys/time.h>
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers. tile implements asm-generic/unistd.h system call
+ * numbers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#define __NR_membarrier 283
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_TILE_H */
--- /dev/null
+#ifndef _URCU_ARCH_UNKNOWN_H
+#define _URCU_ARCH_UNKNOWN_H
+
+/*
+ * arch_unknown.h: #error to prevent build on unknown architectures.
+ *
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* See configure.ac for the list of recognized architectures. */
+#error "Cannot build: unrecognized architecture detected."
+
+#endif /* _URCU_ARCH_UNKNOWN_H */
--- /dev/null
+#ifndef _URCU_ARCH_X86_H
+#define _URCU_ARCH_X86_H
+
+/*
+ * arch_x86.h: trivial definitions for the x86 architecture.
+ *
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/config.h>
+#include <urcu/syscall-compat.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CAA_CACHE_LINE_SIZE 128
+
+#ifdef CONFIG_RCU_HAVE_FENCE
+#define cmm_mb() __asm__ __volatile__ ("mfence":::"memory")
+
+/*
+ * Define cmm_rmb/cmm_wmb to "strict" barriers that may be needed when
+ * using SSE or working with I/O areas. cmm_smp_rmb/cmm_smp_wmb are
+ * only compiler barriers, which is enough for general use.
+ */
+#define cmm_rmb() __asm__ __volatile__ ("lfence":::"memory")
+#define cmm_wmb() __asm__ __volatile__ ("sfence"::: "memory")
+#define cmm_smp_rmb() cmm_barrier()
+#define cmm_smp_wmb() cmm_barrier()
+#else
+/*
+ * We leave smp_rmb/smp_wmb as full barriers for processors that do not have
+ * fence instructions.
+ *
+ * An empty cmm_smp_rmb() may not be enough on old PentiumPro multiprocessor
+ * systems, due to an erratum. The Linux kernel says that "Even distro
+ * kernels should think twice before enabling this", but for now let's
+ * be conservative and leave the full barrier on 32-bit processors. Also,
+ * IDT WinChip supports weak store ordering, and the kernel may enable it
+ * under our feet; cmm_smp_wmb() ceases to be a nop for these processors.
+ */
+#if (CAA_BITS_PER_LONG == 32)
+#define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
+#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
+#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
+#else
+#define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
+#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
+#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
+#endif
+#endif
+
+#define caa_cpu_relax() __asm__ __volatile__ ("rep; nop" : : : "memory")
+
+#define HAS_CAA_GET_CYCLES
+
+#define rdtscll(val) \
+ do { \
+ unsigned int __a, __d; \
+ __asm__ __volatile__ ("rdtsc" : "=a" (__a), "=d" (__d)); \
+ (val) = ((unsigned long long)__a) \
+ | (((unsigned long long)__d) << 32); \
+ } while(0)
+
+typedef uint64_t caa_cycles_t;
+
+static inline caa_cycles_t caa_get_cycles(void)
+{
+ caa_cycles_t ret = 0;
+
+ rdtscll(ret);
+ return ret;
+}
+
+/*
+ * On Linux, define the membarrier system call number if not yet available in
+ * the system headers.
+ */
+#if (defined(__linux__) && !defined(__NR_membarrier))
+#if (CAA_BITS_PER_LONG == 32)
+#define __NR_membarrier 375
+#else
+#define __NR_membarrier 324
+#endif
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/arch/generic.h>
+
+#endif /* _URCU_ARCH_X86_H */
--- /dev/null
+#ifndef _URCU_CDS_H
+#define _URCU_CDS_H
+
+/*
+ * urcu/cds.h
+ *
+ * Userspace RCU library - Concurrent Data Structures
+ *
+ * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/hlist.h>
+#include <urcu/list.h>
+#include <urcu/rcuhlist.h>
+#include <urcu/rculist.h>
+#include <urcu/rculfqueue.h>
+#include <urcu/rculfstack.h>
+#include <urcu/rculfhash.h>
+#include <urcu/wfqueue.h>
+#include <urcu/wfcqueue.h>
+#include <urcu/wfstack.h>
+#include <urcu/lfstack.h>
+
+#endif /* _URCU_CDS_H */
--- /dev/null
+#ifndef _URCU_COMPILER_H
+#define _URCU_COMPILER_H
+
+/*
+ * compiler.h
+ *
+ * Compiler definitions.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+
+#include <stddef.h> /* for offsetof */
+
+#define caa_likely(x) __builtin_expect(!!(x), 1)
+#define caa_unlikely(x) __builtin_expect(!!(x), 0)
+
+#define cmm_barrier() __asm__ __volatile__ ("" : : : "memory")
+
+/*
+ * Instruct the compiler to perform only a single access to a variable
+ * (prohibits merging and refetching). The compiler is also forbidden to reorder
+ * successive instances of CMM_ACCESS_ONCE(), but only when the compiler is aware of
+ * particular ordering. Compiler ordering can be ensured, for example, by
+ * putting two CMM_ACCESS_ONCE() in separate C statements.
+ *
+ * This macro does absolutely -nothing- to prevent the CPU from reordering,
+ * merging, or refetching absolutely anything at any time. Its main intended
+ * use is to mediate communication between process-level code and irq/NMI
+ * handlers, all running on the same CPU.
+ */
+#define CMM_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
+
+#ifndef caa_max
+#define caa_max(a,b) ((a)>(b)?(a):(b))
+#endif
+
+#ifndef caa_min
+#define caa_min(a,b) ((a)<(b)?(a):(b))
+#endif
+
+#if defined(__SIZEOF_LONG__)
+#define CAA_BITS_PER_LONG (__SIZEOF_LONG__ * 8)
+#elif defined(_LP64)
+#define CAA_BITS_PER_LONG 64
+#else
+#define CAA_BITS_PER_LONG 32
+#endif
+
+/*
+ * caa_container_of - Get the address of an object containing a field.
+ *
+ * @ptr: pointer to the field.
+ * @type: type of the object.
+ * @member: name of the field within the object.
+ */
+#define caa_container_of(ptr, type, member) \
+ __extension__ \
+ ({ \
+ const __typeof__(((type *) NULL)->member) * __ptr = (ptr); \
+ (type *)((char *)__ptr - offsetof(type, member)); \
+ })
+
+#define CAA_BUILD_BUG_ON_ZERO(cond) (sizeof(struct { int:-!!(cond); }))
+#define CAA_BUILD_BUG_ON(cond) ((void)CAA_BUILD_BUG_ON_ZERO(cond))
+
+/*
+ * __rcu is an annotation that documents RCU pointer accesses that need
+ * to be protected by a read-side critical section. Eventually, a static
+ * checker will be able to use this annotation to detect incorrect RCU
+ * usage.
+ */
+#define __rcu
+
+#ifdef __cplusplus
+#define URCU_FORCE_CAST(type, arg) (reinterpret_cast<type>(arg))
+#else
+#define URCU_FORCE_CAST(type, arg) ((type) (arg))
+#endif
+
+#define caa_is_signed_type(type) ((type) -1 < (type) 0)
+
+/*
+ * Cast to unsigned long, sign-extending if @v is signed.
+ * Note: casting to a larger type or to same type size keeps the sign of
+ * the expression being cast (see C99 6.3.1.3).
+ */
+#define caa_cast_long_keep_sign(v) ((unsigned long) (v))
+
+#if defined (__GNUC__) \
+ && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 5) \
+ || __GNUC__ >= 5)
+#define CDS_DEPRECATED(msg) \
+ __attribute__((deprecated(msg)))
+#else
+#define CDS_DEPRECATED(msg) \
+ __attribute__((deprecated))
+#endif
+
+#define CAA_ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+
+/*
+ * Don't allow compiling with buggy compiler.
+ */
+
+#ifdef __GNUC__
+# define URCU_GCC_VERSION (__GNUC__ * 10000 \
+ + __GNUC_MINOR__ * 100 \
+ + __GNUC_PATCHLEVEL__)
+
+/*
+ * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58854
+ */
+# ifdef __ARMEL__
+# if URCU_GCC_VERSION >= 40800 && URCU_GCC_VERSION <= 40802
+# error Your gcc version produces clobbered frame accesses
+# endif
+# endif
+#endif
+
+#endif /* _URCU_COMPILER_H */
--- /dev/null
+/* urcu/config.h.in. Manually generated for control over the contained defs. */
+
+/* Defined when on a system that has memory fence instructions. */
+#undef CONFIG_RCU_HAVE_FENCE
+
+/* Defined when on a system with futex support. */
+#undef CONFIG_RCU_HAVE_FUTEX
+
+/* Enable SMP support. With SMP support enabled, uniprocessors are also
+ supported. With SMP support disabled, UP systems work fine, but the
+ behavior of SMP systems is undefined. */
+#undef CONFIG_RCU_SMP
+
+/* Compatibility mode for i386 which lacks cmpxchg instruction. */
+#undef CONFIG_RCU_COMPAT_ARCH
+
+/* Use the dmb instruction is available for use on ARM. */
+#undef CONFIG_RCU_ARM_HAVE_DMB
+
+/* TLS provided by the compiler. */
+#undef CONFIG_RCU_TLS
+
+/* clock_gettime() is detected. */
+#undef CONFIG_RCU_HAVE_CLOCK_GETTIME
--- /dev/null
+#ifndef _URCU_DEBUG_H
+#define _URCU_DEBUG_H
+
+/*
+ * urcu/debug.h
+ *
+ * Userspace RCU debugging facilities.
+ *
+ * Copyright (c) 2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+
+#include <assert.h>
+
+#ifdef DEBUG_RCU
+#define urcu_assert(...) assert(__VA_ARGS__)
+#else
+#define urcu_assert(...)
+#endif
+
+#endif /* _URCU_DEBUG_H */
--- /dev/null
+#ifndef _URCU_FUTEX_H
+#define _URCU_FUTEX_H
+
+/*
+ * urcu-futex.h
+ *
+ * Userspace RCU - sys_futex/compat_futex header.
+ *
+ * Copyright 2011-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/config.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+
+/*
+ * sys_futex compatibility header.
+ * Use *only* *either of* futex_noasync OR futex_async on a given address.
+ *
+ * futex_noasync cannot be executed in signal handlers, but ensures that
+ * it will be put in a wait queue even in compatibility mode.
+ *
+ * futex_async is signal-handler safe for the wakeup. It uses polling
+ * on the wait-side in compatibility mode.
+ *
+ * BEWARE: sys_futex() FUTEX_WAIT may return early if interrupted
+ * (returns EINTR).
+ */
+
+extern int compat_futex_noasync(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3);
+extern int compat_futex_async(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3);
+
+#ifdef CONFIG_RCU_HAVE_FUTEX
+
+#include <unistd.h>
+#include <errno.h>
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+static inline int futex(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ return syscall(__NR_futex, uaddr, op, val, timeout,
+ uaddr2, val3);
+}
+
+static inline int futex_noasync(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ int ret;
+
+ ret = futex(uaddr, op, val, timeout, uaddr2, val3);
+ if (caa_unlikely(ret < 0 && errno == ENOSYS)) {
+ /*
+ * The fallback on ENOSYS is the async-safe version of
+ * the compat futex implementation, because the
+ * async-safe compat implementation allows being used
+ * concurrently with calls to futex(). Indeed, sys_futex
+ * FUTEX_WAIT, on some architectures (mips and parisc),
+ * within a given process, spuriously return ENOSYS due
+ * to signal restart bugs on some kernel versions.
+ */
+ return compat_futex_async(uaddr, op, val, timeout,
+ uaddr2, val3);
+ }
+ return ret;
+
+}
+
+static inline int futex_async(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ int ret;
+
+ ret = futex(uaddr, op, val, timeout, uaddr2, val3);
+ if (caa_unlikely(ret < 0 && errno == ENOSYS)) {
+ return compat_futex_async(uaddr, op, val, timeout,
+ uaddr2, val3);
+ }
+ return ret;
+}
+
+#else
+
+static inline int futex_noasync(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ return compat_futex_noasync(uaddr, op, val, timeout, uaddr2, val3);
+}
+
+static inline int futex_async(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ return compat_futex_async(uaddr, op, val, timeout, uaddr2, val3);
+}
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_FUTEX_H */
--- /dev/null
+#ifndef _KCOMPAT_HLIST_H
+#define _KCOMPAT_HLIST_H
+
+/*
+ * Kernel sourcecode compatible lightweight single pointer list head useful
+ * for implementing hash tables
+ *
+ * Copyright (C) 2009 Novell Inc.
+ *
+ * Author: Jan Blunck <jblunck@suse.de>
+ *
+ * Copyright (C) 2010-2013 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License version 2.1 as
+ * published by the Free Software Foundation.
+ */
+
+#include <stddef.h>
+
+struct cds_hlist_head {
+ struct cds_hlist_node *next;
+};
+
+struct cds_hlist_node {
+ struct cds_hlist_node *next, *prev;
+};
+
+/* Initialize a new list head. */
+static inline
+void CDS_INIT_HLIST_HEAD(struct cds_hlist_head *ptr)
+{
+ ptr->next = NULL;
+}
+
+#define CDS_HLIST_HEAD(name) \
+ struct cds_hlist_head name = { NULL }
+
+#define CDS_HLIST_HEAD_INIT(name) \
+ { .next = NULL }
+
+/* Get typed element from list at a given position. */
+#define cds_hlist_entry(ptr, type, member) \
+ ((type *) ((char *) (ptr) - (unsigned long) (&((type *) 0)->member)))
+
+/* Add new element at the head of the list. */
+static inline
+void cds_hlist_add_head(struct cds_hlist_node *newp,
+ struct cds_hlist_head *head)
+{
+ if (head->next)
+ head->next->prev = newp;
+ newp->next = head->next;
+ newp->prev = (struct cds_hlist_node *) head;
+ head->next = newp;
+}
+
+/* Remove element from list. */
+static inline
+void cds_hlist_del(struct cds_hlist_node *elem)
+{
+ if (elem->next)
+ elem->next->prev = elem->prev;
+ elem->prev->next = elem->next;
+}
+
+#define cds_hlist_for_each(pos, head) \
+ for (pos = (head)->next; pos != NULL; pos = pos->next)
+
+#define cds_hlist_for_each_safe(pos, p, head) \
+ for (pos = (head)->next; \
+ (pos != NULL) && (p = pos->next, 1); \
+ pos = p)
+
+/*
+ * cds_hlist_for_each_entry and cds_hlist_for_each_entry_safe take
+ * respectively 4 and 5 arguments, while the Linux kernel APIs take 3,
+ * and 4. We implement cds_hlist_for_each_entry_2() and
+ * cds_hlist_for_each_entry_safe_2() to follow the Linux kernel APIs.
+ */
+#define cds_hlist_for_each_entry(entry, pos, head, member) \
+ for (pos = (head)->next, \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
+ pos != NULL; \
+ pos = pos->next, \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member))
+
+#define cds_hlist_for_each_entry_safe(entry, pos, p, head, member) \
+ for (pos = (head)->next, \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
+ (pos != NULL) && (p = pos->next, 1); \
+ pos = p, \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member))
+
+#define cds_hlist_for_each_entry_2(entry, head, member) \
+ for (entry = ((head)->next == NULL ? NULL \
+ : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \
+ entry != NULL; \
+ entry = (entry->member.next == NULL ? NULL \
+ : cds_hlist_entry(entry->member.next, __typeof__(*entry), member)))
+
+#define cds_hlist_for_each_entry_safe_2(entry, e, head, member) \
+ for (entry = ((head)->next == NULL ? NULL \
+ : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \
+ (entry != NULL) && (e = (entry->member.next == NULL ? NULL \
+ : cds_hlist_entry(entry->member.next, \
+ __typeof__(*entry), member)), 1); \
+ entry = e)
+
+#endif /* _KCOMPAT_HLIST_H */
--- /dev/null
+#ifndef _URCU_LFSTACK_H
+#define _URCU_LFSTACK_H
+
+/*
+ * lfstack.h
+ *
+ * Userspace RCU library - Lock-Free Stack
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <pthread.h>
+
+/*
+ * Lock-free stack.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * cds_lfs_push __cds_lfs_pop __cds_lfs_pop_all
+ * cds_lfs_push - - -
+ * __cds_lfs_pop - X X
+ * __cds_lfs_pop_all - X -
+ *
+ * cds_lfs_pop_blocking and cds_lfs_pop_all_blocking use an internal
+ * mutex to provide synchronization.
+ */
+
+/*
+ * struct cds_lfs_node is returned by cds_lfs_pop, and also used as
+ * iterator on stack. It is not safe to dereference the node next
+ * pointer when returned by cds_lfs_pop.
+ */
+struct cds_lfs_node {
+ struct cds_lfs_node *next;
+};
+
+/*
+ * struct cds_lfs_head is returned by __cds_lfs_pop_all, and can be used
+ * to begin iteration on the stack. "node" needs to be the first field
+ * of cds_lfs_head, so the end-of-stack pointer value can be used for
+ * both types.
+ */
+struct cds_lfs_head {
+ struct cds_lfs_node node;
+};
+
+struct __cds_lfs_stack {
+ struct cds_lfs_head *head;
+};
+
+struct cds_lfs_stack {
+ struct cds_lfs_head *head;
+ pthread_mutex_t lock;
+};
+
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_lfs_stack and struct __cds_lfs_stack on any of those two
+ * types.
+ */
+typedef union {
+ struct __cds_lfs_stack *_s;
+ struct cds_lfs_stack *s;
+} __attribute__((__transparent_union__)) cds_lfs_stack_ptr_t;
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/lfstack.h>
+
+#define cds_lfs_node_init _cds_lfs_node_init
+#define cds_lfs_init _cds_lfs_init
+#define cds_lfs_destroy _cds_lfs_destroy
+#define __cds_lfs_init ___cds_lfs_init
+#define cds_lfs_empty _cds_lfs_empty
+#define cds_lfs_push _cds_lfs_push
+
+/* Locking performed internally */
+#define cds_lfs_pop_blocking _cds_lfs_pop_blocking
+#define cds_lfs_pop_all_blocking _cds_lfs_pop_all_blocking
+
+/* Synchronize pop with internal mutex */
+#define cds_lfs_pop_lock _cds_lfs_pop_lock
+#define cds_lfs_pop_unlock _cds_lfs_pop_unlock
+
+/* Synchronization ensured by the caller. See synchronization table. */
+#define __cds_lfs_pop ___cds_lfs_pop
+#define __cds_lfs_pop_all ___cds_lfs_pop_all
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * cds_lfs_node_init: initialize lock-free stack node.
+ */
+extern void cds_lfs_node_init(struct cds_lfs_node *node);
+
+/*
+ * cds_lfs_init: initialize lock-free stack (with locking). Pair with
+ * cds_lfs_destroy().
+ */
+extern void cds_lfs_init(struct cds_lfs_stack *s);
+
+/*
+ * cds_lfs_destroy: destroy lock-free stack (with lock). Pair with
+ * cds_lfs_init().
+ */
+extern void cds_lfs_destroy(struct cds_lfs_stack *s);
+
+/*
+ * __cds_lfs_init: initialize lock-free stack (without lock).
+ * Don't pair with any destroy function.
+ */
+extern void __cds_lfs_init(struct __cds_lfs_stack *s);
+
+/*
+ * cds_lfs_empty: return whether lock-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+extern bool cds_lfs_empty(cds_lfs_stack_ptr_t s);
+
+/*
+ * cds_lfs_push: push a node into the stack.
+ *
+ * Does not require any synchronization with other push nor pop.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+extern bool cds_lfs_push(cds_lfs_stack_ptr_t s,
+ struct cds_lfs_node *node);
+
+/*
+ * cds_lfs_pop_blocking: pop a node from the stack.
+ *
+ * Calls __cds_lfs_pop with an internal pop mutex held.
+ */
+extern struct cds_lfs_node *cds_lfs_pop_blocking(struct cds_lfs_stack *s);
+
+/*
+ * cds_lfs_pop_all_blocking: pop all nodes from a stack.
+ *
+ * Calls __cds_lfs_pop_all with an internal pop mutex held.
+ */
+extern struct cds_lfs_head *cds_lfs_pop_all_blocking(struct cds_lfs_stack *s);
+
+/*
+ * cds_lfs_pop_lock: lock stack pop-protection mutex.
+ */
+extern void cds_lfs_pop_lock(struct cds_lfs_stack *s);
+
+/*
+ * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+extern void cds_lfs_pop_unlock(struct cds_lfs_stack *s);
+
+/*
+ * __cds_lfs_pop: pop a node from the stack.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_lfs_pop needs to be synchronized using one of the following
+ * techniques:
+ *
+ * 1) Calling __cds_lfs_pop under rcu read lock critical section.
+ * Both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
+ * grace period to pass before freeing the returned node or pushing
+ * the node back into the stack. It is valid to overwrite the content
+ * of cds_lfs_node immediately after __cds_lfs_pop and
+ * __cds_lfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop
+ * and __cds_lfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
+ * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+extern struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s);
+
+/*
+ * __cds_lfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_lfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_lfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_lfs_pop:
+ *
+ * 1) If __cds_lfs_pop is called under rcu read lock critical section,
+ * both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
+ * grace period to pass before freeing the returned node or pushing
+ * the node back into the stack. It is valid to overwrite the content
+ * of cds_lfs_node immediately after __cds_lfs_pop and
+ * __cds_lfs_pop_all. No RCU read-side critical section is needed
+ * around __cds_lfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
+ * __cds_lfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
+ * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+extern struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s);
+
+#endif /* !_LGPL_SOURCE */
+
+/*
+ * cds_lfs_for_each: Iterate over all nodes returned by
+ * __cds_lfs_pop_all.
+ * @__head: node returned by __cds_lfs_pop_all (struct cds_lfs_head pointer).
+ * @__node: node to use as iterator (struct cds_lfs_node pointer).
+ *
+ * Content written into each node before push is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_lfs_for_each(__head, __node) \
+ for (__node = &__head->node; \
+ __node != NULL; \
+ __node = __node->next)
+
+/*
+ * cds_lfs_for_each_safe: Iterate over all nodes returned by
+ * __cds_lfs_pop_all, safe against node deletion.
+ * @__head: node returned by __cds_lfs_pop_all (struct cds_lfs_head pointer).
+ * @__node: node to use as iterator (struct cds_lfs_node pointer).
+ * @__n: struct cds_lfs_node pointer holding the next pointer (used
+ * internally).
+ *
+ * Content written into each node before push is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_lfs_for_each_safe(__head, __node, __n) \
+ for (__node = &__head->node, __n = (__node ? __node->next : NULL); \
+ __node != NULL; \
+ __node = __n, __n = (__node ? __node->next : NULL))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_LFSTACK_H */
--- /dev/null
+/*
+ * Copyright (C) 2002 Free Software Foundation, Inc.
+ * (originally part of the GNU C Library)
+ * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+ *
+ * Copyright (C) 2009 Pierre-Marc Fournier
+ * Conversion to RCU list.
+ * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _CDS_LIST_H
+#define _CDS_LIST_H 1
+
+/*
+ * The definitions of this file are adopted from those which can be
+ * found in the Linux kernel headers to enable people familiar with the
+ * latter find their way in these sources as well.
+ */
+
+/* Basic type for the double-link list. */
+struct cds_list_head {
+ struct cds_list_head *next, *prev;
+};
+
+/* Define a variable with the head and tail of the list. */
+#define CDS_LIST_HEAD(name) \
+ struct cds_list_head name = { &(name), &(name) }
+
+/* Initialize a new list head. */
+#define CDS_INIT_LIST_HEAD(ptr) \
+ (ptr)->next = (ptr)->prev = (ptr)
+
+#define CDS_LIST_HEAD_INIT(name) { .prev = &(name), .next = &(name) }
+
+/* Add new element at the head of the list. */
+static inline
+void cds_list_add(struct cds_list_head *newp, struct cds_list_head *head)
+{
+ head->next->prev = newp;
+ newp->next = head->next;
+ newp->prev = head;
+ head->next = newp;
+}
+
+/* Add new element at the tail of the list. */
+static inline
+void cds_list_add_tail(struct cds_list_head *newp, struct cds_list_head *head)
+{
+ head->prev->next = newp;
+ newp->next = head;
+ newp->prev = head->prev;
+ head->prev = newp;
+}
+
+/* Remove element from list. */
+static inline
+void __cds_list_del(struct cds_list_head *prev, struct cds_list_head *next)
+{
+ next->prev = prev;
+ prev->next = next;
+}
+
+/* Remove element from list. */
+static inline
+void cds_list_del(struct cds_list_head *elem)
+{
+ __cds_list_del(elem->prev, elem->next);
+}
+
+/* Remove element from list, initializing the element's list pointers. */
+static inline
+void cds_list_del_init(struct cds_list_head *elem)
+{
+ cds_list_del(elem);
+ CDS_INIT_LIST_HEAD(elem);
+}
+
+/* Delete from list, add to another list as head. */
+static inline
+void cds_list_move(struct cds_list_head *elem, struct cds_list_head *head)
+{
+ __cds_list_del(elem->prev, elem->next);
+ cds_list_add(elem, head);
+}
+
+/* Replace an old entry. */
+static inline
+void cds_list_replace(struct cds_list_head *old, struct cds_list_head *_new)
+{
+ _new->next = old->next;
+ _new->prev = old->prev;
+ _new->prev->next = _new;
+ _new->next->prev = _new;
+}
+
+/* Join two lists. */
+static inline
+void cds_list_splice(struct cds_list_head *add, struct cds_list_head *head)
+{
+ /* Do nothing if the list which gets added is empty. */
+ if (add != add->next) {
+ add->next->prev = head;
+ add->prev->next = head->next;
+ head->next->prev = add->prev;
+ head->next = add->next;
+ }
+}
+
+/* Get typed element from list at a given position. */
+#define cds_list_entry(ptr, type, member) \
+ ((type *) ((char *) (ptr) - (unsigned long) (&((type *) 0)->member)))
+
+
+/* Get first entry from a list. */
+#define cds_list_first_entry(ptr, type, member) \
+ cds_list_entry((ptr)->next, type, member)
+
+/* Iterate forward over the elements of the list. */
+#define cds_list_for_each(pos, head) \
+ for (pos = (head)->next; pos != (head); pos = pos->next)
+
+/*
+ * Iterate forward over the elements list. The list elements can be
+ * removed from the list while doing this.
+ */
+#define cds_list_for_each_safe(pos, p, head) \
+ for (pos = (head)->next, p = pos->next; \
+ pos != (head); \
+ pos = p, p = pos->next)
+
+/* Iterate backward over the elements of the list. */
+#define cds_list_for_each_prev(pos, head) \
+ for (pos = (head)->prev; pos != (head); pos = pos->prev)
+
+/*
+ * Iterate backwards over the elements list. The list elements can be
+ * removed from the list while doing this.
+ */
+#define cds_list_for_each_prev_safe(pos, p, head) \
+ for (pos = (head)->prev, p = pos->prev; \
+ pos != (head); \
+ pos = p, p = pos->prev)
+
+#define cds_list_for_each_entry(pos, head, member) \
+ for (pos = cds_list_entry((head)->next, __typeof__(*pos), member); \
+ &pos->member != (head); \
+ pos = cds_list_entry(pos->member.next, __typeof__(*pos), member))
+
+#define cds_list_for_each_entry_reverse(pos, head, member) \
+ for (pos = cds_list_entry((head)->prev, __typeof__(*pos), member); \
+ &pos->member != (head); \
+ pos = cds_list_entry(pos->member.prev, __typeof__(*pos), member))
+
+#define cds_list_for_each_entry_safe(pos, p, head, member) \
+ for (pos = cds_list_entry((head)->next, __typeof__(*pos), member), \
+ p = cds_list_entry(pos->member.next, __typeof__(*pos), member); \
+ &pos->member != (head); \
+ pos = p, p = cds_list_entry(pos->member.next, __typeof__(*pos), member))
+
+/*
+ * Same as cds_list_for_each_entry_safe, but starts from "pos" which should
+ * point to an entry within the list.
+ */
+#define cds_list_for_each_entry_safe_from(pos, p, head, member) \
+ for (p = cds_list_entry(pos->member.next, __typeof__(*pos), member); \
+ &pos->member != (head); \
+ pos = p, p = cds_list_entry(pos->member.next, __typeof__(*pos), member))
+
+static inline
+int cds_list_empty(struct cds_list_head *head)
+{
+ return head == head->next;
+}
+
+static inline
+void cds_list_replace_init(struct cds_list_head *old,
+ struct cds_list_head *_new)
+{
+ struct cds_list_head *head = old->next;
+
+ cds_list_del(old);
+ cds_list_add_tail(_new, head);
+ CDS_INIT_LIST_HEAD(old);
+}
+
+#endif /* _CDS_LIST_H */
--- /dev/null
+#ifndef _URCU_BP_MAP_H
+#define _URCU_BP_MAP_H
+
+/*
+ * urcu-map.h
+ *
+ * Userspace RCU header -- name mapping to allow multiple flavors to be
+ * used in the same executable.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+/* Mapping macros to allow multiple flavors in a single binary. */
+
+#define rcu_read_lock rcu_read_lock_bp
+#define _rcu_read_lock _rcu_read_lock_bp
+#define rcu_read_unlock rcu_read_unlock_bp
+#define _rcu_read_unlock _rcu_read_unlock_bp
+#define rcu_read_ongoing rcu_read_ongoing_bp
+#define _rcu_read_ongoing _rcu_read_ongoing_bp
+#define rcu_register_thread rcu_register_thread_bp
+#define rcu_unregister_thread rcu_unregister_thread_bp
+#define rcu_init rcu_init_bp
+#define rcu_exit rcu_exit_bp
+#define synchronize_rcu synchronize_rcu_bp
+#define rcu_reader rcu_reader_bp
+#define rcu_gp rcu_gp_bp
+
+#define get_cpu_call_rcu_data get_cpu_call_rcu_data_bp
+#define get_call_rcu_thread get_call_rcu_thread_bp
+#define create_call_rcu_data create_call_rcu_data_bp
+#define set_cpu_call_rcu_data set_cpu_call_rcu_data_bp
+#define get_default_call_rcu_data get_default_call_rcu_data_bp
+#define get_call_rcu_data get_call_rcu_data_bp
+#define get_thread_call_rcu_data get_thread_call_rcu_data_bp
+#define set_thread_call_rcu_data set_thread_call_rcu_data_bp
+#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_bp
+#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_bp
+#define call_rcu call_rcu_bp
+#define call_rcu_data_free call_rcu_data_free_bp
+#define call_rcu_before_fork call_rcu_before_fork_bp
+#define call_rcu_after_fork_parent call_rcu_after_fork_parent_bp
+#define call_rcu_after_fork_child call_rcu_after_fork_child_bp
+#define rcu_barrier rcu_barrier_bp
+
+#define defer_rcu defer_rcu_bp
+#define rcu_defer_register_thread rcu_defer_register_thread_bp
+#define rcu_defer_unregister_thread rcu_defer_unregister_thread_bp
+#define rcu_defer_barrier rcu_defer_barrier_bp
+#define rcu_defer_barrier_thread rcu_defer_barrier_thread_bp
+#define rcu_defer_exit rcu_defer_exit_bp
+
+#define rcu_flavor rcu_flavor_bp
+
+#define rcu_yield_active rcu_yield_active_bp
+#define rcu_rand_yield rcu_rand_yield_bp
+
+#endif /* _URCU_BP_MAP_H */
--- /dev/null
+#ifndef _URCU_QSBR_MAP_H
+#define _URCU_QSBR_MAP_H
+
+/*
+ * urcu-map.h
+ *
+ * Userspace RCU header -- name mapping to allow multiple flavors to be
+ * used in the same executable.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+/* Mapping macros to allow multiple flavors in a single binary. */
+
+#define rcu_read_lock rcu_read_lock_qsbr
+#define _rcu_read_lock _rcu_read_lock_qsbr
+#define rcu_read_unlock rcu_read_unlock_qsbr
+#define _rcu_read_unlock _rcu_read_unlock_qsbr
+#define rcu_read_ongoing rcu_read_ongoing_qsbr
+#define _rcu_read_ongoing _rcu_read_ongoing_qsbr
+#define rcu_quiescent_state rcu_quiescent_state_qsbr
+#define _rcu_quiescent_state _rcu_quiescent_state_qsbr
+#define rcu_thread_offline rcu_thread_offline_qsbr
+#define rcu_thread_online rcu_thread_online_qsbr
+#define rcu_register_thread rcu_register_thread_qsbr
+#define rcu_unregister_thread rcu_unregister_thread_qsbr
+#define rcu_exit rcu_exit_qsbr
+#define synchronize_rcu synchronize_rcu_qsbr
+#define rcu_reader rcu_reader_qsbr
+#define rcu_gp rcu_gp_qsbr
+
+#define get_cpu_call_rcu_data get_cpu_call_rcu_data_qsbr
+#define get_call_rcu_thread get_call_rcu_thread_qsbr
+#define create_call_rcu_data create_call_rcu_data_qsbr
+#define set_cpu_call_rcu_data set_cpu_call_rcu_data_qsbr
+#define get_default_call_rcu_data get_default_call_rcu_data_qsbr
+#define get_call_rcu_data get_call_rcu_data_qsbr
+#define get_thread_call_rcu_data get_thread_call_rcu_data_qsbr
+#define set_thread_call_rcu_data set_thread_call_rcu_data_qsbr
+#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_qsbr
+#define call_rcu call_rcu_qsbr
+#define call_rcu_data_free call_rcu_data_free_qsbr
+#define call_rcu_before_fork call_rcu_before_fork_qsbr
+#define call_rcu_after_fork_parent call_rcu_after_fork_parent_qsbr
+#define call_rcu_after_fork_child call_rcu_after_fork_child_qsbr
+#define rcu_barrier rcu_barrier_qsbr
+
+#define defer_rcu defer_rcu_qsbr
+#define rcu_defer_register_thread rcu_defer_register_thread_qsbr
+#define rcu_defer_unregister_thread rcu_defer_unregister_thread_qsbr
+#define rcu_defer_barrier rcu_defer_barrier_qsbr
+#define rcu_defer_barrier_thread rcu_defer_barrier_thread_qsbr
+#define rcu_defer_exit rcu_defer_exit_qsbr
+
+#define rcu_flavor rcu_flavor_qsbr
+
+#endif /* _URCU_QSBR_MAP_H */
--- /dev/null
+#ifndef _URCU_MAP_H
+#define _URCU_MAP_H
+
+/*
+ * urcu-map.h
+ *
+ * Userspace RCU header -- name mapping to allow multiple flavors to be
+ * used in the same executable.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+/* Mapping macros to allow multiple flavors in a single binary. */
+
+#if !defined(RCU_MEMBARRIER) && !defined(RCU_SIGNAL) && !defined(RCU_MB)
+#define RCU_MEMBARRIER
+#endif
+
+#ifdef RCU_MEMBARRIER
+
+#define rcu_read_lock rcu_read_lock_memb
+#define _rcu_read_lock _rcu_read_lock_memb
+#define rcu_read_unlock rcu_read_unlock_memb
+#define _rcu_read_unlock _rcu_read_unlock_memb
+#define rcu_read_ongoing rcu_read_ongoing_memb
+#define _rcu_read_ongoing _rcu_read_ongoing_memb
+#define rcu_register_thread rcu_register_thread_memb
+#define rcu_unregister_thread rcu_unregister_thread_memb
+#define rcu_init rcu_init_memb
+#define rcu_exit rcu_exit_memb
+#define synchronize_rcu synchronize_rcu_memb
+#define rcu_reader rcu_reader_memb
+#define rcu_gp rcu_gp_memb
+
+#define get_cpu_call_rcu_data get_cpu_call_rcu_data_memb
+#define get_call_rcu_thread get_call_rcu_thread_memb
+#define create_call_rcu_data create_call_rcu_data_memb
+#define set_cpu_call_rcu_data set_cpu_call_rcu_data_memb
+#define get_default_call_rcu_data get_default_call_rcu_data_memb
+#define get_call_rcu_data get_call_rcu_data_memb
+#define get_thread_call_rcu_data get_thread_call_rcu_data_memb
+#define set_thread_call_rcu_data set_thread_call_rcu_data_memb
+#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_memb
+#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_memb
+#define call_rcu call_rcu_memb
+#define call_rcu_data_free call_rcu_data_free_memb
+#define call_rcu_before_fork call_rcu_before_fork_memb
+#define call_rcu_after_fork_parent call_rcu_after_fork_parent_memb
+#define call_rcu_after_fork_child call_rcu_after_fork_child_memb
+#define rcu_barrier rcu_barrier_memb
+
+#define defer_rcu defer_rcu_memb
+#define rcu_defer_register_thread rcu_defer_register_thread_memb
+#define rcu_defer_unregister_thread rcu_defer_unregister_thread_memb
+#define rcu_defer_barrier rcu_defer_barrier_memb
+#define rcu_defer_barrier_thread rcu_defer_barrier_thread_memb
+#define rcu_defer_exit rcu_defer_exit_memb
+
+#define rcu_flavor rcu_flavor_memb
+
+/* Specific to MEMBARRIER flavor */
+#define rcu_has_sys_membarrier rcu_has_sys_membarrier_memb
+
+#elif defined(RCU_SIGNAL)
+
+#define rcu_read_lock rcu_read_lock_sig
+#define _rcu_read_lock _rcu_read_lock_sig
+#define rcu_read_unlock rcu_read_unlock_sig
+#define _rcu_read_unlock _rcu_read_unlock_sig
+#define rcu_read_ongoing rcu_read_ongoing_sig
+#define _rcu_read_ongoing _rcu_read_ongoing_sig
+#define rcu_register_thread rcu_register_thread_sig
+#define rcu_unregister_thread rcu_unregister_thread_sig
+#define rcu_init rcu_init_sig
+#define rcu_exit rcu_exit_sig
+#define synchronize_rcu synchronize_rcu_sig
+#define rcu_reader rcu_reader_sig
+#define rcu_gp rcu_gp_sig
+
+#define get_cpu_call_rcu_data get_cpu_call_rcu_data_sig
+#define get_call_rcu_thread get_call_rcu_thread_sig
+#define create_call_rcu_data create_call_rcu_data_sig
+#define set_cpu_call_rcu_data set_cpu_call_rcu_data_sig
+#define get_default_call_rcu_data get_default_call_rcu_data_sig
+#define get_call_rcu_data get_call_rcu_data_sig
+#define get_thread_call_rcu_data get_thread_call_rcu_data_sig
+#define set_thread_call_rcu_data set_thread_call_rcu_data_sig
+#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_sig
+#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_sig
+#define call_rcu call_rcu_sig
+#define call_rcu_data_free call_rcu_data_free_sig
+#define call_rcu_before_fork call_rcu_before_fork_sig
+#define call_rcu_after_fork_parent call_rcu_after_fork_parent_sig
+#define call_rcu_after_fork_child call_rcu_after_fork_child_sig
+#define rcu_barrier rcu_barrier_sig
+
+#define defer_rcu defer_rcu_sig
+#define rcu_defer_register_thread rcu_defer_register_thread_sig
+#define rcu_defer_unregister_thread rcu_defer_unregister_thread_sig
+#define rcu_defer_barrier rcu_defer_barrier_sig
+#define rcu_defer_barrier_thread rcu_defer_barrier_thread_sig
+#define rcu_defer_exit rcu_defer_exit_sig
+
+#define rcu_flavor rcu_flavor_sig
+
+#elif defined(RCU_MB)
+
+#define rcu_read_lock rcu_read_lock_mb
+#define _rcu_read_lock _rcu_read_lock_mb
+#define rcu_read_unlock rcu_read_unlock_mb
+#define _rcu_read_unlock _rcu_read_unlock_mb
+#define rcu_read_ongoing rcu_read_ongoing_mb
+#define _rcu_read_ongoing _rcu_read_ongoing_mb
+#define rcu_register_thread rcu_register_thread_mb
+#define rcu_unregister_thread rcu_unregister_thread_mb
+#define rcu_init rcu_init_mb
+#define rcu_exit rcu_exit_mb
+#define synchronize_rcu synchronize_rcu_mb
+#define rcu_reader rcu_reader_mb
+#define rcu_gp rcu_gp_mb
+
+#define get_cpu_call_rcu_data get_cpu_call_rcu_data_mb
+#define get_call_rcu_thread get_call_rcu_thread_mb
+#define create_call_rcu_data create_call_rcu_data_mb
+#define set_cpu_call_rcu_data set_cpu_call_rcu_data_mb
+#define get_default_call_rcu_data get_default_call_rcu_data_mb
+#define get_call_rcu_data get_call_rcu_data_mb
+#define get_thread_call_rcu_data get_thread_call_rcu_data_mb
+#define set_thread_call_rcu_data set_thread_call_rcu_data_mb
+#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_mb
+#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_mb
+#define call_rcu call_rcu_mb
+#define call_rcu_data_free call_rcu_data_free_mb
+#define call_rcu_before_fork call_rcu_before_fork_mb
+#define call_rcu_after_fork_parent call_rcu_after_fork_parent_mb
+#define call_rcu_after_fork_child call_rcu_after_fork_child_mb
+#define rcu_barrier rcu_barrier_mb
+
+#define defer_rcu defer_rcu_mb
+#define rcu_defer_register_thread rcu_defer_register_thread_mb
+#define rcu_defer_unregister_thread rcu_defer_unregister_thread_mb
+#define rcu_defer_barrier rcu_defer_barrier_mb
+#define rcu_defer_barrier_thread rcu_defer_barrier_thread_mb
+#define rcu_defer_exit rcu_defer_exit_mb
+
+#define rcu_flavor rcu_flavor_mb
+
+#else
+
+#error "Undefined selection"
+
+#endif
+
+#endif /* _URCU_MAP_H */
--- /dev/null
+#ifndef _URCU_RAND_COMPAT_H
+#define _URCU_RAND_COMPAT_H
+
+/*
+ * urcu/rand-compat.h
+ *
+ * Userspace RCU library - rand/rand_r Compatibility Header
+ *
+ * Copyright 1996 - Ulrich Drepper <drepper@cygnus.com >
+ * Copyright 2013 - Pierre-Luc St-Charles <pierre-luc.st-charles@polymtl.ca>
+ *
+ * Note: this file is only used to simplify the code required to
+ * use the 'rand_r(...)' system function across multiple platforms,
+ * which might not always be referenced the same way.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifdef __ANDROID__
+/*
+ * Reentrant random function from POSIX.1c.
+ * Copyright (C) 1996, 1999 Free Software Foundation, Inc.
+ * This file is part of the GNU C Library.
+ * Contributed by Ulrich Drepper <drepper@cygnus.com <mailto:drepper@cygnus.com>>, 1996.
+ */
+static inline int rand_r(unsigned int *seed)
+{
+ unsigned int next = *seed;
+ int result;
+
+ next *= 1103515245;
+ next += 12345;
+ result = (unsigned int) (next / 65536) % 2048;
+
+ next *= 1103515245;
+ next += 12345;
+ result <<= 10;
+ result ^= (unsigned int) (next / 65536) % 1024;
+
+ next *= 1103515245;
+ next += 12345;
+ result <<= 10;
+ result ^= (unsigned int) (next / 65536) % 1024;
+
+ *seed = next;
+
+ return result;
+}
+#endif /* __ANDROID__ */
+
+#endif /* _URCU_RAND_COMPAT_H */
--- /dev/null
+/*
+ * Copyright (C) 2002 Free Software Foundation, Inc.
+ * (originally part of the GNU C Library)
+ * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+ *
+ * Copyright (C) 2009 Pierre-Marc Fournier
+ * Conversion to RCU list.
+ * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _URCU_RCUHLIST_H
+#define _URCU_RCUHLIST_H
+
+#include <urcu/hlist.h>
+#include <urcu/arch.h>
+#include <urcu-pointer.h>
+
+/* Add new element at the head of the list. */
+static inline
+void cds_hlist_add_head_rcu(struct cds_hlist_node *newp,
+ struct cds_hlist_head *head)
+{
+ newp->next = head->next;
+ newp->prev = (struct cds_hlist_node *)head;
+ if (head->next)
+ head->next->prev = newp;
+ rcu_assign_pointer(head->next, newp);
+}
+
+/* Remove element from list. */
+static inline
+void cds_hlist_del_rcu(struct cds_hlist_node *elem)
+{
+ if (elem->next)
+ elem->next->prev = elem->prev;
+ CMM_STORE_SHARED(elem->prev->next, elem->next);
+}
+
+/*
+ * Iterate through elements of the list.
+ * This must be done while rcu_read_lock() is held.
+ */
+#define cds_hlist_for_each_rcu(pos, head) \
+ for (pos = rcu_dereference((head)->next); pos != NULL; \
+ pos = rcu_dereference(pos->next))
+
+/*
+ * cds_hlist_for_each_entry_rcu takes 4 arguments, while the Linux
+ * kernel API only takes 3.
+ * We implement cds_hlist_for_each_entry_rcu_2() to follow the Linux
+ * kernel APIs.
+ */
+#define cds_hlist_for_each_entry_rcu(entry, pos, head, member) \
+ for (pos = rcu_dereference((head)->next), \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
+ pos != NULL; \
+ pos = rcu_dereference(pos->next), \
+ entry = cds_hlist_entry(pos, __typeof__(*entry), member))
+
+#define cds_hlist_for_each_entry_rcu_2(entry, head, member) \
+ for (entry = cds_hlist_entry(rcu_dereference((head)->next), \
+ __typeof__(*entry), member); \
+ &entry->member != NULL; \
+ entry = cds_hlist_entry(rcu_dereference(entry->member.next), \
+ __typeof__(*entry), member))
+
+#endif /* _URCU_RCUHLIST_H */
--- /dev/null
+#ifndef _URCU_RCULFHASH_H
+#define _URCU_RCULFHASH_H
+
+/*
+ * urcu/rculfhash.h
+ *
+ * Userspace RCU library - Lock-Free RCU Hash Table
+ *
+ * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Include this file _after_ including your URCU flavor.
+ */
+
+#include <stdint.h>
+#include <urcu/compiler.h>
+#include <urcu-call-rcu.h>
+#include <urcu-flavor.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * cds_lfht_node: Contains the next pointers and reverse-hash
+ * value required for lookup and traversal of the hash table.
+ *
+ * struct cds_lfht_node should be aligned on 8-bytes boundaries because
+ * the three lower bits are used as flags. It is worth noting that the
+ * information contained within these three bits could be represented on
+ * two bits by re-using the same bit for REMOVAL_OWNER_FLAG and
+ * BUCKET_FLAG. This can be done if we ensure that no iterator nor
+ * updater check the BUCKET_FLAG after it detects that the REMOVED_FLAG
+ * is set. Given the minimum size of struct cds_lfht_node is 8 bytes on
+ * 32-bit architectures, we choose to go for simplicity and reserve
+ * three bits.
+ *
+ * struct cds_lfht_node can be embedded into a structure (as a field).
+ * caa_container_of() can be used to get the structure from the struct
+ * cds_lfht_node after a lookup.
+ *
+ * The structure which embeds it typically holds the key (or key-value
+ * pair) of the object. The caller code is responsible for calculation
+ * of the hash value for cds_lfht APIs.
+ */
+struct cds_lfht_node {
+ struct cds_lfht_node *next; /* ptr | REMOVAL_OWNER_FLAG | BUCKET_FLAG | REMOVED_FLAG */
+ unsigned long reverse_hash;
+} __attribute__((aligned(8)));
+
+/* cds_lfht_iter: Used to track state while traversing a hash chain. */
+struct cds_lfht_iter {
+ struct cds_lfht_node *node, *next;
+};
+
+static inline
+struct cds_lfht_node *cds_lfht_iter_get_node(struct cds_lfht_iter *iter)
+{
+ return iter->node;
+}
+
+struct cds_lfht;
+
+/*
+ * Caution !
+ * Ensure reader and writer threads are registered as urcu readers.
+ */
+
+typedef int (*cds_lfht_match_fct)(struct cds_lfht_node *node, const void *key);
+
+/*
+ * cds_lfht_node_init - initialize a hash table node
+ * @node: the node to initialize.
+ *
+ * This function is kept to be eventually used for debugging purposes
+ * (detection of memory corruption).
+ */
+static inline
+void cds_lfht_node_init(struct cds_lfht_node *node)
+{
+}
+
+/*
+ * Hash table creation flags.
+ */
+enum {
+ CDS_LFHT_AUTO_RESIZE = (1U << 0),
+ CDS_LFHT_ACCOUNTING = (1U << 1),
+};
+
+struct cds_lfht_mm_type {
+ struct cds_lfht *(*alloc_cds_lfht)(unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets);
+ void (*alloc_bucket_table)(struct cds_lfht *ht, unsigned long order);
+ void (*free_bucket_table)(struct cds_lfht *ht, unsigned long order);
+ struct cds_lfht_node *(*bucket_at)(struct cds_lfht *ht,
+ unsigned long index);
+};
+
+extern const struct cds_lfht_mm_type cds_lfht_mm_order;
+extern const struct cds_lfht_mm_type cds_lfht_mm_chunk;
+extern const struct cds_lfht_mm_type cds_lfht_mm_mmap;
+
+/*
+ * _cds_lfht_new - API used by cds_lfht_new wrapper. Do not use directly.
+ */
+extern
+struct cds_lfht *_cds_lfht_new(unsigned long init_size,
+ unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets,
+ int flags,
+ const struct cds_lfht_mm_type *mm,
+ const struct rcu_flavor_struct *flavor,
+ pthread_attr_t *attr);
+
+/*
+ * cds_lfht_new - allocate a hash table.
+ * @init_size: number of buckets to allocate initially. Must be power of two.
+ * @min_nr_alloc_buckets: the minimum number of allocated buckets.
+ * (must be power of two)
+ * @max_nr_buckets: the maximum number of hash table buckets allowed.
+ * (must be power of two, 0 is accepted, means
+ * "infinite")
+ * @flags: hash table creation flags (can be combined with bitwise or: '|').
+ * 0: no flags.
+ * CDS_LFHT_AUTO_RESIZE: automatically resize hash table.
+ * CDS_LFHT_ACCOUNTING: count the number of node addition
+ * and removal in the table
+ * @attr: optional resize worker thread attributes. NULL for default.
+ *
+ * Return NULL on error.
+ * Note: the RCU flavor must be already included before the hash table header.
+ *
+ * The programmer is responsible for ensuring that resize operation has a
+ * priority equal to hash table updater threads. It should be performed by
+ * specifying the appropriate priority in the pthread "attr" argument, and,
+ * for CDS_LFHT_AUTO_RESIZE, by ensuring that call_rcu worker threads also have
+ * this priority level. Having lower priority for call_rcu and resize threads
+ * does not pose any correctness issue, but the resize operations could be
+ * starved by updates, thus leading to long hash table bucket chains.
+ * Threads calling cds_lfht_new are NOT required to be registered RCU
+ * read-side threads. It can be called very early. (e.g. before RCU is
+ * initialized)
+ */
+static inline
+struct cds_lfht *cds_lfht_new(unsigned long init_size,
+ unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets,
+ int flags,
+ pthread_attr_t *attr)
+{
+ return _cds_lfht_new(init_size, min_nr_alloc_buckets, max_nr_buckets,
+ flags, NULL, &rcu_flavor, attr);
+}
+
+/*
+ * cds_lfht_destroy - destroy a hash table.
+ * @ht: the hash table to destroy.
+ * @attr: (output) resize worker thread attributes, as received by cds_lfht_new.
+ * The caller will typically want to free this pointer if dynamically
+ * allocated. The attr point can be NULL if the caller does not
+ * need to be informed of the value passed to cds_lfht_new().
+ *
+ * Return 0 on success, negative error value on error.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * cds_lfht_destroy should *not* be called from a RCU read-side critical
+ * section. It should *not* be called from a call_rcu thread context
+ * neither.
+ */
+extern
+int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr);
+
+/*
+ * cds_lfht_count_nodes - count the number of nodes in the hash table.
+ * @ht: the hash table.
+ * @split_count_before: sample the node count split-counter before traversal.
+ * @count: traverse the hash table, count the number of nodes observed.
+ * @split_count_after: sample the node count split-counter after traversal.
+ *
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ */
+extern
+void cds_lfht_count_nodes(struct cds_lfht *ht,
+ long *split_count_before,
+ unsigned long *count,
+ long *split_count_after);
+
+/*
+ * cds_lfht_lookup - lookup a node by key.
+ * @ht: the hash table.
+ * @hash: the key hash.
+ * @match: the key match function.
+ * @key: the current node key.
+ * @iter: node, if found (output). *iter->node set to NULL if not found.
+ *
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function acts as a rcu_dereference() to read the node pointer.
+ */
+extern
+void cds_lfht_lookup(struct cds_lfht *ht, unsigned long hash,
+ cds_lfht_match_fct match, const void *key,
+ struct cds_lfht_iter *iter);
+
+/*
+ * cds_lfht_next_duplicate - get the next item with same key, after iterator.
+ * @ht: the hash table.
+ * @match: the key match function.
+ * @key: the current node key.
+ * @iter: input: current iterator.
+ * output: node, if found. *iter->node set to NULL if not found.
+ *
+ * Uses an iterator initialized by a lookup or traversal. Important: the
+ * iterator _needs_ to be initialized before calling
+ * cds_lfht_next_duplicate.
+ * Sets *iter-node to the following node with same key.
+ * Sets *iter->node to NULL if no following node exists with same key.
+ * RCU read-side lock must be held across cds_lfht_lookup and
+ * cds_lfht_next calls, and also between cds_lfht_next calls using the
+ * node returned by a previous cds_lfht_next.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function acts as a rcu_dereference() to read the node pointer.
+ */
+extern
+void cds_lfht_next_duplicate(struct cds_lfht *ht,
+ cds_lfht_match_fct match, const void *key,
+ struct cds_lfht_iter *iter);
+
+/*
+ * cds_lfht_first - get the first node in the table.
+ * @ht: the hash table.
+ * @iter: First node, if exists (output). *iter->node set to NULL if not found.
+ *
+ * Output in "*iter". *iter->node set to NULL if table is empty.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function acts as a rcu_dereference() to read the node pointer.
+ */
+extern
+void cds_lfht_first(struct cds_lfht *ht, struct cds_lfht_iter *iter);
+
+/*
+ * cds_lfht_next - get the next node in the table.
+ * @ht: the hash table.
+ * @iter: input: current iterator.
+ * output: next node, if exists. *iter->node set to NULL if not found.
+ *
+ * Input/Output in "*iter". *iter->node set to NULL if *iter was
+ * pointing to the last table node.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function acts as a rcu_dereference() to read the node pointer.
+ */
+extern
+void cds_lfht_next(struct cds_lfht *ht, struct cds_lfht_iter *iter);
+
+/*
+ * cds_lfht_add - add a node to the hash table.
+ * @ht: the hash table.
+ * @hash: the key hash.
+ * @node: the node to add.
+ *
+ * This function supports adding redundant keys into the table.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function issues a full memory barrier before and after its
+ * atomic commit.
+ */
+extern
+void cds_lfht_add(struct cds_lfht *ht, unsigned long hash,
+ struct cds_lfht_node *node);
+
+/*
+ * cds_lfht_add_unique - add a node to hash table, if key is not present.
+ * @ht: the hash table.
+ * @hash: the node's hash.
+ * @match: the key match function.
+ * @key: the node's key.
+ * @node: the node to try adding.
+ *
+ * Return the node added upon success.
+ * Return the unique node already present upon failure. If
+ * cds_lfht_add_unique fails, the node passed as parameter should be
+ * freed by the caller. In this case, the caller does NOT need to wait
+ * for a grace period before freeing or re-using the node.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ *
+ * The semantic of this function is that if only this function is used
+ * to add keys into the table, no duplicated keys should ever be
+ * observable in the table. The same guarantee apply for combination of
+ * add_unique and add_replace (see below).
+ *
+ * Upon success, this function issues a full memory barrier before and
+ * after its atomic commit. Upon failure, this function acts like a
+ * simple lookup operation: it acts as a rcu_dereference() to read the
+ * node pointer. The failure case does not guarantee any other memory
+ * barrier.
+ */
+extern
+struct cds_lfht_node *cds_lfht_add_unique(struct cds_lfht *ht,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *node);
+
+/*
+ * cds_lfht_add_replace - replace or add a node within hash table.
+ * @ht: the hash table.
+ * @hash: the node's hash.
+ * @match: the key match function.
+ * @key: the node's key.
+ * @node: the node to add.
+ *
+ * Return the node replaced upon success. If no node matching the key
+ * was present, return NULL, which also means the operation succeeded.
+ * This replacement operation should never fail.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * After successful replacement, a grace period must be waited for before
+ * freeing or re-using the memory reserved for the returned node.
+ *
+ * The semantic of replacement vs lookups and traversals is the
+ * following: if lookups and traversals are performed between a key
+ * unique insertion and its removal, we guarantee that the lookups and
+ * traversals will always find exactly one instance of the key if it is
+ * replaced concurrently with the lookups.
+ *
+ * Providing this semantic allows us to ensure that replacement-only
+ * schemes will never generate duplicated keys. It also allows us to
+ * guarantee that a combination of add_replace and add_unique updates
+ * will never generate duplicated keys.
+ *
+ * This function issues a full memory barrier before and after its
+ * atomic commit.
+ */
+extern
+struct cds_lfht_node *cds_lfht_add_replace(struct cds_lfht *ht,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *node);
+
+/*
+ * cds_lfht_replace - replace a node pointed to by iter within hash table.
+ * @ht: the hash table.
+ * @old_iter: the iterator position of the node to replace.
+ * @hash: the node's hash.
+ * @match: the key match function.
+ * @key: the node's key.
+ * @new_node: the new node to use as replacement.
+ *
+ * Return 0 if replacement is successful, negative value otherwise.
+ * Replacing a NULL old node or an already removed node will fail with
+ * -ENOENT.
+ * If the hash or value of the node to replace and the new node differ,
+ * this function returns -EINVAL without proceeding to the replacement.
+ * Old node can be looked up with cds_lfht_lookup and cds_lfht_next.
+ * RCU read-side lock must be held between lookup and replacement.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * After successful replacement, a grace period must be waited for before
+ * freeing or re-using the memory reserved for the old node (which can
+ * be accessed with cds_lfht_iter_get_node).
+ *
+ * The semantic of replacement vs lookups is the same as
+ * cds_lfht_add_replace().
+ *
+ * Upon success, this function issues a full memory barrier before and
+ * after its atomic commit. Upon failure, this function does not issue
+ * any memory barrier.
+ */
+extern
+int cds_lfht_replace(struct cds_lfht *ht,
+ struct cds_lfht_iter *old_iter,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *new_node);
+
+/*
+ * cds_lfht_del - remove node pointed to by iterator from hash table.
+ * @ht: the hash table.
+ * @node: the node to delete.
+ *
+ * Return 0 if the node is successfully removed, negative value
+ * otherwise.
+ * Deleting a NULL node or an already removed node will fail with a
+ * negative value.
+ * Node can be looked up with cds_lfht_lookup and cds_lfht_next,
+ * followed by use of cds_lfht_iter_get_node.
+ * RCU read-side lock must be held between lookup and removal.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * After successful removal, a grace period must be waited for before
+ * freeing or re-using the memory reserved for old node (which can be
+ * accessed with cds_lfht_iter_get_node).
+ * Upon success, this function issues a full memory barrier before and
+ * after its atomic commit. Upon failure, this function does not issue
+ * any memory barrier.
+ */
+extern
+int cds_lfht_del(struct cds_lfht *ht, struct cds_lfht_node *node);
+
+/*
+ * cds_lfht_is_node_deleted - query whether a node is removed from hash table.
+ *
+ * Return non-zero if the node is deleted from the hash table, 0
+ * otherwise.
+ * Node can be looked up with cds_lfht_lookup and cds_lfht_next,
+ * followed by use of cds_lfht_iter_get_node.
+ * RCU read-side lock must be held between lookup and call to this
+ * function.
+ * Call with rcu_read_lock held.
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function does not issue any memory barrier.
+ */
+extern
+int cds_lfht_is_node_deleted(struct cds_lfht_node *node);
+
+/*
+ * cds_lfht_resize - Force a hash table resize
+ * @ht: the hash table.
+ * @new_size: update to this hash table size.
+ *
+ * Threads calling this API need to be registered RCU read-side threads.
+ * This function does not (necessarily) issue memory barriers.
+ * cds_lfht_resize should *not* be called from a RCU read-side critical
+ * section.
+ */
+extern
+void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size);
+
+/*
+ * Note: it is safe to perform element removal (del), replacement, or
+ * any hash table update operation during any of the following hash
+ * table traversals.
+ * These functions act as rcu_dereference() to read the node pointers.
+ */
+#define cds_lfht_for_each(ht, iter, node) \
+ for (cds_lfht_first(ht, iter), \
+ node = cds_lfht_iter_get_node(iter); \
+ node != NULL; \
+ cds_lfht_next(ht, iter), \
+ node = cds_lfht_iter_get_node(iter))
+
+#define cds_lfht_for_each_duplicate(ht, hash, match, key, iter, node) \
+ for (cds_lfht_lookup(ht, hash, match, key, iter), \
+ node = cds_lfht_iter_get_node(iter); \
+ node != NULL; \
+ cds_lfht_next_duplicate(ht, match, key, iter), \
+ node = cds_lfht_iter_get_node(iter))
+
+#define cds_lfht_for_each_entry(ht, iter, pos, member) \
+ for (cds_lfht_first(ht, iter), \
+ pos = caa_container_of(cds_lfht_iter_get_node(iter), \
+ __typeof__(*(pos)), member); \
+ cds_lfht_iter_get_node(iter) != NULL; \
+ cds_lfht_next(ht, iter), \
+ pos = caa_container_of(cds_lfht_iter_get_node(iter), \
+ __typeof__(*(pos)), member))
+
+#define cds_lfht_for_each_entry_duplicate(ht, hash, match, key, \
+ iter, pos, member) \
+ for (cds_lfht_lookup(ht, hash, match, key, iter), \
+ pos = caa_container_of(cds_lfht_iter_get_node(iter), \
+ __typeof__(*(pos)), member); \
+ cds_lfht_iter_get_node(iter) != NULL; \
+ cds_lfht_next_duplicate(ht, match, key, iter), \
+ pos = caa_container_of(cds_lfht_iter_get_node(iter), \
+ __typeof__(*(pos)), member))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFHASH_H */
--- /dev/null
+#ifndef _URCU_RCULFQUEUE_H
+#define _URCU_RCULFQUEUE_H
+
+/*
+ * rculfqueue.h
+ *
+ * Userspace RCU library - Lock-Free RCU Queue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <assert.h>
+#include <urcu-call-rcu.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct cds_lfq_queue_rcu;
+
+struct cds_lfq_node_rcu {
+ struct cds_lfq_node_rcu *next;
+ int dummy;
+};
+
+struct cds_lfq_queue_rcu {
+ struct cds_lfq_node_rcu *head, *tail;
+ void (*queue_call_rcu)(struct rcu_head *head,
+ void (*func)(struct rcu_head *head));
+};
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/rculfqueue.h>
+
+#define cds_lfq_node_init_rcu _cds_lfq_node_init_rcu
+#define cds_lfq_init_rcu _cds_lfq_init_rcu
+#define cds_lfq_destroy_rcu _cds_lfq_destroy_rcu
+#define cds_lfq_enqueue_rcu _cds_lfq_enqueue_rcu
+#define cds_lfq_dequeue_rcu _cds_lfq_dequeue_rcu
+
+#else /* !_LGPL_SOURCE */
+
+extern void cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node);
+extern void cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
+ void queue_call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head)));
+/*
+ * The queue should be emptied before calling destroy.
+ *
+ * Return 0 on success, -EPERM if queue is not empty.
+ */
+extern int cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q);
+
+/*
+ * Should be called under rcu read lock critical section.
+ */
+extern void cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
+ struct cds_lfq_node_rcu *node);
+
+/*
+ * Should be called under rcu read lock critical section.
+ *
+ * The caller must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_lfq_node_rcu structure.
+ * Returns NULL if queue is empty.
+ */
+extern
+struct cds_lfq_node_rcu *cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q);
+
+#endif /* !_LGPL_SOURCE */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFQUEUE_H */
--- /dev/null
+#ifndef _URCU_RCULFSTACK_H
+#define _URCU_RCULFSTACK_H
+
+/*
+ * rculfstack.h
+ *
+ * Userspace RCU library - Lock-Free RCU Stack
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/compiler.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CDS_LFS_RCU_DEPRECATED
+#define CDS_LFS_RCU_DEPRECATED \
+ CDS_DEPRECATED("urcu/rculfstack.h is deprecated. Please use urcu/lfstack.h instead.")
+#endif
+
+struct cds_lfs_node_rcu {
+ struct cds_lfs_node_rcu *next;
+};
+
+struct cds_lfs_stack_rcu {
+ struct cds_lfs_node_rcu *head;
+};
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/rculfstack.h>
+
+static inline CDS_LFS_RCU_DEPRECATED
+void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
+{
+ _cds_lfs_node_init_rcu(node);
+}
+
+static inline
+void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
+{
+ _cds_lfs_init_rcu(s);
+}
+
+static inline CDS_LFS_RCU_DEPRECATED
+int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
+ struct cds_lfs_node_rcu *node)
+{
+ return _cds_lfs_push_rcu(s, node);
+}
+
+static inline CDS_LFS_RCU_DEPRECATED
+struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
+{
+ return _cds_lfs_pop_rcu(s);
+}
+
+#else /* !_LGPL_SOURCE */
+
+extern CDS_LFS_RCU_DEPRECATED
+void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node);
+extern CDS_LFS_RCU_DEPRECATED
+void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s);
+extern CDS_LFS_RCU_DEPRECATED
+int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
+ struct cds_lfs_node_rcu *node);
+
+/*
+ * Should be called under rcu read lock critical section.
+ *
+ * The caller must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_lfs_node_rcu structure.
+ * Returns NULL if stack is empty.
+ */
+extern CDS_LFS_RCU_DEPRECATED
+struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s);
+
+#endif /* !_LGPL_SOURCE */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFSTACK_H */
--- /dev/null
+/*
+ * Copyright (C) 2002 Free Software Foundation, Inc.
+ * (originally part of the GNU C Library)
+ * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+ *
+ * Copyright (C) 2009 Pierre-Marc Fournier
+ * Conversion to RCU list.
+ * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef _URCU_RCULIST_H
+#define _URCU_RCULIST_H
+
+#include <urcu/list.h>
+#include <urcu/arch.h>
+#include <urcu-pointer.h>
+
+/* Add new element at the head of the list. */
+static inline
+void cds_list_add_rcu(struct cds_list_head *newp, struct cds_list_head *head)
+{
+ newp->next = head->next;
+ newp->prev = head;
+ head->next->prev = newp;
+ rcu_assign_pointer(head->next, newp);
+}
+
+/* Add new element at the tail of the list. */
+static inline
+void cds_list_add_tail_rcu(struct cds_list_head *newp,
+ struct cds_list_head *head)
+{
+ newp->next = head;
+ newp->prev = head->prev;
+ rcu_assign_pointer(head->prev->next, newp);
+ head->prev = newp;
+}
+
+/*
+ * Replace an old entry atomically with respect to concurrent RCU
+ * traversal. Mutual exclusion against concurrent updates is required
+ * though.
+ */
+static inline
+void cds_list_replace_rcu(struct cds_list_head *old, struct cds_list_head *_new)
+{
+ _new->next = old->next;
+ _new->prev = old->prev;
+ rcu_assign_pointer(_new->prev->next, _new);
+ _new->next->prev = _new;
+}
+
+/* Remove element from list. */
+static inline
+void cds_list_del_rcu(struct cds_list_head *elem)
+{
+ elem->next->prev = elem->prev;
+ CMM_STORE_SHARED(elem->prev->next, elem->next);
+}
+
+/*
+ * Iteration through all elements of the list must be done while rcu_read_lock()
+ * is held.
+ */
+
+/* Iterate forward over the elements of the list. */
+#define cds_list_for_each_rcu(pos, head) \
+ for (pos = rcu_dereference((head)->next); pos != (head); \
+ pos = rcu_dereference(pos->next))
+
+
+/* Iterate through elements of the list. */
+#define cds_list_for_each_entry_rcu(pos, head, member) \
+ for (pos = cds_list_entry(rcu_dereference((head)->next), __typeof__(*pos), member); \
+ &pos->member != (head); \
+ pos = cds_list_entry(rcu_dereference(pos->member.next), __typeof__(*pos), member))
+
+#endif /* _URCU_RCULIST_H */
--- /dev/null
+#ifndef _URCU_REF_H
+#define _URCU_REF_H
+
+/*
+ * Userspace RCU - Reference counting
+ *
+ * Copyright (C) 2009 Novell Inc.
+ * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * Author: Jan Blunck <jblunck@suse.de>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License version 2.1 as
+ * published by the Free Software Foundation.
+ */
+
+#include <assert.h>
+#include <stdbool.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <urcu/uatomic.h>
+
+struct urcu_ref {
+ long refcount; /* ATOMIC */
+};
+
+static inline void urcu_ref_set(struct urcu_ref *ref, long val)
+{
+ uatomic_set(&ref->refcount, val);
+}
+
+static inline void urcu_ref_init(struct urcu_ref *ref)
+{
+ urcu_ref_set(ref, 1);
+}
+
+static inline bool __attribute__((warn_unused_result))
+ urcu_ref_get_safe(struct urcu_ref *ref)
+{
+ long old, _new, res;
+
+ old = uatomic_read(&ref->refcount);
+ for (;;) {
+ if (old == LONG_MAX) {
+ return false; /* Failure. */
+ }
+ _new = old + 1;
+ res = uatomic_cmpxchg(&ref->refcount, old, _new);
+ if (res == old) {
+ return true; /* Success. */
+ }
+ old = res;
+ }
+}
+
+static inline void urcu_ref_get(struct urcu_ref *ref)
+{
+ if (!urcu_ref_get_safe(ref))
+ abort();
+}
+
+static inline void urcu_ref_put(struct urcu_ref *ref,
+ void (*release)(struct urcu_ref *))
+{
+ long res = uatomic_sub_return(&ref->refcount, 1);
+ assert (res >= 0);
+ if (res == 0)
+ release(ref);
+}
+
+/*
+ * urcu_ref_get_unless_zero
+ *
+ * Allows getting a reference atomically if the reference count is not
+ * zero. Returns true if the reference is taken, false otherwise. This
+ * needs to be used in conjunction with another synchronization
+ * technique (e.g. RCU or mutex) to ensure existence of the reference
+ * count. False is also returned in case incrementing the refcount would
+ * result in an overflow.
+ */
+static inline bool urcu_ref_get_unless_zero(struct urcu_ref *ref)
+{
+ long old, _new, res;
+
+ old = uatomic_read(&ref->refcount);
+ for (;;) {
+ if (old == 0 || old == LONG_MAX)
+ return false; /* Failure. */
+ _new = old + 1;
+ res = uatomic_cmpxchg(&ref->refcount, old, _new);
+ if (res == old) {
+ return true; /* Success. */
+ }
+ old = res;
+ }
+}
+
+#endif /* _URCU_REF_H */
--- /dev/null
+#ifndef _URCU_STATIC_LFSTACK_H
+#define _URCU_STATIC_LFSTACK_H
+
+/*
+ * urcu/static/lfstack.h
+ *
+ * Userspace RCU library - Lock-Free Stack
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/lfstack.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdbool.h>
+#include <pthread.h>
+#include <assert.h>
+#include <urcu/uatomic.h>
+#include <urcu-pointer.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Lock-free stack.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * cds_lfs_push __cds_lfs_pop __cds_lfs_pop_all
+ * cds_lfs_push - - -
+ * __cds_lfs_pop - X X
+ * __cds_lfs_pop_all - X -
+ *
+ * cds_lfs_pop_blocking and cds_lfs_pop_all_blocking use an internal
+ * mutex to provide synchronization.
+ */
+
+/*
+ * cds_lfs_node_init: initialize lock-free stack node.
+ */
+static inline
+void _cds_lfs_node_init(struct cds_lfs_node *node)
+{
+}
+
+/*
+ * cds_lfs_init: initialize lock-free stack (with lock). Pair with
+ * cds_lfs_destroy().
+ */
+static inline
+void _cds_lfs_init(struct cds_lfs_stack *s)
+{
+ int ret;
+
+ s->head = NULL;
+ ret = pthread_mutex_init(&s->lock, NULL);
+ assert(!ret);
+}
+
+/*
+ * cds_lfs_destroy: destroy lock-free stack (with lock). Pair with
+ * cds_lfs_init().
+ */
+static inline
+void _cds_lfs_destroy(struct cds_lfs_stack *s)
+{
+ int ret = pthread_mutex_destroy(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * ___cds_lfs_init: initialize lock-free stack (without lock).
+ * Don't pair with any destroy function.
+ */
+static inline
+void ___cds_lfs_init(struct __cds_lfs_stack *s)
+{
+ s->head = NULL;
+}
+
+static inline
+bool ___cds_lfs_empty_head(struct cds_lfs_head *head)
+{
+ return head == NULL;
+}
+
+/*
+ * cds_lfs_empty: return whether lock-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+static inline
+bool _cds_lfs_empty(cds_lfs_stack_ptr_t s)
+{
+ return ___cds_lfs_empty_head(CMM_LOAD_SHARED(s._s->head));
+}
+
+/*
+ * cds_lfs_push: push a node into the stack.
+ *
+ * Does not require any synchronization with other push nor pop.
+ *
+ * Lock-free stack push is not subject to ABA problem, so no need to
+ * take the RCU read-side lock. Even if "head" changes between two
+ * uatomic_cmpxchg() invocations here (being popped, and then pushed
+ * again by one or more concurrent threads), the second
+ * uatomic_cmpxchg() invocation only cares about pushing a new entry at
+ * the head of the stack, ensuring consistency by making sure the new
+ * node->next is the same pointer value as the value replaced as head.
+ * It does not care about the content of the actual next node, so it can
+ * very well be reallocated between the two uatomic_cmpxchg().
+ *
+ * We take the approach of expecting the stack to be usually empty, so
+ * we first try an initial uatomic_cmpxchg() on a NULL old_head, and
+ * retry if the old head was non-NULL (the value read by the first
+ * uatomic_cmpxchg() is used as old head for the following loop). The
+ * upside of this scheme is to minimize the amount of cacheline traffic,
+ * always performing an exclusive cacheline access, rather than doing
+ * non-exclusive followed by exclusive cacheline access (which would be
+ * required if we first read the old head value). This design decision
+ * might be revisited after more thorough benchmarking on various
+ * platforms.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+static inline
+bool _cds_lfs_push(cds_lfs_stack_ptr_t u_s,
+ struct cds_lfs_node *node)
+{
+ struct __cds_lfs_stack *s = u_s._s;
+ struct cds_lfs_head *head = NULL;
+ struct cds_lfs_head *new_head =
+ caa_container_of(node, struct cds_lfs_head, node);
+
+ for (;;) {
+ struct cds_lfs_head *old_head = head;
+
+ /*
+ * node->next is still private at this point, no need to
+ * perform a _CMM_STORE_SHARED().
+ */
+ node->next = &head->node;
+ /*
+ * uatomic_cmpxchg() implicit memory barrier orders earlier
+ * stores to node before publication.
+ */
+ head = uatomic_cmpxchg(&s->head, old_head, new_head);
+ if (old_head == head)
+ break;
+ }
+ return !___cds_lfs_empty_head(head);
+}
+
+/*
+ * __cds_lfs_pop: pop a node from the stack.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_lfs_pop needs to be synchronized using one of the following
+ * techniques:
+ *
+ * 1) Calling __cds_lfs_pop under rcu read lock critical section.
+ * Both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
+ * grace period to pass before freeing the returned node or pushing
+ * the node back into the stack. It is valid to overwrite the content
+ * of cds_lfs_node immediately after __cds_lfs_pop and
+ * __cds_lfs_pop_all. No RCU read-side critical section is needed
+ * around __cds_lfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop
+ * and __cds_lfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
+ * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_lfs_node *___cds_lfs_pop(cds_lfs_stack_ptr_t u_s)
+{
+ struct __cds_lfs_stack *s = u_s._s;
+
+ for (;;) {
+ struct cds_lfs_head *head, *next_head;
+ struct cds_lfs_node *next;
+
+ head = _CMM_LOAD_SHARED(s->head);
+ if (___cds_lfs_empty_head(head))
+ return NULL; /* Empty stack */
+
+ /*
+ * Read head before head->next. Matches the implicit
+ * memory barrier before uatomic_cmpxchg() in
+ * cds_lfs_push.
+ */
+ cmm_smp_read_barrier_depends();
+ next = _CMM_LOAD_SHARED(head->node.next);
+ next_head = caa_container_of(next,
+ struct cds_lfs_head, node);
+ if (uatomic_cmpxchg(&s->head, head, next_head) == head)
+ return &head->node;
+ /* busy-loop if head changed under us */
+ }
+}
+
+/*
+ * __cds_lfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_lfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_lfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_lfs_pop:
+ *
+ * 1) If __cds_lfs_pop is called under rcu read lock critical section,
+ * both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
+ * grace period to pass before freeing the returned node or pushing
+ * the node back into the stack. It is valid to overwrite the content
+ * of cds_lfs_node immediately after __cds_lfs_pop and
+ * __cds_lfs_pop_all. No RCU read-side critical section is needed
+ * around __cds_lfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
+ * __cds_lfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
+ * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_lfs_head *___cds_lfs_pop_all(cds_lfs_stack_ptr_t u_s)
+{
+ struct __cds_lfs_stack *s = u_s._s;
+
+ /*
+ * Implicit memory barrier after uatomic_xchg() matches implicit
+ * memory barrier before uatomic_cmpxchg() in cds_lfs_push. It
+ * ensures that all nodes of the returned list are consistent.
+ * There is no need to issue memory barriers when iterating on
+ * the returned list, because the full memory barrier issued
+ * prior to each uatomic_cmpxchg, which each write to head, are
+ * taking care to order writes to each node prior to the full
+ * memory barrier after this uatomic_xchg().
+ */
+ return uatomic_xchg(&s->head, NULL);
+}
+
+/*
+ * cds_lfs_pop_lock: lock stack pop-protection mutex.
+ */
+static inline void _cds_lfs_pop_lock(struct cds_lfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_lock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+static inline void _cds_lfs_pop_unlock(struct cds_lfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * Call __cds_lfs_pop with an internal pop mutex held.
+ */
+static inline
+struct cds_lfs_node *
+_cds_lfs_pop_blocking(struct cds_lfs_stack *s)
+{
+ struct cds_lfs_node *retnode;
+
+ _cds_lfs_pop_lock(s);
+ retnode = ___cds_lfs_pop(s);
+ _cds_lfs_pop_unlock(s);
+ return retnode;
+}
+
+/*
+ * Call __cds_lfs_pop_all with an internal pop mutex held.
+ */
+static inline
+struct cds_lfs_head *
+_cds_lfs_pop_all_blocking(struct cds_lfs_stack *s)
+{
+ struct cds_lfs_head *rethead;
+
+ _cds_lfs_pop_lock(s);
+ rethead = ___cds_lfs_pop_all(s);
+ _cds_lfs_pop_unlock(s);
+ return rethead;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_LFSTACK_H */
--- /dev/null
+#ifndef _URCU_RCULFQUEUE_STATIC_H
+#define _URCU_RCULFQUEUE_STATIC_H
+
+/*
+ * rculfqueue-static.h
+ *
+ * Userspace RCU library - Lock-Free RCU Queue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu-call-rcu.h>
+#include <urcu/uatomic.h>
+#include <urcu-pointer.h>
+#include <assert.h>
+#include <errno.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct cds_lfq_node_rcu_dummy {
+ struct cds_lfq_node_rcu parent;
+ struct rcu_head head;
+ struct cds_lfq_queue_rcu *q;
+};
+
+/*
+ * Lock-free RCU queue. Enqueue and dequeue operations hold a RCU read
+ * lock to deal with cmpxchg ABA problem. This queue is *not* circular:
+ * head points to the oldest node, tail points to the newest node.
+ * A dummy node is kept to ensure enqueue and dequeue can always proceed
+ * concurrently. Keeping a separate head and tail helps with large
+ * queues: enqueue and dequeue can proceed concurrently without
+ * wrestling for exclusive access to the same variables.
+ *
+ * Dequeue retry if it detects that it would be dequeueing the last node
+ * (it means a dummy node dequeue-requeue is in progress). This ensures
+ * that there is always at least one node in the queue.
+ *
+ * In the dequeue operation, we internally reallocate the dummy node
+ * upon dequeue/requeue and use call_rcu to free the old one after a
+ * grace period.
+ */
+
+static inline
+struct cds_lfq_node_rcu *make_dummy(struct cds_lfq_queue_rcu *q,
+ struct cds_lfq_node_rcu *next)
+{
+ struct cds_lfq_node_rcu_dummy *dummy;
+
+ dummy = malloc(sizeof(struct cds_lfq_node_rcu_dummy));
+ assert(dummy);
+ dummy->parent.next = next;
+ dummy->parent.dummy = 1;
+ dummy->q = q;
+ return &dummy->parent;
+}
+
+static inline
+void free_dummy_cb(struct rcu_head *head)
+{
+ struct cds_lfq_node_rcu_dummy *dummy =
+ caa_container_of(head, struct cds_lfq_node_rcu_dummy, head);
+ free(dummy);
+}
+
+static inline
+void rcu_free_dummy(struct cds_lfq_node_rcu *node)
+{
+ struct cds_lfq_node_rcu_dummy *dummy;
+
+ assert(node->dummy);
+ dummy = caa_container_of(node, struct cds_lfq_node_rcu_dummy, parent);
+ dummy->q->queue_call_rcu(&dummy->head, free_dummy_cb);
+}
+
+static inline
+void free_dummy(struct cds_lfq_node_rcu *node)
+{
+ struct cds_lfq_node_rcu_dummy *dummy;
+
+ assert(node->dummy);
+ dummy = caa_container_of(node, struct cds_lfq_node_rcu_dummy, parent);
+ free(dummy);
+}
+
+static inline
+void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
+{
+ node->next = NULL;
+ node->dummy = 0;
+}
+
+static inline
+void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
+ void queue_call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head)))
+{
+ q->tail = make_dummy(q, NULL);
+ q->head = q->tail;
+ q->queue_call_rcu = queue_call_rcu;
+}
+
+/*
+ * The queue should be emptied before calling destroy.
+ *
+ * Return 0 on success, -EPERM if queue is not empty.
+ */
+static inline
+int _cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q)
+{
+ struct cds_lfq_node_rcu *head;
+
+ head = rcu_dereference(q->head);
+ if (!(head->dummy && head->next == NULL))
+ return -EPERM; /* not empty */
+ free_dummy(head);
+ return 0;
+}
+
+/*
+ * Should be called under rcu read lock critical section.
+ */
+static inline
+void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
+ struct cds_lfq_node_rcu *node)
+{
+ /*
+ * uatomic_cmpxchg() implicit memory barrier orders earlier stores to
+ * node before publication.
+ */
+
+ for (;;) {
+ struct cds_lfq_node_rcu *tail, *next;
+
+ tail = rcu_dereference(q->tail);
+ next = uatomic_cmpxchg(&tail->next, NULL, node);
+ if (next == NULL) {
+ /*
+ * Tail was at the end of queue, we successfully
+ * appended to it. Now move tail (another
+ * enqueue might beat us to it, that's fine).
+ */
+ (void) uatomic_cmpxchg(&q->tail, tail, node);
+ return;
+ } else {
+ /*
+ * Failure to append to current tail.
+ * Help moving tail further and retry.
+ */
+ (void) uatomic_cmpxchg(&q->tail, tail, next);
+ continue;
+ }
+ }
+}
+
+static inline
+void enqueue_dummy(struct cds_lfq_queue_rcu *q)
+{
+ struct cds_lfq_node_rcu *node;
+
+ /* We need to reallocate to protect from ABA. */
+ node = make_dummy(q, NULL);
+ _cds_lfq_enqueue_rcu(q, node);
+}
+
+/*
+ * Should be called under rcu read lock critical section.
+ *
+ * The caller must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_lfq_node_rcu structure.
+ * Returns NULL if queue is empty.
+ */
+static inline
+struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
+{
+ for (;;) {
+ struct cds_lfq_node_rcu *head, *next;
+
+ head = rcu_dereference(q->head);
+ next = rcu_dereference(head->next);
+ if (head->dummy && next == NULL)
+ return NULL; /* empty */
+ /*
+ * We never, ever allow dequeue to get to a state where
+ * the queue is empty (we need at least one node in the
+ * queue). This is ensured by checking if the head next
+ * is NULL, which means we need to enqueue a dummy node
+ * before we can hope dequeuing anything.
+ */
+ if (!next) {
+ enqueue_dummy(q);
+ next = rcu_dereference(head->next);
+ }
+ if (uatomic_cmpxchg(&q->head, head, next) != head)
+ continue; /* Concurrently pushed. */
+ if (head->dummy) {
+ /* Free dummy after grace period. */
+ rcu_free_dummy(head);
+ continue; /* try again */
+ }
+ return head;
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFQUEUE_STATIC_H */
--- /dev/null
+#ifndef _URCU_RCULFSTACK_STATIC_H
+#define _URCU_RCULFSTACK_STATIC_H
+
+/*
+ * rculfstack-static.h
+ *
+ * Userspace RCU library - Lock-Free RCU Stack
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/uatomic.h>
+#include <urcu-pointer.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline
+void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
+{
+}
+
+static inline
+void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
+{
+ s->head = NULL;
+}
+
+/*
+ * Lock-free stack push is not subject to ABA problem, so no need to
+ * take the RCU read-side lock. Even if "head" changes between two
+ * uatomic_cmpxchg() invocations here (being popped, and then pushed
+ * again by one or more concurrent threads), the second
+ * uatomic_cmpxchg() invocation only cares about pushing a new entry at
+ * the head of the stack, ensuring consistency by making sure the new
+ * node->next is the same pointer value as the value replaced as head.
+ * It does not care about the content of the actual next node, so it can
+ * very well be reallocated between the two uatomic_cmpxchg().
+ *
+ * We take the approach of expecting the stack to be usually empty, so
+ * we first try an initial uatomic_cmpxchg() on a NULL old_head, and
+ * retry if the old head was non-NULL (the value read by the first
+ * uatomic_cmpxchg() is used as old head for the following loop). The
+ * upside of this scheme is to minimize the amount of cacheline traffic,
+ * always performing an exclusive cacheline access, rather than doing
+ * non-exclusive followed by exclusive cacheline access (which would be
+ * required if we first read the old head value). This design decision
+ * might be revisited after more throrough benchmarking on various
+ * platforms.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+static inline
+int _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
+ struct cds_lfs_node_rcu *node)
+{
+ struct cds_lfs_node_rcu *head = NULL;
+
+ for (;;) {
+ struct cds_lfs_node_rcu *old_head = head;
+
+ node->next = head;
+ /*
+ * uatomic_cmpxchg() implicit memory barrier orders earlier
+ * stores to node before publication.
+ */
+ head = uatomic_cmpxchg(&s->head, old_head, node);
+ if (old_head == head)
+ break;
+ }
+ return (int) !!((unsigned long) head);
+}
+
+/*
+ * Should be called under rcu read-side lock.
+ *
+ * The caller must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_lfs_node_rcu structure.
+ * Returns NULL if stack is empty.
+ */
+static inline
+struct cds_lfs_node_rcu *
+_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
+{
+ for (;;) {
+ struct cds_lfs_node_rcu *head;
+
+ head = rcu_dereference(s->head);
+ if (head) {
+ struct cds_lfs_node_rcu *next = rcu_dereference(head->next);
+
+ if (uatomic_cmpxchg(&s->head, head, next) == head) {
+ return head;
+ } else {
+ /* Concurrent modification. Retry. */
+ continue;
+ }
+ } else {
+ /* Empty stack */
+ return NULL;
+ }
+ }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_RCULFSTACK_STATIC_H */
--- /dev/null
+#ifndef _URCU_BP_STATIC_H
+#define _URCU_BP_STATIC_H
+
+/*
+ * urcu-bp-static.h
+ *
+ * Userspace RCU header.
+ *
+ * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
+ * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <unistd.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic.h>
+#include <urcu/list.h>
+#include <urcu/tls-compat.h>
+#include <urcu/debug.h>
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rcu_state {
+ RCU_READER_ACTIVE_CURRENT,
+ RCU_READER_ACTIVE_OLD,
+ RCU_READER_INACTIVE,
+};
+
+/*
+ * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a
+ * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
+ */
+#define RCU_GP_COUNT (1UL << 0)
+/* Use the amount of bits equal to half of the architecture long size */
+#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2))
+#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
+
+/*
+ * Used internally by _rcu_read_lock.
+ */
+extern void rcu_bp_register(void);
+
+struct rcu_gp {
+ /*
+ * Global grace period counter.
+ * Contains the current RCU_GP_CTR_PHASE.
+ * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
+ * Written to only by writer with mutex taken.
+ * Read by both writer and readers.
+ */
+ unsigned long ctr;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct rcu_gp rcu_gp;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ unsigned long ctr;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ pthread_t tid;
+ int alloc; /* registry entry allocated */
+};
+
+/*
+ * Bulletproof version keeps a pointer to a registry not part of the TLS.
+ * Adds a pointer dereference on the read-side, but won't require to unregister
+ * the reader thread.
+ */
+extern DECLARE_URCU_TLS(struct rcu_reader *, rcu_reader);
+
+extern int urcu_bp_has_sys_membarrier;
+
+static inline void urcu_bp_smp_mb_slave(void)
+{
+ if (caa_likely(urcu_bp_has_sys_membarrier))
+ cmm_barrier();
+ else
+ cmm_smp_mb();
+}
+
+static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
+{
+ unsigned long v;
+
+ if (ctr == NULL)
+ return RCU_READER_INACTIVE;
+ /*
+ * Make sure both tests below are done on the same version of *value
+ * to insure consistency.
+ */
+ v = CMM_LOAD_SHARED(*ctr);
+ if (!(v & RCU_GP_CTR_NEST_MASK))
+ return RCU_READER_INACTIVE;
+ if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
+ return RCU_READER_ACTIVE_CURRENT;
+ return RCU_READER_ACTIVE_OLD;
+}
+
+/*
+ * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as
+ * the per-thread rcu_reader.ctr) has the upper bits containing a count of
+ * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
+ * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
+ * _rcu_read_lock() happen before the subsequent read-side critical section.
+ */
+static inline void _rcu_read_lock_update(unsigned long tmp)
+{
+ if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
+ urcu_bp_smp_mb_slave();
+ } else
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp + RCU_GP_COUNT);
+}
+
+/*
+ * Enter an RCU read-side critical section.
+ *
+ * The first cmm_barrier() call ensures that the compiler does not reorder
+ * the body of _rcu_read_lock() with a mutex.
+ *
+ * This function and its helper are both less than 10 lines long. The
+ * intent is that this function meets the 10-line criterion in LGPL,
+ * allowing this function to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_read_lock(void)
+{
+ unsigned long tmp;
+
+ if (caa_unlikely(!URCU_TLS(rcu_reader)))
+ rcu_bp_register(); /* If not yet registered. */
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+ tmp = URCU_TLS(rcu_reader)->ctr;
+ urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK);
+ _rcu_read_lock_update(tmp);
+}
+
+/*
+ * Exit an RCU read-side critical section. This function is less than
+ * 10 lines of code, and is intended to be usable by non-LGPL code, as
+ * called out in LGPL.
+ */
+static inline void _rcu_read_unlock(void)
+{
+ unsigned long tmp;
+
+ tmp = URCU_TLS(rcu_reader)->ctr;
+ urcu_assert(tmp & RCU_GP_CTR_NEST_MASK);
+ /* Finish using rcu before decrementing the pointer. */
+ urcu_bp_smp_mb_slave();
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp - RCU_GP_COUNT);
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+/*
+ * Returns whether within a RCU read-side critical section.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline int _rcu_read_ongoing(void)
+{
+ if (caa_unlikely(!URCU_TLS(rcu_reader)))
+ rcu_bp_register(); /* If not yet registered. */
+ return URCU_TLS(rcu_reader)->ctr & RCU_GP_CTR_NEST_MASK;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_BP_STATIC_H */
--- /dev/null
+#ifndef _URCU_POINTER_STATIC_H
+#define _URCU_POINTER_STATIC_H
+
+/*
+ * urcu-pointer-static.h
+ *
+ * Userspace RCU header. Operations on pointers.
+ *
+ * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
+ * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable
+ * into a RCU read-side critical section. The pointer can later be safely
+ * dereferenced within the critical section.
+ *
+ * This ensures that the pointer copy is invariant thorough the whole critical
+ * section.
+ *
+ * Inserts memory barriers on architectures that require them (currently only
+ * Alpha) and documents which pointers are protected by RCU.
+ *
+ * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative
+ * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
+ * data read before the pointer read by speculating the value of the pointer.
+ * Correct ordering is ensured because the pointer is read as a volatile access.
+ * This acts as a global side-effect operation, which forbids reordering of
+ * dependent memory operations. Note that such concern about dependency-breaking
+ * optimizations will eventually be taken care of by the "memory_order_consume"
+ * addition to forthcoming C++ standard.
+ *
+ * Should match rcu_assign_pointer() or rcu_xchg_pointer().
+ *
+ * This macro is less than 10 lines long. The intent is that this macro
+ * meets the 10-line criterion in LGPL, allowing this function to be
+ * expanded directly in non-LGPL code.
+ */
+#define _rcu_dereference(p) \
+ __extension__ \
+ ({ \
+ __typeof__(p) _________p1 = CMM_LOAD_SHARED(p); \
+ cmm_smp_read_barrier_depends(); \
+ (_________p1); \
+ })
+
+/**
+ * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
+ * is as expected by "old". If succeeds, returns the previous pointer to the
+ * data structure, which can be safely freed after waiting for a quiescent state
+ * using synchronize_rcu(). If fails (unexpected value), returns old (which
+ * should not be freed !).
+ *
+ * This macro is less than 10 lines long. The intent is that this macro
+ * meets the 10-line criterion in LGPL, allowing this function to be
+ * expanded directly in non-LGPL code.
+ */
+#define _rcu_cmpxchg_pointer(p, old, _new) \
+ __extension__ \
+ ({ \
+ __typeof__(*p) _________pold = (old); \
+ __typeof__(*p) _________pnew = (_new); \
+ if (!__builtin_constant_p(_new) || \
+ ((_new) != NULL)) \
+ cmm_wmb(); \
+ uatomic_cmpxchg(p, _________pold, _________pnew); \
+ })
+
+/**
+ * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
+ * pointer to the data structure, which can be safely freed after waiting for a
+ * quiescent state using synchronize_rcu().
+ *
+ * This macro is less than 10 lines long. The intent is that this macro
+ * meets the 10-line criterion in LGPL, allowing this function to be
+ * expanded directly in non-LGPL code.
+ */
+#define _rcu_xchg_pointer(p, v) \
+ __extension__ \
+ ({ \
+ __typeof__(*p) _________pv = (v); \
+ if (!__builtin_constant_p(v) || \
+ ((v) != NULL)) \
+ cmm_wmb(); \
+ uatomic_xchg(p, _________pv); \
+ })
+
+
+#define _rcu_set_pointer(p, v) \
+ do { \
+ __typeof__(*p) _________pv = (v); \
+ if (!__builtin_constant_p(v) || \
+ ((v) != NULL)) \
+ cmm_wmb(); \
+ uatomic_set(p, _________pv); \
+ } while (0)
+
+/**
+ * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure
+ * meant to be read by RCU read-side critical sections. Returns the assigned
+ * value.
+ *
+ * Documents which pointers will be dereferenced by RCU read-side critical
+ * sections and adds the required memory barriers on architectures requiring
+ * them. It also makes sure the compiler does not reorder code initializing the
+ * data structure before its publication.
+ *
+ * Should match rcu_dereference().
+ *
+ * This macro is less than 10 lines long. The intent is that this macro
+ * meets the 10-line criterion in LGPL, allowing this function to be
+ * expanded directly in non-LGPL code.
+ */
+#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_POINTER_STATIC_H */
--- /dev/null
+#ifndef _URCU_QSBR_STATIC_H
+#define _URCU_QSBR_STATIC_H
+
+/*
+ * urcu-qsbr-static.h
+ *
+ * Userspace RCU QSBR header.
+ *
+ * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
+ * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <limits.h>
+#include <unistd.h>
+#include <stdint.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic.h>
+#include <urcu/list.h>
+#include <urcu/futex.h>
+#include <urcu/tls-compat.h>
+#include <urcu/debug.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+enum rcu_state {
+ RCU_READER_ACTIVE_CURRENT,
+ RCU_READER_ACTIVE_OLD,
+ RCU_READER_INACTIVE,
+};
+
+#define RCU_GP_ONLINE (1UL << 0)
+#define RCU_GP_CTR (1UL << 1)
+
+struct rcu_gp {
+ /*
+ * Global quiescent period counter with low-order bits unused.
+ * Using a int rather than a char to eliminate false register
+ * dependencies causing stalls on some architectures.
+ */
+ unsigned long ctr;
+
+ int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct rcu_gp rcu_gp;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ unsigned long ctr;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ int waiting;
+ pthread_t tid;
+ /* Reader registered flag, for internal checks. */
+ unsigned int registered:1;
+};
+
+extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
+
+/*
+ * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
+ */
+static inline void wake_up_gp(void)
+{
+ if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
+ cmm_smp_mb();
+ if (uatomic_read(&rcu_gp.futex) != -1)
+ return;
+ uatomic_set(&rcu_gp.futex, 0);
+ /*
+ * Ignoring return value until we can make this function
+ * return something (because urcu_die() is not publicly
+ * exposed).
+ */
+ (void) futex_noasync(&rcu_gp.futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0);
+ }
+}
+
+static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
+{
+ unsigned long v;
+
+ v = CMM_LOAD_SHARED(*ctr);
+ if (!v)
+ return RCU_READER_INACTIVE;
+ if (v == rcu_gp.ctr)
+ return RCU_READER_ACTIVE_CURRENT;
+ return RCU_READER_ACTIVE_OLD;
+}
+
+/*
+ * Enter an RCU read-side critical section.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_read_lock(void)
+{
+ urcu_assert(URCU_TLS(rcu_reader).ctr);
+}
+
+/*
+ * Exit an RCU read-side critical section.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_read_unlock(void)
+{
+ urcu_assert(URCU_TLS(rcu_reader).ctr);
+}
+
+/*
+ * Returns whether within a RCU read-side critical section.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline int _rcu_read_ongoing(void)
+{
+ return URCU_TLS(rcu_reader).ctr;
+}
+
+/*
+ * This is a helper function for _rcu_quiescent_state().
+ * The first cmm_smp_mb() ensures memory accesses in the prior read-side
+ * critical sections are not reordered with store to
+ * URCU_TLS(rcu_reader).ctr, and ensures that mutexes held within an
+ * offline section that would happen to end with this
+ * rcu_quiescent_state() call are not reordered with
+ * store to URCU_TLS(rcu_reader).ctr.
+ */
+static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
+{
+ cmm_smp_mb();
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, gp_ctr);
+ cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
+ wake_up_gp();
+ cmm_smp_mb();
+}
+
+/*
+ * Inform RCU of a quiescent state.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ *
+ * We skip the memory barriers and gp store if our local ctr already
+ * matches the global rcu_gp.ctr value: this is OK because a prior
+ * _rcu_quiescent_state() or _rcu_thread_online() already updated it
+ * within our thread, so we have no quiescent state to report.
+ */
+static inline void _rcu_quiescent_state(void)
+{
+ unsigned long gp_ctr;
+
+ urcu_assert(URCU_TLS(rcu_reader).registered);
+ if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp.ctr)) == URCU_TLS(rcu_reader).ctr)
+ return;
+ _rcu_quiescent_state_update_and_wakeup(gp_ctr);
+}
+
+/*
+ * Take a thread offline, prohibiting it from entering further RCU
+ * read-side critical sections.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_thread_offline(void)
+{
+ urcu_assert(URCU_TLS(rcu_reader).registered);
+ cmm_smp_mb();
+ CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
+ cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
+ wake_up_gp();
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+/*
+ * Bring a thread online, allowing it to once again enter RCU
+ * read-side critical sections.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_thread_online(void)
+{
+ urcu_assert(URCU_TLS(rcu_reader).registered);
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp.ctr));
+ cmm_smp_mb();
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_QSBR_STATIC_H */
--- /dev/null
+#ifndef _URCU_STATIC_H
+#define _URCU_STATIC_H
+
+/*
+ * urcu-static.h
+ *
+ * Userspace RCU header.
+ *
+ * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
+ * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <stdint.h>
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/system.h>
+#include <urcu/uatomic.h>
+#include <urcu/list.h>
+#include <urcu/futex.h>
+#include <urcu/tls-compat.h>
+#include <urcu/rand-compat.h>
+#include <urcu/debug.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Default is RCU_MEMBARRIER */
+#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL)
+#define RCU_MEMBARRIER
+#endif
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+/*
+ * The signal number used by the RCU library can be overridden with
+ * -DSIGRCU= when compiling the library.
+ * Provide backward compatibility for liburcu 0.3.x SIGURCU.
+ */
+#ifdef SIGURCU
+#define SIGRCU SIGURCU
+#endif
+
+#ifndef SIGRCU
+#define SIGRCU SIGUSR1
+#endif
+
+enum rcu_state {
+ RCU_READER_ACTIVE_CURRENT,
+ RCU_READER_ACTIVE_OLD,
+ RCU_READER_INACTIVE,
+};
+
+/*
+ * Slave barriers are only guaranteed to be ordered wrt master barriers.
+ *
+ * The pair ordering is detailed as (O: ordered, X: not ordered) :
+ * slave master
+ * slave X O
+ * master O O
+ */
+
+#ifdef RCU_MEMBARRIER
+extern int rcu_has_sys_membarrier;
+
+static inline void smp_mb_slave(void)
+{
+ if (caa_likely(rcu_has_sys_membarrier))
+ cmm_barrier();
+ else
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_MB
+static inline void smp_mb_slave(void)
+{
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_SIGNAL
+static inline void smp_mb_slave(void)
+{
+ cmm_barrier();
+}
+#endif
+
+/*
+ * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use
+ * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
+ */
+#define RCU_GP_COUNT (1UL << 0)
+/* Use the amount of bits equal to half of the architecture long size */
+#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))
+#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
+
+struct rcu_gp {
+ /*
+ * Global grace period counter.
+ * Contains the current RCU_GP_CTR_PHASE.
+ * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
+ * Written to only by writer with mutex taken.
+ * Read by both writer and readers.
+ */
+ unsigned long ctr;
+
+ int32_t futex;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+extern struct rcu_gp rcu_gp;
+
+struct rcu_reader {
+ /* Data used by both reader and synchronize_rcu() */
+ unsigned long ctr;
+ char need_mb;
+ /* Data used for registry */
+ struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+ pthread_t tid;
+ /* Reader registered flag, for internal checks. */
+ unsigned int registered:1;
+};
+
+extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
+
+/*
+ * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
+ */
+static inline void wake_up_gp(void)
+{
+ if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
+ uatomic_set(&rcu_gp.futex, 0);
+ /*
+ * Ignoring return value until we can make this function
+ * return something (because urcu_die() is not publicly
+ * exposed).
+ */
+ (void) futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0);
+ }
+}
+
+static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
+{
+ unsigned long v;
+
+ /*
+ * Make sure both tests below are done on the same version of *value
+ * to insure consistency.
+ */
+ v = CMM_LOAD_SHARED(*ctr);
+ if (!(v & RCU_GP_CTR_NEST_MASK))
+ return RCU_READER_INACTIVE;
+ if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
+ return RCU_READER_ACTIVE_CURRENT;
+ return RCU_READER_ACTIVE_OLD;
+}
+
+/*
+ * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as
+ * the per-thread rcu_reader.ctr) has the upper bits containing a count of
+ * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
+ * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
+ * _rcu_read_lock() happen before the subsequent read-side critical section.
+ */
+static inline void _rcu_read_lock_update(unsigned long tmp)
+{
+ if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
+ smp_mb_slave();
+ } else
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
+}
+
+/*
+ * Enter an RCU read-side critical section.
+ *
+ * The first cmm_barrier() call ensures that the compiler does not reorder
+ * the body of _rcu_read_lock() with a mutex.
+ *
+ * This function and its helper are both less than 10 lines long. The
+ * intent is that this function meets the 10-line criterion in LGPL,
+ * allowing this function to be invoked directly from non-LGPL code.
+ */
+static inline void _rcu_read_lock(void)
+{
+ unsigned long tmp;
+
+ urcu_assert(URCU_TLS(rcu_reader).registered);
+ cmm_barrier();
+ tmp = URCU_TLS(rcu_reader).ctr;
+ urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK);
+ _rcu_read_lock_update(tmp);
+}
+
+/*
+ * This is a helper function for _rcu_read_unlock().
+ *
+ * The first smp_mb_slave() call ensures that the critical section is
+ * seen to precede the store to rcu_reader.ctr.
+ * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
+ * before reading the update-side futex.
+ */
+static inline void _rcu_read_unlock_update_and_wakeup(unsigned long tmp)
+{
+ if (caa_likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) {
+ smp_mb_slave();
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT);
+ smp_mb_slave();
+ wake_up_gp();
+ } else
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT);
+}
+
+/*
+ * Exit an RCU read-side crtical section. Both this function and its
+ * helper are smaller than 10 lines of code, and are intended to be
+ * usable by non-LGPL code, as called out in LGPL.
+ */
+static inline void _rcu_read_unlock(void)
+{
+ unsigned long tmp;
+
+ urcu_assert(URCU_TLS(rcu_reader).registered);
+ tmp = URCU_TLS(rcu_reader).ctr;
+ urcu_assert(tmp & RCU_GP_CTR_NEST_MASK);
+ _rcu_read_unlock_update_and_wakeup(tmp);
+ cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
+}
+
+/*
+ * Returns whether within a RCU read-side critical section.
+ *
+ * This function is less than 10 lines long. The intent is that this
+ * function meets the 10-line criterion for LGPL, allowing this function
+ * to be invoked directly from non-LGPL code.
+ */
+static inline int _rcu_read_ongoing(void)
+{
+ return URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_H */
--- /dev/null
+#ifndef _URCU_WFCQUEUE_STATIC_H
+#define _URCU_WFCQUEUE_STATIC_H
+
+/*
+ * urcu/static/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfcqueue.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ *
+ * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ * [1] [2] [3] [4] [5] [6]
+ * [1] - - - - - -
+ * [2] - - - - - -
+ * [3] - - X X X X
+ * [4] - - X - X X
+ * [5] - - X X - -
+ * [6] - - X X - -
+ *
+ * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
+ *
+ * For convenience, cds_wfcq_dequeue_blocking() and
+ * cds_wfcq_splice_blocking() hold the dequeue lock.
+ *
+ * Besides locking, mutual exclusion of dequeue, splice and iteration
+ * can be ensured by performing all of those operations from a single
+ * thread, without requiring any lock.
+ */
+
+#define WFCQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
+#define WFCQ_WAIT 10 /* Wait 10 ms if being set */
+
+/*
+ * cds_wfcq_node_init: initialize wait-free queue node.
+ */
+static inline void _cds_wfcq_node_init(struct cds_wfcq_node *node)
+{
+ node->next = NULL;
+}
+
+/*
+ * cds_wfcq_init: initialize wait-free queue (with lock). Pair with
+ * cds_wfcq_destroy().
+ */
+static inline void _cds_wfcq_init(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ int ret;
+
+ /* Set queue head and tail */
+ _cds_wfcq_node_init(&head->node);
+ tail->p = &head->node;
+ ret = pthread_mutex_init(&head->lock, NULL);
+ assert(!ret);
+}
+
+/*
+ * cds_wfcq_destroy: destroy wait-free queue (with lock). Pair with
+ * cds_wfcq_init().
+ */
+static inline void _cds_wfcq_destroy(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ int ret = pthread_mutex_destroy(&head->lock);
+ assert(!ret);
+}
+
+/*
+ * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
+ * pair with any destroy function.
+ */
+static inline void ___cds_wfcq_init(struct __cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ /* Set queue head and tail */
+ _cds_wfcq_node_init(&head->node);
+ tail->p = &head->node;
+}
+
+/*
+ * cds_wfcq_empty: return whether wait-free queue is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ *
+ * We perform the test on head->node.next to check if the queue is
+ * possibly empty, but we confirm this by checking if the tail pointer
+ * points to the head node because the tail pointer is the linearisation
+ * point of the enqueuers. Just checking the head next pointer could
+ * make a queue appear empty if an enqueuer is preempted for a long time
+ * between xchg() and setting the previous node's next pointer.
+ */
+static inline bool _cds_wfcq_empty(cds_wfcq_head_ptr_t u_head,
+ struct cds_wfcq_tail *tail)
+{
+ struct __cds_wfcq_head *head = u_head._h;
+ /*
+ * Queue is empty if no node is pointed by head->node.next nor
+ * tail->p. Even though the tail->p check is sufficient to find
+ * out of the queue is empty, we first check head->node.next as a
+ * common case to ensure that dequeuers do not frequently access
+ * enqueuer's tail->p cache line.
+ */
+ return CMM_LOAD_SHARED(head->node.next) == NULL
+ && CMM_LOAD_SHARED(tail->p) == &head->node;
+}
+
+static inline void _cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ int ret;
+
+ ret = pthread_mutex_lock(&head->lock);
+ assert(!ret);
+}
+
+static inline void _cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(&head->lock);
+ assert(!ret);
+}
+
+static inline bool ___cds_wfcq_append(cds_wfcq_head_ptr_t u_head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *new_head,
+ struct cds_wfcq_node *new_tail)
+{
+ struct __cds_wfcq_head *head = u_head._h;
+ struct cds_wfcq_node *old_tail;
+
+ /*
+ * Implicit memory barrier before uatomic_xchg() orders earlier
+ * stores to data structure containing node and setting
+ * node->next to NULL before publication.
+ */
+ old_tail = uatomic_xchg(&tail->p, new_tail);
+
+ /*
+ * Implicit memory barrier after uatomic_xchg() orders store to
+ * q->tail before store to old_tail->next.
+ *
+ * At this point, dequeuers see a NULL tail->p->next, which
+ * indicates that the queue is being appended to. The following
+ * store will append "node" to the queue from a dequeuer
+ * perspective.
+ */
+ CMM_STORE_SHARED(old_tail->next, new_head);
+ /*
+ * Return false if queue was empty prior to adding the node,
+ * else return true.
+ */
+ return old_tail != &head->node;
+}
+
+/*
+ * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
+ *
+ * Issues a full memory barrier before enqueue. No mutual exclusion is
+ * required.
+ *
+ * Returns false if the queue was empty prior to adding the node.
+ * Returns true otherwise.
+ */
+static inline bool _cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *new_tail)
+{
+ return ___cds_wfcq_append(head, tail, new_tail, new_tail);
+}
+
+/*
+ * ___cds_wfcq_busy_wait: adaptative busy-wait.
+ *
+ * Returns 1 if nonblocking and needs to block, 0 otherwise.
+ */
+static inline bool
+___cds_wfcq_busy_wait(int *attempt, int blocking)
+{
+ if (!blocking)
+ return 1;
+ if (++(*attempt) >= WFCQ_ADAPT_ATTEMPTS) {
+ (void) poll(NULL, 0, WFCQ_WAIT); /* Wait for 10ms */
+ *attempt = 0;
+ } else {
+ caa_cpu_relax();
+ }
+ return 0;
+}
+
+/*
+ * Waiting for enqueuer to complete enqueue and return the next node.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_node_sync_next(struct cds_wfcq_node *node, int blocking)
+{
+ struct cds_wfcq_node *next;
+ int attempt = 0;
+
+ /*
+ * Adaptative busy-looping waiting for enqueuer to complete enqueue.
+ */
+ while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ if (___cds_wfcq_busy_wait(&attempt, blocking))
+ return CDS_WFCQ_WOULDBLOCK;
+ }
+
+ return next;
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_first(cds_wfcq_head_ptr_t u_head,
+ struct cds_wfcq_tail *tail,
+ int blocking)
+{
+ struct __cds_wfcq_head *head = u_head._h;
+ struct cds_wfcq_node *node;
+
+ if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail))
+ return NULL;
+ node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+ /* Load head->node.next before loading node's content */
+ cmm_smp_read_barrier_depends();
+ return node;
+}
+
+/*
+ * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if queue is empty, first node otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_blocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_first(head, tail, 1);
+}
+
+
+/*
+ * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_first_nonblocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_first(head, tail, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_next(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node,
+ int blocking)
+{
+ struct cds_wfcq_node *next;
+
+ /*
+ * Even though the following tail->p check is sufficient to find
+ * out if we reached the end of the queue, we first check
+ * node->next as a common case to ensure that iteration on nodes
+ * do not frequently access enqueuer's tail->p cache line.
+ */
+ if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ /* Load node->next before tail->p */
+ cmm_smp_rmb();
+ if (CMM_LOAD_SHARED(tail->p) == node)
+ return NULL;
+ next = ___cds_wfcq_node_sync_next(node, blocking);
+ }
+ /* Load node->next before loading next's content */
+ cmm_smp_read_barrier_depends();
+ return next;
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros in urcu/wfqueue.h:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of queue, non-NULL next queue node
+ * otherwise.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_blocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node)
+{
+ return ___cds_wfcq_next(head, tail, node, 1);
+}
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_next_nonblocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node)
+{
+ return ___cds_wfcq_next(head, tail, node, 0);
+}
+
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state(cds_wfcq_head_ptr_t u_head,
+ struct cds_wfcq_tail *tail,
+ int *state,
+ int blocking)
+{
+ struct __cds_wfcq_head *head = u_head._h;
+ struct cds_wfcq_node *node, *next;
+
+ if (state)
+ *state = 0;
+
+ if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail)) {
+ return NULL;
+ }
+
+ node = ___cds_wfcq_node_sync_next(&head->node, blocking);
+ if (!blocking && node == CDS_WFCQ_WOULDBLOCK) {
+ return CDS_WFCQ_WOULDBLOCK;
+ }
+
+ if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ /*
+ * @node is probably the only node in the queue.
+ * Try to move the tail to &q->head.
+ * q->head.next is set to NULL here, and stays
+ * NULL if the cmpxchg succeeds. Should the
+ * cmpxchg fail due to a concurrent enqueue, the
+ * q->head.next will be set to the next node.
+ * The implicit memory barrier before
+ * uatomic_cmpxchg() orders load node->next
+ * before loading q->tail.
+ * The implicit memory barrier before uatomic_cmpxchg
+ * orders load q->head.next before loading node's
+ * content.
+ */
+ _cds_wfcq_node_init(&head->node);
+ if (uatomic_cmpxchg(&tail->p, node, &head->node) == node) {
+ if (state)
+ *state |= CDS_WFCQ_STATE_LAST;
+ return node;
+ }
+ next = ___cds_wfcq_node_sync_next(node, blocking);
+ /*
+ * In nonblocking mode, if we would need to block to
+ * get node's next, set the head next node pointer
+ * (currently NULL) back to its original value.
+ */
+ if (!blocking && next == CDS_WFCQ_WOULDBLOCK) {
+ head->node.next = node;
+ return CDS_WFCQ_WOULDBLOCK;
+ }
+ }
+
+ /*
+ * Move queue head forward.
+ */
+ head->node.next = next;
+
+ /* Load q->head.next before loading node's content */
+ cmm_smp_read_barrier_depends();
+ return node;
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_blocking: dequeue node from queue, with state.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail, int *state)
+{
+ return ___cds_wfcq_dequeue_with_state(head, tail, state, 1);
+}
+
+/*
+ * ___cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_blocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_blocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_dequeue_with_state_nonblocking: dequeue node, with state.
+ *
+ * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
+ * if it needs to block.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_with_state_nonblocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail, int *state)
+{
+ return ___cds_wfcq_dequeue_with_state(head, tail, state, 0);
+}
+
+/*
+ * ___cds_wfcq_dequeue_nonblocking: dequeue node from queue.
+ *
+ * Same as __cds_wfcq_dequeue_with_state_nonblocking, but without saving
+ * state.
+ */
+static inline struct cds_wfcq_node *
+___cds_wfcq_dequeue_nonblocking(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, NULL);
+}
+
+/*
+ * __cds_wfcq_splice: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice(
+ cds_wfcq_head_ptr_t u_dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t u_src_q_head,
+ struct cds_wfcq_tail *src_q_tail,
+ int blocking)
+{
+ struct __cds_wfcq_head *dest_q_head = u_dest_q_head._h;
+ struct __cds_wfcq_head *src_q_head = u_src_q_head._h;
+ struct cds_wfcq_node *head, *tail;
+ int attempt = 0;
+
+ /*
+ * Initial emptiness check to speed up cases where queue is
+ * empty: only require loads to check if queue is empty.
+ */
+ if (_cds_wfcq_empty(__cds_wfcq_head_cast(src_q_head), src_q_tail))
+ return CDS_WFCQ_RET_SRC_EMPTY;
+
+ for (;;) {
+ /*
+ * Open-coded _cds_wfcq_empty() by testing result of
+ * uatomic_xchg, as well as tail pointer vs head node
+ * address.
+ */
+ head = uatomic_xchg(&src_q_head->node.next, NULL);
+ if (head)
+ break; /* non-empty */
+ if (CMM_LOAD_SHARED(src_q_tail->p) == &src_q_head->node)
+ return CDS_WFCQ_RET_SRC_EMPTY;
+ if (___cds_wfcq_busy_wait(&attempt, blocking))
+ return CDS_WFCQ_RET_WOULDBLOCK;
+ }
+
+ /*
+ * Memory barrier implied before uatomic_xchg() orders store to
+ * src_q->head before store to src_q->tail. This is required by
+ * concurrent enqueue on src_q, which exchanges the tail before
+ * updating the previous tail's next pointer.
+ */
+ tail = uatomic_xchg(&src_q_tail->p, &src_q_head->node);
+
+ /*
+ * Append the spliced content of src_q into dest_q. Does not
+ * require mutual exclusion on dest_q (wait-free).
+ */
+ if (___cds_wfcq_append(__cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+ head, tail))
+ return CDS_WFCQ_RET_DEST_NON_EMPTY;
+ else
+ return CDS_WFCQ_RET_DEST_EMPTY;
+}
+
+/*
+ * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_blocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+ src_q_head, src_q_tail, 1);
+}
+
+/*
+ * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Same as __cds_wfcq_splice_blocking, but returns
+ * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
+ */
+static inline enum cds_wfcq_ret
+___cds_wfcq_splice_nonblocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
+ src_q_head, src_q_tail, 0);
+}
+
+/*
+ * cds_wfcq_dequeue_with_state_blocking: dequeue a node from a wait-free queue.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_splice_blocking and dequeue lock is
+ * ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail, int *state)
+{
+ struct cds_wfcq_node *retval;
+
+ _cds_wfcq_dequeue_lock(head, tail);
+ retval = ___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_cast(head),
+ tail, state);
+ _cds_wfcq_dequeue_unlock(head, tail);
+ return retval;
+}
+
+/*
+ * cds_wfcq_dequeue_blocking: dequeue node from queue.
+ *
+ * Same as cds_wfcq_dequeue_blocking, but without saving state.
+ */
+static inline struct cds_wfcq_node *
+_cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ return _cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
+}
+
+/*
+ * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
+ * ensured.
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+static inline enum cds_wfcq_ret
+_cds_wfcq_splice_blocking(
+ struct cds_wfcq_head *dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ struct cds_wfcq_head *src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ enum cds_wfcq_ret ret;
+
+ _cds_wfcq_dequeue_lock(src_q_head, src_q_tail);
+ ret = ___cds_wfcq_splice_blocking(cds_wfcq_head_cast(dest_q_head), dest_q_tail,
+ cds_wfcq_head_cast(src_q_head), src_q_tail);
+ _cds_wfcq_dequeue_unlock(src_q_head, src_q_tail);
+ return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_STATIC_H */
--- /dev/null
+#ifndef _URCU_WFQUEUE_STATIC_H
+#define _URCU_WFQUEUE_STATIC_H
+
+/*
+ * wfqueue-static.h
+ *
+ * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking
+ * dynamically with the userspace rcu library.
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Queue with wait-free enqueue/blocking dequeue.
+ * This implementation adds a dummy head node when the queue is empty to ensure
+ * we can always update the queue locklessly.
+ *
+ * Inspired from half-wait-free/half-blocking queue implementation done by
+ * Paul E. McKenney.
+ */
+
+#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
+#define WFQ_WAIT 10 /* Wait 10 ms if being set */
+
+static inline void _cds_wfq_node_init(struct cds_wfq_node *node)
+{
+ node->next = NULL;
+}
+
+static inline void _cds_wfq_init(struct cds_wfq_queue *q)
+{
+ int ret;
+
+ _cds_wfq_node_init(&q->dummy);
+ /* Set queue head and tail */
+ q->head = &q->dummy;
+ q->tail = &q->dummy.next;
+ ret = pthread_mutex_init(&q->lock, NULL);
+ assert(!ret);
+}
+
+static inline void _cds_wfq_destroy(struct cds_wfq_queue *q)
+{
+ int ret = pthread_mutex_destroy(&q->lock);
+ assert(!ret);
+}
+
+static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q,
+ struct cds_wfq_node *node)
+{
+ struct cds_wfq_node **old_tail;
+
+ /*
+ * uatomic_xchg() implicit memory barrier orders earlier stores to data
+ * structure containing node and setting node->next to NULL before
+ * publication.
+ */
+ old_tail = uatomic_xchg(&q->tail, &node->next);
+ /*
+ * At this point, dequeuers see a NULL old_tail->next, which indicates
+ * that the queue is being appended to. The following store will append
+ * "node" to the queue from a dequeuer perspective.
+ */
+ CMM_STORE_SHARED(*old_tail, node);
+}
+
+/*
+ * Waiting for enqueuer to complete enqueue and return the next node
+ */
+static inline struct cds_wfq_node *
+___cds_wfq_node_sync_next(struct cds_wfq_node *node)
+{
+ struct cds_wfq_node *next;
+ int attempt = 0;
+
+ /*
+ * Adaptative busy-looping waiting for enqueuer to complete enqueue.
+ */
+ while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ if (++attempt >= WFQ_ADAPT_ATTEMPTS) {
+ (void) poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */
+ attempt = 0;
+ } else
+ caa_cpu_relax();
+ }
+
+ return next;
+}
+
+/*
+ * It is valid to reuse and free a dequeued node immediately.
+ *
+ * No need to go on a waitqueue here, as there is no possible state in which the
+ * list could cause dequeue to busy-loop needlessly while waiting for another
+ * thread to be scheduled. The queue appears empty until tail->next is set by
+ * enqueue.
+ */
+static inline struct cds_wfq_node *
+___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ struct cds_wfq_node *node, *next;
+
+ /*
+ * Queue is empty if it only contains the dummy node.
+ */
+ if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next)
+ return NULL;
+ node = q->head;
+
+ next = ___cds_wfq_node_sync_next(node);
+
+ /*
+ * Move queue head forward.
+ */
+ q->head = next;
+ /*
+ * Requeue dummy node if we just dequeued it.
+ */
+ if (node == &q->dummy) {
+ _cds_wfq_node_init(node);
+ _cds_wfq_enqueue(q, node);
+ return ___cds_wfq_dequeue_blocking(q);
+ }
+ return node;
+}
+
+static inline struct cds_wfq_node *
+_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ struct cds_wfq_node *retnode;
+ int ret;
+
+ ret = pthread_mutex_lock(&q->lock);
+ assert(!ret);
+ retnode = ___cds_wfq_dequeue_blocking(q);
+ ret = pthread_mutex_unlock(&q->lock);
+ assert(!ret);
+ return retnode;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFQUEUE_STATIC_H */
--- /dev/null
+#ifndef _URCU_STATIC_WFSTACK_H
+#define _URCU_STATIC_WFSTACK_H
+
+/*
+ * urcu/static/wfstack.h
+ *
+ * Userspace RCU library - Stack with with wait-free push, blocking traversal.
+ *
+ * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfstack.h for
+ * linking dynamically with the userspace rcu library.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <poll.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CDS_WFS_END ((void *) 0x1UL)
+#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */
+#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ * cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ * iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * cds_wfs_push __cds_wfs_pop __cds_wfs_pop_all
+ * cds_wfs_push - - -
+ * __cds_wfs_pop - X X
+ * __cds_wfs_pop_all - X -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+/*
+ * cds_wfs_node_init: initialize wait-free stack node.
+ */
+static inline
+void _cds_wfs_node_init(struct cds_wfs_node *node)
+{
+ node->next = NULL;
+}
+
+/*
+ * __cds_wfs_init: initialize wait-free stack. Don't pair with
+ * any destroy function.
+ */
+static inline void ___cds_wfs_init(struct __cds_wfs_stack *s)
+{
+ s->head = CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_init: initialize wait-free stack. Pair with
+ * cds_wfs_destroy().
+ */
+static inline
+void _cds_wfs_init(struct cds_wfs_stack *s)
+{
+ int ret;
+
+ s->head = CDS_WFS_END;
+ ret = pthread_mutex_init(&s->lock, NULL);
+ assert(!ret);
+}
+
+/*
+ * cds_wfs_destroy: destroy wait-free stack. Pair with
+ * cds_wfs_init().
+ */
+static inline
+void _cds_wfs_destroy(struct cds_wfs_stack *s)
+{
+ int ret = pthread_mutex_destroy(&s->lock);
+ assert(!ret);
+}
+
+static inline bool ___cds_wfs_end(void *node)
+{
+ return node == CDS_WFS_END;
+}
+
+/*
+ * cds_wfs_empty: return whether wait-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+static inline bool _cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
+{
+ struct __cds_wfs_stack *s = u_stack._s;
+
+ return ___cds_wfs_end(CMM_LOAD_SHARED(s->head));
+}
+
+/*
+ * cds_wfs_push: push a node into the stack.
+ *
+ * Issues a full memory barrier before push. No mutual exclusion is
+ * required.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+static inline
+int _cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
+{
+ struct __cds_wfs_stack *s = u_stack._s;
+ struct cds_wfs_head *old_head, *new_head;
+
+ assert(node->next == NULL);
+ new_head = caa_container_of(node, struct cds_wfs_head, node);
+ /*
+ * uatomic_xchg() implicit memory barrier orders earlier stores
+ * to node (setting it to NULL) before publication.
+ */
+ old_head = uatomic_xchg(&s->head, new_head);
+ /*
+ * At this point, dequeuers see a NULL node->next, they should
+ * busy-wait until node->next is set to old_head.
+ */
+ CMM_STORE_SHARED(node->next, &old_head->node);
+ return !___cds_wfs_end(old_head);
+}
+
+/*
+ * Waiting for push to complete enqueue and return the next node.
+ */
+static inline struct cds_wfs_node *
+___cds_wfs_node_sync_next(struct cds_wfs_node *node, int blocking)
+{
+ struct cds_wfs_node *next;
+ int attempt = 0;
+
+ /*
+ * Adaptative busy-looping waiting for push to complete.
+ */
+ while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
+ if (!blocking)
+ return CDS_WFS_WOULDBLOCK;
+ if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
+ (void) poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */
+ attempt = 0;
+ } else {
+ caa_cpu_relax();
+ }
+ }
+
+ return next;
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop(cds_wfs_stack_ptr_t u_stack, int *state, int blocking)
+{
+ struct cds_wfs_head *head, *new_head;
+ struct cds_wfs_node *next;
+ struct __cds_wfs_stack *s = u_stack._s;
+
+ if (state)
+ *state = 0;
+ for (;;) {
+ head = CMM_LOAD_SHARED(s->head);
+ if (___cds_wfs_end(head)) {
+ return NULL;
+ }
+ next = ___cds_wfs_node_sync_next(&head->node, blocking);
+ if (!blocking && next == CDS_WFS_WOULDBLOCK) {
+ return CDS_WFS_WOULDBLOCK;
+ }
+ new_head = caa_container_of(next, struct cds_wfs_head, node);
+ if (uatomic_cmpxchg(&s->head, head, new_head) == head) {
+ if (state && ___cds_wfs_end(new_head))
+ *state |= CDS_WFS_STATE_LAST;
+ return &head->node;
+ }
+ if (!blocking) {
+ return CDS_WFS_WOULDBLOCK;
+ }
+ /* busy-loop if head changed under us */
+ }
+}
+
+/*
+ * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_wfs_pop_blocking needs to be synchronized using one of the
+ * following techniques:
+ *
+ * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
+ * section. The caller must wait for a grace period to pass before
+ * freeing the returned node or modifying the cds_wfs_node structure.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+ return ___cds_wfs_pop(u_stack, state, 1);
+}
+
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
+{
+ return ___cds_wfs_pop_with_state_blocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_with_state_blocking, but returns
+ * CDS_WFS_WOULDBLOCK if it needs to block.
+ *
+ * "state" saves state flags atomically sampled with pop operation.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state)
+{
+ return ___cds_wfs_pop(u_stack, state, 0);
+}
+
+/*
+ * __cds_wfs_pop_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
+ * it needs to block.
+ */
+static inline
+struct cds_wfs_node *
+___cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
+{
+ return ___cds_wfs_pop_with_state_nonblocking(u_stack, NULL);
+}
+
+/*
+ * __cds_wfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_wfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_wfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_wfs_pop_blocking:
+ *
+ * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
+ * section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
+ * must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_wfs_node structure. However, no RCU
+ * read-side critical section is needed around __cds_wfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_wfs_head *
+___cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
+{
+ struct __cds_wfs_stack *s = u_stack._s;
+ struct cds_wfs_head *head;
+
+ /*
+ * Implicit memory barrier after uatomic_xchg() matches implicit
+ * memory barrier before uatomic_xchg() in cds_wfs_push. It
+ * ensures that all nodes of the returned list are consistent.
+ * There is no need to issue memory barriers when iterating on
+ * the returned list, because the full memory barrier issued
+ * prior to each uatomic_cmpxchg, which each write to head, are
+ * taking care to order writes to each node prior to the full
+ * memory barrier after this uatomic_xchg().
+ */
+ head = uatomic_xchg(&s->head, CDS_WFS_END);
+ if (___cds_wfs_end(head))
+ return NULL;
+ return head;
+}
+
+/*
+ * cds_wfs_pop_lock: lock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_lock(struct cds_wfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_lock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+static inline void _cds_wfs_pop_unlock(struct cds_wfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * Call __cds_wfs_pop_with_state_blocking with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
+{
+ struct cds_wfs_node *retnode;
+
+ _cds_wfs_pop_lock(s);
+ retnode = ___cds_wfs_pop_with_state_blocking(s, state);
+ _cds_wfs_pop_unlock(s);
+ return retnode;
+}
+
+/*
+ * Call _cds_wfs_pop_with_state_blocking without saving any state.
+ */
+static inline
+struct cds_wfs_node *
+_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+ return _cds_wfs_pop_with_state_blocking(s, NULL);
+}
+
+/*
+ * Call __cds_wfs_pop_all with an internal pop mutex held.
+ */
+static inline
+struct cds_wfs_head *
+_cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
+{
+ struct cds_wfs_head *rethead;
+
+ _cds_wfs_pop_lock(s);
+ rethead = ___cds_wfs_pop_all(s);
+ _cds_wfs_pop_unlock(s);
+ return rethead;
+}
+
+/*
+ * cds_wfs_first: get first node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if popped stack is empty, top stack node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_first(struct cds_wfs_head *head)
+{
+ if (___cds_wfs_end(head))
+ return NULL;
+ return &head->node;
+}
+
+static inline struct cds_wfs_node *
+___cds_wfs_next(struct cds_wfs_node *node, int blocking)
+{
+ struct cds_wfs_node *next;
+
+ next = ___cds_wfs_node_sync_next(node, blocking);
+ /*
+ * CDS_WFS_WOULDBLOCK != CSD_WFS_END, so we can check for end
+ * even if ___cds_wfs_node_sync_next returns CDS_WFS_WOULDBLOCK,
+ * and still return CDS_WFS_WOULDBLOCK.
+ */
+ if (___cds_wfs_end(next))
+ return NULL;
+ return next;
+}
+
+/*
+ * cds_wfs_next_blocking: get next node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of popped stack, non-NULL next stack
+ * node otherwise.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_blocking(struct cds_wfs_node *node)
+{
+ return ___cds_wfs_next(node, 1);
+}
+
+
+/*
+ * cds_wfs_next_nonblocking: get next node of a popped stack.
+ *
+ * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
+ * needs to block.
+ */
+static inline struct cds_wfs_node *
+_cds_wfs_next_nonblocking(struct cds_wfs_node *node)
+{
+ return ___cds_wfs_next(node, 0);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_STATIC_WFSTACK_H */
--- /dev/null
+#ifndef _URCU_SYSCALL_COMPAT_H
+#define _URCU_SYSCALL_COMPAT_H
+
+/*
+ * urcu/syscall-compat.h
+ *
+ * Userspace RCU library - Syscall Compatibility Header
+ *
+ * Copyright 2013 - Pierre-Luc St-Charles <pierre-luc.st-charles@polymtl.ca>
+ *
+ * Note: this file is only used to simplify the code required to
+ * include the 'syscall.h' system header across multiple platforms,
+ * which might not always be located at the same place (or needed at all).
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#if defined(__ANDROID__) || defined(__sun__) || defined(__GNU__)
+#include <sys/syscall.h>
+#elif defined(__linux__) || defined(__GLIBC__)
+#include <syscall.h>
+
+#elif defined(__CYGWIN__) || defined(__APPLE__)
+/* Don't include anything on Cygwin or MacOSX. */
+
+#else
+#error "Add platform support to urcu/syscall-compat.h"
+#endif
+
+#endif /* _URCU_SYSCALL_COMPAT_H */
--- /dev/null
+#ifndef _URCU_SYSTEM_H
+#define _URCU_SYSTEM_H
+
+/*
+ * system.h
+ *
+ * System definitions.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+/*
+ * Identify a shared load. A cmm_smp_rmc() or cmm_smp_mc() should come
+ * before the load.
+ */
+#define _CMM_LOAD_SHARED(p) CMM_ACCESS_ONCE(p)
+
+/*
+ * Load a data from shared memory, doing a cache flush if required.
+ */
+#define CMM_LOAD_SHARED(p) \
+ __extension__ \
+ ({ \
+ cmm_smp_rmc(); \
+ _CMM_LOAD_SHARED(p); \
+ })
+
+/*
+ * Identify a shared store. A cmm_smp_wmc() or cmm_smp_mc() should
+ * follow the store.
+ */
+#define _CMM_STORE_SHARED(x, v) __extension__ ({ CMM_ACCESS_ONCE(x) = (v); })
+
+/*
+ * Store v into x, where x is located in shared memory. Performs the
+ * required cache flush after writing. Returns v.
+ */
+#define CMM_STORE_SHARED(x, v) \
+ __extension__ \
+ ({ \
+ __typeof__(x) _v = _CMM_STORE_SHARED(x, v); \
+ cmm_smp_wmc(); \
+ _v = _v; /* Work around clang "unused result" */ \
+ })
+
+#endif /* _URCU_SYSTEM_H */
--- /dev/null
+#ifndef _URCU_TLS_COMPAT_H
+#define _URCU_TLS_COMPAT_H
+
+/*
+ * urcu/tls-compat.h
+ *
+ * Userspace RCU library - Thread-Local Storage Compatibility Header
+ *
+ * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdlib.h>
+#include <urcu/config.h>
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef CONFIG_RCU_TLS /* Based on ax_tls.m4 */
+
+/*
+ * Hint: How to define/declare TLS variables of compound types
+ * such as array or function pointers?
+ *
+ * Answer: Use typedef to assign a type_name to the compound type.
+ * Example: Define a TLS variable which is an int array with len=4:
+ *
+ * typedef int my_int_array_type[4];
+ * DEFINE_URCU_TLS(my_int_array_type, var_name);
+ *
+ * Another example:
+ * typedef void (*call_rcu_flavor)(struct rcu_head *, XXXX);
+ * DECLARE_URCU_TLS(call_rcu_flavor, p_call_rcu);
+ *
+ * NOTE: URCU_TLS() is NOT async-signal-safe, you can't use it
+ * inside any function which can be called from signal handler.
+ *
+ * But if pthread_getspecific() is async-signal-safe in your
+ * platform, you can make URCU_TLS() async-signal-safe via:
+ * ensuring the first call to URCU_TLS() of a given TLS variable of
+ * all threads is called earliest from a non-signal handler function.
+ *
+ * Example: In any thread, the first call of URCU_TLS(rcu_reader)
+ * is called from rcu_register_thread(), so we can ensure all later
+ * URCU_TLS(rcu_reader) in any thread is async-signal-safe.
+ *
+ * Moreover, URCU_TLS variables should not be touched from signal
+ * handlers setup with with sigaltstack(2).
+ */
+
+# define DECLARE_URCU_TLS(type, name) \
+ CONFIG_RCU_TLS type name
+
+# define DEFINE_URCU_TLS(type, name) \
+ CONFIG_RCU_TLS type name
+
+# define URCU_TLS(name) (name)
+
+#else /* #ifndef CONFIG_RCU_TLS */
+
+/*
+ * The *_1() macros ensure macro parameters are expanded.
+ */
+
+# include <pthread.h>
+
+struct urcu_tls {
+ pthread_key_t key;
+ pthread_mutex_t init_mutex;
+ int init_done;
+};
+
+# define DECLARE_URCU_TLS_1(type, name) \
+ type *__tls_access_ ## name(void)
+# define DECLARE_URCU_TLS(type, name) \
+ DECLARE_URCU_TLS_1(type, name)
+
+/*
+ * Note: we don't free memory at process exit, since it will be dealt
+ * with by the OS.
+ */
+# define DEFINE_URCU_TLS_1(type, name) \
+ type *__tls_access_ ## name(void) \
+ { \
+ static struct urcu_tls __tls_ ## name = { \
+ .init_mutex = PTHREAD_MUTEX_INITIALIZER,\
+ .init_done = 0, \
+ }; \
+ void *__tls_p; \
+ if (!__tls_ ## name.init_done) { \
+ /* Mutex to protect concurrent init */ \
+ pthread_mutex_lock(&__tls_ ## name.init_mutex); \
+ if (!__tls_ ## name.init_done) { \
+ (void) pthread_key_create(&__tls_ ## name.key, \
+ free); \
+ cmm_smp_wmb(); /* create key before write init_done */ \
+ __tls_ ## name.init_done = 1; \
+ } \
+ pthread_mutex_unlock(&__tls_ ## name.init_mutex); \
+ } \
+ cmm_smp_rmb(); /* read init_done before getting key */ \
+ __tls_p = pthread_getspecific(__tls_ ## name.key); \
+ if (caa_unlikely(__tls_p == NULL)) { \
+ __tls_p = calloc(1, sizeof(type)); \
+ (void) pthread_setspecific(__tls_ ## name.key, \
+ __tls_p); \
+ } \
+ return __tls_p; \
+ }
+
+# define DEFINE_URCU_TLS(type, name) \
+ DEFINE_URCU_TLS_1(type, name)
+
+# define URCU_TLS_1(name) (*__tls_access_ ## name())
+
+# define URCU_TLS(name) URCU_TLS_1(name)
+
+#endif /* #else #ifndef CONFIG_RCU_TLS */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_TLS_COMPAT_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_AARCH64_H
+#define _URCU_ARCH_UATOMIC_AARCH64_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UATOMIC_HAS_ATOMIC_BYTE
+#define UATOMIC_HAS_ATOMIC_SHORT
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_AARCH64_H */
--- /dev/null
+#ifndef _URCU_UATOMIC_ARCH_ALPHA_H
+#define _URCU_UATOMIC_ARCH_ALPHA_H
+
+/*
+ * Atomic exchange operations for the Alpha architecture. Let GCC do it.
+ *
+ * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_UATOMIC_ARCH_ALPHA_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_ARM_H
+#define _URCU_ARCH_UATOMIC_ARM_H
+
+/*
+ * Atomics for ARM. This approach is usable on kernels back to 2.6.15.
+ *
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* xchg */
+#define uatomic_xchg(addr, v) __sync_lock_test_and_set(addr, v)
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_ARM_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_GCC_H
+#define _URCU_ARCH_UATOMIC_GCC_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * If your platform doesn't have a full set of atomics, you will need
+ * a separate uatomic_arch_*.h file for your architecture. Otherwise,
+ * just rely on the definitions in uatomic/generic.h.
+ */
+#define UATOMIC_HAS_ATOMIC_BYTE
+#define UATOMIC_HAS_ATOMIC_SHORT
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_GCC_H */
--- /dev/null
+#ifndef _URCU_UATOMIC_GENERIC_H
+#define _URCU_UATOMIC_GENERIC_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ * Copyright (c) 2010 Paolo Bonzini
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <stdint.h>
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef uatomic_set
+#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
+#endif
+
+#ifndef uatomic_read
+#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
+#endif
+
+#if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
+static inline __attribute__((always_inline))
+void _uatomic_link_error(void)
+{
+#ifdef ILLEGAL_INSTR
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__(ILLEGAL_INSTR);
+#else
+ __builtin_trap();
+#endif
+}
+
+#else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
+extern void _uatomic_link_error(void);
+#endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
+
+/* cmpxchg */
+
+#ifndef uatomic_cmpxchg
+static inline __attribute__((always_inline))
+unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
+ _new);
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
+ _new);
+#endif
+ case 4:
+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
+ _new);
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
+ _new);
+#endif
+ }
+ _uatomic_link_error();
+ return 0;
+}
+
+
+#define uatomic_cmpxchg(addr, old, _new) \
+ ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new),\
+ sizeof(*(addr))))
+
+
+/* uatomic_and */
+
+#ifndef uatomic_and
+static inline __attribute__((always_inline))
+void _uatomic_and(void *addr, unsigned long val,
+ int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ __sync_and_and_fetch_1((uint8_t *) addr, val);
+ return;
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ __sync_and_and_fetch_2((uint16_t *) addr, val);
+ return;
+#endif
+ case 4:
+ __sync_and_and_fetch_4((uint32_t *) addr, val);
+ return;
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ __sync_and_and_fetch_8((uint64_t *) addr, val);
+ return;
+#endif
+ }
+ _uatomic_link_error();
+}
+
+#define uatomic_and(addr, v) \
+ (_uatomic_and((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
+
+#endif
+
+/* uatomic_or */
+
+#ifndef uatomic_or
+static inline __attribute__((always_inline))
+void _uatomic_or(void *addr, unsigned long val,
+ int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ __sync_or_and_fetch_1((uint8_t *) addr, val);
+ return;
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ __sync_or_and_fetch_2((uint16_t *) addr, val);
+ return;
+#endif
+ case 4:
+ __sync_or_and_fetch_4((uint32_t *) addr, val);
+ return;
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ __sync_or_and_fetch_8((uint64_t *) addr, val);
+ return;
+#endif
+ }
+ _uatomic_link_error();
+ return;
+}
+
+#define uatomic_or(addr, v) \
+ (_uatomic_or((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
+
+#endif
+
+
+/* uatomic_add_return */
+
+#ifndef uatomic_add_return
+static inline __attribute__((always_inline))
+unsigned long _uatomic_add_return(void *addr, unsigned long val,
+ int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ return __sync_add_and_fetch_1((uint8_t *) addr, val);
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ return __sync_add_and_fetch_2((uint16_t *) addr, val);
+#endif
+ case 4:
+ return __sync_add_and_fetch_4((uint32_t *) addr, val);
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ return __sync_add_and_fetch_8((uint64_t *) addr, val);
+#endif
+ }
+ _uatomic_link_error();
+ return 0;
+}
+
+
+#define uatomic_add_return(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_add_return((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#endif /* #ifndef uatomic_add_return */
+
+#ifndef uatomic_xchg
+/* xchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ {
+ uint8_t old;
+
+ do {
+ old = uatomic_read((uint8_t *) addr);
+ } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
+ old, val));
+
+ return old;
+ }
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ {
+ uint16_t old;
+
+ do {
+ old = uatomic_read((uint16_t *) addr);
+ } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
+ old, val));
+
+ return old;
+ }
+#endif
+ case 4:
+ {
+ uint32_t old;
+
+ do {
+ old = uatomic_read((uint32_t *) addr);
+ } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
+ old, val));
+
+ return old;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ uint64_t old;
+
+ do {
+ old = uatomic_read((uint64_t *) addr);
+ } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
+ old, val));
+
+ return old;
+ }
+#endif
+ }
+ _uatomic_link_error();
+ return 0;
+}
+
+#define uatomic_xchg(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_exchange((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#endif /* #ifndef uatomic_xchg */
+
+#else /* #ifndef uatomic_cmpxchg */
+
+#ifndef uatomic_and
+/* uatomic_and */
+
+static inline __attribute__((always_inline))
+void _uatomic_and(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ {
+ uint8_t old, oldt;
+
+ oldt = uatomic_read((uint8_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
+ } while (oldt != old);
+
+ return;
+ }
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ {
+ uint16_t old, oldt;
+
+ oldt = uatomic_read((uint16_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
+ } while (oldt != old);
+ }
+#endif
+ case 4:
+ {
+ uint32_t old, oldt;
+
+ oldt = uatomic_read((uint32_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
+ } while (oldt != old);
+
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ uint64_t old, oldt;
+
+ oldt = uatomic_read((uint64_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
+ } while (oldt != old);
+
+ return;
+ }
+#endif
+ }
+ _uatomic_link_error();
+}
+
+#define uatomic_and(addr, v) \
+ (_uatomic_and((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
+
+#endif /* #ifndef uatomic_and */
+
+#ifndef uatomic_or
+/* uatomic_or */
+
+static inline __attribute__((always_inline))
+void _uatomic_or(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ {
+ uint8_t old, oldt;
+
+ oldt = uatomic_read((uint8_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
+ } while (oldt != old);
+
+ return;
+ }
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ {
+ uint16_t old, oldt;
+
+ oldt = uatomic_read((uint16_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
+ } while (oldt != old);
+
+ return;
+ }
+#endif
+ case 4:
+ {
+ uint32_t old, oldt;
+
+ oldt = uatomic_read((uint32_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
+ } while (oldt != old);
+
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ uint64_t old, oldt;
+
+ oldt = uatomic_read((uint64_t *) addr);
+ do {
+ old = oldt;
+ oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
+ } while (oldt != old);
+
+ return;
+ }
+#endif
+ }
+ _uatomic_link_error();
+}
+
+#define uatomic_or(addr, v) \
+ (_uatomic_or((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
+
+#endif /* #ifndef uatomic_or */
+
+#ifndef uatomic_add_return
+/* uatomic_add_return */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ {
+ uint8_t old, oldt;
+
+ oldt = uatomic_read((uint8_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint8_t *) addr,
+ old, old + val);
+ } while (oldt != old);
+
+ return old + val;
+ }
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ {
+ uint16_t old, oldt;
+
+ oldt = uatomic_read((uint16_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint16_t *) addr,
+ old, old + val);
+ } while (oldt != old);
+
+ return old + val;
+ }
+#endif
+ case 4:
+ {
+ uint32_t old, oldt;
+
+ oldt = uatomic_read((uint32_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint32_t *) addr,
+ old, old + val);
+ } while (oldt != old);
+
+ return old + val;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ uint64_t old, oldt;
+
+ oldt = uatomic_read((uint64_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint64_t *) addr,
+ old, old + val);
+ } while (oldt != old);
+
+ return old + val;
+ }
+#endif
+ }
+ _uatomic_link_error();
+ return 0;
+}
+
+#define uatomic_add_return(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_add_return((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#endif /* #ifndef uatomic_add_return */
+
+#ifndef uatomic_xchg
+/* xchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+#ifdef UATOMIC_HAS_ATOMIC_BYTE
+ case 1:
+ {
+ uint8_t old, oldt;
+
+ oldt = uatomic_read((uint8_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint8_t *) addr,
+ old, val);
+ } while (oldt != old);
+
+ return old;
+ }
+#endif
+#ifdef UATOMIC_HAS_ATOMIC_SHORT
+ case 2:
+ {
+ uint16_t old, oldt;
+
+ oldt = uatomic_read((uint16_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint16_t *) addr,
+ old, val);
+ } while (oldt != old);
+
+ return old;
+ }
+#endif
+ case 4:
+ {
+ uint32_t old, oldt;
+
+ oldt = uatomic_read((uint32_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint32_t *) addr,
+ old, val);
+ } while (oldt != old);
+
+ return old;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ uint64_t old, oldt;
+
+ oldt = uatomic_read((uint64_t *) addr);
+ do {
+ old = oldt;
+ oldt = uatomic_cmpxchg((uint64_t *) addr,
+ old, val);
+ } while (oldt != old);
+
+ return old;
+ }
+#endif
+ }
+ _uatomic_link_error();
+ return 0;
+}
+
+#define uatomic_xchg(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_exchange((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+#endif /* #ifndef uatomic_xchg */
+
+#endif /* #else #ifndef uatomic_cmpxchg */
+
+/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
+
+#ifndef uatomic_add
+#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
+#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
+#endif
+
+#define uatomic_sub_return(addr, v) \
+ uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
+#define uatomic_sub(addr, v) \
+ uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
+#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
+#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
+
+#ifndef uatomic_inc
+#define uatomic_inc(addr) uatomic_add((addr), 1)
+#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
+#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
+#endif
+
+#ifndef uatomic_dec
+#define uatomic_dec(addr) uatomic_add((addr), -1)
+#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
+#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_UATOMIC_GENERIC_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_HPPA_H
+#define _URCU_ARCH_UATOMIC_HPPA_H
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#define UATOMIC_HAS_ATOMIC_SHORT
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_HPPA_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_IA64_H
+#define _URCU_ARCH_UATOMIC_IA64_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UATOMIC_HAS_ATOMIC_BYTE
+#define UATOMIC_HAS_ATOMIC_SHORT
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_IA64_H */
--- /dev/null
+#ifndef _URCU_UATOMIC_ARCH_MIPS_H
+#define _URCU_UATOMIC_ARCH_MIPS_H
+
+/*
+ * Atomic exchange operations for the MIPS architecture. Let GCC do it.
+ *
+ * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_UATOMIC_ARCH_MIPS_H */
--- /dev/null
+#ifndef _URCU_UATOMIC_ARCH_NIOS2_H
+#define _URCU_UATOMIC_ARCH_NIOS2_H
+
+/*
+ * Atomic exchange operations for the NIOS2 architecture. Let GCC do it.
+ *
+ * Copyright (c) 2016 Marek Vasut <marex@denx.de>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_UATOMIC_ARCH_NIOS2_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_PPC_H
+#define _URCU_ARCH_UATOMIC_PPC_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ILLEGAL_INSTR ".long 0xd00d00"
+
+/*
+ * Providing sequential consistency semantic with respect to other
+ * instructions for cmpxchg and add_return family of atomic primitives.
+ *
+ * This is achieved with:
+ * lwsync (prior loads can be reordered after following load)
+ * lwarx
+ * stwcx.
+ * test if success (retry)
+ * sync
+ *
+ * Explanation of the sequential consistency provided by this scheme
+ * from Paul E. McKenney:
+ *
+ * The reason we can get away with the lwsync before is that if a prior
+ * store reorders with the lwarx, then you have to store to the atomic
+ * variable from some other CPU to detect it.
+ *
+ * And if you do that, the lwarx will lose its reservation, so the stwcx
+ * will fail. The atomic operation will retry, so that the caller won't be
+ * able to see the misordering.
+ */
+
+/* xchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+ case 4:
+ {
+ unsigned int result;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
+ "stwcx. %2,0,%1\n" /* else store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ : "=&r"(result)
+ : "r"(addr), "r"(val)
+ : "memory", "cc");
+
+ return result;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long result;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
+ "stdcx. %2,0,%1\n" /* else store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ : "=&r"(result)
+ : "r"(addr), "r"(val)
+ : "memory", "cc");
+
+ return result;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__(ILLEGAL_INSTR);
+ return 0;
+}
+
+#define uatomic_xchg(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_exchange((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+/* cmpxchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ switch (len) {
+ case 4:
+ {
+ unsigned int old_val;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
+ "cmpw %0,%3\n" /* if load is not equal to */
+ "bne 2f\n" /* old, fail */
+ "stwcx. %2,0,%1\n" /* else store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ "2:\n"
+ : "=&r"(old_val)
+ : "r"(addr), "r"((unsigned int)_new),
+ "r"((unsigned int)old)
+ : "memory", "cc");
+
+ return old_val;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long old_val;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
+ "cmpd %0,%3\n" /* if load is not equal to */
+ "bne 2f\n" /* old, fail */
+ "stdcx. %2,0,%1\n" /* else store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ "2:\n"
+ : "=&r"(old_val)
+ : "r"(addr), "r"((unsigned long)_new),
+ "r"((unsigned long)old)
+ : "memory", "cc");
+
+ return old_val;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__(ILLEGAL_INSTR);
+ return 0;
+}
+
+
+#define uatomic_cmpxchg(addr, old, _new) \
+ ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new),\
+ sizeof(*(addr))))
+
+/* uatomic_add_return */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_add_return(void *addr, unsigned long val,
+ int len)
+{
+ switch (len) {
+ case 4:
+ {
+ unsigned int result;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
+ "add %0,%2,%0\n" /* add val to value loaded */
+ "stwcx. %0,0,%1\n" /* store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ : "=&r"(result)
+ : "r"(addr), "r"(val)
+ : "memory", "cc");
+
+ return result;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long result;
+
+ __asm__ __volatile__(
+ LWSYNC_OPCODE
+ "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
+ "add %0,%2,%0\n" /* add val to value loaded */
+ "stdcx. %0,0,%1\n" /* store conditional */
+ "bne- 1b\n" /* retry if lost reservation */
+ "sync\n"
+ : "=&r"(result)
+ : "r"(addr), "r"(val)
+ : "memory", "cc");
+
+ return result;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__(ILLEGAL_INSTR);
+ return 0;
+}
+
+
+#define uatomic_add_return(addr, v) \
+ ((__typeof__(*(addr))) _uatomic_add_return((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_PPC_H */
--- /dev/null
+#ifndef _URCU_UATOMIC_ARCH_S390_H
+#define _URCU_UATOMIC_ARCH_S390_H
+
+/*
+ * Atomic exchange operations for the S390 architecture. Based on information
+ * taken from the Principles of Operation Appendix A "Conditional Swapping
+ * Instructions (CS, CDS)".
+ *
+ * Copyright (c) 2009 Novell, Inc.
+ * Author: Jan Blunck <jblunck@suse.de>
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
+#define COMPILER_HAVE_SHORT_MEM_OPERAND
+#endif
+
+/*
+ * MEMOP assembler operand rules:
+ * - op refer to MEMOP_IN operand
+ * - MEMOP_IN can expand to more than a single operand. Use it at the end of
+ * operand list only.
+ */
+
+#ifdef COMPILER_HAVE_SHORT_MEM_OPERAND
+
+#define MEMOP_OUT(addr) "=Q" (*(addr))
+#define MEMOP_IN(addr) "Q" (*(addr))
+#define MEMOP_REF(op) #op /* op refer to MEMOP_IN operand */
+
+#else /* !COMPILER_HAVE_SHORT_MEM_OPERAND */
+
+#define MEMOP_OUT(addr) "=m" (*(addr))
+#define MEMOP_IN(addr) "a" (addr), "m" (*(addr))
+#define MEMOP_REF(op) "0(" #op ")" /* op refer to MEMOP_IN operand */
+
+#endif /* !COMPILER_HAVE_SHORT_MEM_OPERAND */
+
+struct __uatomic_dummy {
+ unsigned long v[10];
+};
+#define __hp(x) ((struct __uatomic_dummy *)(x))
+
+/* xchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_exchange(volatile void *addr, unsigned long val, int len)
+{
+ switch (len) {
+ case 4:
+ {
+ unsigned int old_val;
+
+ __asm__ __volatile__(
+ "0: cs %0,%2," MEMOP_REF(%3) "\n"
+ " brc 4,0b\n"
+ : "=&r" (old_val), MEMOP_OUT (__hp(addr))
+ : "r" (val), MEMOP_IN (__hp(addr))
+ : "memory", "cc");
+ return old_val;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long old_val;
+
+ __asm__ __volatile__(
+ "0: csg %0,%2," MEMOP_REF(%3) "\n"
+ " brc 4,0b\n"
+ : "=&r" (old_val), MEMOP_OUT (__hp(addr))
+ : "r" (val), MEMOP_IN (__hp(addr))
+ : "memory", "cc");
+ return old_val;
+ }
+#endif
+ default:
+ __asm__ __volatile__(".long 0xd00d00");
+ }
+
+ return 0;
+}
+
+#define uatomic_xchg(addr, v) \
+ (__typeof__(*(addr))) _uatomic_exchange((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr)))
+
+/* cmpxchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ switch (len) {
+ case 4:
+ {
+ unsigned int old_val = (unsigned int)old;
+
+ __asm__ __volatile__(
+ " cs %0,%2," MEMOP_REF(%3) "\n"
+ : "+r" (old_val), MEMOP_OUT (__hp(addr))
+ : "r" (_new), MEMOP_IN (__hp(addr))
+ : "memory", "cc");
+ return old_val;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ " csg %0,%2," MEMOP_REF(%3) "\n"
+ : "+r" (old), MEMOP_OUT (__hp(addr))
+ : "r" (_new), MEMOP_IN (__hp(addr))
+ : "memory", "cc");
+ return old;
+ }
+#endif
+ default:
+ __asm__ __volatile__(".long 0xd00d00");
+ }
+
+ return 0;
+}
+
+#define uatomic_cmpxchg(addr, old, _new) \
+ (__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new),\
+ sizeof(*(addr)))
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_UATOMIC_ARCH_S390_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_SPARC64_H
+#define _URCU_ARCH_UATOMIC_SPARC64_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* cmpxchg */
+
+static inline __attribute__((always_inline))
+unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ switch (len) {
+ case 4:
+ {
+ __asm__ __volatile__ (
+ "membar #StoreLoad | #LoadLoad\n\t"
+ "cas [%1],%2,%0\n\t"
+ "membar #StoreLoad | #StoreStore\n\t"
+ : "+&r" (_new)
+ : "r" (addr), "r" (old)
+ : "memory");
+
+ return _new;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__ (
+ "membar #StoreLoad | #LoadLoad\n\t"
+ "casx [%1],%2,%0\n\t"
+ "membar #StoreLoad | #StoreStore\n\t"
+ : "+&r" (_new)
+ : "r" (addr), "r" (old)
+ : "memory");
+
+ return _new;
+ }
+#endif
+ }
+ __builtin_trap();
+ return 0;
+}
+
+
+#define uatomic_cmpxchg(addr, old, _new) \
+ ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new), \
+ sizeof(*(addr))))
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_PPC_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_TILE_H
+#define _URCU_ARCH_UATOMIC_TILE_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009-2015 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define UATOMIC_HAS_ATOMIC_BYTE
+#define UATOMIC_HAS_ATOMIC_SHORT
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_TILE_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_UNKNOWN_H
+#define _URCU_ARCH_UATOMIC_UNKNOWN_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
+ * (Adapted from uatomic_arch_ppc.h)
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ */
+
+/* See configure.ac for the list of recognized architectures. */
+#error "Cannot build: unrecognized architecture detected."
+
+#endif /* _URCU_ARCH_UATOMIC_UNKNOWN_H */
--- /dev/null
+#ifndef _URCU_ARCH_UATOMIC_X86_H
+#define _URCU_ARCH_UATOMIC_X86_H
+
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
+ * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2009 Mathieu Desnoyers
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ * Code inspired from libuatomic_ops-1.2, inherited in part from the
+ * Boehm-Demers-Weiser conservative garbage collector.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/system.h>
+
+#define UATOMIC_HAS_ATOMIC_BYTE
+#define UATOMIC_HAS_ATOMIC_SHORT
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Derived from AO_compare_and_swap() and AO_test_and_set_full().
+ */
+
+struct __uatomic_dummy {
+ unsigned long v[10];
+};
+#define __hp(x) ((struct __uatomic_dummy *)(x))
+
+#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
+
+/* cmpxchg */
+
+static inline __attribute__((always_inline))
+unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ unsigned char result = old;
+
+ __asm__ __volatile__(
+ "lock; cmpxchgb %2, %1"
+ : "+a"(result), "+m"(*__hp(addr))
+ : "q"((unsigned char)_new)
+ : "memory");
+ return result;
+ }
+ case 2:
+ {
+ unsigned short result = old;
+
+ __asm__ __volatile__(
+ "lock; cmpxchgw %2, %1"
+ : "+a"(result), "+m"(*__hp(addr))
+ : "r"((unsigned short)_new)
+ : "memory");
+ return result;
+ }
+ case 4:
+ {
+ unsigned int result = old;
+
+ __asm__ __volatile__(
+ "lock; cmpxchgl %2, %1"
+ : "+a"(result), "+m"(*__hp(addr))
+ : "r"((unsigned int)_new)
+ : "memory");
+ return result;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long result = old;
+
+ __asm__ __volatile__(
+ "lock; cmpxchgq %2, %1"
+ : "+a"(result), "+m"(*__hp(addr))
+ : "r"((unsigned long)_new)
+ : "memory");
+ return result;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return 0;
+}
+
+#define _uatomic_cmpxchg(addr, old, _new) \
+ ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new),\
+ sizeof(*(addr))))
+
+/* xchg */
+
+static inline __attribute__((always_inline))
+unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
+{
+ /* Note: the "xchg" instruction does not need a "lock" prefix. */
+ switch (len) {
+ case 1:
+ {
+ unsigned char result;
+ __asm__ __volatile__(
+ "xchgb %0, %1"
+ : "=q"(result), "+m"(*__hp(addr))
+ : "0" ((unsigned char)val)
+ : "memory");
+ return result;
+ }
+ case 2:
+ {
+ unsigned short result;
+ __asm__ __volatile__(
+ "xchgw %0, %1"
+ : "=r"(result), "+m"(*__hp(addr))
+ : "0" ((unsigned short)val)
+ : "memory");
+ return result;
+ }
+ case 4:
+ {
+ unsigned int result;
+ __asm__ __volatile__(
+ "xchgl %0, %1"
+ : "=r"(result), "+m"(*__hp(addr))
+ : "0" ((unsigned int)val)
+ : "memory");
+ return result;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long result;
+ __asm__ __volatile__(
+ "xchgq %0, %1"
+ : "=r"(result), "+m"(*__hp(addr))
+ : "0" ((unsigned long)val)
+ : "memory");
+ return result;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return 0;
+}
+
+#define _uatomic_xchg(addr, v) \
+ ((__typeof__(*(addr))) __uatomic_exchange((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+/* uatomic_add_return */
+
+static inline __attribute__((always_inline))
+unsigned long __uatomic_add_return(void *addr, unsigned long val,
+ int len)
+{
+ switch (len) {
+ case 1:
+ {
+ unsigned char result = val;
+
+ __asm__ __volatile__(
+ "lock; xaddb %1, %0"
+ : "+m"(*__hp(addr)), "+q" (result)
+ :
+ : "memory");
+ return result + (unsigned char)val;
+ }
+ case 2:
+ {
+ unsigned short result = val;
+
+ __asm__ __volatile__(
+ "lock; xaddw %1, %0"
+ : "+m"(*__hp(addr)), "+r" (result)
+ :
+ : "memory");
+ return result + (unsigned short)val;
+ }
+ case 4:
+ {
+ unsigned int result = val;
+
+ __asm__ __volatile__(
+ "lock; xaddl %1, %0"
+ : "+m"(*__hp(addr)), "+r" (result)
+ :
+ : "memory");
+ return result + (unsigned int)val;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ unsigned long result = val;
+
+ __asm__ __volatile__(
+ "lock; xaddq %1, %0"
+ : "+m"(*__hp(addr)), "+r" (result)
+ :
+ : "memory");
+ return result + (unsigned long)val;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return 0;
+}
+
+#define _uatomic_add_return(addr, v) \
+ ((__typeof__(*(addr))) __uatomic_add_return((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+/* uatomic_and */
+
+static inline __attribute__((always_inline))
+void __uatomic_and(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ __asm__ __volatile__(
+ "lock; andb %1, %0"
+ : "=m"(*__hp(addr))
+ : "iq" ((unsigned char)val)
+ : "memory");
+ return;
+ }
+ case 2:
+ {
+ __asm__ __volatile__(
+ "lock; andw %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned short)val)
+ : "memory");
+ return;
+ }
+ case 4:
+ {
+ __asm__ __volatile__(
+ "lock; andl %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned int)val)
+ : "memory");
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ "lock; andq %1, %0"
+ : "=m"(*__hp(addr))
+ : "er" ((unsigned long)val)
+ : "memory");
+ return;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return;
+}
+
+#define _uatomic_and(addr, v) \
+ (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
+
+/* uatomic_or */
+
+static inline __attribute__((always_inline))
+void __uatomic_or(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ __asm__ __volatile__(
+ "lock; orb %1, %0"
+ : "=m"(*__hp(addr))
+ : "iq" ((unsigned char)val)
+ : "memory");
+ return;
+ }
+ case 2:
+ {
+ __asm__ __volatile__(
+ "lock; orw %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned short)val)
+ : "memory");
+ return;
+ }
+ case 4:
+ {
+ __asm__ __volatile__(
+ "lock; orl %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned int)val)
+ : "memory");
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ "lock; orq %1, %0"
+ : "=m"(*__hp(addr))
+ : "er" ((unsigned long)val)
+ : "memory");
+ return;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return;
+}
+
+#define _uatomic_or(addr, v) \
+ (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
+
+/* uatomic_add */
+
+static inline __attribute__((always_inline))
+void __uatomic_add(void *addr, unsigned long val, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ __asm__ __volatile__(
+ "lock; addb %1, %0"
+ : "=m"(*__hp(addr))
+ : "iq" ((unsigned char)val)
+ : "memory");
+ return;
+ }
+ case 2:
+ {
+ __asm__ __volatile__(
+ "lock; addw %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned short)val)
+ : "memory");
+ return;
+ }
+ case 4:
+ {
+ __asm__ __volatile__(
+ "lock; addl %1, %0"
+ : "=m"(*__hp(addr))
+ : "ir" ((unsigned int)val)
+ : "memory");
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ "lock; addq %1, %0"
+ : "=m"(*__hp(addr))
+ : "er" ((unsigned long)val)
+ : "memory");
+ return;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return;
+}
+
+#define _uatomic_add(addr, v) \
+ (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
+
+
+/* uatomic_inc */
+
+static inline __attribute__((always_inline))
+void __uatomic_inc(void *addr, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ __asm__ __volatile__(
+ "lock; incb %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+ case 2:
+ {
+ __asm__ __volatile__(
+ "lock; incw %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+ case 4:
+ {
+ __asm__ __volatile__(
+ "lock; incl %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ "lock; incq %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+#endif
+ }
+ /* generate an illegal instruction. Cannot catch this with linker tricks
+ * when optimizations are disabled. */
+ __asm__ __volatile__("ud2");
+ return;
+}
+
+#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
+
+/* uatomic_dec */
+
+static inline __attribute__((always_inline))
+void __uatomic_dec(void *addr, int len)
+{
+ switch (len) {
+ case 1:
+ {
+ __asm__ __volatile__(
+ "lock; decb %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+ case 2:
+ {
+ __asm__ __volatile__(
+ "lock; decw %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+ case 4:
+ {
+ __asm__ __volatile__(
+ "lock; decl %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+#if (CAA_BITS_PER_LONG == 64)
+ case 8:
+ {
+ __asm__ __volatile__(
+ "lock; decq %0"
+ : "=m"(*__hp(addr))
+ :
+ : "memory");
+ return;
+ }
+#endif
+ }
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ return;
+}
+
+#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
+
+#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
+extern int __rcu_cas_avail;
+extern int __rcu_cas_init(void);
+
+#define UATOMIC_COMPAT(insn) \
+ ((caa_likely(__rcu_cas_avail > 0)) \
+ ? (_uatomic_##insn) \
+ : ((caa_unlikely(__rcu_cas_avail < 0) \
+ ? ((__rcu_cas_init() > 0) \
+ ? (_uatomic_##insn) \
+ : (compat_uatomic_##insn)) \
+ : (compat_uatomic_##insn))))
+
+/*
+ * We leave the return value so we don't break the ABI, but remove the
+ * return value from the API.
+ */
+extern unsigned long _compat_uatomic_set(void *addr,
+ unsigned long _new, int len);
+#define compat_uatomic_set(addr, _new) \
+ ((void) _compat_uatomic_set((addr), \
+ caa_cast_long_keep_sign(_new), \
+ sizeof(*(addr))))
+
+
+extern unsigned long _compat_uatomic_xchg(void *addr,
+ unsigned long _new, int len);
+#define compat_uatomic_xchg(addr, _new) \
+ ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
+ caa_cast_long_keep_sign(_new), \
+ sizeof(*(addr))))
+
+extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len);
+#define compat_uatomic_cmpxchg(addr, old, _new) \
+ ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
+ caa_cast_long_keep_sign(old), \
+ caa_cast_long_keep_sign(_new), \
+ sizeof(*(addr))))
+
+extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
+#define compat_uatomic_and(addr, v) \
+ (_compat_uatomic_and((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
+#define compat_uatomic_or(addr, v) \
+ (_compat_uatomic_or((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+extern unsigned long _compat_uatomic_add_return(void *addr,
+ unsigned long _new, int len);
+#define compat_uatomic_add_return(addr, v) \
+ ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
+ caa_cast_long_keep_sign(v), \
+ sizeof(*(addr))))
+
+#define compat_uatomic_add(addr, v) \
+ ((void)compat_uatomic_add_return((addr), (v)))
+#define compat_uatomic_inc(addr) \
+ (compat_uatomic_add((addr), 1))
+#define compat_uatomic_dec(addr) \
+ (compat_uatomic_add((addr), -1))
+
+#else
+#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
+#endif
+
+/* Read is atomic even in compat mode */
+#define uatomic_set(addr, v) \
+ UATOMIC_COMPAT(set(addr, v))
+
+#define uatomic_cmpxchg(addr, old, _new) \
+ UATOMIC_COMPAT(cmpxchg(addr, old, _new))
+#define uatomic_xchg(addr, v) \
+ UATOMIC_COMPAT(xchg(addr, v))
+
+#define uatomic_and(addr, v) \
+ UATOMIC_COMPAT(and(addr, v))
+#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
+
+#define uatomic_or(addr, v) \
+ UATOMIC_COMPAT(or(addr, v))
+#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
+
+#define uatomic_add_return(addr, v) \
+ UATOMIC_COMPAT(add_return(addr, v))
+
+#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
+#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
+
+#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
+#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
+
+#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
+#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
+#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu/uatomic/generic.h>
+
+#endif /* _URCU_ARCH_UATOMIC_X86_H */
--- /dev/null
+#warning "urcu/uatomic_arch.h is deprecated. Please include urcu/uatomic.h instead."
+#include <urcu/uatomic.h>
--- /dev/null
+#warning "urcu/urcu-futex.h is deprecated. Please include urcu/futex.h instead."
+#include <urcu/futex.h>
--- /dev/null
+#warning "urcu/urcu_ref.h is deprecated. Please include urcu/ref.h instead."
+#include <urcu/ref.h>
--- /dev/null
+#ifndef _URCU_WFCQUEUE_H
+#define _URCU_WFCQUEUE_H
+
+/*
+ * urcu/wfcqueue.h
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Concurrent queue with wait-free enqueue/blocking dequeue.
+ *
+ * This queue has been designed and implemented collaboratively by
+ * Mathieu Desnoyers and Lai Jiangshan. Inspired from
+ * half-wait-free/half-blocking queue implementation done by Paul E.
+ * McKenney.
+ */
+
+#define CDS_WFCQ_WOULDBLOCK ((struct cds_wfcq_node *) -1UL)
+
+enum cds_wfcq_ret {
+ CDS_WFCQ_RET_WOULDBLOCK = -1,
+ CDS_WFCQ_RET_DEST_EMPTY = 0,
+ CDS_WFCQ_RET_DEST_NON_EMPTY = 1,
+ CDS_WFCQ_RET_SRC_EMPTY = 2,
+};
+
+enum cds_wfcq_state {
+ CDS_WFCQ_STATE_LAST = (1U << 0),
+};
+
+struct cds_wfcq_node {
+ struct cds_wfcq_node *next;
+};
+
+/*
+ * Do not put head and tail on the same cache-line if concurrent
+ * enqueue/dequeue are expected from many CPUs. This eliminates
+ * false-sharing between enqueue and dequeue.
+ */
+struct __cds_wfcq_head {
+ struct cds_wfcq_node node;
+};
+
+struct cds_wfcq_head {
+ struct cds_wfcq_node node;
+ pthread_mutex_t lock;
+};
+
+#ifndef __cplusplus
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfcq_head and struct __cds_wfcq_head on any of those two
+ * types.
+ */
+typedef union {
+ struct __cds_wfcq_head *_h;
+ struct cds_wfcq_head *h;
+} __attribute__((__transparent_union__)) cds_wfcq_head_ptr_t;
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct __cds_wfcq_head *__cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+ return head;
+}
+
+/*
+ * This static inline is only present for compatibility with C++. It is
+ * effect-less in C.
+ */
+static inline struct cds_wfcq_head *cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+ return head;
+}
+#else /* #ifndef __cplusplus */
+
+/* C++ ignores transparent union. */
+typedef union {
+ struct __cds_wfcq_head *_h;
+ struct cds_wfcq_head *h;
+} cds_wfcq_head_ptr_t;
+
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t __cds_wfcq_head_cast(struct __cds_wfcq_head *head)
+{
+ cds_wfcq_head_ptr_t ret = { ._h = head };
+ return ret;
+}
+/* C++ ignores transparent union. Requires an explicit conversion. */
+static inline cds_wfcq_head_ptr_t cds_wfcq_head_cast(struct cds_wfcq_head *head)
+{
+ cds_wfcq_head_ptr_t ret = { .h = head };
+ return ret;
+}
+#endif /* #else #ifndef __cplusplus */
+
+struct cds_wfcq_tail {
+ struct cds_wfcq_node *p;
+};
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/wfcqueue.h>
+
+#define cds_wfcq_node_init _cds_wfcq_node_init
+#define cds_wfcq_init _cds_wfcq_init
+#define __cds_wfcq_init ___cds_wfcq_init
+#define cds_wfcq_destroy _cds_wfcq_destroy
+#define cds_wfcq_empty _cds_wfcq_empty
+#define cds_wfcq_enqueue _cds_wfcq_enqueue
+
+/* Dequeue locking */
+#define cds_wfcq_dequeue_lock _cds_wfcq_dequeue_lock
+#define cds_wfcq_dequeue_unlock _cds_wfcq_dequeue_unlock
+
+/* Locking performed within cds_wfcq calls. */
+#define cds_wfcq_dequeue_blocking _cds_wfcq_dequeue_blocking
+#define cds_wfcq_dequeue_with_state_blocking \
+ _cds_wfcq_dequeue_with_state_blocking
+#define cds_wfcq_splice_blocking _cds_wfcq_splice_blocking
+#define cds_wfcq_first_blocking _cds_wfcq_first_blocking
+#define cds_wfcq_next_blocking _cds_wfcq_next_blocking
+
+/* Locking ensured by caller by holding cds_wfcq_dequeue_lock() */
+#define __cds_wfcq_dequeue_blocking ___cds_wfcq_dequeue_blocking
+#define __cds_wfcq_dequeue_with_state_blocking \
+ ___cds_wfcq_dequeue_with_state_blocking
+#define __cds_wfcq_splice_blocking ___cds_wfcq_splice_blocking
+#define __cds_wfcq_first_blocking ___cds_wfcq_first_blocking
+#define __cds_wfcq_next_blocking ___cds_wfcq_next_blocking
+
+/*
+ * Locking ensured by caller by holding cds_wfcq_dequeue_lock().
+ * Non-blocking: deque, first, next return CDS_WFCQ_WOULDBLOCK if they
+ * need to block. splice returns nonzero if it needs to block.
+ */
+#define __cds_wfcq_dequeue_nonblocking ___cds_wfcq_dequeue_nonblocking
+#define __cds_wfcq_dequeue_with_state_nonblocking \
+ ___cds_wfcq_dequeue_with_state_nonblocking
+#define __cds_wfcq_splice_nonblocking ___cds_wfcq_splice_nonblocking
+#define __cds_wfcq_first_nonblocking ___cds_wfcq_first_nonblocking
+#define __cds_wfcq_next_nonblocking ___cds_wfcq_next_nonblocking
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * Legend:
+ * [1] cds_wfcq_enqueue
+ * [2] __cds_wfcq_splice (destination queue)
+ * [3] __cds_wfcq_dequeue
+ * [4] __cds_wfcq_splice (source queue)
+ * [5] __cds_wfcq_first
+ * [6] __cds_wfcq_next
+ *
+ * [1] [2] [3] [4] [5] [6]
+ * [1] - - - - - -
+ * [2] - - - - - -
+ * [3] - - X X X X
+ * [4] - - X - X X
+ * [5] - - X X - -
+ * [6] - - X X - -
+ *
+ * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
+ *
+ * For convenience, cds_wfcq_dequeue_blocking() and
+ * cds_wfcq_splice_blocking() hold the dequeue lock.
+ *
+ * Besides locking, mutual exclusion of dequeue, splice and iteration
+ * can be ensured by performing all of those operations from a single
+ * thread, without requiring any lock.
+ */
+
+/*
+ * cds_wfcq_node_init: initialize wait-free queue node.
+ */
+extern void cds_wfcq_node_init(struct cds_wfcq_node *node);
+
+/*
+ * cds_wfcq_init: initialize wait-free queue. Pair with
+ * cds_wfcq_destroy().
+ */
+extern void cds_wfcq_init(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_destroy: destroy wait-free queue. Pair with
+ * cds_wfcq_init().
+ */
+extern void cds_wfcq_destroy(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
+ * pair with any destroy function.
+ */
+extern void __cds_wfcq_init(struct __cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_empty: return whether wait-free queue is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+extern bool cds_wfcq_empty(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_dequeue_lock: take the dequeue mutual exclusion lock.
+ */
+extern void cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_dequeue_unlock: release the dequeue mutual exclusion lock.
+ */
+extern void cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
+ *
+ * Issues a full memory barrier before enqueue. No mutual exclusion is
+ * required.
+ *
+ * Returns false if the queue was empty prior to adding the node.
+ * Returns true otherwise.
+ */
+extern bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node);
+
+/*
+ * cds_wfcq_dequeue_blocking: dequeue a node from a wait-free queue.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
+ * ensured.
+ */
+extern struct cds_wfcq_node *cds_wfcq_dequeue_blocking(
+ struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * cds_wfcq_dequeue_with_state_blocking: dequeue with state.
+ *
+ * Same as cds_wfcq_dequeue_blocking, but saves whether dequeueing the
+ * last node of the queue into state (CDS_WFCQ_STATE_LAST).
+ */
+extern struct cds_wfcq_node *cds_wfcq_dequeue_with_state_blocking(
+ struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail,
+ int *state);
+
+/*
+ * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
+ * ensured.
+ *
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue.
+ */
+extern enum cds_wfcq_ret cds_wfcq_splice_blocking(
+ struct cds_wfcq_head *dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ struct cds_wfcq_head *src_q_head,
+ struct cds_wfcq_tail *src_q_tail);
+
+/*
+ * __cds_wfcq_dequeue_blocking: dequeue a node from a wait-free queue.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * It is valid to reuse and free a dequeued node immediately.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_dequeue_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * __cds_wfcq_dequeue_with_state_blocking: dequeue with state.
+ *
+ * Same as __cds_wfcq_dequeue_blocking, but saves whether dequeueing the
+ * last node of the queue into state (CDS_WFCQ_STATE_LAST).
+ */
+extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ int *state);
+
+/*
+ * __cds_wfcq_dequeue_nonblocking: dequeue a node from a wait-free queue.
+ *
+ * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
+ * if it needs to block.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * __cds_wfcq_dequeue_with_state_blocking: dequeue with state.
+ *
+ * Same as __cds_wfcq_dequeue_nonblocking, but saves whether dequeueing
+ * the last node of the queue into state (CDS_WFCQ_STATE_LAST).
+ */
+extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ int *state);
+
+/*
+ * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Dequeue all nodes from src_q.
+ * dest_q must be already initialized.
+ * Mutual exclusion for src_q should be ensured by the caller as
+ * specified in the "Synchronisation table".
+ * Returns enum cds_wfcq_ret which indicates the state of the src or
+ * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
+ */
+extern enum cds_wfcq_ret __cds_wfcq_splice_blocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail);
+
+/*
+ * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
+ *
+ * Same as __cds_wfcq_splice_blocking, but returns
+ * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
+ */
+extern enum cds_wfcq_ret __cds_wfcq_splice_nonblocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail);
+
+/*
+ * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if queue is empty, first node otherwise.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_first_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_first_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail);
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ *
+ * Used by for-like iteration macros:
+ * __cds_wfcq_for_each_blocking()
+ * __cds_wfcq_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of queue, non-NULL next queue node
+ * otherwise.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_next_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node);
+
+/*
+ * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
+ *
+ * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
+ * it needs to block.
+ */
+extern struct cds_wfcq_node *__cds_wfcq_next_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node);
+
+#endif /* !_LGPL_SOURCE */
+
+/*
+ * __cds_wfcq_for_each_blocking: Iterate over all nodes in a queue,
+ * without dequeuing them.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking(head, tail, node) \
+ for (node = __cds_wfcq_first_blocking(head, tail); \
+ node != NULL; \
+ node = __cds_wfcq_next_blocking(head, tail, node))
+
+/*
+ * __cds_wfcq_for_each_blocking_safe: Iterate over all nodes in a queue,
+ * without dequeuing them. Safe against deletion.
+ * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
+ * @tail: tail of the queue (struct cds_wfcq_tail pointer).
+ * @node: iterator on the queue (struct cds_wfcq_node pointer).
+ * @n: struct cds_wfcq_node pointer holding the next pointer (used
+ * internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ * Dequeue/splice/iteration mutual exclusion should be ensured by the
+ * caller.
+ */
+#define __cds_wfcq_for_each_blocking_safe(head, tail, node, n) \
+ for (node = __cds_wfcq_first_blocking(head, tail), \
+ n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL); \
+ node != NULL; \
+ node = n, n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFCQUEUE_H */
--- /dev/null
+#ifndef _URCU_WFQUEUE_H
+#define _URCU_WFQUEUE_H
+
+/*
+ * wfqueue.h
+ *
+ * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <urcu/compiler.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CDS_WFQ_DEPRECATED
+#define CDS_WFQ_DEPRECATED \
+ CDS_DEPRECATED("urcu/wfqueue.h is deprecated. Please use urcu/wfcqueue.h instead.")
+#endif
+
+/*
+ * Queue with wait-free enqueue/blocking dequeue.
+ * This implementation adds a dummy head node when the queue is empty to ensure
+ * we can always update the queue locklessly.
+ *
+ * Inspired from half-wait-free/half-blocking queue implementation done by
+ * Paul E. McKenney.
+ */
+
+struct cds_wfq_node {
+ struct cds_wfq_node *next;
+};
+
+struct cds_wfq_queue {
+ struct cds_wfq_node *head, **tail;
+ struct cds_wfq_node dummy; /* Dummy node */
+ pthread_mutex_t lock;
+};
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/wfqueue.h>
+
+static inline CDS_WFQ_DEPRECATED
+void cds_wfq_node_init(struct cds_wfq_node *node)
+{
+ _cds_wfq_node_init(node);
+}
+
+static inline CDS_WFQ_DEPRECATED
+void cds_wfq_init(struct cds_wfq_queue *q)
+{
+ _cds_wfq_init(q);
+}
+
+static inline CDS_WFQ_DEPRECATED
+void cds_wfq_destroy(struct cds_wfq_queue *q)
+{
+ _cds_wfq_destroy(q);
+}
+
+static inline CDS_WFQ_DEPRECATED
+void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node)
+{
+ _cds_wfq_enqueue(q, node);
+}
+
+static inline CDS_WFQ_DEPRECATED
+struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ return ___cds_wfq_dequeue_blocking(q);
+}
+
+static inline CDS_WFQ_DEPRECATED
+struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ return _cds_wfq_dequeue_blocking(q);
+}
+
+#else /* !_LGPL_SOURCE */
+
+extern CDS_WFQ_DEPRECATED
+void cds_wfq_node_init(struct cds_wfq_node *node);
+
+extern CDS_WFQ_DEPRECATED
+void cds_wfq_init(struct cds_wfq_queue *q);
+
+extern CDS_WFQ_DEPRECATED
+void cds_wfq_destroy(struct cds_wfq_queue *q);
+
+extern CDS_WFQ_DEPRECATED
+void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node);
+
+/* __cds_wfq_dequeue_blocking: caller ensures mutual exclusion between dequeues */
+extern CDS_WFQ_DEPRECATED
+struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q);
+
+extern CDS_WFQ_DEPRECATED
+struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q);
+
+#endif /* !_LGPL_SOURCE */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_WFQUEUE_H */
--- /dev/null
+#ifndef _URCU_WFSTACK_H
+#define _URCU_WFSTACK_H
+
+/*
+ * urcu/wfstack.h
+ *
+ * Userspace RCU library - Stack with wait-free push, blocking traversal.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <pthread.h>
+#include <assert.h>
+#include <stdbool.h>
+#include <urcu/compiler.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Stack with wait-free push, blocking traversal.
+ *
+ * Stack implementing push, pop, pop_all operations, as well as iterator
+ * on the stack head returned by pop_all.
+ *
+ * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
+ * cds_wfs_first.
+ * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
+ * iteration on stack head returned by pop_all.
+ *
+ * Synchronization table:
+ *
+ * External synchronization techniques described in the API below is
+ * required between pairs marked with "X". No external synchronization
+ * required between pairs marked with "-".
+ *
+ * cds_wfs_push __cds_wfs_pop __cds_wfs_pop_all
+ * cds_wfs_push - - -
+ * __cds_wfs_pop - X X
+ * __cds_wfs_pop_all - X -
+ *
+ * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
+ * synchronization.
+ */
+
+#define CDS_WFS_WOULDBLOCK ((void *) -1UL)
+
+enum cds_wfs_state {
+ CDS_WFS_STATE_LAST = (1U << 0),
+};
+
+/*
+ * struct cds_wfs_node is returned by __cds_wfs_pop, and also used as
+ * iterator on stack. It is not safe to dereference the node next
+ * pointer when returned by __cds_wfs_pop_blocking.
+ */
+struct cds_wfs_node {
+ struct cds_wfs_node *next;
+};
+
+/*
+ * struct cds_wfs_head is returned by __cds_wfs_pop_all, and can be used
+ * to begin iteration on the stack. "node" needs to be the first field of
+ * cds_wfs_head, so the end-of-stack pointer value can be used for both
+ * types.
+ */
+struct cds_wfs_head {
+ struct cds_wfs_node node;
+};
+
+struct __cds_wfs_stack {
+ struct cds_wfs_head *head;
+};
+
+struct cds_wfs_stack {
+ struct cds_wfs_head *head;
+ pthread_mutex_t lock;
+};
+
+/*
+ * The transparent union allows calling functions that work on both
+ * struct cds_wfs_stack and struct __cds_wfs_stack on any of those two
+ * types.
+ */
+typedef union {
+ struct __cds_wfs_stack *_s;
+ struct cds_wfs_stack *s;
+} __attribute__((__transparent_union__)) cds_wfs_stack_ptr_t;
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/wfstack.h>
+
+#define cds_wfs_node_init _cds_wfs_node_init
+#define cds_wfs_init _cds_wfs_init
+#define cds_wfs_destroy _cds_wfs_destroy
+#define __cds_wfs_init ___cds_wfs_init
+#define cds_wfs_empty _cds_wfs_empty
+#define cds_wfs_push _cds_wfs_push
+
+/* Locking performed internally */
+#define cds_wfs_pop_blocking _cds_wfs_pop_blocking
+#define cds_wfs_pop_with_state_blocking _cds_wfs_pop_with_state_blocking
+#define cds_wfs_pop_all_blocking _cds_wfs_pop_all_blocking
+
+/*
+ * For iteration on cds_wfs_head returned by __cds_wfs_pop_all or
+ * cds_wfs_pop_all_blocking.
+ */
+#define cds_wfs_first _cds_wfs_first
+#define cds_wfs_next_blocking _cds_wfs_next_blocking
+#define cds_wfs_next_nonblocking _cds_wfs_next_nonblocking
+
+/* Pop locking with internal mutex */
+#define cds_wfs_pop_lock _cds_wfs_pop_lock
+#define cds_wfs_pop_unlock _cds_wfs_pop_unlock
+
+/* Synchronization ensured by the caller. See synchronization table. */
+#define __cds_wfs_pop_blocking ___cds_wfs_pop_blocking
+#define __cds_wfs_pop_with_state_blocking \
+ ___cds_wfs_pop_with_state_blocking
+#define __cds_wfs_pop_nonblocking ___cds_wfs_pop_nonblocking
+#define __cds_wfs_pop_with_state_nonblocking \
+ ___cds_wfs_pop_with_state_nonblocking
+#define __cds_wfs_pop_all ___cds_wfs_pop_all
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * cds_wfs_node_init: initialize wait-free stack node.
+ */
+extern void cds_wfs_node_init(struct cds_wfs_node *node);
+
+/*
+ * cds_wfs_init: initialize wait-free stack (with lock). Pair with
+ * cds_wfs_destroy().
+ */
+extern void cds_wfs_init(struct cds_wfs_stack *s);
+
+/*
+ * cds_wfs_destroy: destroy wait-free stack (with lock). Pair with
+ * cds_wfs_init().
+ */
+extern void cds_wfs_destroy(struct cds_wfs_stack *s);
+
+/*
+ * __cds_wfs_init: initialize wait-free stack (no lock). Don't pair with
+ * any destroy function.
+ */
+extern void __cds_wfs_init(struct __cds_wfs_stack *s);
+
+/*
+ * cds_wfs_empty: return whether wait-free stack is empty.
+ *
+ * No memory barrier is issued. No mutual exclusion is required.
+ */
+extern bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack);
+
+/*
+ * cds_wfs_push: push a node into the stack.
+ *
+ * Issues a full memory barrier before push. No mutual exclusion is
+ * required.
+ *
+ * Returns 0 if the stack was empty prior to adding the node.
+ * Returns non-zero otherwise.
+ */
+extern int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node);
+
+/*
+ * cds_wfs_pop_blocking: pop a node from the stack.
+ *
+ * Calls __cds_wfs_pop_blocking with an internal pop mutex held.
+ */
+extern struct cds_wfs_node *cds_wfs_pop_blocking(struct cds_wfs_stack *s);
+
+/*
+ * cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
+ *
+ * Same as cds_wfs_pop_blocking, but stores whether the stack was
+ * empty into state (CDS_WFS_STATE_LAST).
+ */
+extern struct cds_wfs_node *
+ cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state);
+
+/*
+ * cds_wfs_pop_all_blocking: pop all nodes from a stack.
+ *
+ * Calls __cds_wfs_pop_all with an internal pop mutex held.
+ */
+extern struct cds_wfs_head *cds_wfs_pop_all_blocking(struct cds_wfs_stack *s);
+
+/*
+ * cds_wfs_first: get first node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if popped stack is empty, top stack node otherwise.
+ */
+extern struct cds_wfs_node *cds_wfs_first(struct cds_wfs_head *head);
+
+/*
+ * cds_wfs_next_blocking: get next node of a popped stack.
+ *
+ * Content written into the node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ *
+ * Used by for-like iteration macros in urcu/wfstack.h:
+ * cds_wfs_for_each_blocking()
+ * cds_wfs_for_each_blocking_safe()
+ *
+ * Returns NULL if reached end of popped stack, non-NULL next stack
+ * node otherwise.
+ */
+extern struct cds_wfs_node *cds_wfs_next_blocking(struct cds_wfs_node *node);
+
+/*
+ * cds_wfs_next_nonblocking: get next node of a popped stack.
+ *
+ * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
+ * needs to block.
+ */
+extern struct cds_wfs_node *cds_wfs_next_nonblocking(struct cds_wfs_node *node);
+
+/*
+ * cds_wfs_pop_lock: lock stack pop-protection mutex.
+ */
+extern void cds_wfs_pop_lock(struct cds_wfs_stack *s);
+
+/*
+ * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+extern void cds_wfs_pop_unlock(struct cds_wfs_stack *s);
+
+/*
+ * __cds_wfs_pop_blocking: pop a node from the stack.
+ *
+ * Returns NULL if stack is empty.
+ *
+ * __cds_wfs_pop_blocking needs to be synchronized using one of the
+ * following techniques:
+ *
+ * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
+ * section. The caller must wait for a grace period to pass before
+ * freeing the returned node or modifying the cds_wfs_node structure.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+extern struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack);
+
+/*
+ * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
+ *
+ * Same as __cds_wfs_pop_blocking, but stores whether the stack was
+ * empty into state (CDS_WFS_STATE_LAST).
+ */
+extern struct cds_wfs_node *
+ __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack,
+ int *state);
+
+/*
+ * __cds_wfs_pop_nonblocking: pop a node from the stack.
+ *
+ * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
+ * it needs to block.
+ */
+extern struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack);
+
+/*
+ * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack, with state.
+ *
+ * Same as __cds_wfs_pop_nonblocking, but stores whether the stack was
+ * empty into state (CDS_WFS_STATE_LAST).
+ */
+extern struct cds_wfs_node *
+ __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack,
+ int *state);
+
+/*
+ * __cds_wfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_wfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_wfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_wfs_pop_blocking:
+ *
+ * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
+ * section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
+ * must wait for a grace period to pass before freeing the returned
+ * node or modifying the cds_wfs_node structure. However, no RCU
+ * read-side critical section is needed around __cds_wfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect
+ * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
+ * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+extern struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack);
+
+#endif /* !_LGPL_SOURCE */
+
+#ifdef __cplusplus
+}
+#endif
+
+/*
+ * cds_wfs_for_each_blocking: Iterate over all nodes returned by
+ * __cds_wfs_pop_all().
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking(head, node) \
+ for (node = cds_wfs_first(head); \
+ node != NULL; \
+ node = cds_wfs_next_blocking(node))
+
+/*
+ * cds_wfs_for_each_blocking_safe: Iterate over all nodes returned by
+ * __cds_wfs_pop_all(). Safe against deletion.
+ * @head: head of the queue (struct cds_wfs_head pointer).
+ * @node: iterator (struct cds_wfs_node pointer).
+ * @n: struct cds_wfs_node pointer holding the next pointer (used
+ * internally).
+ *
+ * Content written into each node before enqueue is guaranteed to be
+ * consistent, but no other memory ordering is ensured.
+ */
+#define cds_wfs_for_each_blocking_safe(head, node, n) \
+ for (node = cds_wfs_first(head), \
+ n = (node ? cds_wfs_next_blocking(node) : NULL); \
+ node != NULL; \
+ node = n, n = (node ? cds_wfs_next_blocking(node) : NULL))
+
+#endif /* _URCU_WFSTACK_H */
+++ /dev/null
-/*
- * lfstack.c
- *
- * Userspace RCU library - Lock-Free Stack
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu/lfstack.h"
-#define _LGPL_SOURCE
-#include "urcu/static/lfstack.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void cds_lfs_node_init(struct cds_lfs_node *node)
-{
- _cds_lfs_node_init(node);
-}
-
-void cds_lfs_init(struct cds_lfs_stack *s)
-{
- _cds_lfs_init(s);
-}
-
-void cds_lfs_destroy(struct cds_lfs_stack *s)
-{
- _cds_lfs_destroy(s);
-}
-
-void __cds_lfs_init(struct __cds_lfs_stack *s)
-{
- ___cds_lfs_init(s);
-}
-
-bool cds_lfs_empty(cds_lfs_stack_ptr_t s)
-{
- return _cds_lfs_empty(s);
-}
-
-bool cds_lfs_push(cds_lfs_stack_ptr_t s, struct cds_lfs_node *node)
-{
- return _cds_lfs_push(s, node);
-}
-
-struct cds_lfs_node *cds_lfs_pop_blocking(struct cds_lfs_stack *s)
-{
- return _cds_lfs_pop_blocking(s);
-}
-
-struct cds_lfs_head *cds_lfs_pop_all_blocking(struct cds_lfs_stack *s)
-{
- return _cds_lfs_pop_all_blocking(s);
-}
-
-void cds_lfs_pop_lock(struct cds_lfs_stack *s)
-{
- _cds_lfs_pop_lock(s);
-}
-
-void cds_lfs_pop_unlock(struct cds_lfs_stack *s)
-{
- _cds_lfs_pop_unlock(s);
-}
-
-struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s)
-{
- return ___cds_lfs_pop(s);
-}
-
-struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s)
-{
- return ___cds_lfs_pop_all(s);
-}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU BulletProof
-Description: A userspace RCU (read-copy-update) library, bulletproof version
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu-bp
-Cflags: -I${includedir}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU Concurrent Data Structures
-Description: Data structures leveraging RCU and atomic operations to provide efficient concurrency-aware storage
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu-cds
-Cflags: -I${includedir}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU Memory barriers
-Description: A userspace RCU (read-copy-update) library, memory barriers version
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu-mb
-Cflags: -I${includedir}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU QSBR
-Description: A userspace RCU (read-copy-update) library, quiescent state version
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu-qsbr
-Cflags: -I${includedir}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU signal
-Description: A userspace RCU (read-copy-update) library, signal version
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu-signal
-Cflags: -I${includedir}
+++ /dev/null
-prefix=@prefix@
-exec_prefix=@exec_prefix@
-libdir=@libdir@
-includedir=@includedir@
-
-Name: Userspace RCU
-Description: A userspace RCU (read-copy-update) library, standard version
-Version: @PACKAGE_VERSION@
-Requires:
-Libs: -L${libdir} -lurcu
-Cflags: -I${includedir}
+++ /dev/null
-#ifndef _URCU_RCULFHASH_INTERNAL_H
-#define _URCU_RCULFHASH_INTERNAL_H
-
-/*
- * urcu/rculfhash-internal.h
- *
- * Internal header for Lock-Free RCU Hash Table
- *
- * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/rculfhash.h>
-#include <stdio.h>
-
-#ifdef DEBUG
-#define dbg_printf(fmt, args...) printf("[debug rculfhash] " fmt, ## args)
-#else
-#define dbg_printf(fmt, args...) \
-do { \
- /* do nothing but check printf format */ \
- if (0) \
- printf("[debug rculfhash] " fmt, ## args); \
-} while (0)
-#endif
-
-#if (CAA_BITS_PER_LONG == 32)
-#define MAX_TABLE_ORDER 32
-#else
-#define MAX_TABLE_ORDER 64
-#endif
-
-#define MAX_CHUNK_TABLE (1UL << 10)
-
-#ifndef min
-#define min(a, b) ((a) < (b) ? (a) : (b))
-#endif
-
-#ifndef max
-#define max(a, b) ((a) > (b) ? (a) : (b))
-#endif
-
-struct ht_items_count;
-
-/*
- * cds_lfht: Top-level data structure representing a lock-free hash
- * table. Defined in the implementation file to make it be an opaque
- * cookie to users.
- *
- * The fields used in fast-paths are placed near the end of the
- * structure, because we need to have a variable-sized union to contain
- * the mm plugin fields, which are used in the fast path.
- */
-struct cds_lfht {
- /* Initial configuration items */
- unsigned long max_nr_buckets;
- const struct cds_lfht_mm_type *mm; /* memory management plugin */
- const struct rcu_flavor_struct *flavor; /* RCU flavor */
-
- long count; /* global approximate item count */
-
- /*
- * We need to put the work threads offline (QSBR) when taking this
- * mutex, because we use synchronize_rcu within this mutex critical
- * section, which waits on read-side critical sections, and could
- * therefore cause grace-period deadlock if we hold off RCU G.P.
- * completion.
- */
- pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */
- pthread_attr_t *resize_attr; /* Resize threads attributes */
- unsigned int in_progress_resize, in_progress_destroy;
- unsigned long resize_target;
- int resize_initiated;
-
- /*
- * Variables needed for add and remove fast-paths.
- */
- int flags;
- unsigned long min_alloc_buckets_order;
- unsigned long min_nr_alloc_buckets;
- struct ht_items_count *split_count; /* split item count */
-
- /*
- * Variables needed for the lookup, add and remove fast-paths.
- */
- unsigned long size; /* always a power of 2, shared (RCU) */
- /*
- * bucket_at pointer is kept here to skip the extra level of
- * dereference needed to get to "mm" (this is a fast-path).
- */
- struct cds_lfht_node *(*bucket_at)(struct cds_lfht *ht,
- unsigned long index);
- /*
- * Dynamic length "tbl_chunk" needs to be at the end of
- * cds_lfht.
- */
- union {
- /*
- * Contains the per order-index-level bucket node table.
- * The size of each bucket node table is half the number
- * of hashes contained in this order (except for order 0).
- * The minimum allocation buckets size parameter allows
- * combining the bucket node arrays of the lowermost
- * levels to improve cache locality for small index orders.
- */
- struct cds_lfht_node *tbl_order[MAX_TABLE_ORDER];
-
- /*
- * Contains the bucket node chunks. The size of each
- * bucket node chunk is ->min_alloc_size (we avoid to
- * allocate chunks with different size). Chunks improve
- * cache locality for small index orders, and are more
- * friendly with environments where allocation of large
- * contiguous memory areas is challenging due to memory
- * fragmentation concerns or inability to use virtual
- * memory addressing.
- */
- struct cds_lfht_node *tbl_chunk[0];
-
- /*
- * Memory mapping with room for all possible buckets.
- * Their memory is allocated when needed.
- */
- struct cds_lfht_node *tbl_mmap;
- };
- /*
- * End of variables needed for the lookup, add and remove
- * fast-paths.
- */
-};
-
-extern unsigned int cds_lfht_fls_ulong(unsigned long x);
-extern int cds_lfht_get_count_order_ulong(unsigned long x);
-
-#ifdef POISON_FREE
-#define poison_free(ptr) \
- do { \
- if (ptr) { \
- memset(ptr, 0x42, sizeof(*(ptr))); \
- free(ptr); \
- } \
- } while (0)
-#else
-#define poison_free(ptr) free(ptr)
-#endif
-
-static inline
-struct cds_lfht *__default_alloc_cds_lfht(
- const struct cds_lfht_mm_type *mm,
- unsigned long cds_lfht_size,
- unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets)
-{
- struct cds_lfht *ht;
-
- ht = calloc(1, cds_lfht_size);
- assert(ht);
-
- ht->mm = mm;
- ht->bucket_at = mm->bucket_at;
- ht->min_nr_alloc_buckets = min_nr_alloc_buckets;
- ht->min_alloc_buckets_order =
- cds_lfht_get_count_order_ulong(min_nr_alloc_buckets);
- ht->max_nr_buckets = max_nr_buckets;
-
- return ht;
-}
-
-#endif /* _URCU_RCULFHASH_INTERNAL_H */
+++ /dev/null
-/*
- * rculfhash-mm-chunk.c
- *
- * Chunk based memory management for Lock-Free RCU Hash Table
- *
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stddef.h>
-#include <rculfhash-internal.h>
-
-static
-void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0) {
- ht->tbl_chunk[0] = calloc(ht->min_nr_alloc_buckets,
- sizeof(struct cds_lfht_node));
- assert(ht->tbl_chunk[0]);
- } else if (order > ht->min_alloc_buckets_order) {
- unsigned long i, len = 1UL << (order - 1 - ht->min_alloc_buckets_order);
-
- for (i = len; i < 2 * len; i++) {
- ht->tbl_chunk[i] = calloc(ht->min_nr_alloc_buckets,
- sizeof(struct cds_lfht_node));
- assert(ht->tbl_chunk[i]);
- }
- }
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-/*
- * cds_lfht_free_bucket_table() should be called with decreasing order.
- * When cds_lfht_free_bucket_table(0) is called, it means the whole
- * lfht is destroyed.
- */
-static
-void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0)
- poison_free(ht->tbl_chunk[0]);
- else if (order > ht->min_alloc_buckets_order) {
- unsigned long i, len = 1UL << (order - 1 - ht->min_alloc_buckets_order);
-
- for (i = len; i < 2 * len; i++)
- poison_free(ht->tbl_chunk[i]);
- }
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-static
-struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
-{
- unsigned long chunk, offset;
-
- chunk = index >> ht->min_alloc_buckets_order;
- offset = index & (ht->min_nr_alloc_buckets - 1);
- return &ht->tbl_chunk[chunk][offset];
-}
-
-static
-struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets)
-{
- unsigned long nr_chunks, cds_lfht_size;
-
- min_nr_alloc_buckets = max(min_nr_alloc_buckets,
- max_nr_buckets / MAX_CHUNK_TABLE);
- nr_chunks = max_nr_buckets / min_nr_alloc_buckets;
- cds_lfht_size = offsetof(struct cds_lfht, tbl_chunk) +
- sizeof(struct cds_lfht_node *) * nr_chunks;
- cds_lfht_size = max(cds_lfht_size, sizeof(struct cds_lfht));
-
- return __default_alloc_cds_lfht(
- &cds_lfht_mm_chunk, cds_lfht_size,
- min_nr_alloc_buckets, max_nr_buckets);
-}
-
-const struct cds_lfht_mm_type cds_lfht_mm_chunk = {
- .alloc_cds_lfht = alloc_cds_lfht,
- .alloc_bucket_table = cds_lfht_alloc_bucket_table,
- .free_bucket_table = cds_lfht_free_bucket_table,
- .bucket_at = bucket_at,
-};
+++ /dev/null
-/*
- * rculfhash-mm-mmap.c
- *
- * mmap/reservation based memory management for Lock-Free RCU Hash Table
- *
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <unistd.h>
-#include <sys/mman.h>
-#include "rculfhash-internal.h"
-
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-/* reserve inaccessible memory space without allocation any memory */
-static void *memory_map(size_t length)
-{
- void *ret = mmap(NULL, length, PROT_NONE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-
- assert(ret != MAP_FAILED);
- return ret;
-}
-
-static void memory_unmap(void *ptr, size_t length)
-{
- int ret __attribute__((unused));
-
- ret = munmap(ptr, length);
-
- assert(ret == 0);
-}
-
-static void memory_populate(void *ptr, size_t length)
-{
- void *ret __attribute__((unused));
-
- ret = mmap(ptr, length, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-
- assert(ret == ptr);
-}
-
-/*
- * Discard garbage memory and avoid system save it when try to swap it out.
- * Make it still reserved, inaccessible.
- */
-static void memory_discard(void *ptr, size_t length)
-{
- void *ret __attribute__((unused));
-
- ret = mmap(ptr, length, PROT_NONE,
- MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-
- assert(ret == ptr);
-}
-
-static
-void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0) {
- if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
- /* small table */
- ht->tbl_mmap = calloc(ht->max_nr_buckets,
- sizeof(*ht->tbl_mmap));
- assert(ht->tbl_mmap);
- return;
- }
- /* large table */
- ht->tbl_mmap = memory_map(ht->max_nr_buckets
- * sizeof(*ht->tbl_mmap));
- memory_populate(ht->tbl_mmap,
- ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
- } else if (order > ht->min_alloc_buckets_order) {
- /* large table */
- unsigned long len = 1UL << (order - 1);
-
- assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
- memory_populate(ht->tbl_mmap + len,
- len * sizeof(*ht->tbl_mmap));
- }
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-/*
- * cds_lfht_free_bucket_table() should be called with decreasing order.
- * When cds_lfht_free_bucket_table(0) is called, it means the whole
- * lfht is destroyed.
- */
-static
-void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0) {
- if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
- /* small table */
- poison_free(ht->tbl_mmap);
- return;
- }
- /* large table */
- memory_unmap(ht->tbl_mmap,
- ht->max_nr_buckets * sizeof(*ht->tbl_mmap));
- } else if (order > ht->min_alloc_buckets_order) {
- /* large table */
- unsigned long len = 1UL << (order - 1);
-
- assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
- memory_discard(ht->tbl_mmap + len, len * sizeof(*ht->tbl_mmap));
- }
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-static
-struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
-{
- return &ht->tbl_mmap[index];
-}
-
-static
-struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets)
-{
- unsigned long page_bucket_size;
-
- page_bucket_size = getpagesize() / sizeof(struct cds_lfht_node);
- if (max_nr_buckets <= page_bucket_size) {
- /* small table */
- min_nr_alloc_buckets = max_nr_buckets;
- } else {
- /* large table */
- min_nr_alloc_buckets = max(min_nr_alloc_buckets,
- page_bucket_size);
- }
-
- return __default_alloc_cds_lfht(
- &cds_lfht_mm_mmap, sizeof(struct cds_lfht),
- min_nr_alloc_buckets, max_nr_buckets);
-}
-
-const struct cds_lfht_mm_type cds_lfht_mm_mmap = {
- .alloc_cds_lfht = alloc_cds_lfht,
- .alloc_bucket_table = cds_lfht_alloc_bucket_table,
- .free_bucket_table = cds_lfht_free_bucket_table,
- .bucket_at = bucket_at,
-};
+++ /dev/null
-/*
- * rculfhash-mm-order.c
- *
- * Order based memory management for Lock-Free RCU Hash Table
- *
- * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <rculfhash-internal.h>
-
-static
-void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0) {
- ht->tbl_order[0] = calloc(ht->min_nr_alloc_buckets,
- sizeof(struct cds_lfht_node));
- assert(ht->tbl_order[0]);
- } else if (order > ht->min_alloc_buckets_order) {
- ht->tbl_order[order] = calloc(1UL << (order -1),
- sizeof(struct cds_lfht_node));
- assert(ht->tbl_order[order]);
- }
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-/*
- * cds_lfht_free_bucket_table() should be called with decreasing order.
- * When cds_lfht_free_bucket_table(0) is called, it means the whole
- * lfht is destroyed.
- */
-static
-void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- if (order == 0)
- poison_free(ht->tbl_order[0]);
- else if (order > ht->min_alloc_buckets_order)
- poison_free(ht->tbl_order[order]);
- /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
-}
-
-static
-struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
-{
- unsigned long order;
-
- if (index < ht->min_nr_alloc_buckets) {
- dbg_printf("bucket index %lu order 0 aridx 0\n", index);
- return &ht->tbl_order[0][index];
- }
- /*
- * equivalent to cds_lfht_get_count_order_ulong(index + 1), but
- * optimizes away the non-existing 0 special-case for
- * cds_lfht_get_count_order_ulong.
- */
- order = cds_lfht_fls_ulong(index);
- dbg_printf("bucket index %lu order %lu aridx %lu\n",
- index, order, index & ((1UL << (order - 1)) - 1));
- return &ht->tbl_order[order][index & ((1UL << (order - 1)) - 1)];
-}
-
-static
-struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets)
-{
- return __default_alloc_cds_lfht(
- &cds_lfht_mm_order, sizeof(struct cds_lfht),
- min_nr_alloc_buckets, max_nr_buckets);
-}
-
-const struct cds_lfht_mm_type cds_lfht_mm_order = {
- .alloc_cds_lfht = alloc_cds_lfht,
- .alloc_bucket_table = cds_lfht_alloc_bucket_table,
- .free_bucket_table = cds_lfht_free_bucket_table,
- .bucket_at = bucket_at,
-};
+++ /dev/null
-/*
- * rculfhash.c
- *
- * Userspace RCU library - Lock-Free Resizable RCU Hash Table
- *
- * Copyright 2010-2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/*
- * Based on the following articles:
- * - Ori Shalev and Nir Shavit. Split-ordered lists: Lock-free
- * extensible hash tables. J. ACM 53, 3 (May 2006), 379-405.
- * - Michael, M. M. High performance dynamic lock-free hash tables
- * and list-based sets. In Proceedings of the fourteenth annual ACM
- * symposium on Parallel algorithms and architectures, ACM Press,
- * (2002), 73-82.
- *
- * Some specificities of this Lock-Free Resizable RCU Hash Table
- * implementation:
- *
- * - RCU read-side critical section allows readers to perform hash
- * table lookups, as well as traversals, and use the returned objects
- * safely by allowing memory reclaim to take place only after a grace
- * period.
- * - Add and remove operations are lock-free, and do not need to
- * allocate memory. They need to be executed within RCU read-side
- * critical section to ensure the objects they read are valid and to
- * deal with the cmpxchg ABA problem.
- * - add and add_unique operations are supported. add_unique checks if
- * the node key already exists in the hash table. It ensures not to
- * populate a duplicate key if the node key already exists in the hash
- * table.
- * - The resize operation executes concurrently with
- * add/add_unique/add_replace/remove/lookup/traversal.
- * - Hash table nodes are contained within a split-ordered list. This
- * list is ordered by incrementing reversed-bits-hash value.
- * - An index of bucket nodes is kept. These bucket nodes are the hash
- * table "buckets". These buckets are internal nodes that allow to
- * perform a fast hash lookup, similarly to a skip list. These
- * buckets are chained together in the split-ordered list, which
- * allows recursive expansion by inserting new buckets between the
- * existing buckets. The split-ordered list allows adding new buckets
- * between existing buckets as the table needs to grow.
- * - The resize operation for small tables only allows expanding the
- * hash table. It is triggered automatically by detecting long chains
- * in the add operation.
- * - The resize operation for larger tables (and available through an
- * API) allows both expanding and shrinking the hash table.
- * - Split-counters are used to keep track of the number of
- * nodes within the hash table for automatic resize triggering.
- * - Resize operation initiated by long chain detection is executed by a
- * call_rcu thread, which keeps lock-freedom of add and remove.
- * - Resize operations are protected by a mutex.
- * - The removal operation is split in two parts: first, a "removed"
- * flag is set in the next pointer within the node to remove. Then,
- * a "garbage collection" is performed in the bucket containing the
- * removed node (from the start of the bucket up to the removed node).
- * All encountered nodes with "removed" flag set in their next
- * pointers are removed from the linked-list. If the cmpxchg used for
- * removal fails (due to concurrent garbage-collection or concurrent
- * add), we retry from the beginning of the bucket. This ensures that
- * the node with "removed" flag set is removed from the hash table
- * (not visible to lookups anymore) before the RCU read-side critical
- * section held across removal ends. Furthermore, this ensures that
- * the node with "removed" flag set is removed from the linked-list
- * before its memory is reclaimed. After setting the "removal" flag,
- * only the thread which removal is the first to set the "removal
- * owner" flag (with an xchg) into a node's next pointer is considered
- * to have succeeded its removal (and thus owns the node to reclaim).
- * Because we garbage-collect starting from an invariant node (the
- * start-of-bucket bucket node) up to the "removed" node (or find a
- * reverse-hash that is higher), we are sure that a successful
- * traversal of the chain leads to a chain that is present in the
- * linked-list (the start node is never removed) and that it does not
- * contain the "removed" node anymore, even if concurrent delete/add
- * operations are changing the structure of the list concurrently.
- * - The add operations perform garbage collection of buckets if they
- * encounter nodes with removed flag set in the bucket where they want
- * to add their new node. This ensures lock-freedom of add operation by
- * helping the remover unlink nodes from the list rather than to wait
- * for it do to so.
- * - There are three memory backends for the hash table buckets: the
- * "order table", the "chunks", and the "mmap".
- * - These bucket containers contain a compact version of the hash table
- * nodes.
- * - The RCU "order table":
- * - has a first level table indexed by log2(hash index) which is
- * copied and expanded by the resize operation. This order table
- * allows finding the "bucket node" tables.
- * - There is one bucket node table per hash index order. The size of
- * each bucket node table is half the number of hashes contained in
- * this order (except for order 0).
- * - The RCU "chunks" is best suited for close interaction with a page
- * allocator. It uses a linear array as index to "chunks" containing
- * each the same number of buckets.
- * - The RCU "mmap" memory backend uses a single memory map to hold
- * all buckets.
- * - synchronize_rcu is used to garbage-collect the old bucket node table.
- *
- * Ordering Guarantees:
- *
- * To discuss these guarantees, we first define "read" operation as any
- * of the the basic cds_lfht_lookup, cds_lfht_next_duplicate,
- * cds_lfht_first, cds_lfht_next operation, as well as
- * cds_lfht_add_unique (failure).
- *
- * We define "read traversal" operation as any of the following
- * group of operations
- * - cds_lfht_lookup followed by iteration with cds_lfht_next_duplicate
- * (and/or cds_lfht_next, although less common).
- * - cds_lfht_add_unique (failure) followed by iteration with
- * cds_lfht_next_duplicate (and/or cds_lfht_next, although less
- * common).
- * - cds_lfht_first followed iteration with cds_lfht_next (and/or
- * cds_lfht_next_duplicate, although less common).
- *
- * We define "write" operations as any of cds_lfht_add, cds_lfht_replace,
- * cds_lfht_add_unique (success), cds_lfht_add_replace, cds_lfht_del.
- *
- * When cds_lfht_add_unique succeeds (returns the node passed as
- * parameter), it acts as a "write" operation. When cds_lfht_add_unique
- * fails (returns a node different from the one passed as parameter), it
- * acts as a "read" operation. A cds_lfht_add_unique failure is a
- * cds_lfht_lookup "read" operation, therefore, any ordering guarantee
- * referring to "lookup" imply any of "lookup" or cds_lfht_add_unique
- * (failure).
- *
- * We define "prior" and "later" node as nodes observable by reads and
- * read traversals respectively before and after a write or sequence of
- * write operations.
- *
- * Hash-table operations are often cascaded, for example, the pointer
- * returned by a cds_lfht_lookup() might be passed to a cds_lfht_next(),
- * whose return value might in turn be passed to another hash-table
- * operation. This entire cascaded series of operations must be enclosed
- * by a pair of matching rcu_read_lock() and rcu_read_unlock()
- * operations.
- *
- * The following ordering guarantees are offered by this hash table:
- *
- * A.1) "read" after "write": if there is ordering between a write and a
- * later read, then the read is guaranteed to see the write or some
- * later write.
- * A.2) "read traversal" after "write": given that there is dependency
- * ordering between reads in a "read traversal", if there is
- * ordering between a write and the first read of the traversal,
- * then the "read traversal" is guaranteed to see the write or
- * some later write.
- * B.1) "write" after "read": if there is ordering between a read and a
- * later write, then the read will never see the write.
- * B.2) "write" after "read traversal": given that there is dependency
- * ordering between reads in a "read traversal", if there is
- * ordering between the last read of the traversal and a later
- * write, then the "read traversal" will never see the write.
- * C) "write" while "read traversal": if a write occurs during a "read
- * traversal", the traversal may, or may not, see the write.
- * D.1) "write" after "write": if there is ordering between a write and
- * a later write, then the later write is guaranteed to see the
- * effects of the first write.
- * D.2) Concurrent "write" pairs: The system will assign an arbitrary
- * order to any pair of concurrent conflicting writes.
- * Non-conflicting writes (for example, to different keys) are
- * unordered.
- * E) If a grace period separates a "del" or "replace" operation
- * and a subsequent operation, then that subsequent operation is
- * guaranteed not to see the removed item.
- * F) Uniqueness guarantee: given a hash table that does not contain
- * duplicate items for a given key, there will only be one item in
- * the hash table after an arbitrary sequence of add_unique and/or
- * add_replace operations. Note, however, that a pair of
- * concurrent read operations might well access two different items
- * with that key.
- * G.1) If a pair of lookups for a given key are ordered (e.g. by a
- * memory barrier), then the second lookup will return the same
- * node as the previous lookup, or some later node.
- * G.2) A "read traversal" that starts after the end of a prior "read
- * traversal" (ordered by memory barriers) is guaranteed to see the
- * same nodes as the previous traversal, or some later nodes.
- * G.3) Concurrent "read" pairs: concurrent reads are unordered. For
- * example, if a pair of reads to the same key run concurrently
- * with an insertion of that same key, the reads remain unordered
- * regardless of their return values. In other words, you cannot
- * rely on the values returned by the reads to deduce ordering.
- *
- * Progress guarantees:
- *
- * * Reads are wait-free. These operations always move forward in the
- * hash table linked list, and this list has no loop.
- * * Writes are lock-free. Any retry loop performed by a write operation
- * is triggered by progress made within another update operation.
- *
- * Bucket node tables:
- *
- * hash table hash table the last all bucket node tables
- * order size bucket node 0 1 2 3 4 5 6(index)
- * table size
- * 0 1 1 1
- * 1 2 1 1 1
- * 2 4 2 1 1 2
- * 3 8 4 1 1 2 4
- * 4 16 8 1 1 2 4 8
- * 5 32 16 1 1 2 4 8 16
- * 6 64 32 1 1 2 4 8 16 32
- *
- * When growing/shrinking, we only focus on the last bucket node table
- * which size is (!order ? 1 : (1 << (order -1))).
- *
- * Example for growing/shrinking:
- * grow hash table from order 5 to 6: init the index=6 bucket node table
- * shrink hash table from order 6 to 5: fini the index=6 bucket node table
- *
- * A bit of ascii art explanation:
- *
- * The order index is the off-by-one compared to the actual power of 2
- * because we use index 0 to deal with the 0 special-case.
- *
- * This shows the nodes for a small table ordered by reversed bits:
- *
- * bits reverse
- * 0 000 000
- * 4 100 001
- * 2 010 010
- * 6 110 011
- * 1 001 100
- * 5 101 101
- * 3 011 110
- * 7 111 111
- *
- * This shows the nodes in order of non-reversed bits, linked by
- * reversed-bit order.
- *
- * order bits reverse
- * 0 0 000 000
- * 1 | 1 001 100 <-
- * 2 | | 2 010 010 <- |
- * | | | 3 011 110 | <- |
- * 3 -> | | | 4 100 001 | |
- * -> | | 5 101 101 |
- * -> | 6 110 011
- * -> 7 111 111
- */
-
-#define _LGPL_SOURCE
-#define _GNU_SOURCE
-#include <stdlib.h>
-#include <errno.h>
-#include <assert.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <string.h>
-#include <sched.h>
-#include <unistd.h>
-
-#include "config.h"
-#include "compat-getcpu.h"
-#include <urcu-pointer.h>
-#include <urcu-call-rcu.h>
-#include <urcu-flavor.h>
-#include <urcu/arch.h>
-#include <urcu/uatomic.h>
-#include <urcu/compiler.h>
-#include <urcu/rculfhash.h>
-#include <rculfhash-internal.h>
-#include <stdio.h>
-#include <pthread.h>
-
-/*
- * Split-counters lazily update the global counter each 1024
- * addition/removal. It automatically keeps track of resize required.
- * We use the bucket length as indicator for need to expand for small
- * tables and machines lacking per-cpu data support.
- */
-#define COUNT_COMMIT_ORDER 10
-#define DEFAULT_SPLIT_COUNT_MASK 0xFUL
-#define CHAIN_LEN_TARGET 1
-#define CHAIN_LEN_RESIZE_THRESHOLD 3
-
-/*
- * Define the minimum table size.
- */
-#define MIN_TABLE_ORDER 0
-#define MIN_TABLE_SIZE (1UL << MIN_TABLE_ORDER)
-
-/*
- * Minimum number of bucket nodes to touch per thread to parallelize grow/shrink.
- */
-#define MIN_PARTITION_PER_THREAD_ORDER 12
-#define MIN_PARTITION_PER_THREAD (1UL << MIN_PARTITION_PER_THREAD_ORDER)
-
-/*
- * The removed flag needs to be updated atomically with the pointer.
- * It indicates that no node must attach to the node scheduled for
- * removal, and that node garbage collection must be performed.
- * The bucket flag does not require to be updated atomically with the
- * pointer, but it is added as a pointer low bit flag to save space.
- * The "removal owner" flag is used to detect which of the "del"
- * operation that has set the "removed flag" gets to return the removed
- * node to its caller. Note that the replace operation does not need to
- * iteract with the "removal owner" flag, because it validates that
- * the "removed" flag is not set before performing its cmpxchg.
- */
-#define REMOVED_FLAG (1UL << 0)
-#define BUCKET_FLAG (1UL << 1)
-#define REMOVAL_OWNER_FLAG (1UL << 2)
-#define FLAGS_MASK ((1UL << 3) - 1)
-
-/* Value of the end pointer. Should not interact with flags. */
-#define END_VALUE NULL
-
-/*
- * ht_items_count: Split-counters counting the number of node addition
- * and removal in the table. Only used if the CDS_LFHT_ACCOUNTING flag
- * is set at hash table creation.
- *
- * These are free-running counters, never reset to zero. They count the
- * number of add/remove, and trigger every (1 << COUNT_COMMIT_ORDER)
- * operations to update the global counter. We choose a power-of-2 value
- * for the trigger to deal with 32 or 64-bit overflow of the counter.
- */
-struct ht_items_count {
- unsigned long add, del;
-} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
-
-/*
- * rcu_resize_work: Contains arguments passed to RCU worker thread
- * responsible for performing lazy resize.
- */
-struct rcu_resize_work {
- struct rcu_head head;
- struct cds_lfht *ht;
-};
-
-/*
- * partition_resize_work: Contains arguments passed to worker threads
- * executing the hash table resize on partitions of the hash table
- * assigned to each processor's worker thread.
- */
-struct partition_resize_work {
- pthread_t thread_id;
- struct cds_lfht *ht;
- unsigned long i, start, len;
- void (*fct)(struct cds_lfht *ht, unsigned long i,
- unsigned long start, unsigned long len);
-};
-
-/*
- * Algorithm to reverse bits in a word by lookup table, extended to
- * 64-bit words.
- * Source:
- * http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
- * Originally from Public Domain.
- */
-
-static const uint8_t BitReverseTable256[256] =
-{
-#define R2(n) (n), (n) + 2*64, (n) + 1*64, (n) + 3*64
-#define R4(n) R2(n), R2((n) + 2*16), R2((n) + 1*16), R2((n) + 3*16)
-#define R6(n) R4(n), R4((n) + 2*4 ), R4((n) + 1*4 ), R4((n) + 3*4 )
- R6(0), R6(2), R6(1), R6(3)
-};
-#undef R2
-#undef R4
-#undef R6
-
-static
-uint8_t bit_reverse_u8(uint8_t v)
-{
- return BitReverseTable256[v];
-}
-
-#if (CAA_BITS_PER_LONG == 32)
-static
-uint32_t bit_reverse_u32(uint32_t v)
-{
- return ((uint32_t) bit_reverse_u8(v) << 24) |
- ((uint32_t) bit_reverse_u8(v >> 8) << 16) |
- ((uint32_t) bit_reverse_u8(v >> 16) << 8) |
- ((uint32_t) bit_reverse_u8(v >> 24));
-}
-#else
-static
-uint64_t bit_reverse_u64(uint64_t v)
-{
- return ((uint64_t) bit_reverse_u8(v) << 56) |
- ((uint64_t) bit_reverse_u8(v >> 8) << 48) |
- ((uint64_t) bit_reverse_u8(v >> 16) << 40) |
- ((uint64_t) bit_reverse_u8(v >> 24) << 32) |
- ((uint64_t) bit_reverse_u8(v >> 32) << 24) |
- ((uint64_t) bit_reverse_u8(v >> 40) << 16) |
- ((uint64_t) bit_reverse_u8(v >> 48) << 8) |
- ((uint64_t) bit_reverse_u8(v >> 56));
-}
-#endif
-
-static
-unsigned long bit_reverse_ulong(unsigned long v)
-{
-#if (CAA_BITS_PER_LONG == 32)
- return bit_reverse_u32(v);
-#else
- return bit_reverse_u64(v);
-#endif
-}
-
-/*
- * fls: returns the position of the most significant bit.
- * Returns 0 if no bit is set, else returns the position of the most
- * significant bit (from 1 to 32 on 32-bit, from 1 to 64 on 64-bit).
- */
-#if defined(__i386) || defined(__x86_64)
-static inline
-unsigned int fls_u32(uint32_t x)
-{
- int r;
-
- __asm__ ("bsrl %1,%0\n\t"
- "jnz 1f\n\t"
- "movl $-1,%0\n\t"
- "1:\n\t"
- : "=r" (r) : "rm" (x));
- return r + 1;
-}
-#define HAS_FLS_U32
-#endif
-
-#if defined(__x86_64)
-static inline
-unsigned int fls_u64(uint64_t x)
-{
- long r;
-
- __asm__ ("bsrq %1,%0\n\t"
- "jnz 1f\n\t"
- "movq $-1,%0\n\t"
- "1:\n\t"
- : "=r" (r) : "rm" (x));
- return r + 1;
-}
-#define HAS_FLS_U64
-#endif
-
-#ifndef HAS_FLS_U64
-static __attribute__((unused))
-unsigned int fls_u64(uint64_t x)
-{
- unsigned int r = 64;
-
- if (!x)
- return 0;
-
- if (!(x & 0xFFFFFFFF00000000ULL)) {
- x <<= 32;
- r -= 32;
- }
- if (!(x & 0xFFFF000000000000ULL)) {
- x <<= 16;
- r -= 16;
- }
- if (!(x & 0xFF00000000000000ULL)) {
- x <<= 8;
- r -= 8;
- }
- if (!(x & 0xF000000000000000ULL)) {
- x <<= 4;
- r -= 4;
- }
- if (!(x & 0xC000000000000000ULL)) {
- x <<= 2;
- r -= 2;
- }
- if (!(x & 0x8000000000000000ULL)) {
- x <<= 1;
- r -= 1;
- }
- return r;
-}
-#endif
-
-#ifndef HAS_FLS_U32
-static __attribute__((unused))
-unsigned int fls_u32(uint32_t x)
-{
- unsigned int r = 32;
-
- if (!x)
- return 0;
- if (!(x & 0xFFFF0000U)) {
- x <<= 16;
- r -= 16;
- }
- if (!(x & 0xFF000000U)) {
- x <<= 8;
- r -= 8;
- }
- if (!(x & 0xF0000000U)) {
- x <<= 4;
- r -= 4;
- }
- if (!(x & 0xC0000000U)) {
- x <<= 2;
- r -= 2;
- }
- if (!(x & 0x80000000U)) {
- x <<= 1;
- r -= 1;
- }
- return r;
-}
-#endif
-
-unsigned int cds_lfht_fls_ulong(unsigned long x)
-{
-#if (CAA_BITS_PER_LONG == 32)
- return fls_u32(x);
-#else
- return fls_u64(x);
-#endif
-}
-
-/*
- * Return the minimum order for which x <= (1UL << order).
- * Return -1 if x is 0.
- */
-int cds_lfht_get_count_order_u32(uint32_t x)
-{
- if (!x)
- return -1;
-
- return fls_u32(x - 1);
-}
-
-/*
- * Return the minimum order for which x <= (1UL << order).
- * Return -1 if x is 0.
- */
-int cds_lfht_get_count_order_ulong(unsigned long x)
-{
- if (!x)
- return -1;
-
- return cds_lfht_fls_ulong(x - 1);
-}
-
-static
-void cds_lfht_resize_lazy_grow(struct cds_lfht *ht, unsigned long size, int growth);
-
-static
-void cds_lfht_resize_lazy_count(struct cds_lfht *ht, unsigned long size,
- unsigned long count);
-
-static long nr_cpus_mask = -1;
-static long split_count_mask = -1;
-static int split_count_order = -1;
-
-#if defined(HAVE_SYSCONF)
-static void ht_init_nr_cpus_mask(void)
-{
- long maxcpus;
-
- maxcpus = sysconf(_SC_NPROCESSORS_CONF);
- if (maxcpus <= 0) {
- nr_cpus_mask = -2;
- return;
- }
- /*
- * round up number of CPUs to next power of two, so we
- * can use & for modulo.
- */
- maxcpus = 1UL << cds_lfht_get_count_order_ulong(maxcpus);
- nr_cpus_mask = maxcpus - 1;
-}
-#else /* #if defined(HAVE_SYSCONF) */
-static void ht_init_nr_cpus_mask(void)
-{
- nr_cpus_mask = -2;
-}
-#endif /* #else #if defined(HAVE_SYSCONF) */
-
-static
-void alloc_split_items_count(struct cds_lfht *ht)
-{
- if (nr_cpus_mask == -1) {
- ht_init_nr_cpus_mask();
- if (nr_cpus_mask < 0)
- split_count_mask = DEFAULT_SPLIT_COUNT_MASK;
- else
- split_count_mask = nr_cpus_mask;
- split_count_order =
- cds_lfht_get_count_order_ulong(split_count_mask + 1);
- }
-
- assert(split_count_mask >= 0);
-
- if (ht->flags & CDS_LFHT_ACCOUNTING) {
- ht->split_count = calloc(split_count_mask + 1,
- sizeof(struct ht_items_count));
- assert(ht->split_count);
- } else {
- ht->split_count = NULL;
- }
-}
-
-static
-void free_split_items_count(struct cds_lfht *ht)
-{
- poison_free(ht->split_count);
-}
-
-static
-int ht_get_split_count_index(unsigned long hash)
-{
- int cpu;
-
- assert(split_count_mask >= 0);
- cpu = urcu_sched_getcpu();
- if (caa_unlikely(cpu < 0))
- return hash & split_count_mask;
- else
- return cpu & split_count_mask;
-}
-
-static
-void ht_count_add(struct cds_lfht *ht, unsigned long size, unsigned long hash)
-{
- unsigned long split_count;
- int index;
- long count;
-
- if (caa_unlikely(!ht->split_count))
- return;
- index = ht_get_split_count_index(hash);
- split_count = uatomic_add_return(&ht->split_count[index].add, 1);
- if (caa_likely(split_count & ((1UL << COUNT_COMMIT_ORDER) - 1)))
- return;
- /* Only if number of add multiple of 1UL << COUNT_COMMIT_ORDER */
-
- dbg_printf("add split count %lu\n", split_count);
- count = uatomic_add_return(&ht->count,
- 1UL << COUNT_COMMIT_ORDER);
- if (caa_likely(count & (count - 1)))
- return;
- /* Only if global count is power of 2 */
-
- if ((count >> CHAIN_LEN_RESIZE_THRESHOLD) < size)
- return;
- dbg_printf("add set global %ld\n", count);
- cds_lfht_resize_lazy_count(ht, size,
- count >> (CHAIN_LEN_TARGET - 1));
-}
-
-static
-void ht_count_del(struct cds_lfht *ht, unsigned long size, unsigned long hash)
-{
- unsigned long split_count;
- int index;
- long count;
-
- if (caa_unlikely(!ht->split_count))
- return;
- index = ht_get_split_count_index(hash);
- split_count = uatomic_add_return(&ht->split_count[index].del, 1);
- if (caa_likely(split_count & ((1UL << COUNT_COMMIT_ORDER) - 1)))
- return;
- /* Only if number of deletes multiple of 1UL << COUNT_COMMIT_ORDER */
-
- dbg_printf("del split count %lu\n", split_count);
- count = uatomic_add_return(&ht->count,
- -(1UL << COUNT_COMMIT_ORDER));
- if (caa_likely(count & (count - 1)))
- return;
- /* Only if global count is power of 2 */
-
- if ((count >> CHAIN_LEN_RESIZE_THRESHOLD) >= size)
- return;
- dbg_printf("del set global %ld\n", count);
- /*
- * Don't shrink table if the number of nodes is below a
- * certain threshold.
- */
- if (count < (1UL << COUNT_COMMIT_ORDER) * (split_count_mask + 1))
- return;
- cds_lfht_resize_lazy_count(ht, size,
- count >> (CHAIN_LEN_TARGET - 1));
-}
-
-static
-void check_resize(struct cds_lfht *ht, unsigned long size, uint32_t chain_len)
-{
- unsigned long count;
-
- if (!(ht->flags & CDS_LFHT_AUTO_RESIZE))
- return;
- count = uatomic_read(&ht->count);
- /*
- * Use bucket-local length for small table expand and for
- * environments lacking per-cpu data support.
- */
- if (count >= (1UL << (COUNT_COMMIT_ORDER + split_count_order)))
- return;
- if (chain_len > 100)
- dbg_printf("WARNING: large chain length: %u.\n",
- chain_len);
- if (chain_len >= CHAIN_LEN_RESIZE_THRESHOLD) {
- int growth;
-
- /*
- * Ideal growth calculated based on chain length.
- */
- growth = cds_lfht_get_count_order_u32(chain_len
- - (CHAIN_LEN_TARGET - 1));
- if ((ht->flags & CDS_LFHT_ACCOUNTING)
- && (size << growth)
- >= (1UL << (COUNT_COMMIT_ORDER
- + split_count_order))) {
- /*
- * If ideal growth expands the hash table size
- * beyond the "small hash table" sizes, use the
- * maximum small hash table size to attempt
- * expanding the hash table. This only applies
- * when node accounting is available, otherwise
- * the chain length is used to expand the hash
- * table in every case.
- */
- growth = COUNT_COMMIT_ORDER + split_count_order
- - cds_lfht_get_count_order_ulong(size);
- if (growth <= 0)
- return;
- }
- cds_lfht_resize_lazy_grow(ht, size, growth);
- }
-}
-
-static
-struct cds_lfht_node *clear_flag(struct cds_lfht_node *node)
-{
- return (struct cds_lfht_node *) (((unsigned long) node) & ~FLAGS_MASK);
-}
-
-static
-int is_removed(struct cds_lfht_node *node)
-{
- return ((unsigned long) node) & REMOVED_FLAG;
-}
-
-static
-int is_bucket(struct cds_lfht_node *node)
-{
- return ((unsigned long) node) & BUCKET_FLAG;
-}
-
-static
-struct cds_lfht_node *flag_bucket(struct cds_lfht_node *node)
-{
- return (struct cds_lfht_node *) (((unsigned long) node) | BUCKET_FLAG);
-}
-
-static
-int is_removal_owner(struct cds_lfht_node *node)
-{
- return ((unsigned long) node) & REMOVAL_OWNER_FLAG;
-}
-
-static
-struct cds_lfht_node *flag_removal_owner(struct cds_lfht_node *node)
-{
- return (struct cds_lfht_node *) (((unsigned long) node) | REMOVAL_OWNER_FLAG);
-}
-
-static
-struct cds_lfht_node *flag_removed_or_removal_owner(struct cds_lfht_node *node)
-{
- return (struct cds_lfht_node *) (((unsigned long) node) | REMOVED_FLAG | REMOVAL_OWNER_FLAG);
-}
-
-static
-struct cds_lfht_node *get_end(void)
-{
- return (struct cds_lfht_node *) END_VALUE;
-}
-
-static
-int is_end(struct cds_lfht_node *node)
-{
- return clear_flag(node) == (struct cds_lfht_node *) END_VALUE;
-}
-
-static
-unsigned long _uatomic_xchg_monotonic_increase(unsigned long *ptr,
- unsigned long v)
-{
- unsigned long old1, old2;
-
- old1 = uatomic_read(ptr);
- do {
- old2 = old1;
- if (old2 >= v)
- return old2;
- } while ((old1 = uatomic_cmpxchg(ptr, old2, v)) != old2);
- return old2;
-}
-
-static
-void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- return ht->mm->alloc_bucket_table(ht, order);
-}
-
-/*
- * cds_lfht_free_bucket_table() should be called with decreasing order.
- * When cds_lfht_free_bucket_table(0) is called, it means the whole
- * lfht is destroyed.
- */
-static
-void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
-{
- return ht->mm->free_bucket_table(ht, order);
-}
-
-static inline
-struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
-{
- return ht->bucket_at(ht, index);
-}
-
-static inline
-struct cds_lfht_node *lookup_bucket(struct cds_lfht *ht, unsigned long size,
- unsigned long hash)
-{
- assert(size > 0);
- return bucket_at(ht, hash & (size - 1));
-}
-
-/*
- * Remove all logically deleted nodes from a bucket up to a certain node key.
- */
-static
-void _cds_lfht_gc_bucket(struct cds_lfht_node *bucket, struct cds_lfht_node *node)
-{
- struct cds_lfht_node *iter_prev, *iter, *next, *new_next;
-
- assert(!is_bucket(bucket));
- assert(!is_removed(bucket));
- assert(!is_removal_owner(bucket));
- assert(!is_bucket(node));
- assert(!is_removed(node));
- assert(!is_removal_owner(node));
- for (;;) {
- iter_prev = bucket;
- /* We can always skip the bucket node initially */
- iter = rcu_dereference(iter_prev->next);
- assert(!is_removed(iter));
- assert(!is_removal_owner(iter));
- assert(iter_prev->reverse_hash <= node->reverse_hash);
- /*
- * We should never be called with bucket (start of chain)
- * and logically removed node (end of path compression
- * marker) being the actual same node. This would be a
- * bug in the algorithm implementation.
- */
- assert(bucket != node);
- for (;;) {
- if (caa_unlikely(is_end(iter)))
- return;
- if (caa_likely(clear_flag(iter)->reverse_hash > node->reverse_hash))
- return;
- next = rcu_dereference(clear_flag(iter)->next);
- if (caa_likely(is_removed(next)))
- break;
- iter_prev = clear_flag(iter);
- iter = next;
- }
- assert(!is_removed(iter));
- assert(!is_removal_owner(iter));
- if (is_bucket(iter))
- new_next = flag_bucket(clear_flag(next));
- else
- new_next = clear_flag(next);
- (void) uatomic_cmpxchg(&iter_prev->next, iter, new_next);
- }
-}
-
-static
-int _cds_lfht_replace(struct cds_lfht *ht, unsigned long size,
- struct cds_lfht_node *old_node,
- struct cds_lfht_node *old_next,
- struct cds_lfht_node *new_node)
-{
- struct cds_lfht_node *bucket, *ret_next;
-
- if (!old_node) /* Return -ENOENT if asked to replace NULL node */
- return -ENOENT;
-
- assert(!is_removed(old_node));
- assert(!is_removal_owner(old_node));
- assert(!is_bucket(old_node));
- assert(!is_removed(new_node));
- assert(!is_removal_owner(new_node));
- assert(!is_bucket(new_node));
- assert(new_node != old_node);
- for (;;) {
- /* Insert after node to be replaced */
- if (is_removed(old_next)) {
- /*
- * Too late, the old node has been removed under us
- * between lookup and replace. Fail.
- */
- return -ENOENT;
- }
- assert(old_next == clear_flag(old_next));
- assert(new_node != old_next);
- /*
- * REMOVAL_OWNER flag is _NEVER_ set before the REMOVED
- * flag. It is either set atomically at the same time
- * (replace) or after (del).
- */
- assert(!is_removal_owner(old_next));
- new_node->next = old_next;
- /*
- * Here is the whole trick for lock-free replace: we add
- * the replacement node _after_ the node we want to
- * replace by atomically setting its next pointer at the
- * same time we set its removal flag. Given that
- * the lookups/get next use an iterator aware of the
- * next pointer, they will either skip the old node due
- * to the removal flag and see the new node, or use
- * the old node, but will not see the new one.
- * This is a replacement of a node with another node
- * that has the same value: we are therefore not
- * removing a value from the hash table. We set both the
- * REMOVED and REMOVAL_OWNER flags atomically so we own
- * the node after successful cmpxchg.
- */
- ret_next = uatomic_cmpxchg(&old_node->next,
- old_next, flag_removed_or_removal_owner(new_node));
- if (ret_next == old_next)
- break; /* We performed the replacement. */
- old_next = ret_next;
- }
-
- /*
- * Ensure that the old node is not visible to readers anymore:
- * lookup for the node, and remove it (along with any other
- * logically removed node) if found.
- */
- bucket = lookup_bucket(ht, size, bit_reverse_ulong(old_node->reverse_hash));
- _cds_lfht_gc_bucket(bucket, new_node);
-
- assert(is_removed(CMM_LOAD_SHARED(old_node->next)));
- return 0;
-}
-
-/*
- * A non-NULL unique_ret pointer uses the "add unique" (or uniquify) add
- * mode. A NULL unique_ret allows creation of duplicate keys.
- */
-static
-void _cds_lfht_add(struct cds_lfht *ht,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- unsigned long size,
- struct cds_lfht_node *node,
- struct cds_lfht_iter *unique_ret,
- int bucket_flag)
-{
- struct cds_lfht_node *iter_prev, *iter, *next, *new_node, *new_next,
- *return_node;
- struct cds_lfht_node *bucket;
-
- assert(!is_bucket(node));
- assert(!is_removed(node));
- assert(!is_removal_owner(node));
- bucket = lookup_bucket(ht, size, hash);
- for (;;) {
- uint32_t chain_len = 0;
-
- /*
- * iter_prev points to the non-removed node prior to the
- * insert location.
- */
- iter_prev = bucket;
- /* We can always skip the bucket node initially */
- iter = rcu_dereference(iter_prev->next);
- assert(iter_prev->reverse_hash <= node->reverse_hash);
- for (;;) {
- if (caa_unlikely(is_end(iter)))
- goto insert;
- if (caa_likely(clear_flag(iter)->reverse_hash > node->reverse_hash))
- goto insert;
-
- /* bucket node is the first node of the identical-hash-value chain */
- if (bucket_flag && clear_flag(iter)->reverse_hash == node->reverse_hash)
- goto insert;
-
- next = rcu_dereference(clear_flag(iter)->next);
- if (caa_unlikely(is_removed(next)))
- goto gc_node;
-
- /* uniquely add */
- if (unique_ret
- && !is_bucket(next)
- && clear_flag(iter)->reverse_hash == node->reverse_hash) {
- struct cds_lfht_iter d_iter = { .node = node, .next = iter, };
-
- /*
- * uniquely adding inserts the node as the first
- * node of the identical-hash-value node chain.
- *
- * This semantic ensures no duplicated keys
- * should ever be observable in the table
- * (including traversing the table node by
- * node by forward iterations)
- */
- cds_lfht_next_duplicate(ht, match, key, &d_iter);
- if (!d_iter.node)
- goto insert;
-
- *unique_ret = d_iter;
- return;
- }
-
- /* Only account for identical reverse hash once */
- if (iter_prev->reverse_hash != clear_flag(iter)->reverse_hash
- && !is_bucket(next))
- check_resize(ht, size, ++chain_len);
- iter_prev = clear_flag(iter);
- iter = next;
- }
-
- insert:
- assert(node != clear_flag(iter));
- assert(!is_removed(iter_prev));
- assert(!is_removal_owner(iter_prev));
- assert(!is_removed(iter));
- assert(!is_removal_owner(iter));
- assert(iter_prev != node);
- if (!bucket_flag)
- node->next = clear_flag(iter);
- else
- node->next = flag_bucket(clear_flag(iter));
- if (is_bucket(iter))
- new_node = flag_bucket(node);
- else
- new_node = node;
- if (uatomic_cmpxchg(&iter_prev->next, iter,
- new_node) != iter) {
- continue; /* retry */
- } else {
- return_node = node;
- goto end;
- }
-
- gc_node:
- assert(!is_removed(iter));
- assert(!is_removal_owner(iter));
- if (is_bucket(iter))
- new_next = flag_bucket(clear_flag(next));
- else
- new_next = clear_flag(next);
- (void) uatomic_cmpxchg(&iter_prev->next, iter, new_next);
- /* retry */
- }
-end:
- if (unique_ret) {
- unique_ret->node = return_node;
- /* unique_ret->next left unset, never used. */
- }
-}
-
-static
-int _cds_lfht_del(struct cds_lfht *ht, unsigned long size,
- struct cds_lfht_node *node)
-{
- struct cds_lfht_node *bucket, *next;
-
- if (!node) /* Return -ENOENT if asked to delete NULL node */
- return -ENOENT;
-
- /* logically delete the node */
- assert(!is_bucket(node));
- assert(!is_removed(node));
- assert(!is_removal_owner(node));
-
- /*
- * We are first checking if the node had previously been
- * logically removed (this check is not atomic with setting the
- * logical removal flag). Return -ENOENT if the node had
- * previously been removed.
- */
- next = CMM_LOAD_SHARED(node->next); /* next is not dereferenced */
- if (caa_unlikely(is_removed(next)))
- return -ENOENT;
- assert(!is_bucket(next));
- /*
- * The del operation semantic guarantees a full memory barrier
- * before the uatomic_or atomic commit of the deletion flag.
- */
- cmm_smp_mb__before_uatomic_or();
- /*
- * We set the REMOVED_FLAG unconditionally. Note that there may
- * be more than one concurrent thread setting this flag.
- * Knowing which wins the race will be known after the garbage
- * collection phase, stay tuned!
- */
- uatomic_or(&node->next, REMOVED_FLAG);
- /* We performed the (logical) deletion. */
-
- /*
- * Ensure that the node is not visible to readers anymore: lookup for
- * the node, and remove it (along with any other logically removed node)
- * if found.
- */
- bucket = lookup_bucket(ht, size, bit_reverse_ulong(node->reverse_hash));
- _cds_lfht_gc_bucket(bucket, node);
-
- assert(is_removed(CMM_LOAD_SHARED(node->next)));
- /*
- * Last phase: atomically exchange node->next with a version
- * having "REMOVAL_OWNER_FLAG" set. If the returned node->next
- * pointer did _not_ have "REMOVAL_OWNER_FLAG" set, we now own
- * the node and win the removal race.
- * It is interesting to note that all "add" paths are forbidden
- * to change the next pointer starting from the point where the
- * REMOVED_FLAG is set, so here using a read, followed by a
- * xchg() suffice to guarantee that the xchg() will ever only
- * set the "REMOVAL_OWNER_FLAG" (or change nothing if the flag
- * was already set).
- */
- if (!is_removal_owner(uatomic_xchg(&node->next,
- flag_removal_owner(node->next))))
- return 0;
- else
- return -ENOENT;
-}
-
-static
-void *partition_resize_thread(void *arg)
-{
- struct partition_resize_work *work = arg;
-
- work->ht->flavor->register_thread();
- work->fct(work->ht, work->i, work->start, work->len);
- work->ht->flavor->unregister_thread();
- return NULL;
-}
-
-static
-void partition_resize_helper(struct cds_lfht *ht, unsigned long i,
- unsigned long len,
- void (*fct)(struct cds_lfht *ht, unsigned long i,
- unsigned long start, unsigned long len))
-{
- unsigned long partition_len, start = 0;
- struct partition_resize_work *work;
- int thread, ret;
- unsigned long nr_threads;
-
- assert(nr_cpus_mask != -1);
- if (nr_cpus_mask < 0 || len < 2 * MIN_PARTITION_PER_THREAD)
- goto fallback;
-
- /*
- * Note: nr_cpus_mask + 1 is always power of 2.
- * We spawn just the number of threads we need to satisfy the minimum
- * partition size, up to the number of CPUs in the system.
- */
- if (nr_cpus_mask > 0) {
- nr_threads = min(nr_cpus_mask + 1,
- len >> MIN_PARTITION_PER_THREAD_ORDER);
- } else {
- nr_threads = 1;
- }
- partition_len = len >> cds_lfht_get_count_order_ulong(nr_threads);
- work = calloc(nr_threads, sizeof(*work));
- if (!work) {
- dbg_printf("error allocating for resize, single-threading\n");
- goto fallback;
- }
- for (thread = 0; thread < nr_threads; thread++) {
- work[thread].ht = ht;
- work[thread].i = i;
- work[thread].len = partition_len;
- work[thread].start = thread * partition_len;
- work[thread].fct = fct;
- ret = pthread_create(&(work[thread].thread_id), ht->resize_attr,
- partition_resize_thread, &work[thread]);
- if (ret == EAGAIN) {
- /*
- * Out of resources: wait and join the threads
- * we've created, then handle leftovers.
- */
- dbg_printf("error spawning for resize, single-threading\n");
- start = work[thread].start;
- len -= start;
- nr_threads = thread;
- break;
- }
- assert(!ret);
- }
- for (thread = 0; thread < nr_threads; thread++) {
- ret = pthread_join(work[thread].thread_id, NULL);
- assert(!ret);
- }
- free(work);
-
- /*
- * A pthread_create failure above will either lead in us having
- * no threads to join or starting at a non-zero offset,
- * fallback to single thread processing of leftovers.
- */
- if (start == 0 && nr_threads > 0)
- return;
-fallback:
- ht->flavor->thread_online();
- fct(ht, i, start, len);
- ht->flavor->thread_offline();
-}
-
-/*
- * Holding RCU read lock to protect _cds_lfht_add against memory
- * reclaim that could be performed by other call_rcu worker threads (ABA
- * problem).
- *
- * When we reach a certain length, we can split this population phase over
- * many worker threads, based on the number of CPUs available in the system.
- * This should therefore take care of not having the expand lagging behind too
- * many concurrent insertion threads by using the scheduler's ability to
- * schedule bucket node population fairly with insertions.
- */
-static
-void init_table_populate_partition(struct cds_lfht *ht, unsigned long i,
- unsigned long start, unsigned long len)
-{
- unsigned long j, size = 1UL << (i - 1);
-
- assert(i > MIN_TABLE_ORDER);
- ht->flavor->read_lock();
- for (j = size + start; j < size + start + len; j++) {
- struct cds_lfht_node *new_node = bucket_at(ht, j);
-
- assert(j >= size && j < (size << 1));
- dbg_printf("init populate: order %lu index %lu hash %lu\n",
- i, j, j);
- new_node->reverse_hash = bit_reverse_ulong(j);
- _cds_lfht_add(ht, j, NULL, NULL, size, new_node, NULL, 1);
- }
- ht->flavor->read_unlock();
-}
-
-static
-void init_table_populate(struct cds_lfht *ht, unsigned long i,
- unsigned long len)
-{
- partition_resize_helper(ht, i, len, init_table_populate_partition);
-}
-
-static
-void init_table(struct cds_lfht *ht,
- unsigned long first_order, unsigned long last_order)
-{
- unsigned long i;
-
- dbg_printf("init table: first_order %lu last_order %lu\n",
- first_order, last_order);
- assert(first_order > MIN_TABLE_ORDER);
- for (i = first_order; i <= last_order; i++) {
- unsigned long len;
-
- len = 1UL << (i - 1);
- dbg_printf("init order %lu len: %lu\n", i, len);
-
- /* Stop expand if the resize target changes under us */
- if (CMM_LOAD_SHARED(ht->resize_target) < (1UL << i))
- break;
-
- cds_lfht_alloc_bucket_table(ht, i);
-
- /*
- * Set all bucket nodes reverse hash values for a level and
- * link all bucket nodes into the table.
- */
- init_table_populate(ht, i, len);
-
- /*
- * Update table size.
- */
- cmm_smp_wmb(); /* populate data before RCU size */
- CMM_STORE_SHARED(ht->size, 1UL << i);
-
- dbg_printf("init new size: %lu\n", 1UL << i);
- if (CMM_LOAD_SHARED(ht->in_progress_destroy))
- break;
- }
-}
-
-/*
- * Holding RCU read lock to protect _cds_lfht_remove against memory
- * reclaim that could be performed by other call_rcu worker threads (ABA
- * problem).
- * For a single level, we logically remove and garbage collect each node.
- *
- * As a design choice, we perform logical removal and garbage collection on a
- * node-per-node basis to simplify this algorithm. We also assume keeping good
- * cache locality of the operation would overweight possible performance gain
- * that could be achieved by batching garbage collection for multiple levels.
- * However, this would have to be justified by benchmarks.
- *
- * Concurrent removal and add operations are helping us perform garbage
- * collection of logically removed nodes. We guarantee that all logically
- * removed nodes have been garbage-collected (unlinked) before call_rcu is
- * invoked to free a hole level of bucket nodes (after a grace period).
- *
- * Logical removal and garbage collection can therefore be done in batch
- * or on a node-per-node basis, as long as the guarantee above holds.
- *
- * When we reach a certain length, we can split this removal over many worker
- * threads, based on the number of CPUs available in the system. This should
- * take care of not letting resize process lag behind too many concurrent
- * updater threads actively inserting into the hash table.
- */
-static
-void remove_table_partition(struct cds_lfht *ht, unsigned long i,
- unsigned long start, unsigned long len)
-{
- unsigned long j, size = 1UL << (i - 1);
-
- assert(i > MIN_TABLE_ORDER);
- ht->flavor->read_lock();
- for (j = size + start; j < size + start + len; j++) {
- struct cds_lfht_node *fini_bucket = bucket_at(ht, j);
- struct cds_lfht_node *parent_bucket = bucket_at(ht, j - size);
-
- assert(j >= size && j < (size << 1));
- dbg_printf("remove entry: order %lu index %lu hash %lu\n",
- i, j, j);
- /* Set the REMOVED_FLAG to freeze the ->next for gc */
- uatomic_or(&fini_bucket->next, REMOVED_FLAG);
- _cds_lfht_gc_bucket(parent_bucket, fini_bucket);
- }
- ht->flavor->read_unlock();
-}
-
-static
-void remove_table(struct cds_lfht *ht, unsigned long i, unsigned long len)
-{
- partition_resize_helper(ht, i, len, remove_table_partition);
-}
-
-/*
- * fini_table() is never called for first_order == 0, which is why
- * free_by_rcu_order == 0 can be used as criterion to know if free must
- * be called.
- */
-static
-void fini_table(struct cds_lfht *ht,
- unsigned long first_order, unsigned long last_order)
-{
- long i;
- unsigned long free_by_rcu_order = 0;
-
- dbg_printf("fini table: first_order %lu last_order %lu\n",
- first_order, last_order);
- assert(first_order > MIN_TABLE_ORDER);
- for (i = last_order; i >= first_order; i--) {
- unsigned long len;
-
- len = 1UL << (i - 1);
- dbg_printf("fini order %ld len: %lu\n", i, len);
-
- /* Stop shrink if the resize target changes under us */
- if (CMM_LOAD_SHARED(ht->resize_target) > (1UL << (i - 1)))
- break;
-
- cmm_smp_wmb(); /* populate data before RCU size */
- CMM_STORE_SHARED(ht->size, 1UL << (i - 1));
-
- /*
- * We need to wait for all add operations to reach Q.S. (and
- * thus use the new table for lookups) before we can start
- * releasing the old bucket nodes. Otherwise their lookup will
- * return a logically removed node as insert position.
- */
- ht->flavor->update_synchronize_rcu();
- if (free_by_rcu_order)
- cds_lfht_free_bucket_table(ht, free_by_rcu_order);
-
- /*
- * Set "removed" flag in bucket nodes about to be removed.
- * Unlink all now-logically-removed bucket node pointers.
- * Concurrent add/remove operation are helping us doing
- * the gc.
- */
- remove_table(ht, i, len);
-
- free_by_rcu_order = i;
-
- dbg_printf("fini new size: %lu\n", 1UL << i);
- if (CMM_LOAD_SHARED(ht->in_progress_destroy))
- break;
- }
-
- if (free_by_rcu_order) {
- ht->flavor->update_synchronize_rcu();
- cds_lfht_free_bucket_table(ht, free_by_rcu_order);
- }
-}
-
-static
-void cds_lfht_create_bucket(struct cds_lfht *ht, unsigned long size)
-{
- struct cds_lfht_node *prev, *node;
- unsigned long order, len, i;
-
- cds_lfht_alloc_bucket_table(ht, 0);
-
- dbg_printf("create bucket: order 0 index 0 hash 0\n");
- node = bucket_at(ht, 0);
- node->next = flag_bucket(get_end());
- node->reverse_hash = 0;
-
- for (order = 1; order < cds_lfht_get_count_order_ulong(size) + 1; order++) {
- len = 1UL << (order - 1);
- cds_lfht_alloc_bucket_table(ht, order);
-
- for (i = 0; i < len; i++) {
- /*
- * Now, we are trying to init the node with the
- * hash=(len+i) (which is also a bucket with the
- * index=(len+i)) and insert it into the hash table,
- * so this node has to be inserted after the bucket
- * with the index=(len+i)&(len-1)=i. And because there
- * is no other non-bucket node nor bucket node with
- * larger index/hash inserted, so the bucket node
- * being inserted should be inserted directly linked
- * after the bucket node with index=i.
- */
- prev = bucket_at(ht, i);
- node = bucket_at(ht, len + i);
-
- dbg_printf("create bucket: order %lu index %lu hash %lu\n",
- order, len + i, len + i);
- node->reverse_hash = bit_reverse_ulong(len + i);
-
- /* insert after prev */
- assert(is_bucket(prev->next));
- node->next = prev->next;
- prev->next = flag_bucket(node);
- }
- }
-}
-
-struct cds_lfht *_cds_lfht_new(unsigned long init_size,
- unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets,
- int flags,
- const struct cds_lfht_mm_type *mm,
- const struct rcu_flavor_struct *flavor,
- pthread_attr_t *attr)
-{
- struct cds_lfht *ht;
- unsigned long order;
-
- /* min_nr_alloc_buckets must be power of two */
- if (!min_nr_alloc_buckets || (min_nr_alloc_buckets & (min_nr_alloc_buckets - 1)))
- return NULL;
-
- /* init_size must be power of two */
- if (!init_size || (init_size & (init_size - 1)))
- return NULL;
-
- /*
- * Memory management plugin default.
- */
- if (!mm) {
- if (CAA_BITS_PER_LONG > 32
- && max_nr_buckets
- && max_nr_buckets <= (1ULL << 32)) {
- /*
- * For 64-bit architectures, with max number of
- * buckets small enough not to use the entire
- * 64-bit memory mapping space (and allowing a
- * fair number of hash table instances), use the
- * mmap allocator, which is faster than the
- * order allocator.
- */
- mm = &cds_lfht_mm_mmap;
- } else {
- /*
- * The fallback is to use the order allocator.
- */
- mm = &cds_lfht_mm_order;
- }
- }
-
- /* max_nr_buckets == 0 for order based mm means infinite */
- if (mm == &cds_lfht_mm_order && !max_nr_buckets)
- max_nr_buckets = 1UL << (MAX_TABLE_ORDER - 1);
-
- /* max_nr_buckets must be power of two */
- if (!max_nr_buckets || (max_nr_buckets & (max_nr_buckets - 1)))
- return NULL;
-
- min_nr_alloc_buckets = max(min_nr_alloc_buckets, MIN_TABLE_SIZE);
- init_size = max(init_size, MIN_TABLE_SIZE);
- max_nr_buckets = max(max_nr_buckets, min_nr_alloc_buckets);
- init_size = min(init_size, max_nr_buckets);
-
- ht = mm->alloc_cds_lfht(min_nr_alloc_buckets, max_nr_buckets);
- assert(ht);
- assert(ht->mm == mm);
- assert(ht->bucket_at == mm->bucket_at);
-
- ht->flags = flags;
- ht->flavor = flavor;
- ht->resize_attr = attr;
- alloc_split_items_count(ht);
- /* this mutex should not nest in read-side C.S. */
- pthread_mutex_init(&ht->resize_mutex, NULL);
- order = cds_lfht_get_count_order_ulong(init_size);
- ht->resize_target = 1UL << order;
- cds_lfht_create_bucket(ht, 1UL << order);
- ht->size = 1UL << order;
- return ht;
-}
-
-void cds_lfht_lookup(struct cds_lfht *ht, unsigned long hash,
- cds_lfht_match_fct match, const void *key,
- struct cds_lfht_iter *iter)
-{
- struct cds_lfht_node *node, *next, *bucket;
- unsigned long reverse_hash, size;
-
- reverse_hash = bit_reverse_ulong(hash);
-
- size = rcu_dereference(ht->size);
- bucket = lookup_bucket(ht, size, hash);
- /* We can always skip the bucket node initially */
- node = rcu_dereference(bucket->next);
- node = clear_flag(node);
- for (;;) {
- if (caa_unlikely(is_end(node))) {
- node = next = NULL;
- break;
- }
- if (caa_unlikely(node->reverse_hash > reverse_hash)) {
- node = next = NULL;
- break;
- }
- next = rcu_dereference(node->next);
- assert(node == clear_flag(node));
- if (caa_likely(!is_removed(next))
- && !is_bucket(next)
- && node->reverse_hash == reverse_hash
- && caa_likely(match(node, key))) {
- break;
- }
- node = clear_flag(next);
- }
- assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
- iter->node = node;
- iter->next = next;
-}
-
-void cds_lfht_next_duplicate(struct cds_lfht *ht, cds_lfht_match_fct match,
- const void *key, struct cds_lfht_iter *iter)
-{
- struct cds_lfht_node *node, *next;
- unsigned long reverse_hash;
-
- node = iter->node;
- reverse_hash = node->reverse_hash;
- next = iter->next;
- node = clear_flag(next);
-
- for (;;) {
- if (caa_unlikely(is_end(node))) {
- node = next = NULL;
- break;
- }
- if (caa_unlikely(node->reverse_hash > reverse_hash)) {
- node = next = NULL;
- break;
- }
- next = rcu_dereference(node->next);
- if (caa_likely(!is_removed(next))
- && !is_bucket(next)
- && caa_likely(match(node, key))) {
- break;
- }
- node = clear_flag(next);
- }
- assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
- iter->node = node;
- iter->next = next;
-}
-
-void cds_lfht_next(struct cds_lfht *ht, struct cds_lfht_iter *iter)
-{
- struct cds_lfht_node *node, *next;
-
- node = clear_flag(iter->next);
- for (;;) {
- if (caa_unlikely(is_end(node))) {
- node = next = NULL;
- break;
- }
- next = rcu_dereference(node->next);
- if (caa_likely(!is_removed(next))
- && !is_bucket(next)) {
- break;
- }
- node = clear_flag(next);
- }
- assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
- iter->node = node;
- iter->next = next;
-}
-
-void cds_lfht_first(struct cds_lfht *ht, struct cds_lfht_iter *iter)
-{
- /*
- * Get next after first bucket node. The first bucket node is the
- * first node of the linked list.
- */
- iter->next = bucket_at(ht, 0)->next;
- cds_lfht_next(ht, iter);
-}
-
-void cds_lfht_add(struct cds_lfht *ht, unsigned long hash,
- struct cds_lfht_node *node)
-{
- unsigned long size;
-
- node->reverse_hash = bit_reverse_ulong(hash);
- size = rcu_dereference(ht->size);
- _cds_lfht_add(ht, hash, NULL, NULL, size, node, NULL, 0);
- ht_count_add(ht, size, hash);
-}
-
-struct cds_lfht_node *cds_lfht_add_unique(struct cds_lfht *ht,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *node)
-{
- unsigned long size;
- struct cds_lfht_iter iter;
-
- node->reverse_hash = bit_reverse_ulong(hash);
- size = rcu_dereference(ht->size);
- _cds_lfht_add(ht, hash, match, key, size, node, &iter, 0);
- if (iter.node == node)
- ht_count_add(ht, size, hash);
- return iter.node;
-}
-
-struct cds_lfht_node *cds_lfht_add_replace(struct cds_lfht *ht,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *node)
-{
- unsigned long size;
- struct cds_lfht_iter iter;
-
- node->reverse_hash = bit_reverse_ulong(hash);
- size = rcu_dereference(ht->size);
- for (;;) {
- _cds_lfht_add(ht, hash, match, key, size, node, &iter, 0);
- if (iter.node == node) {
- ht_count_add(ht, size, hash);
- return NULL;
- }
-
- if (!_cds_lfht_replace(ht, size, iter.node, iter.next, node))
- return iter.node;
- }
-}
-
-int cds_lfht_replace(struct cds_lfht *ht,
- struct cds_lfht_iter *old_iter,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *new_node)
-{
- unsigned long size;
-
- new_node->reverse_hash = bit_reverse_ulong(hash);
- if (!old_iter->node)
- return -ENOENT;
- if (caa_unlikely(old_iter->node->reverse_hash != new_node->reverse_hash))
- return -EINVAL;
- if (caa_unlikely(!match(old_iter->node, key)))
- return -EINVAL;
- size = rcu_dereference(ht->size);
- return _cds_lfht_replace(ht, size, old_iter->node, old_iter->next,
- new_node);
-}
-
-int cds_lfht_del(struct cds_lfht *ht, struct cds_lfht_node *node)
-{
- unsigned long size;
- int ret;
-
- size = rcu_dereference(ht->size);
- ret = _cds_lfht_del(ht, size, node);
- if (!ret) {
- unsigned long hash;
-
- hash = bit_reverse_ulong(node->reverse_hash);
- ht_count_del(ht, size, hash);
- }
- return ret;
-}
-
-int cds_lfht_is_node_deleted(struct cds_lfht_node *node)
-{
- return is_removed(CMM_LOAD_SHARED(node->next));
-}
-
-static
-int cds_lfht_delete_bucket(struct cds_lfht *ht)
-{
- struct cds_lfht_node *node;
- unsigned long order, i, size;
-
- /* Check that the table is empty */
- node = bucket_at(ht, 0);
- do {
- node = clear_flag(node)->next;
- if (!is_bucket(node))
- return -EPERM;
- assert(!is_removed(node));
- assert(!is_removal_owner(node));
- } while (!is_end(node));
- /*
- * size accessed without rcu_dereference because hash table is
- * being destroyed.
- */
- size = ht->size;
- /* Internal sanity check: all nodes left should be buckets */
- for (i = 0; i < size; i++) {
- node = bucket_at(ht, i);
- dbg_printf("delete bucket: index %lu expected hash %lu hash %lu\n",
- i, i, bit_reverse_ulong(node->reverse_hash));
- assert(is_bucket(node->next));
- }
-
- for (order = cds_lfht_get_count_order_ulong(size); (long)order >= 0; order--)
- cds_lfht_free_bucket_table(ht, order);
-
- return 0;
-}
-
-/*
- * Should only be called when no more concurrent readers nor writers can
- * possibly access the table.
- */
-int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr)
-{
- int ret, was_online;
-
- /* Wait for in-flight resize operations to complete */
- _CMM_STORE_SHARED(ht->in_progress_destroy, 1);
- cmm_smp_mb(); /* Store destroy before load resize */
- was_online = ht->flavor->read_ongoing();
- if (was_online)
- ht->flavor->thread_offline();
- /* Calling with RCU read-side held is an error. */
- if (ht->flavor->read_ongoing()) {
- ret = -EINVAL;
- if (was_online)
- ht->flavor->thread_online();
- goto end;
- }
- while (uatomic_read(&ht->in_progress_resize))
- poll(NULL, 0, 100); /* wait for 100ms */
- if (was_online)
- ht->flavor->thread_online();
- ret = cds_lfht_delete_bucket(ht);
- if (ret)
- return ret;
- free_split_items_count(ht);
- if (attr)
- *attr = ht->resize_attr;
- ret = pthread_mutex_destroy(&ht->resize_mutex);
- if (ret)
- ret = -EBUSY;
- poison_free(ht);
-end:
- return ret;
-}
-
-void cds_lfht_count_nodes(struct cds_lfht *ht,
- long *approx_before,
- unsigned long *count,
- long *approx_after)
-{
- struct cds_lfht_node *node, *next;
- unsigned long nr_bucket = 0, nr_removed = 0;
-
- *approx_before = 0;
- if (ht->split_count) {
- int i;
-
- for (i = 0; i < split_count_mask + 1; i++) {
- *approx_before += uatomic_read(&ht->split_count[i].add);
- *approx_before -= uatomic_read(&ht->split_count[i].del);
- }
- }
-
- *count = 0;
-
- /* Count non-bucket nodes in the table */
- node = bucket_at(ht, 0);
- do {
- next = rcu_dereference(node->next);
- if (is_removed(next)) {
- if (!is_bucket(next))
- (nr_removed)++;
- else
- (nr_bucket)++;
- } else if (!is_bucket(next))
- (*count)++;
- else
- (nr_bucket)++;
- node = clear_flag(next);
- } while (!is_end(node));
- dbg_printf("number of logically removed nodes: %lu\n", nr_removed);
- dbg_printf("number of bucket nodes: %lu\n", nr_bucket);
- *approx_after = 0;
- if (ht->split_count) {
- int i;
-
- for (i = 0; i < split_count_mask + 1; i++) {
- *approx_after += uatomic_read(&ht->split_count[i].add);
- *approx_after -= uatomic_read(&ht->split_count[i].del);
- }
- }
-}
-
-/* called with resize mutex held */
-static
-void _do_cds_lfht_grow(struct cds_lfht *ht,
- unsigned long old_size, unsigned long new_size)
-{
- unsigned long old_order, new_order;
-
- old_order = cds_lfht_get_count_order_ulong(old_size);
- new_order = cds_lfht_get_count_order_ulong(new_size);
- dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n",
- old_size, old_order, new_size, new_order);
- assert(new_size > old_size);
- init_table(ht, old_order + 1, new_order);
-}
-
-/* called with resize mutex held */
-static
-void _do_cds_lfht_shrink(struct cds_lfht *ht,
- unsigned long old_size, unsigned long new_size)
-{
- unsigned long old_order, new_order;
-
- new_size = max(new_size, MIN_TABLE_SIZE);
- old_order = cds_lfht_get_count_order_ulong(old_size);
- new_order = cds_lfht_get_count_order_ulong(new_size);
- dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n",
- old_size, old_order, new_size, new_order);
- assert(new_size < old_size);
-
- /* Remove and unlink all bucket nodes to remove. */
- fini_table(ht, new_order + 1, old_order);
-}
-
-
-/* called with resize mutex held */
-static
-void _do_cds_lfht_resize(struct cds_lfht *ht)
-{
- unsigned long new_size, old_size;
-
- /*
- * Resize table, re-do if the target size has changed under us.
- */
- do {
- assert(uatomic_read(&ht->in_progress_resize));
- if (CMM_LOAD_SHARED(ht->in_progress_destroy))
- break;
- ht->resize_initiated = 1;
- old_size = ht->size;
- new_size = CMM_LOAD_SHARED(ht->resize_target);
- if (old_size < new_size)
- _do_cds_lfht_grow(ht, old_size, new_size);
- else if (old_size > new_size)
- _do_cds_lfht_shrink(ht, old_size, new_size);
- ht->resize_initiated = 0;
- /* write resize_initiated before read resize_target */
- cmm_smp_mb();
- } while (ht->size != CMM_LOAD_SHARED(ht->resize_target));
-}
-
-static
-unsigned long resize_target_grow(struct cds_lfht *ht, unsigned long new_size)
-{
- return _uatomic_xchg_monotonic_increase(&ht->resize_target, new_size);
-}
-
-static
-void resize_target_update_count(struct cds_lfht *ht,
- unsigned long count)
-{
- count = max(count, MIN_TABLE_SIZE);
- count = min(count, ht->max_nr_buckets);
- uatomic_set(&ht->resize_target, count);
-}
-
-void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size)
-{
- int was_online;
-
- was_online = ht->flavor->read_ongoing();
- if (was_online)
- ht->flavor->thread_offline();
- /* Calling with RCU read-side held is an error. */
- if (ht->flavor->read_ongoing()) {
- static int print_once;
-
- if (!CMM_LOAD_SHARED(print_once))
- fprintf(stderr, "[error] rculfhash: cds_lfht_resize "
- "called with RCU read-side lock held.\n");
- CMM_STORE_SHARED(print_once, 1);
- assert(0);
- goto end;
- }
- resize_target_update_count(ht, new_size);
- CMM_STORE_SHARED(ht->resize_initiated, 1);
- pthread_mutex_lock(&ht->resize_mutex);
- _do_cds_lfht_resize(ht);
- pthread_mutex_unlock(&ht->resize_mutex);
-end:
- if (was_online)
- ht->flavor->thread_online();
-}
-
-static
-void do_resize_cb(struct rcu_head *head)
-{
- struct rcu_resize_work *work =
- caa_container_of(head, struct rcu_resize_work, head);
- struct cds_lfht *ht = work->ht;
-
- ht->flavor->thread_offline();
- pthread_mutex_lock(&ht->resize_mutex);
- _do_cds_lfht_resize(ht);
- pthread_mutex_unlock(&ht->resize_mutex);
- ht->flavor->thread_online();
- poison_free(work);
- cmm_smp_mb(); /* finish resize before decrement */
- uatomic_dec(&ht->in_progress_resize);
-}
-
-static
-void __cds_lfht_resize_lazy_launch(struct cds_lfht *ht)
-{
- struct rcu_resize_work *work;
-
- /* Store resize_target before read resize_initiated */
- cmm_smp_mb();
- if (!CMM_LOAD_SHARED(ht->resize_initiated)) {
- uatomic_inc(&ht->in_progress_resize);
- cmm_smp_mb(); /* increment resize count before load destroy */
- if (CMM_LOAD_SHARED(ht->in_progress_destroy)) {
- uatomic_dec(&ht->in_progress_resize);
- return;
- }
- work = malloc(sizeof(*work));
- if (work == NULL) {
- dbg_printf("error allocating resize work, bailing out\n");
- uatomic_dec(&ht->in_progress_resize);
- return;
- }
- work->ht = ht;
- ht->flavor->update_call_rcu(&work->head, do_resize_cb);
- CMM_STORE_SHARED(ht->resize_initiated, 1);
- }
-}
-
-static
-void cds_lfht_resize_lazy_grow(struct cds_lfht *ht, unsigned long size, int growth)
-{
- unsigned long target_size = size << growth;
-
- target_size = min(target_size, ht->max_nr_buckets);
- if (resize_target_grow(ht, target_size) >= target_size)
- return;
-
- __cds_lfht_resize_lazy_launch(ht);
-}
-
-/*
- * We favor grow operations over shrink. A shrink operation never occurs
- * if a grow operation is queued for lazy execution. A grow operation
- * cancels any pending shrink lazy execution.
- */
-static
-void cds_lfht_resize_lazy_count(struct cds_lfht *ht, unsigned long size,
- unsigned long count)
-{
- if (!(ht->flags & CDS_LFHT_AUTO_RESIZE))
- return;
- count = max(count, MIN_TABLE_SIZE);
- count = min(count, ht->max_nr_buckets);
- if (count == size)
- return; /* Already the right size, no resize needed */
- if (count > size) { /* lazy grow */
- if (resize_target_grow(ht, count) >= count)
- return;
- } else { /* lazy shrink */
- for (;;) {
- unsigned long s;
-
- s = uatomic_cmpxchg(&ht->resize_target, size, count);
- if (s == size)
- break; /* no resize needed */
- if (s > size)
- return; /* growing is/(was just) in progress */
- if (s <= count)
- return; /* some other thread do shrink */
- size = s;
- }
- }
- __cds_lfht_resize_lazy_launch(ht);
-}
+++ /dev/null
-/*
- * rculfqueue.c
- *
- * Userspace RCU library - Lock-Free RCU Queue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu/rculfqueue.h"
-#define _LGPL_SOURCE
-#include "urcu/static/rculfqueue.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
-{
- _cds_lfq_node_init_rcu(node);
-}
-
-void cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
- void queue_call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head)))
-{
- _cds_lfq_init_rcu(q, queue_call_rcu);
-}
-
-int cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q)
-{
- return _cds_lfq_destroy_rcu(q);
-}
-
-void cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q, struct cds_lfq_node_rcu *node)
-{
- _cds_lfq_enqueue_rcu(q, node);
-}
-
-struct cds_lfq_node_rcu *
-cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
-{
- return _cds_lfq_dequeue_rcu(q);
-}
+++ /dev/null
-/*
- * rculfstack.c
- *
- * Userspace RCU library - Lock-Free RCU Stack
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Remove deprecation warnings from LGPL wrapper build. */
-#define CDS_LFS_RCU_DEPRECATED
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu/rculfstack.h"
-#define _LGPL_SOURCE
-#include "urcu/static/rculfstack.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-
-void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
-{
- _cds_lfs_node_init_rcu(node);
-}
-
-void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
-{
- _cds_lfs_init_rcu(s);
-}
-
-int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
- struct cds_lfs_node_rcu *node)
-{
- return _cds_lfs_push_rcu(s, node);
-}
-
-struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
-{
- return _cds_lfs_pop_rcu(s);
-}
--- /dev/null
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src
+
+#Add the -version-info directly here since we are only building
+# library that use the version-info
+AM_LDFLAGS=-version-info $(URCU_LIBRARY_VERSION)
+if USE_CYGWIN
+AM_LDFLAGS+=-no-undefined
+endif
+AM_CFLAGS=-Wall
+
+include_HEADERS = urcu.h urcu-bp.h urcu-call-rcu.h urcu-defer.h \
+ urcu-pointer.h urcu-qsbr.h urcu-flavor.h
+
+dist_noinst_HEADERS = urcu-die.h urcu-wait.h compat-getcpu.h
+
+
+if COMPAT_ARCH
+COMPAT=compat_arch_@ARCHTYPE@.c
+else
+COMPAT=
+endif
+
+COMPAT+=compat_futex.c
+
+RCULFHASH = rculfhash.c rculfhash-mm-order.c rculfhash-mm-chunk.c \
+ rculfhash-mm-mmap.c
+
+lib_LTLIBRARIES = liburcu-common.la \
+ liburcu.la liburcu-qsbr.la \
+ liburcu-mb.la liburcu-signal.la liburcu-bp.la \
+ liburcu-cds.la
+
+#
+# liburcu-common contains wait-free queues (needed by call_rcu) as well
+# as futex fallbacks.
+#
+liburcu_common_la_SOURCES = wfqueue.c wfcqueue.c wfstack.c $(COMPAT)
+
+liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_la_LIBADD = liburcu-common.la
+
+liburcu_qsbr_la_SOURCES = urcu-qsbr.c urcu-pointer.c $(COMPAT)
+liburcu_qsbr_la_LIBADD = liburcu-common.la
+
+liburcu_mb_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_mb_la_CFLAGS = -DRCU_MB
+liburcu_mb_la_LIBADD = liburcu-common.la
+
+liburcu_signal_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT)
+liburcu_signal_la_CFLAGS = -DRCU_SIGNAL
+liburcu_signal_la_LIBADD = liburcu-common.la
+
+liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT)
+liburcu_bp_la_LIBADD = liburcu-common.la
+
+liburcu_cds_la_SOURCES = rculfqueue.c rculfstack.c lfstack.c \
+ $(RCULFHASH) $(COMPAT)
+liburcu_cds_la_LIBADD = liburcu-common.la
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = liburcu-cds.pc liburcu.pc liburcu-bp.pc liburcu-qsbr.pc \
+ liburcu-signal.pc liburcu-mb.pc
+
+EXTRA_DIST = compat_arch_x86.c \
+ urcu-call-rcu-impl.h \
+ urcu-defer-impl.h \
+ rculfhash-internal.h
--- /dev/null
+#ifndef _COMPAT_GETCPU_H
+#define _COMPAT_GETCPU_H
+
+/*
+ * compat-getcpu.h
+ *
+ * Copyright (c) 2015 Michael Jeanson <mjeanson@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#if defined(HAVE_SCHED_GETCPU)
+#include <sched.h>
+
+static inline
+int urcu_sched_getcpu(void)
+{
+ return sched_getcpu();
+}
+#elif defined(HAVE_GETCPUID)
+#include <sys/processor.h>
+
+static inline
+int urcu_sched_getcpu(void)
+{
+ return (int) getcpuid();
+}
+#else
+
+static inline
+int urcu_sched_getcpu(void)
+{
+ return -1;
+}
+#endif
+
+#endif /* _COMPAT_GETCPU_H */
--- /dev/null
+/*
+ * compat_arch_x86.c
+ *
+ * Userspace RCU library - x86 compatibility checks
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <urcu/uatomic.h>
+
+/*
+ * Using attribute "weak" for __rcu_cas_avail and
+ * __urcu_x86_compat_mutex. Those are globally visible by the entire
+ * program, even though many shared objects may have their own version.
+ * The first version that gets loaded will be used by the entire
+ * program (executable and all shared objects).
+ */
+
+/*
+ * It does not really matter if the constructor is called before using
+ * the library, as long as the caller checks if __rcu_cas_avail < 0 and calls
+ * compat_arch_init() explicitely if needed.
+ */
+int __attribute__((constructor)) __rcu_cas_init(void);
+
+/*
+ * -1: unknown
+ * 1: available
+ * 0: unavailable
+ */
+__attribute__((weak))
+int __rcu_cas_avail = -1;
+
+__attribute__((weak))
+pthread_mutex_t __urcu_x86_compat_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+/*
+ * get_eflags/set_eflags/compare_and_swap_is_available imported from glibc
+ * 2.3.5. linuxthreads/sysdeps/i386/pt-machine.h.
+ */
+
+static int get_eflags (void)
+{
+ int res;
+ __asm__ __volatile__ ("pushfl; popl %0" : "=r" (res) : );
+ return res;
+}
+
+static void set_eflags (int newflags)
+{
+ __asm__ __volatile__ ("pushl %0; popfl" : : "r" (newflags) : "cc");
+}
+
+static int compare_and_swap_is_available (void)
+{
+ int oldflags = get_eflags ();
+ int changed;
+ /* Flip AC bit in EFLAGS. */
+ set_eflags (oldflags ^ 0x40000);
+ /* See if bit changed. */
+ changed = (get_eflags () ^ oldflags) & 0x40000;
+ /* Restore EFLAGS. */
+ set_eflags (oldflags);
+ /* If the AC flag did not change, it's a 386 and it lacks cmpxchg.
+ Otherwise, it's a 486 or above and it has cmpxchg. */
+ return changed != 0;
+}
+
+static void mutex_lock_signal_save(pthread_mutex_t *mutex, sigset_t *oldmask)
+{
+ sigset_t newmask;
+ int ret;
+
+ /* Disable signals */
+ ret = sigfillset(&newmask);
+ assert(!ret);
+ ret = pthread_sigmask(SIG_BLOCK, &newmask, oldmask);
+ assert(!ret);
+ ret = pthread_mutex_lock(&__urcu_x86_compat_mutex);
+ assert(!ret);
+}
+
+static void mutex_lock_signal_restore(pthread_mutex_t *mutex, sigset_t *oldmask)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(&__urcu_x86_compat_mutex);
+ assert(!ret);
+ ret = pthread_sigmask(SIG_SETMASK, oldmask, NULL);
+ assert(!ret);
+}
+
+unsigned long _compat_uatomic_set(void *addr, unsigned long _new, int len)
+{
+ sigset_t mask;
+ unsigned long result;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ *(unsigned char *)addr = (unsigned char)_new;
+ result = *(unsigned char *)addr;
+ break;
+ case 2:
+ *(unsigned short *)addr = (unsigned short)_new;
+ result = *(unsigned short *)addr;
+ break;
+ case 4:
+ *(unsigned int *)addr = (unsigned int)_new;
+ result = *(unsigned int *)addr;
+ break;
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ result = 0;
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+ return result;
+}
+
+unsigned long _compat_uatomic_xchg(void *addr, unsigned long _new, int len)
+{
+ sigset_t mask;
+ unsigned long retval;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ retval = *(unsigned char *)addr;
+ *(unsigned char *)addr = (unsigned char)_new;
+ break;
+ case 2:
+ retval = *(unsigned short *)addr;
+ *(unsigned short *)addr = (unsigned short)_new;
+ break;
+ case 4:
+ retval = *(unsigned int *)addr;
+ *(unsigned int *)addr = (unsigned int)_new;
+ break;
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ retval = 0; /* silence gcc warnings */
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+ return retval;
+}
+
+unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
+ unsigned long _new, int len)
+{
+ unsigned long retval;
+ sigset_t mask;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ {
+ unsigned char result = *(unsigned char *)addr;
+ if (result == (unsigned char)old)
+ *(unsigned char *)addr = (unsigned char)_new;
+ retval = result;
+ break;
+ }
+ case 2:
+ {
+ unsigned short result = *(unsigned short *)addr;
+ if (result == (unsigned short)old)
+ *(unsigned short *)addr = (unsigned short)_new;
+ retval = result;
+ break;
+ }
+ case 4:
+ {
+ unsigned int result = *(unsigned int *)addr;
+ if (result == (unsigned int)old)
+ *(unsigned int *)addr = (unsigned int)_new;
+ retval = result;
+ break;
+ }
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ retval = 0; /* silence gcc warnings */
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+ return retval;
+}
+
+void _compat_uatomic_or(void *addr, unsigned long v, int len)
+{
+ sigset_t mask;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ *(unsigned char *)addr |= (unsigned char)v;
+ break;
+ case 2:
+ *(unsigned short *)addr |= (unsigned short)v;
+ break;
+ case 4:
+ *(unsigned int *)addr |= (unsigned int)v;
+ break;
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+}
+
+void _compat_uatomic_and(void *addr, unsigned long v, int len)
+{
+ sigset_t mask;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ *(unsigned char *)addr &= (unsigned char)v;
+ break;
+ case 2:
+ *(unsigned short *)addr &= (unsigned short)v;
+ break;
+ case 4:
+ *(unsigned int *)addr &= (unsigned int)v;
+ break;
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+}
+
+unsigned long _compat_uatomic_add_return(void *addr, unsigned long v, int len)
+{
+ sigset_t mask;
+ unsigned long result;
+
+ mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask);
+ switch (len) {
+ case 1:
+ *(unsigned char *)addr += (unsigned char)v;
+ result = *(unsigned char *)addr;
+ break;
+ case 2:
+ *(unsigned short *)addr += (unsigned short)v;
+ result = *(unsigned short *)addr;
+ break;
+ case 4:
+ *(unsigned int *)addr += (unsigned int)v;
+ result = *(unsigned int *)addr;
+ break;
+ default:
+ /*
+ * generate an illegal instruction. Cannot catch this with
+ * linker tricks when optimizations are disabled.
+ */
+ result = 0; /* silence gcc warnings */
+ __asm__ __volatile__("ud2");
+ }
+ mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask);
+ return result;
+}
+
+int __rcu_cas_init(void)
+{
+ if (__rcu_cas_avail < 0)
+ __rcu_cas_avail = compare_and_swap_is_available();
+ return __rcu_cas_avail;
+}
--- /dev/null
+/*
+ * compat_futex.c
+ *
+ * Userspace RCU library - sys_futex compatibility code
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <errno.h>
+#include <poll.h>
+#include <stdint.h>
+
+#include <urcu/arch.h>
+#include <urcu/futex.h>
+#include <urcu/system.h>
+
+/*
+ * Using attribute "weak" for __urcu_compat_futex_lock and
+ * __urcu_compat_futex_cond. Those are globally visible by the entire
+ * program, even though many shared objects may have their own version.
+ * The first version that gets loaded will be used by the entire program
+ * (executable and all shared objects).
+ */
+
+__attribute__((weak))
+pthread_mutex_t __urcu_compat_futex_lock = PTHREAD_MUTEX_INITIALIZER;
+__attribute__((weak))
+pthread_cond_t __urcu_compat_futex_cond = PTHREAD_COND_INITIALIZER;
+
+/*
+ * _NOT SIGNAL-SAFE_. pthread_cond is not signal-safe anyway. Though.
+ * For now, timeout, uaddr2 and val3 are unused.
+ * Waiter will relinquish the CPU until woken up.
+ */
+
+int compat_futex_noasync(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ int ret = 0, lockret;
+
+ /*
+ * Check if NULL. Don't let users expect that they are taken into
+ * account.
+ */
+ assert(!timeout);
+ assert(!uaddr2);
+ assert(!val3);
+
+ /*
+ * memory barriers to serialize with the previous uaddr modification.
+ */
+ cmm_smp_mb();
+
+ lockret = pthread_mutex_lock(&__urcu_compat_futex_lock);
+ if (lockret) {
+ errno = lockret;
+ ret = -1;
+ goto end;
+ }
+ switch (op) {
+ case FUTEX_WAIT:
+ /*
+ * Wait until *uaddr is changed to something else than "val".
+ * Comparing *uaddr content against val figures out which
+ * thread has been awakened.
+ */
+ while (CMM_LOAD_SHARED(*uaddr) == val)
+ pthread_cond_wait(&__urcu_compat_futex_cond,
+ &__urcu_compat_futex_lock);
+ break;
+ case FUTEX_WAKE:
+ /*
+ * Each wake is sending a broadcast, thus attempting wakeup of
+ * all awaiting threads, independently of their respective
+ * uaddr.
+ */
+ pthread_cond_broadcast(&__urcu_compat_futex_cond);
+ break;
+ default:
+ errno = EINVAL;
+ ret = -1;
+ }
+ lockret = pthread_mutex_unlock(&__urcu_compat_futex_lock);
+ if (lockret) {
+ errno = lockret;
+ ret = -1;
+ }
+end:
+ return ret;
+}
+
+/*
+ * _ASYNC SIGNAL-SAFE_.
+ * For now, timeout, uaddr2 and val3 are unused.
+ * Waiter will busy-loop trying to read the condition.
+ * It is OK to use compat_futex_async() on a futex address on which
+ * futex() WAKE operations are also performed.
+ */
+
+int compat_futex_async(int32_t *uaddr, int op, int32_t val,
+ const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
+{
+ int ret = 0;
+
+ /*
+ * Check if NULL. Don't let users expect that they are taken into
+ * account.
+ */
+ assert(!timeout);
+ assert(!uaddr2);
+ assert(!val3);
+
+ /*
+ * Ensure previous memory operations on uaddr have completed.
+ */
+ cmm_smp_mb();
+
+ switch (op) {
+ case FUTEX_WAIT:
+ while (CMM_LOAD_SHARED(*uaddr) == val) {
+ if (poll(NULL, 0, 10) < 0) {
+ ret = -1;
+ /* Keep poll errno. Caller handles EINTR. */
+ goto end;
+ }
+ }
+ break;
+ case FUTEX_WAKE:
+ break;
+ default:
+ errno = EINVAL;
+ ret = -1;
+ }
+end:
+ return ret;
+}
--- /dev/null
+/*
+ * lfstack.c
+ *
+ * Userspace RCU library - Lock-Free Stack
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu/lfstack.h"
+#define _LGPL_SOURCE
+#include "urcu/static/lfstack.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void cds_lfs_node_init(struct cds_lfs_node *node)
+{
+ _cds_lfs_node_init(node);
+}
+
+void cds_lfs_init(struct cds_lfs_stack *s)
+{
+ _cds_lfs_init(s);
+}
+
+void cds_lfs_destroy(struct cds_lfs_stack *s)
+{
+ _cds_lfs_destroy(s);
+}
+
+void __cds_lfs_init(struct __cds_lfs_stack *s)
+{
+ ___cds_lfs_init(s);
+}
+
+bool cds_lfs_empty(cds_lfs_stack_ptr_t s)
+{
+ return _cds_lfs_empty(s);
+}
+
+bool cds_lfs_push(cds_lfs_stack_ptr_t s, struct cds_lfs_node *node)
+{
+ return _cds_lfs_push(s, node);
+}
+
+struct cds_lfs_node *cds_lfs_pop_blocking(struct cds_lfs_stack *s)
+{
+ return _cds_lfs_pop_blocking(s);
+}
+
+struct cds_lfs_head *cds_lfs_pop_all_blocking(struct cds_lfs_stack *s)
+{
+ return _cds_lfs_pop_all_blocking(s);
+}
+
+void cds_lfs_pop_lock(struct cds_lfs_stack *s)
+{
+ _cds_lfs_pop_lock(s);
+}
+
+void cds_lfs_pop_unlock(struct cds_lfs_stack *s)
+{
+ _cds_lfs_pop_unlock(s);
+}
+
+struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s)
+{
+ return ___cds_lfs_pop(s);
+}
+
+struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s)
+{
+ return ___cds_lfs_pop_all(s);
+}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU BulletProof
+Description: A userspace RCU (read-copy-update) library, bulletproof version
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu-bp
+Cflags: -I${includedir}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU Concurrent Data Structures
+Description: Data structures leveraging RCU and atomic operations to provide efficient concurrency-aware storage
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu-cds
+Cflags: -I${includedir}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU Memory barriers
+Description: A userspace RCU (read-copy-update) library, memory barriers version
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu-mb
+Cflags: -I${includedir}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU QSBR
+Description: A userspace RCU (read-copy-update) library, quiescent state version
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu-qsbr
+Cflags: -I${includedir}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU signal
+Description: A userspace RCU (read-copy-update) library, signal version
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu-signal
+Cflags: -I${includedir}
--- /dev/null
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Userspace RCU
+Description: A userspace RCU (read-copy-update) library, standard version
+Version: @PACKAGE_VERSION@
+Requires:
+Libs: -L${libdir} -lurcu
+Cflags: -I${includedir}
--- /dev/null
+#ifndef _URCU_RCULFHASH_INTERNAL_H
+#define _URCU_RCULFHASH_INTERNAL_H
+
+/*
+ * urcu/rculfhash-internal.h
+ *
+ * Internal header for Lock-Free RCU Hash Table
+ *
+ * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/rculfhash.h>
+#include <stdio.h>
+
+#ifdef DEBUG
+#define dbg_printf(fmt, args...) printf("[debug rculfhash] " fmt, ## args)
+#else
+#define dbg_printf(fmt, args...) \
+do { \
+ /* do nothing but check printf format */ \
+ if (0) \
+ printf("[debug rculfhash] " fmt, ## args); \
+} while (0)
+#endif
+
+#if (CAA_BITS_PER_LONG == 32)
+#define MAX_TABLE_ORDER 32
+#else
+#define MAX_TABLE_ORDER 64
+#endif
+
+#define MAX_CHUNK_TABLE (1UL << 10)
+
+#ifndef min
+#define min(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef max
+#define max(a, b) ((a) > (b) ? (a) : (b))
+#endif
+
+struct ht_items_count;
+
+/*
+ * cds_lfht: Top-level data structure representing a lock-free hash
+ * table. Defined in the implementation file to make it be an opaque
+ * cookie to users.
+ *
+ * The fields used in fast-paths are placed near the end of the
+ * structure, because we need to have a variable-sized union to contain
+ * the mm plugin fields, which are used in the fast path.
+ */
+struct cds_lfht {
+ /* Initial configuration items */
+ unsigned long max_nr_buckets;
+ const struct cds_lfht_mm_type *mm; /* memory management plugin */
+ const struct rcu_flavor_struct *flavor; /* RCU flavor */
+
+ long count; /* global approximate item count */
+
+ /*
+ * We need to put the work threads offline (QSBR) when taking this
+ * mutex, because we use synchronize_rcu within this mutex critical
+ * section, which waits on read-side critical sections, and could
+ * therefore cause grace-period deadlock if we hold off RCU G.P.
+ * completion.
+ */
+ pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */
+ pthread_attr_t *resize_attr; /* Resize threads attributes */
+ unsigned int in_progress_resize, in_progress_destroy;
+ unsigned long resize_target;
+ int resize_initiated;
+
+ /*
+ * Variables needed for add and remove fast-paths.
+ */
+ int flags;
+ unsigned long min_alloc_buckets_order;
+ unsigned long min_nr_alloc_buckets;
+ struct ht_items_count *split_count; /* split item count */
+
+ /*
+ * Variables needed for the lookup, add and remove fast-paths.
+ */
+ unsigned long size; /* always a power of 2, shared (RCU) */
+ /*
+ * bucket_at pointer is kept here to skip the extra level of
+ * dereference needed to get to "mm" (this is a fast-path).
+ */
+ struct cds_lfht_node *(*bucket_at)(struct cds_lfht *ht,
+ unsigned long index);
+ /*
+ * Dynamic length "tbl_chunk" needs to be at the end of
+ * cds_lfht.
+ */
+ union {
+ /*
+ * Contains the per order-index-level bucket node table.
+ * The size of each bucket node table is half the number
+ * of hashes contained in this order (except for order 0).
+ * The minimum allocation buckets size parameter allows
+ * combining the bucket node arrays of the lowermost
+ * levels to improve cache locality for small index orders.
+ */
+ struct cds_lfht_node *tbl_order[MAX_TABLE_ORDER];
+
+ /*
+ * Contains the bucket node chunks. The size of each
+ * bucket node chunk is ->min_alloc_size (we avoid to
+ * allocate chunks with different size). Chunks improve
+ * cache locality for small index orders, and are more
+ * friendly with environments where allocation of large
+ * contiguous memory areas is challenging due to memory
+ * fragmentation concerns or inability to use virtual
+ * memory addressing.
+ */
+ struct cds_lfht_node *tbl_chunk[0];
+
+ /*
+ * Memory mapping with room for all possible buckets.
+ * Their memory is allocated when needed.
+ */
+ struct cds_lfht_node *tbl_mmap;
+ };
+ /*
+ * End of variables needed for the lookup, add and remove
+ * fast-paths.
+ */
+};
+
+extern unsigned int cds_lfht_fls_ulong(unsigned long x);
+extern int cds_lfht_get_count_order_ulong(unsigned long x);
+
+#ifdef POISON_FREE
+#define poison_free(ptr) \
+ do { \
+ if (ptr) { \
+ memset(ptr, 0x42, sizeof(*(ptr))); \
+ free(ptr); \
+ } \
+ } while (0)
+#else
+#define poison_free(ptr) free(ptr)
+#endif
+
+static inline
+struct cds_lfht *__default_alloc_cds_lfht(
+ const struct cds_lfht_mm_type *mm,
+ unsigned long cds_lfht_size,
+ unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets)
+{
+ struct cds_lfht *ht;
+
+ ht = calloc(1, cds_lfht_size);
+ assert(ht);
+
+ ht->mm = mm;
+ ht->bucket_at = mm->bucket_at;
+ ht->min_nr_alloc_buckets = min_nr_alloc_buckets;
+ ht->min_alloc_buckets_order =
+ cds_lfht_get_count_order_ulong(min_nr_alloc_buckets);
+ ht->max_nr_buckets = max_nr_buckets;
+
+ return ht;
+}
+
+#endif /* _URCU_RCULFHASH_INTERNAL_H */
--- /dev/null
+/*
+ * rculfhash-mm-chunk.c
+ *
+ * Chunk based memory management for Lock-Free RCU Hash Table
+ *
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stddef.h>
+#include <rculfhash-internal.h>
+
+static
+void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0) {
+ ht->tbl_chunk[0] = calloc(ht->min_nr_alloc_buckets,
+ sizeof(struct cds_lfht_node));
+ assert(ht->tbl_chunk[0]);
+ } else if (order > ht->min_alloc_buckets_order) {
+ unsigned long i, len = 1UL << (order - 1 - ht->min_alloc_buckets_order);
+
+ for (i = len; i < 2 * len; i++) {
+ ht->tbl_chunk[i] = calloc(ht->min_nr_alloc_buckets,
+ sizeof(struct cds_lfht_node));
+ assert(ht->tbl_chunk[i]);
+ }
+ }
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+/*
+ * cds_lfht_free_bucket_table() should be called with decreasing order.
+ * When cds_lfht_free_bucket_table(0) is called, it means the whole
+ * lfht is destroyed.
+ */
+static
+void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0)
+ poison_free(ht->tbl_chunk[0]);
+ else if (order > ht->min_alloc_buckets_order) {
+ unsigned long i, len = 1UL << (order - 1 - ht->min_alloc_buckets_order);
+
+ for (i = len; i < 2 * len; i++)
+ poison_free(ht->tbl_chunk[i]);
+ }
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+static
+struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
+{
+ unsigned long chunk, offset;
+
+ chunk = index >> ht->min_alloc_buckets_order;
+ offset = index & (ht->min_nr_alloc_buckets - 1);
+ return &ht->tbl_chunk[chunk][offset];
+}
+
+static
+struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets)
+{
+ unsigned long nr_chunks, cds_lfht_size;
+
+ min_nr_alloc_buckets = max(min_nr_alloc_buckets,
+ max_nr_buckets / MAX_CHUNK_TABLE);
+ nr_chunks = max_nr_buckets / min_nr_alloc_buckets;
+ cds_lfht_size = offsetof(struct cds_lfht, tbl_chunk) +
+ sizeof(struct cds_lfht_node *) * nr_chunks;
+ cds_lfht_size = max(cds_lfht_size, sizeof(struct cds_lfht));
+
+ return __default_alloc_cds_lfht(
+ &cds_lfht_mm_chunk, cds_lfht_size,
+ min_nr_alloc_buckets, max_nr_buckets);
+}
+
+const struct cds_lfht_mm_type cds_lfht_mm_chunk = {
+ .alloc_cds_lfht = alloc_cds_lfht,
+ .alloc_bucket_table = cds_lfht_alloc_bucket_table,
+ .free_bucket_table = cds_lfht_free_bucket_table,
+ .bucket_at = bucket_at,
+};
--- /dev/null
+/*
+ * rculfhash-mm-mmap.c
+ *
+ * mmap/reservation based memory management for Lock-Free RCU Hash Table
+ *
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <unistd.h>
+#include <sys/mman.h>
+#include "rculfhash-internal.h"
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+/* reserve inaccessible memory space without allocation any memory */
+static void *memory_map(size_t length)
+{
+ void *ret = mmap(NULL, length, PROT_NONE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ assert(ret != MAP_FAILED);
+ return ret;
+}
+
+static void memory_unmap(void *ptr, size_t length)
+{
+ int ret __attribute__((unused));
+
+ ret = munmap(ptr, length);
+
+ assert(ret == 0);
+}
+
+static void memory_populate(void *ptr, size_t length)
+{
+ void *ret __attribute__((unused));
+
+ ret = mmap(ptr, length, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ assert(ret == ptr);
+}
+
+/*
+ * Discard garbage memory and avoid system save it when try to swap it out.
+ * Make it still reserved, inaccessible.
+ */
+static void memory_discard(void *ptr, size_t length)
+{
+ void *ret __attribute__((unused));
+
+ ret = mmap(ptr, length, PROT_NONE,
+ MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ assert(ret == ptr);
+}
+
+static
+void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0) {
+ if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
+ /* small table */
+ ht->tbl_mmap = calloc(ht->max_nr_buckets,
+ sizeof(*ht->tbl_mmap));
+ assert(ht->tbl_mmap);
+ return;
+ }
+ /* large table */
+ ht->tbl_mmap = memory_map(ht->max_nr_buckets
+ * sizeof(*ht->tbl_mmap));
+ memory_populate(ht->tbl_mmap,
+ ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
+ } else if (order > ht->min_alloc_buckets_order) {
+ /* large table */
+ unsigned long len = 1UL << (order - 1);
+
+ assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
+ memory_populate(ht->tbl_mmap + len,
+ len * sizeof(*ht->tbl_mmap));
+ }
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+/*
+ * cds_lfht_free_bucket_table() should be called with decreasing order.
+ * When cds_lfht_free_bucket_table(0) is called, it means the whole
+ * lfht is destroyed.
+ */
+static
+void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0) {
+ if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
+ /* small table */
+ poison_free(ht->tbl_mmap);
+ return;
+ }
+ /* large table */
+ memory_unmap(ht->tbl_mmap,
+ ht->max_nr_buckets * sizeof(*ht->tbl_mmap));
+ } else if (order > ht->min_alloc_buckets_order) {
+ /* large table */
+ unsigned long len = 1UL << (order - 1);
+
+ assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
+ memory_discard(ht->tbl_mmap + len, len * sizeof(*ht->tbl_mmap));
+ }
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+static
+struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
+{
+ return &ht->tbl_mmap[index];
+}
+
+static
+struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets)
+{
+ unsigned long page_bucket_size;
+
+ page_bucket_size = getpagesize() / sizeof(struct cds_lfht_node);
+ if (max_nr_buckets <= page_bucket_size) {
+ /* small table */
+ min_nr_alloc_buckets = max_nr_buckets;
+ } else {
+ /* large table */
+ min_nr_alloc_buckets = max(min_nr_alloc_buckets,
+ page_bucket_size);
+ }
+
+ return __default_alloc_cds_lfht(
+ &cds_lfht_mm_mmap, sizeof(struct cds_lfht),
+ min_nr_alloc_buckets, max_nr_buckets);
+}
+
+const struct cds_lfht_mm_type cds_lfht_mm_mmap = {
+ .alloc_cds_lfht = alloc_cds_lfht,
+ .alloc_bucket_table = cds_lfht_alloc_bucket_table,
+ .free_bucket_table = cds_lfht_free_bucket_table,
+ .bucket_at = bucket_at,
+};
--- /dev/null
+/*
+ * rculfhash-mm-order.c
+ *
+ * Order based memory management for Lock-Free RCU Hash Table
+ *
+ * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <rculfhash-internal.h>
+
+static
+void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0) {
+ ht->tbl_order[0] = calloc(ht->min_nr_alloc_buckets,
+ sizeof(struct cds_lfht_node));
+ assert(ht->tbl_order[0]);
+ } else if (order > ht->min_alloc_buckets_order) {
+ ht->tbl_order[order] = calloc(1UL << (order -1),
+ sizeof(struct cds_lfht_node));
+ assert(ht->tbl_order[order]);
+ }
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+/*
+ * cds_lfht_free_bucket_table() should be called with decreasing order.
+ * When cds_lfht_free_bucket_table(0) is called, it means the whole
+ * lfht is destroyed.
+ */
+static
+void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ if (order == 0)
+ poison_free(ht->tbl_order[0]);
+ else if (order > ht->min_alloc_buckets_order)
+ poison_free(ht->tbl_order[order]);
+ /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
+}
+
+static
+struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
+{
+ unsigned long order;
+
+ if (index < ht->min_nr_alloc_buckets) {
+ dbg_printf("bucket index %lu order 0 aridx 0\n", index);
+ return &ht->tbl_order[0][index];
+ }
+ /*
+ * equivalent to cds_lfht_get_count_order_ulong(index + 1), but
+ * optimizes away the non-existing 0 special-case for
+ * cds_lfht_get_count_order_ulong.
+ */
+ order = cds_lfht_fls_ulong(index);
+ dbg_printf("bucket index %lu order %lu aridx %lu\n",
+ index, order, index & ((1UL << (order - 1)) - 1));
+ return &ht->tbl_order[order][index & ((1UL << (order - 1)) - 1)];
+}
+
+static
+struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets)
+{
+ return __default_alloc_cds_lfht(
+ &cds_lfht_mm_order, sizeof(struct cds_lfht),
+ min_nr_alloc_buckets, max_nr_buckets);
+}
+
+const struct cds_lfht_mm_type cds_lfht_mm_order = {
+ .alloc_cds_lfht = alloc_cds_lfht,
+ .alloc_bucket_table = cds_lfht_alloc_bucket_table,
+ .free_bucket_table = cds_lfht_free_bucket_table,
+ .bucket_at = bucket_at,
+};
--- /dev/null
+/*
+ * rculfhash.c
+ *
+ * Userspace RCU library - Lock-Free Resizable RCU Hash Table
+ *
+ * Copyright 2010-2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/*
+ * Based on the following articles:
+ * - Ori Shalev and Nir Shavit. Split-ordered lists: Lock-free
+ * extensible hash tables. J. ACM 53, 3 (May 2006), 379-405.
+ * - Michael, M. M. High performance dynamic lock-free hash tables
+ * and list-based sets. In Proceedings of the fourteenth annual ACM
+ * symposium on Parallel algorithms and architectures, ACM Press,
+ * (2002), 73-82.
+ *
+ * Some specificities of this Lock-Free Resizable RCU Hash Table
+ * implementation:
+ *
+ * - RCU read-side critical section allows readers to perform hash
+ * table lookups, as well as traversals, and use the returned objects
+ * safely by allowing memory reclaim to take place only after a grace
+ * period.
+ * - Add and remove operations are lock-free, and do not need to
+ * allocate memory. They need to be executed within RCU read-side
+ * critical section to ensure the objects they read are valid and to
+ * deal with the cmpxchg ABA problem.
+ * - add and add_unique operations are supported. add_unique checks if
+ * the node key already exists in the hash table. It ensures not to
+ * populate a duplicate key if the node key already exists in the hash
+ * table.
+ * - The resize operation executes concurrently with
+ * add/add_unique/add_replace/remove/lookup/traversal.
+ * - Hash table nodes are contained within a split-ordered list. This
+ * list is ordered by incrementing reversed-bits-hash value.
+ * - An index of bucket nodes is kept. These bucket nodes are the hash
+ * table "buckets". These buckets are internal nodes that allow to
+ * perform a fast hash lookup, similarly to a skip list. These
+ * buckets are chained together in the split-ordered list, which
+ * allows recursive expansion by inserting new buckets between the
+ * existing buckets. The split-ordered list allows adding new buckets
+ * between existing buckets as the table needs to grow.
+ * - The resize operation for small tables only allows expanding the
+ * hash table. It is triggered automatically by detecting long chains
+ * in the add operation.
+ * - The resize operation for larger tables (and available through an
+ * API) allows both expanding and shrinking the hash table.
+ * - Split-counters are used to keep track of the number of
+ * nodes within the hash table for automatic resize triggering.
+ * - Resize operation initiated by long chain detection is executed by a
+ * call_rcu thread, which keeps lock-freedom of add and remove.
+ * - Resize operations are protected by a mutex.
+ * - The removal operation is split in two parts: first, a "removed"
+ * flag is set in the next pointer within the node to remove. Then,
+ * a "garbage collection" is performed in the bucket containing the
+ * removed node (from the start of the bucket up to the removed node).
+ * All encountered nodes with "removed" flag set in their next
+ * pointers are removed from the linked-list. If the cmpxchg used for
+ * removal fails (due to concurrent garbage-collection or concurrent
+ * add), we retry from the beginning of the bucket. This ensures that
+ * the node with "removed" flag set is removed from the hash table
+ * (not visible to lookups anymore) before the RCU read-side critical
+ * section held across removal ends. Furthermore, this ensures that
+ * the node with "removed" flag set is removed from the linked-list
+ * before its memory is reclaimed. After setting the "removal" flag,
+ * only the thread which removal is the first to set the "removal
+ * owner" flag (with an xchg) into a node's next pointer is considered
+ * to have succeeded its removal (and thus owns the node to reclaim).
+ * Because we garbage-collect starting from an invariant node (the
+ * start-of-bucket bucket node) up to the "removed" node (or find a
+ * reverse-hash that is higher), we are sure that a successful
+ * traversal of the chain leads to a chain that is present in the
+ * linked-list (the start node is never removed) and that it does not
+ * contain the "removed" node anymore, even if concurrent delete/add
+ * operations are changing the structure of the list concurrently.
+ * - The add operations perform garbage collection of buckets if they
+ * encounter nodes with removed flag set in the bucket where they want
+ * to add their new node. This ensures lock-freedom of add operation by
+ * helping the remover unlink nodes from the list rather than to wait
+ * for it do to so.
+ * - There are three memory backends for the hash table buckets: the
+ * "order table", the "chunks", and the "mmap".
+ * - These bucket containers contain a compact version of the hash table
+ * nodes.
+ * - The RCU "order table":
+ * - has a first level table indexed by log2(hash index) which is
+ * copied and expanded by the resize operation. This order table
+ * allows finding the "bucket node" tables.
+ * - There is one bucket node table per hash index order. The size of
+ * each bucket node table is half the number of hashes contained in
+ * this order (except for order 0).
+ * - The RCU "chunks" is best suited for close interaction with a page
+ * allocator. It uses a linear array as index to "chunks" containing
+ * each the same number of buckets.
+ * - The RCU "mmap" memory backend uses a single memory map to hold
+ * all buckets.
+ * - synchronize_rcu is used to garbage-collect the old bucket node table.
+ *
+ * Ordering Guarantees:
+ *
+ * To discuss these guarantees, we first define "read" operation as any
+ * of the the basic cds_lfht_lookup, cds_lfht_next_duplicate,
+ * cds_lfht_first, cds_lfht_next operation, as well as
+ * cds_lfht_add_unique (failure).
+ *
+ * We define "read traversal" operation as any of the following
+ * group of operations
+ * - cds_lfht_lookup followed by iteration with cds_lfht_next_duplicate
+ * (and/or cds_lfht_next, although less common).
+ * - cds_lfht_add_unique (failure) followed by iteration with
+ * cds_lfht_next_duplicate (and/or cds_lfht_next, although less
+ * common).
+ * - cds_lfht_first followed iteration with cds_lfht_next (and/or
+ * cds_lfht_next_duplicate, although less common).
+ *
+ * We define "write" operations as any of cds_lfht_add, cds_lfht_replace,
+ * cds_lfht_add_unique (success), cds_lfht_add_replace, cds_lfht_del.
+ *
+ * When cds_lfht_add_unique succeeds (returns the node passed as
+ * parameter), it acts as a "write" operation. When cds_lfht_add_unique
+ * fails (returns a node different from the one passed as parameter), it
+ * acts as a "read" operation. A cds_lfht_add_unique failure is a
+ * cds_lfht_lookup "read" operation, therefore, any ordering guarantee
+ * referring to "lookup" imply any of "lookup" or cds_lfht_add_unique
+ * (failure).
+ *
+ * We define "prior" and "later" node as nodes observable by reads and
+ * read traversals respectively before and after a write or sequence of
+ * write operations.
+ *
+ * Hash-table operations are often cascaded, for example, the pointer
+ * returned by a cds_lfht_lookup() might be passed to a cds_lfht_next(),
+ * whose return value might in turn be passed to another hash-table
+ * operation. This entire cascaded series of operations must be enclosed
+ * by a pair of matching rcu_read_lock() and rcu_read_unlock()
+ * operations.
+ *
+ * The following ordering guarantees are offered by this hash table:
+ *
+ * A.1) "read" after "write": if there is ordering between a write and a
+ * later read, then the read is guaranteed to see the write or some
+ * later write.
+ * A.2) "read traversal" after "write": given that there is dependency
+ * ordering between reads in a "read traversal", if there is
+ * ordering between a write and the first read of the traversal,
+ * then the "read traversal" is guaranteed to see the write or
+ * some later write.
+ * B.1) "write" after "read": if there is ordering between a read and a
+ * later write, then the read will never see the write.
+ * B.2) "write" after "read traversal": given that there is dependency
+ * ordering between reads in a "read traversal", if there is
+ * ordering between the last read of the traversal and a later
+ * write, then the "read traversal" will never see the write.
+ * C) "write" while "read traversal": if a write occurs during a "read
+ * traversal", the traversal may, or may not, see the write.
+ * D.1) "write" after "write": if there is ordering between a write and
+ * a later write, then the later write is guaranteed to see the
+ * effects of the first write.
+ * D.2) Concurrent "write" pairs: The system will assign an arbitrary
+ * order to any pair of concurrent conflicting writes.
+ * Non-conflicting writes (for example, to different keys) are
+ * unordered.
+ * E) If a grace period separates a "del" or "replace" operation
+ * and a subsequent operation, then that subsequent operation is
+ * guaranteed not to see the removed item.
+ * F) Uniqueness guarantee: given a hash table that does not contain
+ * duplicate items for a given key, there will only be one item in
+ * the hash table after an arbitrary sequence of add_unique and/or
+ * add_replace operations. Note, however, that a pair of
+ * concurrent read operations might well access two different items
+ * with that key.
+ * G.1) If a pair of lookups for a given key are ordered (e.g. by a
+ * memory barrier), then the second lookup will return the same
+ * node as the previous lookup, or some later node.
+ * G.2) A "read traversal" that starts after the end of a prior "read
+ * traversal" (ordered by memory barriers) is guaranteed to see the
+ * same nodes as the previous traversal, or some later nodes.
+ * G.3) Concurrent "read" pairs: concurrent reads are unordered. For
+ * example, if a pair of reads to the same key run concurrently
+ * with an insertion of that same key, the reads remain unordered
+ * regardless of their return values. In other words, you cannot
+ * rely on the values returned by the reads to deduce ordering.
+ *
+ * Progress guarantees:
+ *
+ * * Reads are wait-free. These operations always move forward in the
+ * hash table linked list, and this list has no loop.
+ * * Writes are lock-free. Any retry loop performed by a write operation
+ * is triggered by progress made within another update operation.
+ *
+ * Bucket node tables:
+ *
+ * hash table hash table the last all bucket node tables
+ * order size bucket node 0 1 2 3 4 5 6(index)
+ * table size
+ * 0 1 1 1
+ * 1 2 1 1 1
+ * 2 4 2 1 1 2
+ * 3 8 4 1 1 2 4
+ * 4 16 8 1 1 2 4 8
+ * 5 32 16 1 1 2 4 8 16
+ * 6 64 32 1 1 2 4 8 16 32
+ *
+ * When growing/shrinking, we only focus on the last bucket node table
+ * which size is (!order ? 1 : (1 << (order -1))).
+ *
+ * Example for growing/shrinking:
+ * grow hash table from order 5 to 6: init the index=6 bucket node table
+ * shrink hash table from order 6 to 5: fini the index=6 bucket node table
+ *
+ * A bit of ascii art explanation:
+ *
+ * The order index is the off-by-one compared to the actual power of 2
+ * because we use index 0 to deal with the 0 special-case.
+ *
+ * This shows the nodes for a small table ordered by reversed bits:
+ *
+ * bits reverse
+ * 0 000 000
+ * 4 100 001
+ * 2 010 010
+ * 6 110 011
+ * 1 001 100
+ * 5 101 101
+ * 3 011 110
+ * 7 111 111
+ *
+ * This shows the nodes in order of non-reversed bits, linked by
+ * reversed-bit order.
+ *
+ * order bits reverse
+ * 0 0 000 000
+ * 1 | 1 001 100 <-
+ * 2 | | 2 010 010 <- |
+ * | | | 3 011 110 | <- |
+ * 3 -> | | | 4 100 001 | |
+ * -> | | 5 101 101 |
+ * -> | 6 110 011
+ * -> 7 111 111
+ */
+
+#define _LGPL_SOURCE
+#include <stdlib.h>
+#include <errno.h>
+#include <assert.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <sched.h>
+#include <unistd.h>
+
+#include "compat-getcpu.h"
+#include <urcu-pointer.h>
+#include <urcu-call-rcu.h>
+#include <urcu-flavor.h>
+#include <urcu/arch.h>
+#include <urcu/uatomic.h>
+#include <urcu/compiler.h>
+#include <urcu/rculfhash.h>
+#include <rculfhash-internal.h>
+#include <stdio.h>
+#include <pthread.h>
+
+/*
+ * Split-counters lazily update the global counter each 1024
+ * addition/removal. It automatically keeps track of resize required.
+ * We use the bucket length as indicator for need to expand for small
+ * tables and machines lacking per-cpu data support.
+ */
+#define COUNT_COMMIT_ORDER 10
+#define DEFAULT_SPLIT_COUNT_MASK 0xFUL
+#define CHAIN_LEN_TARGET 1
+#define CHAIN_LEN_RESIZE_THRESHOLD 3
+
+/*
+ * Define the minimum table size.
+ */
+#define MIN_TABLE_ORDER 0
+#define MIN_TABLE_SIZE (1UL << MIN_TABLE_ORDER)
+
+/*
+ * Minimum number of bucket nodes to touch per thread to parallelize grow/shrink.
+ */
+#define MIN_PARTITION_PER_THREAD_ORDER 12
+#define MIN_PARTITION_PER_THREAD (1UL << MIN_PARTITION_PER_THREAD_ORDER)
+
+/*
+ * The removed flag needs to be updated atomically with the pointer.
+ * It indicates that no node must attach to the node scheduled for
+ * removal, and that node garbage collection must be performed.
+ * The bucket flag does not require to be updated atomically with the
+ * pointer, but it is added as a pointer low bit flag to save space.
+ * The "removal owner" flag is used to detect which of the "del"
+ * operation that has set the "removed flag" gets to return the removed
+ * node to its caller. Note that the replace operation does not need to
+ * iteract with the "removal owner" flag, because it validates that
+ * the "removed" flag is not set before performing its cmpxchg.
+ */
+#define REMOVED_FLAG (1UL << 0)
+#define BUCKET_FLAG (1UL << 1)
+#define REMOVAL_OWNER_FLAG (1UL << 2)
+#define FLAGS_MASK ((1UL << 3) - 1)
+
+/* Value of the end pointer. Should not interact with flags. */
+#define END_VALUE NULL
+
+/*
+ * ht_items_count: Split-counters counting the number of node addition
+ * and removal in the table. Only used if the CDS_LFHT_ACCOUNTING flag
+ * is set at hash table creation.
+ *
+ * These are free-running counters, never reset to zero. They count the
+ * number of add/remove, and trigger every (1 << COUNT_COMMIT_ORDER)
+ * operations to update the global counter. We choose a power-of-2 value
+ * for the trigger to deal with 32 or 64-bit overflow of the counter.
+ */
+struct ht_items_count {
+ unsigned long add, del;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+/*
+ * rcu_resize_work: Contains arguments passed to RCU worker thread
+ * responsible for performing lazy resize.
+ */
+struct rcu_resize_work {
+ struct rcu_head head;
+ struct cds_lfht *ht;
+};
+
+/*
+ * partition_resize_work: Contains arguments passed to worker threads
+ * executing the hash table resize on partitions of the hash table
+ * assigned to each processor's worker thread.
+ */
+struct partition_resize_work {
+ pthread_t thread_id;
+ struct cds_lfht *ht;
+ unsigned long i, start, len;
+ void (*fct)(struct cds_lfht *ht, unsigned long i,
+ unsigned long start, unsigned long len);
+};
+
+/*
+ * Algorithm to reverse bits in a word by lookup table, extended to
+ * 64-bit words.
+ * Source:
+ * http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
+ * Originally from Public Domain.
+ */
+
+static const uint8_t BitReverseTable256[256] =
+{
+#define R2(n) (n), (n) + 2*64, (n) + 1*64, (n) + 3*64
+#define R4(n) R2(n), R2((n) + 2*16), R2((n) + 1*16), R2((n) + 3*16)
+#define R6(n) R4(n), R4((n) + 2*4 ), R4((n) + 1*4 ), R4((n) + 3*4 )
+ R6(0), R6(2), R6(1), R6(3)
+};
+#undef R2
+#undef R4
+#undef R6
+
+static
+uint8_t bit_reverse_u8(uint8_t v)
+{
+ return BitReverseTable256[v];
+}
+
+#if (CAA_BITS_PER_LONG == 32)
+static
+uint32_t bit_reverse_u32(uint32_t v)
+{
+ return ((uint32_t) bit_reverse_u8(v) << 24) |
+ ((uint32_t) bit_reverse_u8(v >> 8) << 16) |
+ ((uint32_t) bit_reverse_u8(v >> 16) << 8) |
+ ((uint32_t) bit_reverse_u8(v >> 24));
+}
+#else
+static
+uint64_t bit_reverse_u64(uint64_t v)
+{
+ return ((uint64_t) bit_reverse_u8(v) << 56) |
+ ((uint64_t) bit_reverse_u8(v >> 8) << 48) |
+ ((uint64_t) bit_reverse_u8(v >> 16) << 40) |
+ ((uint64_t) bit_reverse_u8(v >> 24) << 32) |
+ ((uint64_t) bit_reverse_u8(v >> 32) << 24) |
+ ((uint64_t) bit_reverse_u8(v >> 40) << 16) |
+ ((uint64_t) bit_reverse_u8(v >> 48) << 8) |
+ ((uint64_t) bit_reverse_u8(v >> 56));
+}
+#endif
+
+static
+unsigned long bit_reverse_ulong(unsigned long v)
+{
+#if (CAA_BITS_PER_LONG == 32)
+ return bit_reverse_u32(v);
+#else
+ return bit_reverse_u64(v);
+#endif
+}
+
+/*
+ * fls: returns the position of the most significant bit.
+ * Returns 0 if no bit is set, else returns the position of the most
+ * significant bit (from 1 to 32 on 32-bit, from 1 to 64 on 64-bit).
+ */
+#if defined(__i386) || defined(__x86_64)
+static inline
+unsigned int fls_u32(uint32_t x)
+{
+ int r;
+
+ __asm__ ("bsrl %1,%0\n\t"
+ "jnz 1f\n\t"
+ "movl $-1,%0\n\t"
+ "1:\n\t"
+ : "=r" (r) : "rm" (x));
+ return r + 1;
+}
+#define HAS_FLS_U32
+#endif
+
+#if defined(__x86_64)
+static inline
+unsigned int fls_u64(uint64_t x)
+{
+ long r;
+
+ __asm__ ("bsrq %1,%0\n\t"
+ "jnz 1f\n\t"
+ "movq $-1,%0\n\t"
+ "1:\n\t"
+ : "=r" (r) : "rm" (x));
+ return r + 1;
+}
+#define HAS_FLS_U64
+#endif
+
+#ifndef HAS_FLS_U64
+static __attribute__((unused))
+unsigned int fls_u64(uint64_t x)
+{
+ unsigned int r = 64;
+
+ if (!x)
+ return 0;
+
+ if (!(x & 0xFFFFFFFF00000000ULL)) {
+ x <<= 32;
+ r -= 32;
+ }
+ if (!(x & 0xFFFF000000000000ULL)) {
+ x <<= 16;
+ r -= 16;
+ }
+ if (!(x & 0xFF00000000000000ULL)) {
+ x <<= 8;
+ r -= 8;
+ }
+ if (!(x & 0xF000000000000000ULL)) {
+ x <<= 4;
+ r -= 4;
+ }
+ if (!(x & 0xC000000000000000ULL)) {
+ x <<= 2;
+ r -= 2;
+ }
+ if (!(x & 0x8000000000000000ULL)) {
+ x <<= 1;
+ r -= 1;
+ }
+ return r;
+}
+#endif
+
+#ifndef HAS_FLS_U32
+static __attribute__((unused))
+unsigned int fls_u32(uint32_t x)
+{
+ unsigned int r = 32;
+
+ if (!x)
+ return 0;
+ if (!(x & 0xFFFF0000U)) {
+ x <<= 16;
+ r -= 16;
+ }
+ if (!(x & 0xFF000000U)) {
+ x <<= 8;
+ r -= 8;
+ }
+ if (!(x & 0xF0000000U)) {
+ x <<= 4;
+ r -= 4;
+ }
+ if (!(x & 0xC0000000U)) {
+ x <<= 2;
+ r -= 2;
+ }
+ if (!(x & 0x80000000U)) {
+ x <<= 1;
+ r -= 1;
+ }
+ return r;
+}
+#endif
+
+unsigned int cds_lfht_fls_ulong(unsigned long x)
+{
+#if (CAA_BITS_PER_LONG == 32)
+ return fls_u32(x);
+#else
+ return fls_u64(x);
+#endif
+}
+
+/*
+ * Return the minimum order for which x <= (1UL << order).
+ * Return -1 if x is 0.
+ */
+int cds_lfht_get_count_order_u32(uint32_t x)
+{
+ if (!x)
+ return -1;
+
+ return fls_u32(x - 1);
+}
+
+/*
+ * Return the minimum order for which x <= (1UL << order).
+ * Return -1 if x is 0.
+ */
+int cds_lfht_get_count_order_ulong(unsigned long x)
+{
+ if (!x)
+ return -1;
+
+ return cds_lfht_fls_ulong(x - 1);
+}
+
+static
+void cds_lfht_resize_lazy_grow(struct cds_lfht *ht, unsigned long size, int growth);
+
+static
+void cds_lfht_resize_lazy_count(struct cds_lfht *ht, unsigned long size,
+ unsigned long count);
+
+static long nr_cpus_mask = -1;
+static long split_count_mask = -1;
+static int split_count_order = -1;
+
+#if defined(HAVE_SYSCONF)
+static void ht_init_nr_cpus_mask(void)
+{
+ long maxcpus;
+
+ maxcpus = sysconf(_SC_NPROCESSORS_CONF);
+ if (maxcpus <= 0) {
+ nr_cpus_mask = -2;
+ return;
+ }
+ /*
+ * round up number of CPUs to next power of two, so we
+ * can use & for modulo.
+ */
+ maxcpus = 1UL << cds_lfht_get_count_order_ulong(maxcpus);
+ nr_cpus_mask = maxcpus - 1;
+}
+#else /* #if defined(HAVE_SYSCONF) */
+static void ht_init_nr_cpus_mask(void)
+{
+ nr_cpus_mask = -2;
+}
+#endif /* #else #if defined(HAVE_SYSCONF) */
+
+static
+void alloc_split_items_count(struct cds_lfht *ht)
+{
+ if (nr_cpus_mask == -1) {
+ ht_init_nr_cpus_mask();
+ if (nr_cpus_mask < 0)
+ split_count_mask = DEFAULT_SPLIT_COUNT_MASK;
+ else
+ split_count_mask = nr_cpus_mask;
+ split_count_order =
+ cds_lfht_get_count_order_ulong(split_count_mask + 1);
+ }
+
+ assert(split_count_mask >= 0);
+
+ if (ht->flags & CDS_LFHT_ACCOUNTING) {
+ ht->split_count = calloc(split_count_mask + 1,
+ sizeof(struct ht_items_count));
+ assert(ht->split_count);
+ } else {
+ ht->split_count = NULL;
+ }
+}
+
+static
+void free_split_items_count(struct cds_lfht *ht)
+{
+ poison_free(ht->split_count);
+}
+
+static
+int ht_get_split_count_index(unsigned long hash)
+{
+ int cpu;
+
+ assert(split_count_mask >= 0);
+ cpu = urcu_sched_getcpu();
+ if (caa_unlikely(cpu < 0))
+ return hash & split_count_mask;
+ else
+ return cpu & split_count_mask;
+}
+
+static
+void ht_count_add(struct cds_lfht *ht, unsigned long size, unsigned long hash)
+{
+ unsigned long split_count;
+ int index;
+ long count;
+
+ if (caa_unlikely(!ht->split_count))
+ return;
+ index = ht_get_split_count_index(hash);
+ split_count = uatomic_add_return(&ht->split_count[index].add, 1);
+ if (caa_likely(split_count & ((1UL << COUNT_COMMIT_ORDER) - 1)))
+ return;
+ /* Only if number of add multiple of 1UL << COUNT_COMMIT_ORDER */
+
+ dbg_printf("add split count %lu\n", split_count);
+ count = uatomic_add_return(&ht->count,
+ 1UL << COUNT_COMMIT_ORDER);
+ if (caa_likely(count & (count - 1)))
+ return;
+ /* Only if global count is power of 2 */
+
+ if ((count >> CHAIN_LEN_RESIZE_THRESHOLD) < size)
+ return;
+ dbg_printf("add set global %ld\n", count);
+ cds_lfht_resize_lazy_count(ht, size,
+ count >> (CHAIN_LEN_TARGET - 1));
+}
+
+static
+void ht_count_del(struct cds_lfht *ht, unsigned long size, unsigned long hash)
+{
+ unsigned long split_count;
+ int index;
+ long count;
+
+ if (caa_unlikely(!ht->split_count))
+ return;
+ index = ht_get_split_count_index(hash);
+ split_count = uatomic_add_return(&ht->split_count[index].del, 1);
+ if (caa_likely(split_count & ((1UL << COUNT_COMMIT_ORDER) - 1)))
+ return;
+ /* Only if number of deletes multiple of 1UL << COUNT_COMMIT_ORDER */
+
+ dbg_printf("del split count %lu\n", split_count);
+ count = uatomic_add_return(&ht->count,
+ -(1UL << COUNT_COMMIT_ORDER));
+ if (caa_likely(count & (count - 1)))
+ return;
+ /* Only if global count is power of 2 */
+
+ if ((count >> CHAIN_LEN_RESIZE_THRESHOLD) >= size)
+ return;
+ dbg_printf("del set global %ld\n", count);
+ /*
+ * Don't shrink table if the number of nodes is below a
+ * certain threshold.
+ */
+ if (count < (1UL << COUNT_COMMIT_ORDER) * (split_count_mask + 1))
+ return;
+ cds_lfht_resize_lazy_count(ht, size,
+ count >> (CHAIN_LEN_TARGET - 1));
+}
+
+static
+void check_resize(struct cds_lfht *ht, unsigned long size, uint32_t chain_len)
+{
+ unsigned long count;
+
+ if (!(ht->flags & CDS_LFHT_AUTO_RESIZE))
+ return;
+ count = uatomic_read(&ht->count);
+ /*
+ * Use bucket-local length for small table expand and for
+ * environments lacking per-cpu data support.
+ */
+ if (count >= (1UL << (COUNT_COMMIT_ORDER + split_count_order)))
+ return;
+ if (chain_len > 100)
+ dbg_printf("WARNING: large chain length: %u.\n",
+ chain_len);
+ if (chain_len >= CHAIN_LEN_RESIZE_THRESHOLD) {
+ int growth;
+
+ /*
+ * Ideal growth calculated based on chain length.
+ */
+ growth = cds_lfht_get_count_order_u32(chain_len
+ - (CHAIN_LEN_TARGET - 1));
+ if ((ht->flags & CDS_LFHT_ACCOUNTING)
+ && (size << growth)
+ >= (1UL << (COUNT_COMMIT_ORDER
+ + split_count_order))) {
+ /*
+ * If ideal growth expands the hash table size
+ * beyond the "small hash table" sizes, use the
+ * maximum small hash table size to attempt
+ * expanding the hash table. This only applies
+ * when node accounting is available, otherwise
+ * the chain length is used to expand the hash
+ * table in every case.
+ */
+ growth = COUNT_COMMIT_ORDER + split_count_order
+ - cds_lfht_get_count_order_ulong(size);
+ if (growth <= 0)
+ return;
+ }
+ cds_lfht_resize_lazy_grow(ht, size, growth);
+ }
+}
+
+static
+struct cds_lfht_node *clear_flag(struct cds_lfht_node *node)
+{
+ return (struct cds_lfht_node *) (((unsigned long) node) & ~FLAGS_MASK);
+}
+
+static
+int is_removed(struct cds_lfht_node *node)
+{
+ return ((unsigned long) node) & REMOVED_FLAG;
+}
+
+static
+int is_bucket(struct cds_lfht_node *node)
+{
+ return ((unsigned long) node) & BUCKET_FLAG;
+}
+
+static
+struct cds_lfht_node *flag_bucket(struct cds_lfht_node *node)
+{
+ return (struct cds_lfht_node *) (((unsigned long) node) | BUCKET_FLAG);
+}
+
+static
+int is_removal_owner(struct cds_lfht_node *node)
+{
+ return ((unsigned long) node) & REMOVAL_OWNER_FLAG;
+}
+
+static
+struct cds_lfht_node *flag_removal_owner(struct cds_lfht_node *node)
+{
+ return (struct cds_lfht_node *) (((unsigned long) node) | REMOVAL_OWNER_FLAG);
+}
+
+static
+struct cds_lfht_node *flag_removed_or_removal_owner(struct cds_lfht_node *node)
+{
+ return (struct cds_lfht_node *) (((unsigned long) node) | REMOVED_FLAG | REMOVAL_OWNER_FLAG);
+}
+
+static
+struct cds_lfht_node *get_end(void)
+{
+ return (struct cds_lfht_node *) END_VALUE;
+}
+
+static
+int is_end(struct cds_lfht_node *node)
+{
+ return clear_flag(node) == (struct cds_lfht_node *) END_VALUE;
+}
+
+static
+unsigned long _uatomic_xchg_monotonic_increase(unsigned long *ptr,
+ unsigned long v)
+{
+ unsigned long old1, old2;
+
+ old1 = uatomic_read(ptr);
+ do {
+ old2 = old1;
+ if (old2 >= v)
+ return old2;
+ } while ((old1 = uatomic_cmpxchg(ptr, old2, v)) != old2);
+ return old2;
+}
+
+static
+void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ return ht->mm->alloc_bucket_table(ht, order);
+}
+
+/*
+ * cds_lfht_free_bucket_table() should be called with decreasing order.
+ * When cds_lfht_free_bucket_table(0) is called, it means the whole
+ * lfht is destroyed.
+ */
+static
+void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
+{
+ return ht->mm->free_bucket_table(ht, order);
+}
+
+static inline
+struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
+{
+ return ht->bucket_at(ht, index);
+}
+
+static inline
+struct cds_lfht_node *lookup_bucket(struct cds_lfht *ht, unsigned long size,
+ unsigned long hash)
+{
+ assert(size > 0);
+ return bucket_at(ht, hash & (size - 1));
+}
+
+/*
+ * Remove all logically deleted nodes from a bucket up to a certain node key.
+ */
+static
+void _cds_lfht_gc_bucket(struct cds_lfht_node *bucket, struct cds_lfht_node *node)
+{
+ struct cds_lfht_node *iter_prev, *iter, *next, *new_next;
+
+ assert(!is_bucket(bucket));
+ assert(!is_removed(bucket));
+ assert(!is_removal_owner(bucket));
+ assert(!is_bucket(node));
+ assert(!is_removed(node));
+ assert(!is_removal_owner(node));
+ for (;;) {
+ iter_prev = bucket;
+ /* We can always skip the bucket node initially */
+ iter = rcu_dereference(iter_prev->next);
+ assert(!is_removed(iter));
+ assert(!is_removal_owner(iter));
+ assert(iter_prev->reverse_hash <= node->reverse_hash);
+ /*
+ * We should never be called with bucket (start of chain)
+ * and logically removed node (end of path compression
+ * marker) being the actual same node. This would be a
+ * bug in the algorithm implementation.
+ */
+ assert(bucket != node);
+ for (;;) {
+ if (caa_unlikely(is_end(iter)))
+ return;
+ if (caa_likely(clear_flag(iter)->reverse_hash > node->reverse_hash))
+ return;
+ next = rcu_dereference(clear_flag(iter)->next);
+ if (caa_likely(is_removed(next)))
+ break;
+ iter_prev = clear_flag(iter);
+ iter = next;
+ }
+ assert(!is_removed(iter));
+ assert(!is_removal_owner(iter));
+ if (is_bucket(iter))
+ new_next = flag_bucket(clear_flag(next));
+ else
+ new_next = clear_flag(next);
+ (void) uatomic_cmpxchg(&iter_prev->next, iter, new_next);
+ }
+}
+
+static
+int _cds_lfht_replace(struct cds_lfht *ht, unsigned long size,
+ struct cds_lfht_node *old_node,
+ struct cds_lfht_node *old_next,
+ struct cds_lfht_node *new_node)
+{
+ struct cds_lfht_node *bucket, *ret_next;
+
+ if (!old_node) /* Return -ENOENT if asked to replace NULL node */
+ return -ENOENT;
+
+ assert(!is_removed(old_node));
+ assert(!is_removal_owner(old_node));
+ assert(!is_bucket(old_node));
+ assert(!is_removed(new_node));
+ assert(!is_removal_owner(new_node));
+ assert(!is_bucket(new_node));
+ assert(new_node != old_node);
+ for (;;) {
+ /* Insert after node to be replaced */
+ if (is_removed(old_next)) {
+ /*
+ * Too late, the old node has been removed under us
+ * between lookup and replace. Fail.
+ */
+ return -ENOENT;
+ }
+ assert(old_next == clear_flag(old_next));
+ assert(new_node != old_next);
+ /*
+ * REMOVAL_OWNER flag is _NEVER_ set before the REMOVED
+ * flag. It is either set atomically at the same time
+ * (replace) or after (del).
+ */
+ assert(!is_removal_owner(old_next));
+ new_node->next = old_next;
+ /*
+ * Here is the whole trick for lock-free replace: we add
+ * the replacement node _after_ the node we want to
+ * replace by atomically setting its next pointer at the
+ * same time we set its removal flag. Given that
+ * the lookups/get next use an iterator aware of the
+ * next pointer, they will either skip the old node due
+ * to the removal flag and see the new node, or use
+ * the old node, but will not see the new one.
+ * This is a replacement of a node with another node
+ * that has the same value: we are therefore not
+ * removing a value from the hash table. We set both the
+ * REMOVED and REMOVAL_OWNER flags atomically so we own
+ * the node after successful cmpxchg.
+ */
+ ret_next = uatomic_cmpxchg(&old_node->next,
+ old_next, flag_removed_or_removal_owner(new_node));
+ if (ret_next == old_next)
+ break; /* We performed the replacement. */
+ old_next = ret_next;
+ }
+
+ /*
+ * Ensure that the old node is not visible to readers anymore:
+ * lookup for the node, and remove it (along with any other
+ * logically removed node) if found.
+ */
+ bucket = lookup_bucket(ht, size, bit_reverse_ulong(old_node->reverse_hash));
+ _cds_lfht_gc_bucket(bucket, new_node);
+
+ assert(is_removed(CMM_LOAD_SHARED(old_node->next)));
+ return 0;
+}
+
+/*
+ * A non-NULL unique_ret pointer uses the "add unique" (or uniquify) add
+ * mode. A NULL unique_ret allows creation of duplicate keys.
+ */
+static
+void _cds_lfht_add(struct cds_lfht *ht,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ unsigned long size,
+ struct cds_lfht_node *node,
+ struct cds_lfht_iter *unique_ret,
+ int bucket_flag)
+{
+ struct cds_lfht_node *iter_prev, *iter, *next, *new_node, *new_next,
+ *return_node;
+ struct cds_lfht_node *bucket;
+
+ assert(!is_bucket(node));
+ assert(!is_removed(node));
+ assert(!is_removal_owner(node));
+ bucket = lookup_bucket(ht, size, hash);
+ for (;;) {
+ uint32_t chain_len = 0;
+
+ /*
+ * iter_prev points to the non-removed node prior to the
+ * insert location.
+ */
+ iter_prev = bucket;
+ /* We can always skip the bucket node initially */
+ iter = rcu_dereference(iter_prev->next);
+ assert(iter_prev->reverse_hash <= node->reverse_hash);
+ for (;;) {
+ if (caa_unlikely(is_end(iter)))
+ goto insert;
+ if (caa_likely(clear_flag(iter)->reverse_hash > node->reverse_hash))
+ goto insert;
+
+ /* bucket node is the first node of the identical-hash-value chain */
+ if (bucket_flag && clear_flag(iter)->reverse_hash == node->reverse_hash)
+ goto insert;
+
+ next = rcu_dereference(clear_flag(iter)->next);
+ if (caa_unlikely(is_removed(next)))
+ goto gc_node;
+
+ /* uniquely add */
+ if (unique_ret
+ && !is_bucket(next)
+ && clear_flag(iter)->reverse_hash == node->reverse_hash) {
+ struct cds_lfht_iter d_iter = { .node = node, .next = iter, };
+
+ /*
+ * uniquely adding inserts the node as the first
+ * node of the identical-hash-value node chain.
+ *
+ * This semantic ensures no duplicated keys
+ * should ever be observable in the table
+ * (including traversing the table node by
+ * node by forward iterations)
+ */
+ cds_lfht_next_duplicate(ht, match, key, &d_iter);
+ if (!d_iter.node)
+ goto insert;
+
+ *unique_ret = d_iter;
+ return;
+ }
+
+ /* Only account for identical reverse hash once */
+ if (iter_prev->reverse_hash != clear_flag(iter)->reverse_hash
+ && !is_bucket(next))
+ check_resize(ht, size, ++chain_len);
+ iter_prev = clear_flag(iter);
+ iter = next;
+ }
+
+ insert:
+ assert(node != clear_flag(iter));
+ assert(!is_removed(iter_prev));
+ assert(!is_removal_owner(iter_prev));
+ assert(!is_removed(iter));
+ assert(!is_removal_owner(iter));
+ assert(iter_prev != node);
+ if (!bucket_flag)
+ node->next = clear_flag(iter);
+ else
+ node->next = flag_bucket(clear_flag(iter));
+ if (is_bucket(iter))
+ new_node = flag_bucket(node);
+ else
+ new_node = node;
+ if (uatomic_cmpxchg(&iter_prev->next, iter,
+ new_node) != iter) {
+ continue; /* retry */
+ } else {
+ return_node = node;
+ goto end;
+ }
+
+ gc_node:
+ assert(!is_removed(iter));
+ assert(!is_removal_owner(iter));
+ if (is_bucket(iter))
+ new_next = flag_bucket(clear_flag(next));
+ else
+ new_next = clear_flag(next);
+ (void) uatomic_cmpxchg(&iter_prev->next, iter, new_next);
+ /* retry */
+ }
+end:
+ if (unique_ret) {
+ unique_ret->node = return_node;
+ /* unique_ret->next left unset, never used. */
+ }
+}
+
+static
+int _cds_lfht_del(struct cds_lfht *ht, unsigned long size,
+ struct cds_lfht_node *node)
+{
+ struct cds_lfht_node *bucket, *next;
+
+ if (!node) /* Return -ENOENT if asked to delete NULL node */
+ return -ENOENT;
+
+ /* logically delete the node */
+ assert(!is_bucket(node));
+ assert(!is_removed(node));
+ assert(!is_removal_owner(node));
+
+ /*
+ * We are first checking if the node had previously been
+ * logically removed (this check is not atomic with setting the
+ * logical removal flag). Return -ENOENT if the node had
+ * previously been removed.
+ */
+ next = CMM_LOAD_SHARED(node->next); /* next is not dereferenced */
+ if (caa_unlikely(is_removed(next)))
+ return -ENOENT;
+ assert(!is_bucket(next));
+ /*
+ * The del operation semantic guarantees a full memory barrier
+ * before the uatomic_or atomic commit of the deletion flag.
+ */
+ cmm_smp_mb__before_uatomic_or();
+ /*
+ * We set the REMOVED_FLAG unconditionally. Note that there may
+ * be more than one concurrent thread setting this flag.
+ * Knowing which wins the race will be known after the garbage
+ * collection phase, stay tuned!
+ */
+ uatomic_or(&node->next, REMOVED_FLAG);
+ /* We performed the (logical) deletion. */
+
+ /*
+ * Ensure that the node is not visible to readers anymore: lookup for
+ * the node, and remove it (along with any other logically removed node)
+ * if found.
+ */
+ bucket = lookup_bucket(ht, size, bit_reverse_ulong(node->reverse_hash));
+ _cds_lfht_gc_bucket(bucket, node);
+
+ assert(is_removed(CMM_LOAD_SHARED(node->next)));
+ /*
+ * Last phase: atomically exchange node->next with a version
+ * having "REMOVAL_OWNER_FLAG" set. If the returned node->next
+ * pointer did _not_ have "REMOVAL_OWNER_FLAG" set, we now own
+ * the node and win the removal race.
+ * It is interesting to note that all "add" paths are forbidden
+ * to change the next pointer starting from the point where the
+ * REMOVED_FLAG is set, so here using a read, followed by a
+ * xchg() suffice to guarantee that the xchg() will ever only
+ * set the "REMOVAL_OWNER_FLAG" (or change nothing if the flag
+ * was already set).
+ */
+ if (!is_removal_owner(uatomic_xchg(&node->next,
+ flag_removal_owner(node->next))))
+ return 0;
+ else
+ return -ENOENT;
+}
+
+static
+void *partition_resize_thread(void *arg)
+{
+ struct partition_resize_work *work = arg;
+
+ work->ht->flavor->register_thread();
+ work->fct(work->ht, work->i, work->start, work->len);
+ work->ht->flavor->unregister_thread();
+ return NULL;
+}
+
+static
+void partition_resize_helper(struct cds_lfht *ht, unsigned long i,
+ unsigned long len,
+ void (*fct)(struct cds_lfht *ht, unsigned long i,
+ unsigned long start, unsigned long len))
+{
+ unsigned long partition_len, start = 0;
+ struct partition_resize_work *work;
+ int thread, ret;
+ unsigned long nr_threads;
+
+ assert(nr_cpus_mask != -1);
+ if (nr_cpus_mask < 0 || len < 2 * MIN_PARTITION_PER_THREAD)
+ goto fallback;
+
+ /*
+ * Note: nr_cpus_mask + 1 is always power of 2.
+ * We spawn just the number of threads we need to satisfy the minimum
+ * partition size, up to the number of CPUs in the system.
+ */
+ if (nr_cpus_mask > 0) {
+ nr_threads = min(nr_cpus_mask + 1,
+ len >> MIN_PARTITION_PER_THREAD_ORDER);
+ } else {
+ nr_threads = 1;
+ }
+ partition_len = len >> cds_lfht_get_count_order_ulong(nr_threads);
+ work = calloc(nr_threads, sizeof(*work));
+ if (!work) {
+ dbg_printf("error allocating for resize, single-threading\n");
+ goto fallback;
+ }
+ for (thread = 0; thread < nr_threads; thread++) {
+ work[thread].ht = ht;
+ work[thread].i = i;
+ work[thread].len = partition_len;
+ work[thread].start = thread * partition_len;
+ work[thread].fct = fct;
+ ret = pthread_create(&(work[thread].thread_id), ht->resize_attr,
+ partition_resize_thread, &work[thread]);
+ if (ret == EAGAIN) {
+ /*
+ * Out of resources: wait and join the threads
+ * we've created, then handle leftovers.
+ */
+ dbg_printf("error spawning for resize, single-threading\n");
+ start = work[thread].start;
+ len -= start;
+ nr_threads = thread;
+ break;
+ }
+ assert(!ret);
+ }
+ for (thread = 0; thread < nr_threads; thread++) {
+ ret = pthread_join(work[thread].thread_id, NULL);
+ assert(!ret);
+ }
+ free(work);
+
+ /*
+ * A pthread_create failure above will either lead in us having
+ * no threads to join or starting at a non-zero offset,
+ * fallback to single thread processing of leftovers.
+ */
+ if (start == 0 && nr_threads > 0)
+ return;
+fallback:
+ ht->flavor->thread_online();
+ fct(ht, i, start, len);
+ ht->flavor->thread_offline();
+}
+
+/*
+ * Holding RCU read lock to protect _cds_lfht_add against memory
+ * reclaim that could be performed by other call_rcu worker threads (ABA
+ * problem).
+ *
+ * When we reach a certain length, we can split this population phase over
+ * many worker threads, based on the number of CPUs available in the system.
+ * This should therefore take care of not having the expand lagging behind too
+ * many concurrent insertion threads by using the scheduler's ability to
+ * schedule bucket node population fairly with insertions.
+ */
+static
+void init_table_populate_partition(struct cds_lfht *ht, unsigned long i,
+ unsigned long start, unsigned long len)
+{
+ unsigned long j, size = 1UL << (i - 1);
+
+ assert(i > MIN_TABLE_ORDER);
+ ht->flavor->read_lock();
+ for (j = size + start; j < size + start + len; j++) {
+ struct cds_lfht_node *new_node = bucket_at(ht, j);
+
+ assert(j >= size && j < (size << 1));
+ dbg_printf("init populate: order %lu index %lu hash %lu\n",
+ i, j, j);
+ new_node->reverse_hash = bit_reverse_ulong(j);
+ _cds_lfht_add(ht, j, NULL, NULL, size, new_node, NULL, 1);
+ }
+ ht->flavor->read_unlock();
+}
+
+static
+void init_table_populate(struct cds_lfht *ht, unsigned long i,
+ unsigned long len)
+{
+ partition_resize_helper(ht, i, len, init_table_populate_partition);
+}
+
+static
+void init_table(struct cds_lfht *ht,
+ unsigned long first_order, unsigned long last_order)
+{
+ unsigned long i;
+
+ dbg_printf("init table: first_order %lu last_order %lu\n",
+ first_order, last_order);
+ assert(first_order > MIN_TABLE_ORDER);
+ for (i = first_order; i <= last_order; i++) {
+ unsigned long len;
+
+ len = 1UL << (i - 1);
+ dbg_printf("init order %lu len: %lu\n", i, len);
+
+ /* Stop expand if the resize target changes under us */
+ if (CMM_LOAD_SHARED(ht->resize_target) < (1UL << i))
+ break;
+
+ cds_lfht_alloc_bucket_table(ht, i);
+
+ /*
+ * Set all bucket nodes reverse hash values for a level and
+ * link all bucket nodes into the table.
+ */
+ init_table_populate(ht, i, len);
+
+ /*
+ * Update table size.
+ */
+ cmm_smp_wmb(); /* populate data before RCU size */
+ CMM_STORE_SHARED(ht->size, 1UL << i);
+
+ dbg_printf("init new size: %lu\n", 1UL << i);
+ if (CMM_LOAD_SHARED(ht->in_progress_destroy))
+ break;
+ }
+}
+
+/*
+ * Holding RCU read lock to protect _cds_lfht_remove against memory
+ * reclaim that could be performed by other call_rcu worker threads (ABA
+ * problem).
+ * For a single level, we logically remove and garbage collect each node.
+ *
+ * As a design choice, we perform logical removal and garbage collection on a
+ * node-per-node basis to simplify this algorithm. We also assume keeping good
+ * cache locality of the operation would overweight possible performance gain
+ * that could be achieved by batching garbage collection for multiple levels.
+ * However, this would have to be justified by benchmarks.
+ *
+ * Concurrent removal and add operations are helping us perform garbage
+ * collection of logically removed nodes. We guarantee that all logically
+ * removed nodes have been garbage-collected (unlinked) before call_rcu is
+ * invoked to free a hole level of bucket nodes (after a grace period).
+ *
+ * Logical removal and garbage collection can therefore be done in batch
+ * or on a node-per-node basis, as long as the guarantee above holds.
+ *
+ * When we reach a certain length, we can split this removal over many worker
+ * threads, based on the number of CPUs available in the system. This should
+ * take care of not letting resize process lag behind too many concurrent
+ * updater threads actively inserting into the hash table.
+ */
+static
+void remove_table_partition(struct cds_lfht *ht, unsigned long i,
+ unsigned long start, unsigned long len)
+{
+ unsigned long j, size = 1UL << (i - 1);
+
+ assert(i > MIN_TABLE_ORDER);
+ ht->flavor->read_lock();
+ for (j = size + start; j < size + start + len; j++) {
+ struct cds_lfht_node *fini_bucket = bucket_at(ht, j);
+ struct cds_lfht_node *parent_bucket = bucket_at(ht, j - size);
+
+ assert(j >= size && j < (size << 1));
+ dbg_printf("remove entry: order %lu index %lu hash %lu\n",
+ i, j, j);
+ /* Set the REMOVED_FLAG to freeze the ->next for gc */
+ uatomic_or(&fini_bucket->next, REMOVED_FLAG);
+ _cds_lfht_gc_bucket(parent_bucket, fini_bucket);
+ }
+ ht->flavor->read_unlock();
+}
+
+static
+void remove_table(struct cds_lfht *ht, unsigned long i, unsigned long len)
+{
+ partition_resize_helper(ht, i, len, remove_table_partition);
+}
+
+/*
+ * fini_table() is never called for first_order == 0, which is why
+ * free_by_rcu_order == 0 can be used as criterion to know if free must
+ * be called.
+ */
+static
+void fini_table(struct cds_lfht *ht,
+ unsigned long first_order, unsigned long last_order)
+{
+ long i;
+ unsigned long free_by_rcu_order = 0;
+
+ dbg_printf("fini table: first_order %lu last_order %lu\n",
+ first_order, last_order);
+ assert(first_order > MIN_TABLE_ORDER);
+ for (i = last_order; i >= first_order; i--) {
+ unsigned long len;
+
+ len = 1UL << (i - 1);
+ dbg_printf("fini order %ld len: %lu\n", i, len);
+
+ /* Stop shrink if the resize target changes under us */
+ if (CMM_LOAD_SHARED(ht->resize_target) > (1UL << (i - 1)))
+ break;
+
+ cmm_smp_wmb(); /* populate data before RCU size */
+ CMM_STORE_SHARED(ht->size, 1UL << (i - 1));
+
+ /*
+ * We need to wait for all add operations to reach Q.S. (and
+ * thus use the new table for lookups) before we can start
+ * releasing the old bucket nodes. Otherwise their lookup will
+ * return a logically removed node as insert position.
+ */
+ ht->flavor->update_synchronize_rcu();
+ if (free_by_rcu_order)
+ cds_lfht_free_bucket_table(ht, free_by_rcu_order);
+
+ /*
+ * Set "removed" flag in bucket nodes about to be removed.
+ * Unlink all now-logically-removed bucket node pointers.
+ * Concurrent add/remove operation are helping us doing
+ * the gc.
+ */
+ remove_table(ht, i, len);
+
+ free_by_rcu_order = i;
+
+ dbg_printf("fini new size: %lu\n", 1UL << i);
+ if (CMM_LOAD_SHARED(ht->in_progress_destroy))
+ break;
+ }
+
+ if (free_by_rcu_order) {
+ ht->flavor->update_synchronize_rcu();
+ cds_lfht_free_bucket_table(ht, free_by_rcu_order);
+ }
+}
+
+static
+void cds_lfht_create_bucket(struct cds_lfht *ht, unsigned long size)
+{
+ struct cds_lfht_node *prev, *node;
+ unsigned long order, len, i;
+
+ cds_lfht_alloc_bucket_table(ht, 0);
+
+ dbg_printf("create bucket: order 0 index 0 hash 0\n");
+ node = bucket_at(ht, 0);
+ node->next = flag_bucket(get_end());
+ node->reverse_hash = 0;
+
+ for (order = 1; order < cds_lfht_get_count_order_ulong(size) + 1; order++) {
+ len = 1UL << (order - 1);
+ cds_lfht_alloc_bucket_table(ht, order);
+
+ for (i = 0; i < len; i++) {
+ /*
+ * Now, we are trying to init the node with the
+ * hash=(len+i) (which is also a bucket with the
+ * index=(len+i)) and insert it into the hash table,
+ * so this node has to be inserted after the bucket
+ * with the index=(len+i)&(len-1)=i. And because there
+ * is no other non-bucket node nor bucket node with
+ * larger index/hash inserted, so the bucket node
+ * being inserted should be inserted directly linked
+ * after the bucket node with index=i.
+ */
+ prev = bucket_at(ht, i);
+ node = bucket_at(ht, len + i);
+
+ dbg_printf("create bucket: order %lu index %lu hash %lu\n",
+ order, len + i, len + i);
+ node->reverse_hash = bit_reverse_ulong(len + i);
+
+ /* insert after prev */
+ assert(is_bucket(prev->next));
+ node->next = prev->next;
+ prev->next = flag_bucket(node);
+ }
+ }
+}
+
+struct cds_lfht *_cds_lfht_new(unsigned long init_size,
+ unsigned long min_nr_alloc_buckets,
+ unsigned long max_nr_buckets,
+ int flags,
+ const struct cds_lfht_mm_type *mm,
+ const struct rcu_flavor_struct *flavor,
+ pthread_attr_t *attr)
+{
+ struct cds_lfht *ht;
+ unsigned long order;
+
+ /* min_nr_alloc_buckets must be power of two */
+ if (!min_nr_alloc_buckets || (min_nr_alloc_buckets & (min_nr_alloc_buckets - 1)))
+ return NULL;
+
+ /* init_size must be power of two */
+ if (!init_size || (init_size & (init_size - 1)))
+ return NULL;
+
+ /*
+ * Memory management plugin default.
+ */
+ if (!mm) {
+ if (CAA_BITS_PER_LONG > 32
+ && max_nr_buckets
+ && max_nr_buckets <= (1ULL << 32)) {
+ /*
+ * For 64-bit architectures, with max number of
+ * buckets small enough not to use the entire
+ * 64-bit memory mapping space (and allowing a
+ * fair number of hash table instances), use the
+ * mmap allocator, which is faster than the
+ * order allocator.
+ */
+ mm = &cds_lfht_mm_mmap;
+ } else {
+ /*
+ * The fallback is to use the order allocator.
+ */
+ mm = &cds_lfht_mm_order;
+ }
+ }
+
+ /* max_nr_buckets == 0 for order based mm means infinite */
+ if (mm == &cds_lfht_mm_order && !max_nr_buckets)
+ max_nr_buckets = 1UL << (MAX_TABLE_ORDER - 1);
+
+ /* max_nr_buckets must be power of two */
+ if (!max_nr_buckets || (max_nr_buckets & (max_nr_buckets - 1)))
+ return NULL;
+
+ min_nr_alloc_buckets = max(min_nr_alloc_buckets, MIN_TABLE_SIZE);
+ init_size = max(init_size, MIN_TABLE_SIZE);
+ max_nr_buckets = max(max_nr_buckets, min_nr_alloc_buckets);
+ init_size = min(init_size, max_nr_buckets);
+
+ ht = mm->alloc_cds_lfht(min_nr_alloc_buckets, max_nr_buckets);
+ assert(ht);
+ assert(ht->mm == mm);
+ assert(ht->bucket_at == mm->bucket_at);
+
+ ht->flags = flags;
+ ht->flavor = flavor;
+ ht->resize_attr = attr;
+ alloc_split_items_count(ht);
+ /* this mutex should not nest in read-side C.S. */
+ pthread_mutex_init(&ht->resize_mutex, NULL);
+ order = cds_lfht_get_count_order_ulong(init_size);
+ ht->resize_target = 1UL << order;
+ cds_lfht_create_bucket(ht, 1UL << order);
+ ht->size = 1UL << order;
+ return ht;
+}
+
+void cds_lfht_lookup(struct cds_lfht *ht, unsigned long hash,
+ cds_lfht_match_fct match, const void *key,
+ struct cds_lfht_iter *iter)
+{
+ struct cds_lfht_node *node, *next, *bucket;
+ unsigned long reverse_hash, size;
+
+ reverse_hash = bit_reverse_ulong(hash);
+
+ size = rcu_dereference(ht->size);
+ bucket = lookup_bucket(ht, size, hash);
+ /* We can always skip the bucket node initially */
+ node = rcu_dereference(bucket->next);
+ node = clear_flag(node);
+ for (;;) {
+ if (caa_unlikely(is_end(node))) {
+ node = next = NULL;
+ break;
+ }
+ if (caa_unlikely(node->reverse_hash > reverse_hash)) {
+ node = next = NULL;
+ break;
+ }
+ next = rcu_dereference(node->next);
+ assert(node == clear_flag(node));
+ if (caa_likely(!is_removed(next))
+ && !is_bucket(next)
+ && node->reverse_hash == reverse_hash
+ && caa_likely(match(node, key))) {
+ break;
+ }
+ node = clear_flag(next);
+ }
+ assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
+ iter->node = node;
+ iter->next = next;
+}
+
+void cds_lfht_next_duplicate(struct cds_lfht *ht, cds_lfht_match_fct match,
+ const void *key, struct cds_lfht_iter *iter)
+{
+ struct cds_lfht_node *node, *next;
+ unsigned long reverse_hash;
+
+ node = iter->node;
+ reverse_hash = node->reverse_hash;
+ next = iter->next;
+ node = clear_flag(next);
+
+ for (;;) {
+ if (caa_unlikely(is_end(node))) {
+ node = next = NULL;
+ break;
+ }
+ if (caa_unlikely(node->reverse_hash > reverse_hash)) {
+ node = next = NULL;
+ break;
+ }
+ next = rcu_dereference(node->next);
+ if (caa_likely(!is_removed(next))
+ && !is_bucket(next)
+ && caa_likely(match(node, key))) {
+ break;
+ }
+ node = clear_flag(next);
+ }
+ assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
+ iter->node = node;
+ iter->next = next;
+}
+
+void cds_lfht_next(struct cds_lfht *ht, struct cds_lfht_iter *iter)
+{
+ struct cds_lfht_node *node, *next;
+
+ node = clear_flag(iter->next);
+ for (;;) {
+ if (caa_unlikely(is_end(node))) {
+ node = next = NULL;
+ break;
+ }
+ next = rcu_dereference(node->next);
+ if (caa_likely(!is_removed(next))
+ && !is_bucket(next)) {
+ break;
+ }
+ node = clear_flag(next);
+ }
+ assert(!node || !is_bucket(CMM_LOAD_SHARED(node->next)));
+ iter->node = node;
+ iter->next = next;
+}
+
+void cds_lfht_first(struct cds_lfht *ht, struct cds_lfht_iter *iter)
+{
+ /*
+ * Get next after first bucket node. The first bucket node is the
+ * first node of the linked list.
+ */
+ iter->next = bucket_at(ht, 0)->next;
+ cds_lfht_next(ht, iter);
+}
+
+void cds_lfht_add(struct cds_lfht *ht, unsigned long hash,
+ struct cds_lfht_node *node)
+{
+ unsigned long size;
+
+ node->reverse_hash = bit_reverse_ulong(hash);
+ size = rcu_dereference(ht->size);
+ _cds_lfht_add(ht, hash, NULL, NULL, size, node, NULL, 0);
+ ht_count_add(ht, size, hash);
+}
+
+struct cds_lfht_node *cds_lfht_add_unique(struct cds_lfht *ht,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *node)
+{
+ unsigned long size;
+ struct cds_lfht_iter iter;
+
+ node->reverse_hash = bit_reverse_ulong(hash);
+ size = rcu_dereference(ht->size);
+ _cds_lfht_add(ht, hash, match, key, size, node, &iter, 0);
+ if (iter.node == node)
+ ht_count_add(ht, size, hash);
+ return iter.node;
+}
+
+struct cds_lfht_node *cds_lfht_add_replace(struct cds_lfht *ht,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *node)
+{
+ unsigned long size;
+ struct cds_lfht_iter iter;
+
+ node->reverse_hash = bit_reverse_ulong(hash);
+ size = rcu_dereference(ht->size);
+ for (;;) {
+ _cds_lfht_add(ht, hash, match, key, size, node, &iter, 0);
+ if (iter.node == node) {
+ ht_count_add(ht, size, hash);
+ return NULL;
+ }
+
+ if (!_cds_lfht_replace(ht, size, iter.node, iter.next, node))
+ return iter.node;
+ }
+}
+
+int cds_lfht_replace(struct cds_lfht *ht,
+ struct cds_lfht_iter *old_iter,
+ unsigned long hash,
+ cds_lfht_match_fct match,
+ const void *key,
+ struct cds_lfht_node *new_node)
+{
+ unsigned long size;
+
+ new_node->reverse_hash = bit_reverse_ulong(hash);
+ if (!old_iter->node)
+ return -ENOENT;
+ if (caa_unlikely(old_iter->node->reverse_hash != new_node->reverse_hash))
+ return -EINVAL;
+ if (caa_unlikely(!match(old_iter->node, key)))
+ return -EINVAL;
+ size = rcu_dereference(ht->size);
+ return _cds_lfht_replace(ht, size, old_iter->node, old_iter->next,
+ new_node);
+}
+
+int cds_lfht_del(struct cds_lfht *ht, struct cds_lfht_node *node)
+{
+ unsigned long size;
+ int ret;
+
+ size = rcu_dereference(ht->size);
+ ret = _cds_lfht_del(ht, size, node);
+ if (!ret) {
+ unsigned long hash;
+
+ hash = bit_reverse_ulong(node->reverse_hash);
+ ht_count_del(ht, size, hash);
+ }
+ return ret;
+}
+
+int cds_lfht_is_node_deleted(struct cds_lfht_node *node)
+{
+ return is_removed(CMM_LOAD_SHARED(node->next));
+}
+
+static
+int cds_lfht_delete_bucket(struct cds_lfht *ht)
+{
+ struct cds_lfht_node *node;
+ unsigned long order, i, size;
+
+ /* Check that the table is empty */
+ node = bucket_at(ht, 0);
+ do {
+ node = clear_flag(node)->next;
+ if (!is_bucket(node))
+ return -EPERM;
+ assert(!is_removed(node));
+ assert(!is_removal_owner(node));
+ } while (!is_end(node));
+ /*
+ * size accessed without rcu_dereference because hash table is
+ * being destroyed.
+ */
+ size = ht->size;
+ /* Internal sanity check: all nodes left should be buckets */
+ for (i = 0; i < size; i++) {
+ node = bucket_at(ht, i);
+ dbg_printf("delete bucket: index %lu expected hash %lu hash %lu\n",
+ i, i, bit_reverse_ulong(node->reverse_hash));
+ assert(is_bucket(node->next));
+ }
+
+ for (order = cds_lfht_get_count_order_ulong(size); (long)order >= 0; order--)
+ cds_lfht_free_bucket_table(ht, order);
+
+ return 0;
+}
+
+/*
+ * Should only be called when no more concurrent readers nor writers can
+ * possibly access the table.
+ */
+int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr)
+{
+ int ret, was_online;
+
+ /* Wait for in-flight resize operations to complete */
+ _CMM_STORE_SHARED(ht->in_progress_destroy, 1);
+ cmm_smp_mb(); /* Store destroy before load resize */
+ was_online = ht->flavor->read_ongoing();
+ if (was_online)
+ ht->flavor->thread_offline();
+ /* Calling with RCU read-side held is an error. */
+ if (ht->flavor->read_ongoing()) {
+ ret = -EINVAL;
+ if (was_online)
+ ht->flavor->thread_online();
+ goto end;
+ }
+ while (uatomic_read(&ht->in_progress_resize))
+ poll(NULL, 0, 100); /* wait for 100ms */
+ if (was_online)
+ ht->flavor->thread_online();
+ ret = cds_lfht_delete_bucket(ht);
+ if (ret)
+ return ret;
+ free_split_items_count(ht);
+ if (attr)
+ *attr = ht->resize_attr;
+ ret = pthread_mutex_destroy(&ht->resize_mutex);
+ if (ret)
+ ret = -EBUSY;
+ poison_free(ht);
+end:
+ return ret;
+}
+
+void cds_lfht_count_nodes(struct cds_lfht *ht,
+ long *approx_before,
+ unsigned long *count,
+ long *approx_after)
+{
+ struct cds_lfht_node *node, *next;
+ unsigned long nr_bucket = 0, nr_removed = 0;
+
+ *approx_before = 0;
+ if (ht->split_count) {
+ int i;
+
+ for (i = 0; i < split_count_mask + 1; i++) {
+ *approx_before += uatomic_read(&ht->split_count[i].add);
+ *approx_before -= uatomic_read(&ht->split_count[i].del);
+ }
+ }
+
+ *count = 0;
+
+ /* Count non-bucket nodes in the table */
+ node = bucket_at(ht, 0);
+ do {
+ next = rcu_dereference(node->next);
+ if (is_removed(next)) {
+ if (!is_bucket(next))
+ (nr_removed)++;
+ else
+ (nr_bucket)++;
+ } else if (!is_bucket(next))
+ (*count)++;
+ else
+ (nr_bucket)++;
+ node = clear_flag(next);
+ } while (!is_end(node));
+ dbg_printf("number of logically removed nodes: %lu\n", nr_removed);
+ dbg_printf("number of bucket nodes: %lu\n", nr_bucket);
+ *approx_after = 0;
+ if (ht->split_count) {
+ int i;
+
+ for (i = 0; i < split_count_mask + 1; i++) {
+ *approx_after += uatomic_read(&ht->split_count[i].add);
+ *approx_after -= uatomic_read(&ht->split_count[i].del);
+ }
+ }
+}
+
+/* called with resize mutex held */
+static
+void _do_cds_lfht_grow(struct cds_lfht *ht,
+ unsigned long old_size, unsigned long new_size)
+{
+ unsigned long old_order, new_order;
+
+ old_order = cds_lfht_get_count_order_ulong(old_size);
+ new_order = cds_lfht_get_count_order_ulong(new_size);
+ dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n",
+ old_size, old_order, new_size, new_order);
+ assert(new_size > old_size);
+ init_table(ht, old_order + 1, new_order);
+}
+
+/* called with resize mutex held */
+static
+void _do_cds_lfht_shrink(struct cds_lfht *ht,
+ unsigned long old_size, unsigned long new_size)
+{
+ unsigned long old_order, new_order;
+
+ new_size = max(new_size, MIN_TABLE_SIZE);
+ old_order = cds_lfht_get_count_order_ulong(old_size);
+ new_order = cds_lfht_get_count_order_ulong(new_size);
+ dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n",
+ old_size, old_order, new_size, new_order);
+ assert(new_size < old_size);
+
+ /* Remove and unlink all bucket nodes to remove. */
+ fini_table(ht, new_order + 1, old_order);
+}
+
+
+/* called with resize mutex held */
+static
+void _do_cds_lfht_resize(struct cds_lfht *ht)
+{
+ unsigned long new_size, old_size;
+
+ /*
+ * Resize table, re-do if the target size has changed under us.
+ */
+ do {
+ assert(uatomic_read(&ht->in_progress_resize));
+ if (CMM_LOAD_SHARED(ht->in_progress_destroy))
+ break;
+ ht->resize_initiated = 1;
+ old_size = ht->size;
+ new_size = CMM_LOAD_SHARED(ht->resize_target);
+ if (old_size < new_size)
+ _do_cds_lfht_grow(ht, old_size, new_size);
+ else if (old_size > new_size)
+ _do_cds_lfht_shrink(ht, old_size, new_size);
+ ht->resize_initiated = 0;
+ /* write resize_initiated before read resize_target */
+ cmm_smp_mb();
+ } while (ht->size != CMM_LOAD_SHARED(ht->resize_target));
+}
+
+static
+unsigned long resize_target_grow(struct cds_lfht *ht, unsigned long new_size)
+{
+ return _uatomic_xchg_monotonic_increase(&ht->resize_target, new_size);
+}
+
+static
+void resize_target_update_count(struct cds_lfht *ht,
+ unsigned long count)
+{
+ count = max(count, MIN_TABLE_SIZE);
+ count = min(count, ht->max_nr_buckets);
+ uatomic_set(&ht->resize_target, count);
+}
+
+void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size)
+{
+ int was_online;
+
+ was_online = ht->flavor->read_ongoing();
+ if (was_online)
+ ht->flavor->thread_offline();
+ /* Calling with RCU read-side held is an error. */
+ if (ht->flavor->read_ongoing()) {
+ static int print_once;
+
+ if (!CMM_LOAD_SHARED(print_once))
+ fprintf(stderr, "[error] rculfhash: cds_lfht_resize "
+ "called with RCU read-side lock held.\n");
+ CMM_STORE_SHARED(print_once, 1);
+ assert(0);
+ goto end;
+ }
+ resize_target_update_count(ht, new_size);
+ CMM_STORE_SHARED(ht->resize_initiated, 1);
+ pthread_mutex_lock(&ht->resize_mutex);
+ _do_cds_lfht_resize(ht);
+ pthread_mutex_unlock(&ht->resize_mutex);
+end:
+ if (was_online)
+ ht->flavor->thread_online();
+}
+
+static
+void do_resize_cb(struct rcu_head *head)
+{
+ struct rcu_resize_work *work =
+ caa_container_of(head, struct rcu_resize_work, head);
+ struct cds_lfht *ht = work->ht;
+
+ ht->flavor->thread_offline();
+ pthread_mutex_lock(&ht->resize_mutex);
+ _do_cds_lfht_resize(ht);
+ pthread_mutex_unlock(&ht->resize_mutex);
+ ht->flavor->thread_online();
+ poison_free(work);
+ cmm_smp_mb(); /* finish resize before decrement */
+ uatomic_dec(&ht->in_progress_resize);
+}
+
+static
+void __cds_lfht_resize_lazy_launch(struct cds_lfht *ht)
+{
+ struct rcu_resize_work *work;
+
+ /* Store resize_target before read resize_initiated */
+ cmm_smp_mb();
+ if (!CMM_LOAD_SHARED(ht->resize_initiated)) {
+ uatomic_inc(&ht->in_progress_resize);
+ cmm_smp_mb(); /* increment resize count before load destroy */
+ if (CMM_LOAD_SHARED(ht->in_progress_destroy)) {
+ uatomic_dec(&ht->in_progress_resize);
+ return;
+ }
+ work = malloc(sizeof(*work));
+ if (work == NULL) {
+ dbg_printf("error allocating resize work, bailing out\n");
+ uatomic_dec(&ht->in_progress_resize);
+ return;
+ }
+ work->ht = ht;
+ ht->flavor->update_call_rcu(&work->head, do_resize_cb);
+ CMM_STORE_SHARED(ht->resize_initiated, 1);
+ }
+}
+
+static
+void cds_lfht_resize_lazy_grow(struct cds_lfht *ht, unsigned long size, int growth)
+{
+ unsigned long target_size = size << growth;
+
+ target_size = min(target_size, ht->max_nr_buckets);
+ if (resize_target_grow(ht, target_size) >= target_size)
+ return;
+
+ __cds_lfht_resize_lazy_launch(ht);
+}
+
+/*
+ * We favor grow operations over shrink. A shrink operation never occurs
+ * if a grow operation is queued for lazy execution. A grow operation
+ * cancels any pending shrink lazy execution.
+ */
+static
+void cds_lfht_resize_lazy_count(struct cds_lfht *ht, unsigned long size,
+ unsigned long count)
+{
+ if (!(ht->flags & CDS_LFHT_AUTO_RESIZE))
+ return;
+ count = max(count, MIN_TABLE_SIZE);
+ count = min(count, ht->max_nr_buckets);
+ if (count == size)
+ return; /* Already the right size, no resize needed */
+ if (count > size) { /* lazy grow */
+ if (resize_target_grow(ht, count) >= count)
+ return;
+ } else { /* lazy shrink */
+ for (;;) {
+ unsigned long s;
+
+ s = uatomic_cmpxchg(&ht->resize_target, size, count);
+ if (s == size)
+ break; /* no resize needed */
+ if (s > size)
+ return; /* growing is/(was just) in progress */
+ if (s <= count)
+ return; /* some other thread do shrink */
+ size = s;
+ }
+ }
+ __cds_lfht_resize_lazy_launch(ht);
+}
--- /dev/null
+/*
+ * rculfqueue.c
+ *
+ * Userspace RCU library - Lock-Free RCU Queue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu/rculfqueue.h"
+#define _LGPL_SOURCE
+#include "urcu/static/rculfqueue.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
+{
+ _cds_lfq_node_init_rcu(node);
+}
+
+void cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
+ void queue_call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head)))
+{
+ _cds_lfq_init_rcu(q, queue_call_rcu);
+}
+
+int cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q)
+{
+ return _cds_lfq_destroy_rcu(q);
+}
+
+void cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q, struct cds_lfq_node_rcu *node)
+{
+ _cds_lfq_enqueue_rcu(q, node);
+}
+
+struct cds_lfq_node_rcu *
+cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
+{
+ return _cds_lfq_dequeue_rcu(q);
+}
--- /dev/null
+/*
+ * rculfstack.c
+ *
+ * Userspace RCU library - Lock-Free RCU Stack
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Remove deprecation warnings from LGPL wrapper build. */
+#define CDS_LFS_RCU_DEPRECATED
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu/rculfstack.h"
+#define _LGPL_SOURCE
+#include "urcu/static/rculfstack.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+
+void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
+{
+ _cds_lfs_node_init_rcu(node);
+}
+
+void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
+{
+ _cds_lfs_init_rcu(s);
+}
+
+int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
+ struct cds_lfs_node_rcu *node)
+{
+ return _cds_lfs_push_rcu(s, node);
+}
+
+struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
+{
+ return _cds_lfs_pop_rcu(s);
+}
--- /dev/null
+/*
+ * urcu-bp.c
+ *
+ * Userspace RCU library, "bulletproof" version.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#define _LGPL_SOURCE
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <poll.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+#include "urcu/arch.h"
+#include "urcu/wfcqueue.h"
+#include "urcu/map/urcu-bp.h"
+#include "urcu/static/urcu-bp.h"
+#include "urcu-pointer.h"
+#include "urcu/tls-compat.h"
+
+#include "urcu-die.h"
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu-bp.h"
+#define _LGPL_SOURCE
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+#ifdef __linux__
+static
+void *mremap_wrapper(void *old_address, size_t old_size,
+ size_t new_size, int flags)
+{
+ return mremap(old_address, old_size, new_size, flags);
+}
+#else
+
+#define MREMAP_MAYMOVE 1
+#define MREMAP_FIXED 2
+
+/*
+ * mremap wrapper for non-Linux systems not allowing MAYMOVE.
+ * This is not generic.
+*/
+static
+void *mremap_wrapper(void *old_address, size_t old_size,
+ size_t new_size, int flags)
+{
+ assert(!(flags & MREMAP_MAYMOVE));
+
+ return MAP_FAILED;
+}
+#endif
+
+/* Sleep delay in ms */
+#define RCU_SLEEP_DELAY_MS 10
+#define INIT_NR_THREADS 8
+#define ARENA_INIT_ALLOC \
+ sizeof(struct registry_chunk) \
+ + INIT_NR_THREADS * sizeof(struct rcu_reader)
+
+/*
+ * Active attempts to check for reader Q.S. before calling sleep().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+static
+int rcu_bp_refcount;
+
+/* If the headers do not support membarrier system call, fall back smp_mb. */
+#ifdef __NR_membarrier
+# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
+#else
+# define membarrier(...) -ENOSYS
+#endif
+
+enum membarrier_cmd {
+ MEMBARRIER_CMD_QUERY = 0,
+ MEMBARRIER_CMD_SHARED = (1 << 0),
+};
+
+static
+void __attribute__((constructor)) rcu_bp_init(void);
+static
+void __attribute__((destructor)) rcu_bp_exit(void);
+
+int urcu_bp_has_sys_membarrier;
+
+/*
+ * rcu_gp_lock ensures mutual exclusion between threads calling
+ * synchronize_rcu().
+ */
+static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
+/*
+ * rcu_registry_lock ensures mutual exclusion between threads
+ * registering and unregistering themselves to/from the registry, and
+ * with threads reading that registry from synchronize_rcu(). However,
+ * this lock is not held all the way through the completion of awaiting
+ * for the grace period. It is sporadically released between iterations
+ * on the registry.
+ * rcu_registry_lock may nest inside rcu_gp_lock.
+ */
+static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
+static int initialized;
+
+static pthread_key_t urcu_bp_key;
+
+struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
+
+/*
+ * Pointer to registry elements. Written to only by each individual reader. Read
+ * by both the reader and the writers.
+ */
+DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
+
+static CDS_LIST_HEAD(registry);
+
+struct registry_chunk {
+ size_t data_len; /* data length */
+ size_t used; /* amount of data used */
+ struct cds_list_head node; /* chunk_list node */
+ char data[];
+};
+
+struct registry_arena {
+ struct cds_list_head chunk_list;
+};
+
+static struct registry_arena registry_arena = {
+ .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
+};
+
+/* Saved fork signal mask, protected by rcu_gp_lock */
+static sigset_t saved_fork_signal_mask;
+
+static void mutex_lock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+#ifndef DISTRUST_SIGNALS_EXTREME
+ ret = pthread_mutex_lock(mutex);
+ if (ret)
+ urcu_die(ret);
+#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
+ while ((ret = pthread_mutex_trylock(mutex)) != 0) {
+ if (ret != EBUSY && ret != EINTR)
+ urcu_die(ret);
+ poll(NULL,0,10);
+ }
+#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
+}
+
+static void mutex_unlock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(mutex);
+ if (ret)
+ urcu_die(ret);
+}
+
+static void smp_mb_master(void)
+{
+ if (caa_likely(urcu_bp_has_sys_membarrier))
+ (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
+ else
+ cmm_smp_mb();
+}
+
+/*
+ * Always called with rcu_registry lock held. Releases this lock between
+ * iterations and grabs it again. Holds the lock when it returns.
+ */
+static void wait_for_readers(struct cds_list_head *input_readers,
+ struct cds_list_head *cur_snap_readers,
+ struct cds_list_head *qsreaders)
+{
+ unsigned int wait_loops = 0;
+ struct rcu_reader *index, *tmp;
+
+ /*
+ * Wait for each thread URCU_TLS(rcu_reader).ctr to either
+ * indicate quiescence (not nested), or observe the current
+ * rcu_gp.ctr value.
+ */
+ for (;;) {
+ if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
+ wait_loops++;
+
+ cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
+ switch (rcu_reader_state(&index->ctr)) {
+ case RCU_READER_ACTIVE_CURRENT:
+ if (cur_snap_readers) {
+ cds_list_move(&index->node,
+ cur_snap_readers);
+ break;
+ }
+ /* Fall-through */
+ case RCU_READER_INACTIVE:
+ cds_list_move(&index->node, qsreaders);
+ break;
+ case RCU_READER_ACTIVE_OLD:
+ /*
+ * Old snapshot. Leaving node in
+ * input_readers will make us busy-loop
+ * until the snapshot becomes current or
+ * the reader becomes inactive.
+ */
+ break;
+ }
+ }
+
+ if (cds_list_empty(input_readers)) {
+ break;
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
+ (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
+ else
+ caa_cpu_relax();
+ /* Re-lock the registry lock before the next loop. */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+}
+
+void synchronize_rcu(void)
+{
+ CDS_LIST_HEAD(cur_snap_readers);
+ CDS_LIST_HEAD(qsreaders);
+ sigset_t newmask, oldmask;
+ int ret;
+
+ ret = sigfillset(&newmask);
+ assert(!ret);
+ ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
+ assert(!ret);
+
+ mutex_lock(&rcu_gp_lock);
+
+ mutex_lock(&rcu_registry_lock);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /* All threads should read qparity before accessing data structure
+ * where new ptr points to. */
+ /* Write new ptr before changing the qparity */
+ smp_mb_master();
+
+ /*
+ * Wait for readers to observe original parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /* Switch parity: 0 -> 1, 1 -> 0 */
+ CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
+
+ /*
+ * Must commit qparity update to memory before waiting for other parity
+ * quiescent state. Failure to do so could result in the writer waiting
+ * forever while new readers are always accessing data (no progress).
+ * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
+ */
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /*
+ * Wait for readers to observe new parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
+
+ /*
+ * Put quiescent reader list back into registry.
+ */
+ cds_list_splice(&qsreaders, ®istry);
+
+ /*
+ * Finish waiting for reader threads before letting the old ptr being
+ * freed.
+ */
+ smp_mb_master();
+out:
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+ ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+ assert(!ret);
+}
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void rcu_read_lock(void)
+{
+ _rcu_read_lock();
+}
+
+void rcu_read_unlock(void)
+{
+ _rcu_read_unlock();
+}
+
+int rcu_read_ongoing(void)
+{
+ return _rcu_read_ongoing();
+}
+
+/*
+ * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
+ * Else, try expanding the last chunk. If this fails, allocate a new
+ * chunk twice as big as the last chunk.
+ * Memory used by chunks _never_ moves. A chunk could theoretically be
+ * freed when all "used" slots are released, but we don't do it at this
+ * point.
+ */
+static
+void expand_arena(struct registry_arena *arena)
+{
+ struct registry_chunk *new_chunk, *last_chunk;
+ size_t old_chunk_len, new_chunk_len;
+
+ /* No chunk. */
+ if (cds_list_empty(&arena->chunk_list)) {
+ assert(ARENA_INIT_ALLOC >=
+ sizeof(struct registry_chunk)
+ + sizeof(struct rcu_reader));
+ new_chunk_len = ARENA_INIT_ALLOC;
+ new_chunk = mmap(NULL, new_chunk_len,
+ PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ if (new_chunk == MAP_FAILED)
+ abort();
+ memset(new_chunk, 0, new_chunk_len);
+ new_chunk->data_len =
+ new_chunk_len - sizeof(struct registry_chunk);
+ cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
+ return; /* We're done. */
+ }
+
+ /* Try expanding last chunk. */
+ last_chunk = cds_list_entry(arena->chunk_list.prev,
+ struct registry_chunk, node);
+ old_chunk_len =
+ last_chunk->data_len + sizeof(struct registry_chunk);
+ new_chunk_len = old_chunk_len << 1;
+
+ /* Don't allow memory mapping to move, just expand. */
+ new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
+ new_chunk_len, 0);
+ if (new_chunk != MAP_FAILED) {
+ /* Should not have moved. */
+ assert(new_chunk == last_chunk);
+ memset((char *) last_chunk + old_chunk_len, 0,
+ new_chunk_len - old_chunk_len);
+ last_chunk->data_len =
+ new_chunk_len - sizeof(struct registry_chunk);
+ return; /* We're done. */
+ }
+
+ /* Remap did not succeed, we need to add a new chunk. */
+ new_chunk = mmap(NULL, new_chunk_len,
+ PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE,
+ -1, 0);
+ if (new_chunk == MAP_FAILED)
+ abort();
+ memset(new_chunk, 0, new_chunk_len);
+ new_chunk->data_len =
+ new_chunk_len - sizeof(struct registry_chunk);
+ cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
+}
+
+static
+struct rcu_reader *arena_alloc(struct registry_arena *arena)
+{
+ struct registry_chunk *chunk;
+ struct rcu_reader *rcu_reader_reg;
+ int expand_done = 0; /* Only allow to expand once per alloc */
+ size_t len = sizeof(struct rcu_reader);
+
+retry:
+ cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
+ if (chunk->data_len - chunk->used < len)
+ continue;
+ /* Find spot */
+ for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
+ rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
+ rcu_reader_reg++) {
+ if (!rcu_reader_reg->alloc) {
+ rcu_reader_reg->alloc = 1;
+ chunk->used += len;
+ return rcu_reader_reg;
+ }
+ }
+ }
+
+ if (!expand_done) {
+ expand_arena(arena);
+ expand_done = 1;
+ goto retry;
+ }
+
+ return NULL;
+}
+
+/* Called with signals off and mutex locked */
+static
+void add_thread(void)
+{
+ struct rcu_reader *rcu_reader_reg;
+ int ret;
+
+ rcu_reader_reg = arena_alloc(®istry_arena);
+ if (!rcu_reader_reg)
+ abort();
+ ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
+ if (ret)
+ abort();
+
+ /* Add to registry */
+ rcu_reader_reg->tid = pthread_self();
+ assert(rcu_reader_reg->ctr == 0);
+ cds_list_add(&rcu_reader_reg->node, ®istry);
+ /*
+ * Reader threads are pointing to the reader registry. This is
+ * why its memory should never be relocated.
+ */
+ URCU_TLS(rcu_reader) = rcu_reader_reg;
+}
+
+/* Called with mutex locked */
+static
+void cleanup_thread(struct registry_chunk *chunk,
+ struct rcu_reader *rcu_reader_reg)
+{
+ rcu_reader_reg->ctr = 0;
+ cds_list_del(&rcu_reader_reg->node);
+ rcu_reader_reg->tid = 0;
+ rcu_reader_reg->alloc = 0;
+ chunk->used -= sizeof(struct rcu_reader);
+}
+
+static
+struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
+{
+ struct registry_chunk *chunk;
+
+ cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) {
+ if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
+ continue;
+ if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
+ continue;
+ return chunk;
+ }
+ return NULL;
+}
+
+/* Called with signals off and mutex locked */
+static
+void remove_thread(struct rcu_reader *rcu_reader_reg)
+{
+ cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
+ URCU_TLS(rcu_reader) = NULL;
+}
+
+/* Disable signals, take mutex, add to registry */
+void rcu_bp_register(void)
+{
+ sigset_t newmask, oldmask;
+ int ret;
+
+ ret = sigfillset(&newmask);
+ if (ret)
+ abort();
+ ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
+ if (ret)
+ abort();
+
+ /*
+ * Check if a signal concurrently registered our thread since
+ * the check in rcu_read_lock().
+ */
+ if (URCU_TLS(rcu_reader))
+ goto end;
+
+ /*
+ * Take care of early registration before urcu_bp constructor.
+ */
+ rcu_bp_init();
+
+ mutex_lock(&rcu_registry_lock);
+ add_thread();
+ mutex_unlock(&rcu_registry_lock);
+end:
+ ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+ if (ret)
+ abort();
+}
+
+/* Disable signals, take mutex, remove from registry */
+static
+void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
+{
+ sigset_t newmask, oldmask;
+ int ret;
+
+ ret = sigfillset(&newmask);
+ if (ret)
+ abort();
+ ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
+ if (ret)
+ abort();
+
+ mutex_lock(&rcu_registry_lock);
+ remove_thread(rcu_reader_reg);
+ mutex_unlock(&rcu_registry_lock);
+ ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+ if (ret)
+ abort();
+ rcu_bp_exit();
+}
+
+/*
+ * Remove thread from the registry when it exits, and flag it as
+ * destroyed so garbage collection can take care of it.
+ */
+static
+void urcu_bp_thread_exit_notifier(void *rcu_key)
+{
+ rcu_bp_unregister(rcu_key);
+}
+
+static
+void rcu_bp_init(void)
+{
+ mutex_lock(&init_lock);
+ if (!rcu_bp_refcount++) {
+ int ret;
+
+ ret = pthread_key_create(&urcu_bp_key,
+ urcu_bp_thread_exit_notifier);
+ if (ret)
+ abort();
+ ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
+ if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
+ urcu_bp_has_sys_membarrier = 1;
+ }
+ initialized = 1;
+ }
+ mutex_unlock(&init_lock);
+}
+
+static
+void rcu_bp_exit(void)
+{
+ mutex_lock(&init_lock);
+ if (!--rcu_bp_refcount) {
+ struct registry_chunk *chunk, *tmp;
+ int ret;
+
+ cds_list_for_each_entry_safe(chunk, tmp,
+ ®istry_arena.chunk_list, node) {
+ munmap(chunk, chunk->data_len
+ + sizeof(struct registry_chunk));
+ }
+ CDS_INIT_LIST_HEAD(®istry_arena.chunk_list);
+ ret = pthread_key_delete(urcu_bp_key);
+ if (ret)
+ abort();
+ }
+ mutex_unlock(&init_lock);
+}
+
+/*
+ * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
+ * sure we fork() don't race with a concurrent thread executing with
+ * any of those locks held. This ensures that the registry and data
+ * protected by rcu_gp_lock are in a coherent state in the child.
+ */
+void rcu_bp_before_fork(void)
+{
+ sigset_t newmask, oldmask;
+ int ret;
+
+ ret = sigfillset(&newmask);
+ assert(!ret);
+ ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
+ assert(!ret);
+ mutex_lock(&rcu_gp_lock);
+ mutex_lock(&rcu_registry_lock);
+ saved_fork_signal_mask = oldmask;
+}
+
+void rcu_bp_after_fork_parent(void)
+{
+ sigset_t oldmask;
+ int ret;
+
+ oldmask = saved_fork_signal_mask;
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+ ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+ assert(!ret);
+}
+
+/*
+ * Prune all entries from registry except our own thread. Fits the Linux
+ * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
+ */
+static
+void urcu_bp_prune_registry(void)
+{
+ struct registry_chunk *chunk;
+ struct rcu_reader *rcu_reader_reg;
+
+ cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) {
+ for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
+ rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
+ rcu_reader_reg++) {
+ if (!rcu_reader_reg->alloc)
+ continue;
+ if (rcu_reader_reg->tid == pthread_self())
+ continue;
+ cleanup_thread(chunk, rcu_reader_reg);
+ }
+ }
+}
+
+void rcu_bp_after_fork_child(void)
+{
+ sigset_t oldmask;
+ int ret;
+
+ urcu_bp_prune_registry();
+ oldmask = saved_fork_signal_mask;
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+ ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+ assert(!ret);
+}
+
+void *rcu_dereference_sym_bp(void *p)
+{
+ return _rcu_dereference(p);
+}
+
+void *rcu_set_pointer_sym_bp(void **p, void *v)
+{
+ cmm_wmb();
+ uatomic_set(p, v);
+ return v;
+}
+
+void *rcu_xchg_pointer_sym_bp(void **p, void *v)
+{
+ cmm_wmb();
+ return uatomic_xchg(p, v);
+}
+
+void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
+{
+ cmm_wmb();
+ return uatomic_cmpxchg(p, old, _new);
+}
+
+DEFINE_RCU_FLAVOR(rcu_flavor);
+
+#include "urcu-call-rcu-impl.h"
+#include "urcu-defer-impl.h"
--- /dev/null
+#ifndef _URCU_BP_H
+#define _URCU_BP_H
+
+/*
+ * urcu-bp.h
+ *
+ * Userspace RCU header, "bulletproof" version.
+ *
+ * Slower RCU read-side adapted for tracing library. Does not require thread
+ * registration nor unregistration. Also signal-safe.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <urcu/map/urcu-bp.h>
+
+/*
+ * Important !
+ *
+ * Each thread containing read-side critical sections must be registered
+ * with rcu_register_thread() before calling rcu_read_lock().
+ * rcu_unregister_thread() should be called before the thread exits.
+ */
+
+/*
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
+ */
+#include <urcu-pointer.h>
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/urcu-bp.h>
+
+/*
+ * Mappings for static use of the userspace RCU library.
+ * Should only be used in LGPL-compatible code.
+ */
+
+/*
+ * rcu_read_lock()
+ * rcu_read_unlock()
+ *
+ * Mark the beginning and end of a read-side critical section.
+ */
+#define rcu_read_lock_bp _rcu_read_lock
+#define rcu_read_unlock_bp _rcu_read_unlock
+#define rcu_read_ongoing_bp _rcu_read_ongoing
+
+#define rcu_dereference_bp rcu_dereference
+#define rcu_cmpxchg_pointer_bp rcu_cmpxchg_pointer
+#define rcu_xchg_pointer_bp rcu_xchg_pointer
+#define rcu_set_pointer_bp rcu_set_pointer
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ * See LGPL-only urcu/static/urcu-pointer.h for documentation.
+ */
+
+extern void rcu_read_lock(void);
+extern void rcu_read_unlock(void);
+extern int rcu_read_ongoing(void);
+
+extern void *rcu_dereference_sym_bp(void *p);
+#define rcu_dereference_bp(p) \
+ __extension__ \
+ ({ \
+ __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \
+ rcu_dereference_sym_bp(URCU_FORCE_CAST(void *, p))); \
+ (_________p1); \
+ })
+
+extern void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new);
+#define rcu_cmpxchg_pointer_bp(p, old, _new) \
+ __extension__ \
+ ({ \
+ __typeof__(*(p)) _________pold = (old); \
+ __typeof__(*(p)) _________pnew = (_new); \
+ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
+ rcu_cmpxchg_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
+ _________pold, \
+ _________pnew)); \
+ (_________p1); \
+ })
+
+extern void *rcu_xchg_pointer_sym_bp(void **p, void *v);
+#define rcu_xchg_pointer_bp(p, v) \
+ __extension__ \
+ ({ \
+ __typeof__(*(p)) _________pv = (v); \
+ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)),\
+ rcu_xchg_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
+ _________pv)); \
+ (_________p1); \
+ })
+
+extern void *rcu_set_pointer_sym_bp(void **p, void *v);
+#define rcu_set_pointer_bp(p, v) \
+ __extension__ \
+ ({ \
+ __typeof__(*(p)) _________pv = (v); \
+ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
+ rcu_set_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
+ _________pv)); \
+ (_________p1); \
+ })
+
+#endif /* !_LGPL_SOURCE */
+
+extern void synchronize_rcu(void);
+
+/*
+ * rcu_bp_before_fork, rcu_bp_after_fork_parent and rcu_bp_after_fork_child
+ * should be called around fork() system calls when the child process is not
+ * expected to immediately perform an exec(). For pthread users, see
+ * pthread_atfork(3).
+ */
+extern void rcu_bp_before_fork(void);
+extern void rcu_bp_after_fork_parent(void);
+extern void rcu_bp_after_fork_child(void);
+
+/*
+ * In the bulletproof version, the following functions are no-ops.
+ */
+static inline void rcu_register_thread(void)
+{
+}
+
+static inline void rcu_unregister_thread(void)
+{
+}
+
+static inline void rcu_init(void)
+{
+}
+
+/*
+ * Q.S. reporting are no-ops for these URCU flavors.
+ */
+static inline void rcu_quiescent_state(void)
+{
+}
+
+static inline void rcu_thread_offline(void)
+{
+}
+
+static inline void rcu_thread_online(void)
+{
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu-call-rcu.h>
+#include <urcu-defer.h>
+#include <urcu-flavor.h>
+
+#endif /* _URCU_BP_H */
--- /dev/null
+/*
+ * urcu-call-rcu.c
+ *
+ * Userspace RCU library - batch memory reclamation with kernel API
+ *
+ * Copyright (c) 2010 Paul E. McKenney <paulmck@linux.vnet.ibm.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#define _LGPL_SOURCE
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <poll.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <sched.h>
+
+#include "compat-getcpu.h"
+#include "urcu/wfcqueue.h"
+#include "urcu-call-rcu.h"
+#include "urcu-pointer.h"
+#include "urcu/list.h"
+#include "urcu/futex.h"
+#include "urcu/tls-compat.h"
+#include "urcu/ref.h"
+#include "urcu-die.h"
+
+#define SET_AFFINITY_CHECK_PERIOD (1U << 8) /* 256 */
+#define SET_AFFINITY_CHECK_PERIOD_MASK (SET_AFFINITY_CHECK_PERIOD - 1)
+
+/* Data structure that identifies a call_rcu thread. */
+
+struct call_rcu_data {
+ /*
+ * We do not align head on a different cache-line than tail
+ * mainly because call_rcu callback-invocation threads use
+ * batching ("splice") to get an entire list of callbacks, which
+ * effectively empties the queue, and requires to touch the tail
+ * anyway.
+ */
+ struct cds_wfcq_tail cbs_tail;
+ struct cds_wfcq_head cbs_head;
+ unsigned long flags;
+ int32_t futex;
+ unsigned long qlen; /* maintained for debugging. */
+ pthread_t tid;
+ int cpu_affinity;
+ unsigned long gp_count;
+ struct cds_list_head list;
+} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
+
+struct call_rcu_completion {
+ int barrier_count;
+ int32_t futex;
+ struct urcu_ref ref;
+};
+
+struct call_rcu_completion_work {
+ struct rcu_head head;
+ struct call_rcu_completion *completion;
+};
+
+/*
+ * List of all call_rcu_data structures to keep valgrind happy.
+ * Protected by call_rcu_mutex.
+ */
+
+static CDS_LIST_HEAD(call_rcu_data_list);
+
+/* Link a thread using call_rcu() to its call_rcu thread. */
+
+static DEFINE_URCU_TLS(struct call_rcu_data *, thread_call_rcu_data);
+
+/*
+ * Guard call_rcu thread creation and atfork handlers.
+ */
+static pthread_mutex_t call_rcu_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+/* If a given thread does not have its own call_rcu thread, this is default. */
+
+static struct call_rcu_data *default_call_rcu_data;
+
+/*
+ * If the sched_getcpu() and sysconf(_SC_NPROCESSORS_CONF) calls are
+ * available, then we can have call_rcu threads assigned to individual
+ * CPUs rather than only to specific threads.
+ */
+
+#if defined(HAVE_SYSCONF) && (defined(HAVE_SCHED_GETCPU) || defined(HAVE_GETCPUID))
+
+/*
+ * Pointer to array of pointers to per-CPU call_rcu_data structures
+ * and # CPUs. per_cpu_call_rcu_data is a RCU-protected pointer to an
+ * array of RCU-protected pointers to call_rcu_data. call_rcu acts as a
+ * RCU read-side and reads per_cpu_call_rcu_data and the per-cpu pointer
+ * without mutex. The call_rcu_mutex protects updates.
+ */
+
+static struct call_rcu_data **per_cpu_call_rcu_data;
+static long maxcpus;
+
+static void maxcpus_reset(void)
+{
+ maxcpus = 0;
+}
+
+/* Allocate the array if it has not already been allocated. */
+
+static void alloc_cpu_call_rcu_data(void)
+{
+ struct call_rcu_data **p;
+ static int warned = 0;
+
+ if (maxcpus != 0)
+ return;
+ maxcpus = sysconf(_SC_NPROCESSORS_CONF);
+ if (maxcpus <= 0) {
+ return;
+ }
+ p = malloc(maxcpus * sizeof(*per_cpu_call_rcu_data));
+ if (p != NULL) {
+ memset(p, '\0', maxcpus * sizeof(*per_cpu_call_rcu_data));
+ rcu_set_pointer(&per_cpu_call_rcu_data, p);
+ } else {
+ if (!warned) {
+ fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n");
+ }
+ warned = 1;
+ }
+}
+
+#else /* #if defined(HAVE_SYSCONF) && defined(HAVE_SCHED_GETCPU) */
+
+/*
+ * per_cpu_call_rcu_data should be constant, but some functions below, used both
+ * for cases where cpu number is available and not available, assume it it not
+ * constant.
+ */
+static struct call_rcu_data **per_cpu_call_rcu_data = NULL;
+static const long maxcpus = -1;
+
+static void maxcpus_reset(void)
+{
+}
+
+static void alloc_cpu_call_rcu_data(void)
+{
+}
+
+#endif /* #else #if defined(HAVE_SYSCONF) && defined(HAVE_SCHED_GETCPU) */
+
+/* Acquire the specified pthread mutex. */
+
+static void call_rcu_lock(pthread_mutex_t *pmp)
+{
+ int ret;
+
+ ret = pthread_mutex_lock(pmp);
+ if (ret)
+ urcu_die(ret);
+}
+
+/* Release the specified pthread mutex. */
+
+static void call_rcu_unlock(pthread_mutex_t *pmp)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(pmp);
+ if (ret)
+ urcu_die(ret);
+}
+
+/*
+ * Periodically retry setting CPU affinity if we migrate.
+ * Losing affinity can be caused by CPU hotunplug/hotplug, or by
+ * cpuset(7).
+ */
+#if HAVE_SCHED_SETAFFINITY
+static
+int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+{
+ cpu_set_t mask;
+ int ret;
+
+ if (crdp->cpu_affinity < 0)
+ return 0;
+ if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
+ return 0;
+ if (urcu_sched_getcpu() == crdp->cpu_affinity)
+ return 0;
+
+ CPU_ZERO(&mask);
+ CPU_SET(crdp->cpu_affinity, &mask);
+#if SCHED_SETAFFINITY_ARGS == 2
+ ret = sched_setaffinity(0, &mask);
+#else
+ ret = sched_setaffinity(0, sizeof(mask), &mask);
+#endif
+ /*
+ * EINVAL is fine: can be caused by hotunplugged CPUs, or by
+ * cpuset(7). This is why we should always retry if we detect
+ * migration.
+ */
+ if (ret && errno == EINVAL) {
+ ret = 0;
+ errno = 0;
+ }
+ return ret;
+}
+#else
+static
+int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+{
+ return 0;
+}
+#endif
+
+static void call_rcu_wait(struct call_rcu_data *crdp)
+{
+ /* Read call_rcu list before read futex */
+ cmm_smp_mb();
+ if (uatomic_read(&crdp->futex) != -1)
+ return;
+ while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ return;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+}
+
+static void call_rcu_wake_up(struct call_rcu_data *crdp)
+{
+ /* Write to call_rcu list before reading/writing futex */
+ cmm_smp_mb();
+ if (caa_unlikely(uatomic_read(&crdp->futex) == -1)) {
+ uatomic_set(&crdp->futex, 0);
+ if (futex_async(&crdp->futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0) < 0)
+ urcu_die(errno);
+ }
+}
+
+static void call_rcu_completion_wait(struct call_rcu_completion *completion)
+{
+ /* Read completion barrier count before read futex */
+ cmm_smp_mb();
+ if (uatomic_read(&completion->futex) != -1)
+ return;
+ while (futex_async(&completion->futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ return;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+}
+
+static void call_rcu_completion_wake_up(struct call_rcu_completion *completion)
+{
+ /* Write to completion barrier count before reading/writing futex */
+ cmm_smp_mb();
+ if (caa_unlikely(uatomic_read(&completion->futex) == -1)) {
+ uatomic_set(&completion->futex, 0);
+ if (futex_async(&completion->futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0) < 0)
+ urcu_die(errno);
+ }
+}
+
+/* This is the code run by each call_rcu thread. */
+
+static void *call_rcu_thread(void *arg)
+{
+ unsigned long cbcount;
+ struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
+ int rt = !!(uatomic_read(&crdp->flags) & URCU_CALL_RCU_RT);
+
+ if (set_thread_cpu_affinity(crdp))
+ urcu_die(errno);
+
+ /*
+ * If callbacks take a read-side lock, we need to be registered.
+ */
+ rcu_register_thread();
+
+ URCU_TLS(thread_call_rcu_data) = crdp;
+ if (!rt) {
+ uatomic_dec(&crdp->futex);
+ /* Decrement futex before reading call_rcu list */
+ cmm_smp_mb();
+ }
+ for (;;) {
+ struct cds_wfcq_head cbs_tmp_head;
+ struct cds_wfcq_tail cbs_tmp_tail;
+ struct cds_wfcq_node *cbs, *cbs_tmp_n;
+ enum cds_wfcq_ret splice_ret;
+
+ if (set_thread_cpu_affinity(crdp))
+ urcu_die(errno);
+
+ if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
+ /*
+ * Pause requested. Become quiescent: remove
+ * ourself from all global lists, and don't
+ * process any callback. The callback lists may
+ * still be non-empty though.
+ */
+ rcu_unregister_thread();
+ cmm_smp_mb__before_uatomic_or();
+ uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSED);
+ while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) != 0)
+ (void) poll(NULL, 0, 1);
+ uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSED);
+ cmm_smp_mb__after_uatomic_and();
+ rcu_register_thread();
+ }
+
+ cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
+ splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
+ &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
+ assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
+ assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
+ if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
+ synchronize_rcu();
+ cbcount = 0;
+ __cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
+ &cbs_tmp_tail, cbs, cbs_tmp_n) {
+ struct rcu_head *rhp;
+
+ rhp = caa_container_of(cbs,
+ struct rcu_head, next);
+ rhp->func(rhp);
+ cbcount++;
+ }
+ uatomic_sub(&crdp->qlen, cbcount);
+ }
+ if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOP)
+ break;
+ rcu_thread_offline();
+ if (!rt) {
+ if (cds_wfcq_empty(&crdp->cbs_head,
+ &crdp->cbs_tail)) {
+ call_rcu_wait(crdp);
+ (void) poll(NULL, 0, 10);
+ uatomic_dec(&crdp->futex);
+ /*
+ * Decrement futex before reading
+ * call_rcu list.
+ */
+ cmm_smp_mb();
+ } else {
+ (void) poll(NULL, 0, 10);
+ }
+ } else {
+ (void) poll(NULL, 0, 10);
+ }
+ rcu_thread_online();
+ }
+ if (!rt) {
+ /*
+ * Read call_rcu list before write futex.
+ */
+ cmm_smp_mb();
+ uatomic_set(&crdp->futex, 0);
+ }
+ uatomic_or(&crdp->flags, URCU_CALL_RCU_STOPPED);
+ rcu_unregister_thread();
+ return NULL;
+}
+
+/*
+ * Create both a call_rcu thread and the corresponding call_rcu_data
+ * structure, linking the structure in as specified. Caller must hold
+ * call_rcu_mutex.
+ */
+
+static void call_rcu_data_init(struct call_rcu_data **crdpp,
+ unsigned long flags,
+ int cpu_affinity)
+{
+ struct call_rcu_data *crdp;
+ int ret;
+
+ crdp = malloc(sizeof(*crdp));
+ if (crdp == NULL)
+ urcu_die(errno);
+ memset(crdp, '\0', sizeof(*crdp));
+ cds_wfcq_init(&crdp->cbs_head, &crdp->cbs_tail);
+ crdp->qlen = 0;
+ crdp->futex = 0;
+ crdp->flags = flags;
+ cds_list_add(&crdp->list, &call_rcu_data_list);
+ crdp->cpu_affinity = cpu_affinity;
+ crdp->gp_count = 0;
+ cmm_smp_mb(); /* Structure initialized before pointer is planted. */
+ *crdpp = crdp;
+ ret = pthread_create(&crdp->tid, NULL, call_rcu_thread, crdp);
+ if (ret)
+ urcu_die(ret);
+}
+
+/*
+ * Return a pointer to the call_rcu_data structure for the specified
+ * CPU, returning NULL if there is none. We cannot automatically
+ * created it because the platform we are running on might not define
+ * urcu_sched_getcpu().
+ *
+ * The call to this function and use of the returned call_rcu_data
+ * should be protected by RCU read-side lock.
+ */
+
+struct call_rcu_data *get_cpu_call_rcu_data(int cpu)
+{
+ static int warned = 0;
+ struct call_rcu_data **pcpu_crdp;
+
+ pcpu_crdp = rcu_dereference(per_cpu_call_rcu_data);
+ if (pcpu_crdp == NULL)
+ return NULL;
+ if (!warned && maxcpus > 0 && (cpu < 0 || maxcpus <= cpu)) {
+ fprintf(stderr, "[error] liburcu: get CPU # out of range\n");
+ warned = 1;
+ }
+ if (cpu < 0 || maxcpus <= cpu)
+ return NULL;
+ return rcu_dereference(pcpu_crdp[cpu]);
+}
+
+/*
+ * Return the tid corresponding to the call_rcu thread whose
+ * call_rcu_data structure is specified.
+ */
+
+pthread_t get_call_rcu_thread(struct call_rcu_data *crdp)
+{
+ return crdp->tid;
+}
+
+/*
+ * Create a call_rcu_data structure (with thread) and return a pointer.
+ */
+
+static struct call_rcu_data *__create_call_rcu_data(unsigned long flags,
+ int cpu_affinity)
+{
+ struct call_rcu_data *crdp;
+
+ call_rcu_data_init(&crdp, flags, cpu_affinity);
+ return crdp;
+}
+
+struct call_rcu_data *create_call_rcu_data(unsigned long flags,
+ int cpu_affinity)
+{
+ struct call_rcu_data *crdp;
+
+ call_rcu_lock(&call_rcu_mutex);
+ crdp = __create_call_rcu_data(flags, cpu_affinity);
+ call_rcu_unlock(&call_rcu_mutex);
+ return crdp;
+}
+
+/*
+ * Set the specified CPU to use the specified call_rcu_data structure.
+ *
+ * Use NULL to remove a CPU's call_rcu_data structure, but it is
+ * the caller's responsibility to dispose of the removed structure.
+ * Use get_cpu_call_rcu_data() to obtain a pointer to the old structure
+ * (prior to NULLing it out, of course).
+ *
+ * The caller must wait for a grace-period to pass between return from
+ * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
+ * previous call rcu data as argument.
+ */
+
+int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp)
+{
+ static int warned = 0;
+
+ call_rcu_lock(&call_rcu_mutex);
+ alloc_cpu_call_rcu_data();
+ if (cpu < 0 || maxcpus <= cpu) {
+ if (!warned) {
+ fprintf(stderr, "[error] liburcu: set CPU # out of range\n");
+ warned = 1;
+ }
+ call_rcu_unlock(&call_rcu_mutex);
+ errno = EINVAL;
+ return -EINVAL;
+ }
+
+ if (per_cpu_call_rcu_data == NULL) {
+ call_rcu_unlock(&call_rcu_mutex);
+ errno = ENOMEM;
+ return -ENOMEM;
+ }
+
+ if (per_cpu_call_rcu_data[cpu] != NULL && crdp != NULL) {
+ call_rcu_unlock(&call_rcu_mutex);
+ errno = EEXIST;
+ return -EEXIST;
+ }
+
+ rcu_set_pointer(&per_cpu_call_rcu_data[cpu], crdp);
+ call_rcu_unlock(&call_rcu_mutex);
+ return 0;
+}
+
+/*
+ * Return a pointer to the default call_rcu_data structure, creating
+ * one if need be. Because we never free call_rcu_data structures,
+ * we don't need to be in an RCU read-side critical section.
+ */
+
+struct call_rcu_data *get_default_call_rcu_data(void)
+{
+ if (default_call_rcu_data != NULL)
+ return rcu_dereference(default_call_rcu_data);
+ call_rcu_lock(&call_rcu_mutex);
+ if (default_call_rcu_data != NULL) {
+ call_rcu_unlock(&call_rcu_mutex);
+ return default_call_rcu_data;
+ }
+ call_rcu_data_init(&default_call_rcu_data, 0, -1);
+ call_rcu_unlock(&call_rcu_mutex);
+ return default_call_rcu_data;
+}
+
+/*
+ * Return the call_rcu_data structure that applies to the currently
+ * running thread. Any call_rcu_data structure assigned specifically
+ * to this thread has first priority, followed by any call_rcu_data
+ * structure assigned to the CPU on which the thread is running,
+ * followed by the default call_rcu_data structure. If there is not
+ * yet a default call_rcu_data structure, one will be created.
+ *
+ * Calls to this function and use of the returned call_rcu_data should
+ * be protected by RCU read-side lock.
+ */
+struct call_rcu_data *get_call_rcu_data(void)
+{
+ struct call_rcu_data *crd;
+
+ if (URCU_TLS(thread_call_rcu_data) != NULL)
+ return URCU_TLS(thread_call_rcu_data);
+
+ if (maxcpus > 0) {
+ crd = get_cpu_call_rcu_data(urcu_sched_getcpu());
+ if (crd)
+ return crd;
+ }
+
+ return get_default_call_rcu_data();
+}
+
+/*
+ * Return a pointer to this task's call_rcu_data if there is one.
+ */
+
+struct call_rcu_data *get_thread_call_rcu_data(void)
+{
+ return URCU_TLS(thread_call_rcu_data);
+}
+
+/*
+ * Set this task's call_rcu_data structure as specified, regardless
+ * of whether or not this task already had one. (This allows switching
+ * to and from real-time call_rcu threads, for example.)
+ *
+ * Use NULL to remove a thread's call_rcu_data structure, but it is
+ * the caller's responsibility to dispose of the removed structure.
+ * Use get_thread_call_rcu_data() to obtain a pointer to the old structure
+ * (prior to NULLing it out, of course).
+ */
+
+void set_thread_call_rcu_data(struct call_rcu_data *crdp)
+{
+ URCU_TLS(thread_call_rcu_data) = crdp;
+}
+
+/*
+ * Create a separate call_rcu thread for each CPU. This does not
+ * replace a pre-existing call_rcu thread -- use the set_cpu_call_rcu_data()
+ * function if you want that behavior. Should be paired with
+ * free_all_cpu_call_rcu_data() to teardown these call_rcu worker
+ * threads.
+ */
+
+int create_all_cpu_call_rcu_data(unsigned long flags)
+{
+ int i;
+ struct call_rcu_data *crdp;
+ int ret;
+
+ call_rcu_lock(&call_rcu_mutex);
+ alloc_cpu_call_rcu_data();
+ call_rcu_unlock(&call_rcu_mutex);
+ if (maxcpus <= 0) {
+ errno = EINVAL;
+ return -EINVAL;
+ }
+ if (per_cpu_call_rcu_data == NULL) {
+ errno = ENOMEM;
+ return -ENOMEM;
+ }
+ for (i = 0; i < maxcpus; i++) {
+ call_rcu_lock(&call_rcu_mutex);
+ if (get_cpu_call_rcu_data(i)) {
+ call_rcu_unlock(&call_rcu_mutex);
+ continue;
+ }
+ crdp = __create_call_rcu_data(flags, i);
+ if (crdp == NULL) {
+ call_rcu_unlock(&call_rcu_mutex);
+ errno = ENOMEM;
+ return -ENOMEM;
+ }
+ call_rcu_unlock(&call_rcu_mutex);
+ if ((ret = set_cpu_call_rcu_data(i, crdp)) != 0) {
+ call_rcu_data_free(crdp);
+
+ /* it has been created by other thread */
+ if (ret == -EEXIST)
+ continue;
+
+ return ret;
+ }
+ }
+ return 0;
+}
+
+/*
+ * Wake up the call_rcu thread corresponding to the specified
+ * call_rcu_data structure.
+ */
+static void wake_call_rcu_thread(struct call_rcu_data *crdp)
+{
+ if (!(_CMM_LOAD_SHARED(crdp->flags) & URCU_CALL_RCU_RT))
+ call_rcu_wake_up(crdp);
+}
+
+static void _call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head),
+ struct call_rcu_data *crdp)
+{
+ cds_wfcq_node_init(&head->next);
+ head->func = func;
+ cds_wfcq_enqueue(&crdp->cbs_head, &crdp->cbs_tail, &head->next);
+ uatomic_inc(&crdp->qlen);
+ wake_call_rcu_thread(crdp);
+}
+
+/*
+ * Schedule a function to be invoked after a following grace period.
+ * This is the only function that must be called -- the others are
+ * only present to allow applications to tune their use of RCU for
+ * maximum performance.
+ *
+ * Note that unless a call_rcu thread has not already been created,
+ * the first invocation of call_rcu() will create one. So, if you
+ * need the first invocation of call_rcu() to be fast, make sure
+ * to create a call_rcu thread first. One way to accomplish this is
+ * "get_call_rcu_data();", and another is create_all_cpu_call_rcu_data().
+ *
+ * call_rcu must be called by registered RCU read-side threads.
+ */
+void call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head))
+{
+ struct call_rcu_data *crdp;
+
+ /* Holding rcu read-side lock across use of per-cpu crdp */
+ _rcu_read_lock();
+ crdp = get_call_rcu_data();
+ _call_rcu(head, func, crdp);
+ _rcu_read_unlock();
+}
+
+/*
+ * Free up the specified call_rcu_data structure, terminating the
+ * associated call_rcu thread. The caller must have previously
+ * removed the call_rcu_data structure from per-thread or per-CPU
+ * usage. For example, set_cpu_call_rcu_data(cpu, NULL) for per-CPU
+ * call_rcu_data structures or set_thread_call_rcu_data(NULL) for
+ * per-thread call_rcu_data structures.
+ *
+ * We silently refuse to free up the default call_rcu_data structure
+ * because that is where we put any leftover callbacks. Note that
+ * the possibility of self-spawning callbacks makes it impossible
+ * to execute all the callbacks in finite time without putting any
+ * newly spawned callbacks somewhere else. The "somewhere else" of
+ * last resort is the default call_rcu_data structure.
+ *
+ * We also silently refuse to free NULL pointers. This simplifies
+ * the calling code.
+ *
+ * The caller must wait for a grace-period to pass between return from
+ * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
+ * previous call rcu data as argument.
+ *
+ * Note: introducing __cds_wfcq_splice_blocking() in this function fixed
+ * a list corruption bug in the 0.7.x series. The equivalent fix
+ * appeared in 0.6.8 for the stable-0.6 branch.
+ */
+void call_rcu_data_free(struct call_rcu_data *crdp)
+{
+ if (crdp == NULL || crdp == default_call_rcu_data) {
+ return;
+ }
+ if ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0) {
+ uatomic_or(&crdp->flags, URCU_CALL_RCU_STOP);
+ wake_call_rcu_thread(crdp);
+ while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
+ (void) poll(NULL, 0, 1);
+ }
+ if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
+ /* Create default call rcu data if need be */
+ (void) get_default_call_rcu_data();
+ __cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
+ &default_call_rcu_data->cbs_tail,
+ &crdp->cbs_head, &crdp->cbs_tail);
+ uatomic_add(&default_call_rcu_data->qlen,
+ uatomic_read(&crdp->qlen));
+ wake_call_rcu_thread(default_call_rcu_data);
+ }
+
+ call_rcu_lock(&call_rcu_mutex);
+ cds_list_del(&crdp->list);
+ call_rcu_unlock(&call_rcu_mutex);
+
+ free(crdp);
+}
+
+/*
+ * Clean up all the per-CPU call_rcu threads.
+ */
+void free_all_cpu_call_rcu_data(void)
+{
+ int cpu;
+ struct call_rcu_data **crdp;
+ static int warned = 0;
+
+ if (maxcpus <= 0)
+ return;
+
+ crdp = malloc(sizeof(*crdp) * maxcpus);
+ if (!crdp) {
+ if (!warned) {
+ fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n");
+ }
+ warned = 1;
+ return;
+ }
+
+ for (cpu = 0; cpu < maxcpus; cpu++) {
+ crdp[cpu] = get_cpu_call_rcu_data(cpu);
+ if (crdp[cpu] == NULL)
+ continue;
+ set_cpu_call_rcu_data(cpu, NULL);
+ }
+ /*
+ * Wait for call_rcu sites acting as RCU readers of the
+ * call_rcu_data to become quiescent.
+ */
+ synchronize_rcu();
+ for (cpu = 0; cpu < maxcpus; cpu++) {
+ if (crdp[cpu] == NULL)
+ continue;
+ call_rcu_data_free(crdp[cpu]);
+ }
+ free(crdp);
+}
+
+static
+void free_completion(struct urcu_ref *ref)
+{
+ struct call_rcu_completion *completion;
+
+ completion = caa_container_of(ref, struct call_rcu_completion, ref);
+ free(completion);
+}
+
+static
+void _rcu_barrier_complete(struct rcu_head *head)
+{
+ struct call_rcu_completion_work *work;
+ struct call_rcu_completion *completion;
+
+ work = caa_container_of(head, struct call_rcu_completion_work, head);
+ completion = work->completion;
+ if (!uatomic_sub_return(&completion->barrier_count, 1))
+ call_rcu_completion_wake_up(completion);
+ urcu_ref_put(&completion->ref, free_completion);
+ free(work);
+}
+
+/*
+ * Wait for all in-flight call_rcu callbacks to complete execution.
+ */
+void rcu_barrier(void)
+{
+ struct call_rcu_data *crdp;
+ struct call_rcu_completion *completion;
+ int count = 0;
+ int was_online;
+
+ /* Put in offline state in QSBR. */
+ was_online = _rcu_read_ongoing();
+ if (was_online)
+ rcu_thread_offline();
+ /*
+ * Calling a rcu_barrier() within a RCU read-side critical
+ * section is an error.
+ */
+ if (_rcu_read_ongoing()) {
+ static int warned = 0;
+
+ if (!warned) {
+ fprintf(stderr, "[error] liburcu: rcu_barrier() called from within RCU read-side critical section.\n");
+ }
+ warned = 1;
+ goto online;
+ }
+
+ completion = calloc(sizeof(*completion), 1);
+ if (!completion)
+ urcu_die(errno);
+
+ call_rcu_lock(&call_rcu_mutex);
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
+ count++;
+
+ /* Referenced by rcu_barrier() and each call_rcu thread. */
+ urcu_ref_set(&completion->ref, count + 1);
+ completion->barrier_count = count;
+
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+ struct call_rcu_completion_work *work;
+
+ work = calloc(sizeof(*work), 1);
+ if (!work)
+ urcu_die(errno);
+ work->completion = completion;
+ _call_rcu(&work->head, _rcu_barrier_complete, crdp);
+ }
+ call_rcu_unlock(&call_rcu_mutex);
+
+ /* Wait for them */
+ for (;;) {
+ uatomic_dec(&completion->futex);
+ /* Decrement futex before reading barrier_count */
+ cmm_smp_mb();
+ if (!uatomic_read(&completion->barrier_count))
+ break;
+ call_rcu_completion_wait(completion);
+ }
+
+ urcu_ref_put(&completion->ref, free_completion);
+
+online:
+ if (was_online)
+ rcu_thread_online();
+}
+
+/*
+ * Acquire the call_rcu_mutex in order to ensure that the child sees
+ * all of the call_rcu() data structures in a consistent state. Ensure
+ * that all call_rcu threads are in a quiescent state across fork.
+ * Suitable for pthread_atfork() and friends.
+ */
+void call_rcu_before_fork(void)
+{
+ struct call_rcu_data *crdp;
+
+ call_rcu_lock(&call_rcu_mutex);
+
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+ uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSE);
+ cmm_smp_mb__after_uatomic_or();
+ wake_call_rcu_thread(crdp);
+ }
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+ while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) == 0)
+ (void) poll(NULL, 0, 1);
+ }
+}
+
+/*
+ * Clean up call_rcu data structures in the parent of a successful fork()
+ * that is not followed by exec() in the child. Suitable for
+ * pthread_atfork() and friends.
+ */
+void call_rcu_after_fork_parent(void)
+{
+ struct call_rcu_data *crdp;
+
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
+ uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSE);
+ cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
+ while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) != 0)
+ (void) poll(NULL, 0, 1);
+ }
+ call_rcu_unlock(&call_rcu_mutex);
+}
+
+/*
+ * Clean up call_rcu data structures in the child of a successful fork()
+ * that is not followed by exec(). Suitable for pthread_atfork() and
+ * friends.
+ */
+void call_rcu_after_fork_child(void)
+{
+ struct call_rcu_data *crdp, *next;
+
+ /* Release the mutex. */
+ call_rcu_unlock(&call_rcu_mutex);
+
+ /* Do nothing when call_rcu() has not been used */
+ if (cds_list_empty(&call_rcu_data_list))
+ return;
+
+ /*
+ * Allocate a new default call_rcu_data structure in order
+ * to get a working call_rcu thread to go with it.
+ */
+ default_call_rcu_data = NULL;
+ (void)get_default_call_rcu_data();
+
+ /* Cleanup call_rcu_data pointers before use */
+ maxcpus_reset();
+ free(per_cpu_call_rcu_data);
+ rcu_set_pointer(&per_cpu_call_rcu_data, NULL);
+ URCU_TLS(thread_call_rcu_data) = NULL;
+
+ /*
+ * Dispose of all of the rest of the call_rcu_data structures.
+ * Leftover call_rcu callbacks will be merged into the new
+ * default call_rcu thread queue.
+ */
+ cds_list_for_each_entry_safe(crdp, next, &call_rcu_data_list, list) {
+ if (crdp == default_call_rcu_data)
+ continue;
+ uatomic_set(&crdp->flags, URCU_CALL_RCU_STOPPED);
+ call_rcu_data_free(crdp);
+ }
+}
--- /dev/null
+#ifndef _URCU_CALL_RCU_H
+#define _URCU_CALL_RCU_H
+
+/*
+ * urcu-call-rcu.h
+ *
+ * Userspace RCU header - deferred execution
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu-defer.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+
+#include <urcu/wfcqueue.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Note that struct call_rcu_data is opaque to callers. */
+
+struct call_rcu_data;
+
+/* Flag values. */
+
+#define URCU_CALL_RCU_RT (1U << 0)
+#define URCU_CALL_RCU_RUNNING (1U << 1)
+#define URCU_CALL_RCU_STOP (1U << 2)
+#define URCU_CALL_RCU_STOPPED (1U << 3)
+#define URCU_CALL_RCU_PAUSE (1U << 4)
+#define URCU_CALL_RCU_PAUSED (1U << 5)
+
+/*
+ * The rcu_head data structure is placed in the structure to be freed
+ * via call_rcu().
+ */
+
+struct rcu_head {
+ struct cds_wfcq_node next;
+ void (*func)(struct rcu_head *head);
+};
+
+/*
+ * Exported functions
+ *
+ * Important: see rcu-api.md in userspace-rcu documentation for
+ * call_rcu family of functions usage detail, including the surrounding
+ * RCU usage required when using these primitives.
+ */
+
+void call_rcu(struct rcu_head *head,
+ void (*func)(struct rcu_head *head));
+
+struct call_rcu_data *create_call_rcu_data(unsigned long flags,
+ int cpu_affinity);
+void call_rcu_data_free(struct call_rcu_data *crdp);
+
+struct call_rcu_data *get_default_call_rcu_data(void);
+struct call_rcu_data *get_cpu_call_rcu_data(int cpu);
+struct call_rcu_data *get_thread_call_rcu_data(void);
+struct call_rcu_data *get_call_rcu_data(void);
+pthread_t get_call_rcu_thread(struct call_rcu_data *crdp);
+
+void set_thread_call_rcu_data(struct call_rcu_data *crdp);
+int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp);
+
+int create_all_cpu_call_rcu_data(unsigned long flags);
+void free_all_cpu_call_rcu_data(void);
+
+void call_rcu_before_fork(void);
+void call_rcu_after_fork_parent(void);
+void call_rcu_after_fork_child(void);
+
+void rcu_barrier(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_CALL_RCU_H */
--- /dev/null
+#ifndef _URCU_DEFER_IMPL_H
+#define _URCU_DEFER_IMPL_H
+
+/*
+ * urcu-defer-impl.h
+ *
+ * Userspace RCU header - memory reclamation.
+ *
+ * TO BE INCLUDED ONLY FROM URCU LIBRARY CODE. See urcu-defer.h for linking
+ * dynamically with the userspace rcu reclamation library.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <signal.h>
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+#include <poll.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <stdint.h>
+
+#include "urcu/futex.h"
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/uatomic.h>
+#include <urcu/list.h>
+#include <urcu/system.h>
+#include <urcu/tls-compat.h>
+#include "urcu-die.h"
+
+/*
+ * Number of entries in the per-thread defer queue. Must be power of 2.
+ */
+#define DEFER_QUEUE_SIZE (1 << 12)
+#define DEFER_QUEUE_MASK (DEFER_QUEUE_SIZE - 1)
+
+/*
+ * Typically, data is aligned at least on the architecture size.
+ * Use lowest bit to indicate that the current callback is changing.
+ * Assumes that (void *)-2L is not used often. Used to encode non-aligned
+ * functions and non-aligned data using extra space.
+ * We encode the (void *)-2L fct as: -2L, fct, data.
+ * We encode the (void *)-2L data as either:
+ * fct | DQ_FCT_BIT, data (if fct is aligned), or
+ * -2L, fct, data (if fct is not aligned).
+ * Here, DQ_FCT_MARK == ~DQ_FCT_BIT. Required for the test order.
+ */
+#define DQ_FCT_BIT (1 << 0)
+#define DQ_IS_FCT_BIT(x) ((unsigned long)(x) & DQ_FCT_BIT)
+#define DQ_SET_FCT_BIT(x) \
+ (x = (void *)((unsigned long)(x) | DQ_FCT_BIT))
+#define DQ_CLEAR_FCT_BIT(x) \
+ (x = (void *)((unsigned long)(x) & ~DQ_FCT_BIT))
+#define DQ_FCT_MARK ((void *)(~DQ_FCT_BIT))
+
+/*
+ * This code section can only be included in LGPL 2.1 compatible source code.
+ * See below for the function call wrappers which can be used in code meant to
+ * be only linked with the Userspace RCU library. This comes with a small
+ * performance degradation on the read-side due to the added function calls.
+ * This is required to permit relinking with newer versions of the library.
+ */
+
+/*
+ * defer queue.
+ * Contains pointers. Encoded to save space when same callback is often used.
+ * When looking up the next item:
+ * - if DQ_FCT_BIT is set, set the current callback to DQ_CLEAR_FCT_BIT(ptr)
+ * - next element contains pointer to data.
+ * - else if item == DQ_FCT_MARK
+ * - set the current callback to next element ptr
+ * - following next element contains pointer to data.
+ * - else current element contains data
+ */
+struct defer_queue {
+ unsigned long head; /* add element at head */
+ void *last_fct_in; /* last fct pointer encoded */
+ unsigned long tail; /* next element to remove at tail */
+ void *last_fct_out; /* last fct pointer encoded */
+ void **q;
+ /* registry information */
+ unsigned long last_head;
+ struct cds_list_head list; /* list of thread queues */
+};
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#include "urcu-defer.h"
+
+void __attribute__((destructor)) rcu_defer_exit(void);
+
+extern void synchronize_rcu(void);
+
+/*
+ * rcu_defer_mutex nests inside defer_thread_mutex.
+ */
+static pthread_mutex_t rcu_defer_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t defer_thread_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+static int32_t defer_thread_futex;
+static int32_t defer_thread_stop;
+
+/*
+ * Written to only by each individual deferer. Read by both the deferer and
+ * the reclamation tread.
+ */
+static DEFINE_URCU_TLS(struct defer_queue, defer_queue);
+static CDS_LIST_HEAD(registry_defer);
+static pthread_t tid_defer;
+
+static void mutex_lock_defer(pthread_mutex_t *mutex)
+{
+ int ret;
+
+#ifndef DISTRUST_SIGNALS_EXTREME
+ ret = pthread_mutex_lock(mutex);
+ if (ret)
+ urcu_die(ret);
+#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
+ while ((ret = pthread_mutex_trylock(mutex)) != 0) {
+ if (ret != EBUSY && ret != EINTR)
+ urcu_die(ret);
+ (void) poll(NULL,0,10);
+ }
+#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
+}
+
+/*
+ * Wake-up any waiting defer thread. Called from many concurrent threads.
+ */
+static void wake_up_defer(void)
+{
+ if (caa_unlikely(uatomic_read(&defer_thread_futex) == -1)) {
+ uatomic_set(&defer_thread_futex, 0);
+ if (futex_noasync(&defer_thread_futex, FUTEX_WAKE, 1,
+ NULL, NULL, 0) < 0)
+ urcu_die(errno);
+ }
+}
+
+static unsigned long rcu_defer_num_callbacks(void)
+{
+ unsigned long num_items = 0, head;
+ struct defer_queue *index;
+
+ mutex_lock_defer(&rcu_defer_mutex);
+ cds_list_for_each_entry(index, ®istry_defer, list) {
+ head = CMM_LOAD_SHARED(index->head);
+ num_items += head - index->tail;
+ }
+ mutex_unlock(&rcu_defer_mutex);
+ return num_items;
+}
+
+/*
+ * Defer thread waiting. Single thread.
+ */
+static void wait_defer(void)
+{
+ uatomic_dec(&defer_thread_futex);
+ /* Write futex before read queue */
+ /* Write futex before read defer_thread_stop */
+ cmm_smp_mb();
+ if (_CMM_LOAD_SHARED(defer_thread_stop)) {
+ uatomic_set(&defer_thread_futex, 0);
+ pthread_exit(0);
+ }
+ if (rcu_defer_num_callbacks()) {
+ cmm_smp_mb(); /* Read queue before write futex */
+ /* Callbacks are queued, don't wait. */
+ uatomic_set(&defer_thread_futex, 0);
+ } else {
+ cmm_smp_rmb(); /* Read queue before read futex */
+ if (uatomic_read(&defer_thread_futex) != -1)
+ return;
+ while (futex_noasync(&defer_thread_futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ return;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+ }
+}
+
+/*
+ * Must be called after Q.S. is reached.
+ */
+static void rcu_defer_barrier_queue(struct defer_queue *queue,
+ unsigned long head)
+{
+ unsigned long i;
+ void (*fct)(void *p);
+ void *p;
+
+ /*
+ * Tail is only modified when lock is held.
+ * Head is only modified by owner thread.
+ */
+
+ for (i = queue->tail; i != head;) {
+ cmm_smp_rmb(); /* read head before q[]. */
+ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
+ if (caa_unlikely(DQ_IS_FCT_BIT(p))) {
+ DQ_CLEAR_FCT_BIT(p);
+ queue->last_fct_out = p;
+ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
+ } else if (caa_unlikely(p == DQ_FCT_MARK)) {
+ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
+ queue->last_fct_out = p;
+ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
+ }
+ fct = queue->last_fct_out;
+ fct(p);
+ }
+ cmm_smp_mb(); /* push tail after having used q[] */
+ CMM_STORE_SHARED(queue->tail, i);
+}
+
+static void _rcu_defer_barrier_thread(void)
+{
+ unsigned long head, num_items;
+
+ head = URCU_TLS(defer_queue).head;
+ num_items = head - URCU_TLS(defer_queue).tail;
+ if (caa_unlikely(!num_items))
+ return;
+ synchronize_rcu();
+ rcu_defer_barrier_queue(&URCU_TLS(defer_queue), head);
+}
+
+void rcu_defer_barrier_thread(void)
+{
+ mutex_lock_defer(&rcu_defer_mutex);
+ _rcu_defer_barrier_thread();
+ mutex_unlock(&rcu_defer_mutex);
+}
+
+/*
+ * rcu_defer_barrier - Execute all queued rcu callbacks.
+ *
+ * Execute all RCU callbacks queued before rcu_defer_barrier() execution.
+ * All callbacks queued on the local thread prior to a rcu_defer_barrier() call
+ * are guaranteed to be executed.
+ * Callbacks queued by other threads concurrently with rcu_defer_barrier()
+ * execution are not guaranteed to be executed in the current batch (could
+ * be left for the next batch). These callbacks queued by other threads are only
+ * guaranteed to be executed if there is explicit synchronization between
+ * the thread adding to the queue and the thread issuing the defer_barrier call.
+ */
+
+void rcu_defer_barrier(void)
+{
+ struct defer_queue *index;
+ unsigned long num_items = 0;
+
+ if (cds_list_empty(®istry_defer))
+ return;
+
+ mutex_lock_defer(&rcu_defer_mutex);
+ cds_list_for_each_entry(index, ®istry_defer, list) {
+ index->last_head = CMM_LOAD_SHARED(index->head);
+ num_items += index->last_head - index->tail;
+ }
+ if (caa_likely(!num_items)) {
+ /*
+ * We skip the grace period because there are no queued
+ * callbacks to execute.
+ */
+ goto end;
+ }
+ synchronize_rcu();
+ cds_list_for_each_entry(index, ®istry_defer, list)
+ rcu_defer_barrier_queue(index, index->last_head);
+end:
+ mutex_unlock(&rcu_defer_mutex);
+}
+
+/*
+ * _defer_rcu - Queue a RCU callback.
+ */
+static void _defer_rcu(void (*fct)(void *p), void *p)
+{
+ unsigned long head, tail;
+
+ /*
+ * Head is only modified by ourself. Tail can be modified by reclamation
+ * thread.
+ */
+ head = URCU_TLS(defer_queue).head;
+ tail = CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail);
+
+ /*
+ * If queue is full, or reached threshold. Empty queue ourself.
+ * Worse-case: must allow 2 supplementary entries for fct pointer.
+ */
+ if (caa_unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) {
+ assert(head - tail <= DEFER_QUEUE_SIZE);
+ rcu_defer_barrier_thread();
+ assert(head - CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail) == 0);
+ }
+
+ /*
+ * Encode:
+ * if the function is not changed and the data is aligned and it is
+ * not the marker:
+ * store the data
+ * otherwise if the function is aligned and its not the marker:
+ * store the function with DQ_FCT_BIT
+ * store the data
+ * otherwise:
+ * store the marker (DQ_FCT_MARK)
+ * store the function
+ * store the data
+ *
+ * Decode: see the comments before 'struct defer_queue'
+ * or the code in rcu_defer_barrier_queue().
+ */
+ if (caa_unlikely(URCU_TLS(defer_queue).last_fct_in != fct
+ || DQ_IS_FCT_BIT(p)
+ || p == DQ_FCT_MARK)) {
+ URCU_TLS(defer_queue).last_fct_in = fct;
+ if (caa_unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) {
+ _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
+ DQ_FCT_MARK);
+ _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
+ fct);
+ } else {
+ DQ_SET_FCT_BIT(fct);
+ _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
+ fct);
+ }
+ }
+ _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], p);
+ cmm_smp_wmb(); /* Publish new pointer before head */
+ /* Write q[] before head. */
+ CMM_STORE_SHARED(URCU_TLS(defer_queue).head, head);
+ cmm_smp_mb(); /* Write queue head before read futex */
+ /*
+ * Wake-up any waiting defer thread.
+ */
+ wake_up_defer();
+}
+
+static void *thr_defer(void *args)
+{
+ for (;;) {
+ /*
+ * "Be green". Don't wake up the CPU if there is no RCU work
+ * to perform whatsoever. Aims at saving laptop battery life by
+ * leaving the processor in sleep state when idle.
+ */
+ wait_defer();
+ /* Sleeping after wait_defer to let many callbacks enqueue */
+ (void) poll(NULL,0,100); /* wait for 100ms */
+ rcu_defer_barrier();
+ }
+
+ return NULL;
+}
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void defer_rcu(void (*fct)(void *p), void *p)
+{
+ _defer_rcu(fct, p);
+}
+
+static void start_defer_thread(void)
+{
+ int ret;
+
+ ret = pthread_create(&tid_defer, NULL, thr_defer, NULL);
+ assert(!ret);
+}
+
+static void stop_defer_thread(void)
+{
+ int ret;
+ void *tret;
+
+ _CMM_STORE_SHARED(defer_thread_stop, 1);
+ /* Store defer_thread_stop before testing futex */
+ cmm_smp_mb();
+ wake_up_defer();
+
+ ret = pthread_join(tid_defer, &tret);
+ assert(!ret);
+
+ CMM_STORE_SHARED(defer_thread_stop, 0);
+ /* defer thread should always exit when futex value is 0 */
+ assert(uatomic_read(&defer_thread_futex) == 0);
+}
+
+int rcu_defer_register_thread(void)
+{
+ int was_empty;
+
+ assert(URCU_TLS(defer_queue).last_head == 0);
+ assert(URCU_TLS(defer_queue).q == NULL);
+ URCU_TLS(defer_queue).q = malloc(sizeof(void *) * DEFER_QUEUE_SIZE);
+ if (!URCU_TLS(defer_queue).q)
+ return -ENOMEM;
+
+ mutex_lock_defer(&defer_thread_mutex);
+ mutex_lock_defer(&rcu_defer_mutex);
+ was_empty = cds_list_empty(®istry_defer);
+ cds_list_add(&URCU_TLS(defer_queue).list, ®istry_defer);
+ mutex_unlock(&rcu_defer_mutex);
+
+ if (was_empty)
+ start_defer_thread();
+ mutex_unlock(&defer_thread_mutex);
+ return 0;
+}
+
+void rcu_defer_unregister_thread(void)
+{
+ int is_empty;
+
+ mutex_lock_defer(&defer_thread_mutex);
+ mutex_lock_defer(&rcu_defer_mutex);
+ cds_list_del(&URCU_TLS(defer_queue).list);
+ _rcu_defer_barrier_thread();
+ free(URCU_TLS(defer_queue).q);
+ URCU_TLS(defer_queue).q = NULL;
+ is_empty = cds_list_empty(®istry_defer);
+ mutex_unlock(&rcu_defer_mutex);
+
+ if (is_empty)
+ stop_defer_thread();
+ mutex_unlock(&defer_thread_mutex);
+}
+
+void rcu_defer_exit(void)
+{
+ assert(cds_list_empty(®istry_defer));
+}
+
+#endif /* _URCU_DEFER_IMPL_H */
--- /dev/null
+#ifndef _URCU_BATCH_H
+#define _URCU_BATCH_H
+
+/*
+ * urcu-defer.h
+ *
+ * Userspace RCU header - deferred execution
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu-defer.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Note: the defer_rcu() API is currently EXPERIMENTAL. It may change in the
+ * future.
+ *
+ * Important !
+ *
+ * Each thread queuing memory reclamation must be registered with
+ * rcu_defer_register_thread(). rcu_defer_unregister_thread() should be
+ * called before the thread exits.
+ *
+ * *NEVER* use defer_rcu() within a RCU read-side critical section, because this
+ * primitive need to call synchronize_rcu() if the thread queue is full.
+ */
+
+extern void defer_rcu(void (*fct)(void *p), void *p);
+
+/*
+ * Thread registration for reclamation.
+ */
+extern int rcu_defer_register_thread(void);
+extern void rcu_defer_unregister_thread(void);
+extern void rcu_defer_barrier(void);
+extern void rcu_defer_barrier_thread(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_BATCH_H */
--- /dev/null
+#ifndef _URCU_DIE_H
+#define _URCU_DIE_H
+
+/*
+ * urcu-die.h
+ *
+ * Userspace RCU library unrecoverable error handling
+ *
+ * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#define urcu_die(cause) \
+do { \
+ fprintf(stderr, "(" __FILE__ ":%s@%u) Unrecoverable error: %s\n", \
+ __func__, __LINE__, strerror(cause)); \
+ abort(); \
+} while (0)
+
+#endif /* _URCU_DIE_H */
--- /dev/null
+#ifndef _URCU_FLAVOR_H
+#define _URCU_FLAVOR_H
+
+/*
+ * urcu-flavor.h
+ *
+ * Userspace RCU header - rcu flavor declarations
+ *
+ * Copyright (c) 2011 Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rcu_flavor_struct {
+ void (*read_lock)(void);
+ void (*read_unlock)(void);
+ int (*read_ongoing)(void);
+ void (*read_quiescent_state)(void);
+ void (*update_call_rcu)(struct rcu_head *head,
+ void (*func)(struct rcu_head *head));
+ void (*update_synchronize_rcu)(void);
+ void (*update_defer_rcu)(void (*fct)(void *p), void *p);
+
+ void (*thread_offline)(void);
+ void (*thread_online)(void);
+ void (*register_thread)(void);
+ void (*unregister_thread)(void);
+
+ void (*barrier)(void);
+};
+
+#define DEFINE_RCU_FLAVOR(x) \
+const struct rcu_flavor_struct x = { \
+ .read_lock = rcu_read_lock, \
+ .read_unlock = rcu_read_unlock, \
+ .read_ongoing = rcu_read_ongoing, \
+ .read_quiescent_state = rcu_quiescent_state, \
+ .update_call_rcu = call_rcu, \
+ .update_synchronize_rcu = synchronize_rcu, \
+ .update_defer_rcu = defer_rcu, \
+ .thread_offline = rcu_thread_offline, \
+ .thread_online = rcu_thread_online, \
+ .register_thread = rcu_register_thread, \
+ .unregister_thread = rcu_unregister_thread,\
+ .barrier = rcu_barrier, \
+}
+
+extern const struct rcu_flavor_struct rcu_flavor;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_FLAVOR_H */
--- /dev/null
+
+/*
+ * urcu-pointer.c
+ *
+ * library wrappers to be used by non-LGPL compatible source code.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <urcu/uatomic.h>
+
+#include "urcu/static/urcu-pointer.h"
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#include "urcu-pointer.h"
+
+extern void synchronize_rcu(void);
+
+void *rcu_dereference_sym(void *p)
+{
+ return _rcu_dereference(p);
+}
+
+void *rcu_set_pointer_sym(void **p, void *v)
+{
+ cmm_wmb();
+ uatomic_set(p, v);
+ return v;
+}
+
+void *rcu_xchg_pointer_sym(void **p, void *v)
+{
+ cmm_wmb();
+ return uatomic_xchg(p, v);
+}
+
+void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
+{
+ cmm_wmb();
+ return uatomic_cmpxchg(p, old, _new);
+}
--- /dev/null
+#ifndef _URCU_POINTER_H
+#define _URCU_POINTER_H
+
+/*
+ * urcu-pointer.h
+ *
+ * Userspace RCU header. Operations on pointers.
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <urcu/compiler.h>
+#include <urcu/arch.h>
+#include <urcu/uatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)
+
+#include <urcu/static/urcu-pointer.h>
+
+/*
+ * rcu_dereference(ptr)
+ *
+ * Fetch a RCU-protected pointer. Typically used to copy the variable ptr to a
+ * local variable.
+ */
+#define rcu_dereference _rcu_dereference
+
+/*
+ * type *rcu_cmpxchg_pointer(type **ptr, type *new, type *old)
+ * type *rcu_xchg_pointer(type **ptr, type *new)
+ * void rcu_set_pointer(type **ptr, type *new)
+ *
+ * RCU pointer updates.
+ * @ptr: address of the pointer to modify
+ * @new: new pointer value
+ * @old: old pointer value (expected)
+ *
+ * return: old pointer value
+ */
+#define rcu_cmpxchg_pointer _rcu_cmpxchg_pointer
+#define rcu_xchg_pointer _rcu_xchg_pointer
+#define rcu_set_pointer _rcu_set_pointer
+
+#else /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */
+
+extern void *rcu_dereference_sym(void *p);
+#define rcu_dereference(p) \
+ __extension__ \
+ ({ \
+ __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \
+ rcu_dereference_sym(URCU_FORCE_CAST(void *, p))); \
+ (_________p1); \
+ })
+
+extern void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new);
+#define rcu_cmpxchg_pointer(p, old, _new) \
+ __extension__ \
+ ({ \
+ __typeof__(*(p)) _________pold = (old); \
+ __typeof__(*(p)) _________pnew = (_new); \
+ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
+ rcu_cmpxchg_pointer_sym(URCU_FORCE_CAST(void **, p), \
+ _________pold, \
+ _________pnew)); \
+ (_________p1); \
+ })
+
+extern void *rcu_xchg_pointer_sym(void **p, void *v);
+#define rcu_xchg_pointer(p, v) \
+ __extension__ \
+ ({ \
+ __typeof__(*(p)) _________pv = (v); \
+ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
+ rcu_xchg_pointer_sym(URCU_FORCE_CAST(void **, p), \
+ _________pv)); \
+ (_________p1); \
+ })
+
+/*
+ * Note: rcu_set_pointer_sym returns @v because we don't want to break
+ * the ABI. At the API level, rcu_set_pointer() now returns void. Use of
+ * the return value is therefore deprecated, and will cause a build
+ * error.
+ */
+extern void *rcu_set_pointer_sym(void **p, void *v);
+#define rcu_set_pointer(p, v) \
+ do { \
+ __typeof__(*(p)) _________pv = (v); \
+ (void) rcu_set_pointer_sym(URCU_FORCE_CAST(void **, p), \
+ _________pv); \
+ } while (0)
+
+#endif /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */
+
+/*
+ * void rcu_assign_pointer(type *ptr, type *new)
+ *
+ * Same as rcu_set_pointer, but takes the pointer to assign to rather than its
+ * address as first parameter. Provided for compatibility with the Linux kernel
+ * RCU semantic.
+ */
+#define rcu_assign_pointer(p, v) rcu_set_pointer((&p), (v))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _URCU_POINTER_H */
--- /dev/null
+/*
+ * urcu-qsbr.c
+ *
+ * Userspace RCU QSBR library
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#define _LGPL_SOURCE
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <poll.h>
+
+#include "urcu/wfcqueue.h"
+#include "urcu/map/urcu-qsbr.h"
+#define BUILD_QSBR_LIB
+#include "urcu/static/urcu-qsbr.h"
+#include "urcu-pointer.h"
+#include "urcu/tls-compat.h"
+
+#include "urcu-die.h"
+#include "urcu-wait.h"
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu-qsbr.h"
+#define _LGPL_SOURCE
+
+void __attribute__((destructor)) rcu_exit(void);
+
+/*
+ * rcu_gp_lock ensures mutual exclusion between threads calling
+ * synchronize_rcu().
+ */
+static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
+/*
+ * rcu_registry_lock ensures mutual exclusion between threads
+ * registering and unregistering themselves to/from the registry, and
+ * with threads reading that registry from synchronize_rcu(). However,
+ * this lock is not held all the way through the completion of awaiting
+ * for the grace period. It is sporadically released between iterations
+ * on the registry.
+ * rcu_registry_lock may nest inside rcu_gp_lock.
+ */
+static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
+struct rcu_gp rcu_gp = { .ctr = RCU_GP_ONLINE };
+
+/*
+ * Active attempts to check for reader Q.S. before calling futex().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+/*
+ * Written to only by each individual reader. Read by both the reader and the
+ * writers.
+ */
+DEFINE_URCU_TLS(struct rcu_reader, rcu_reader);
+
+static CDS_LIST_HEAD(registry);
+
+/*
+ * Queue keeping threads awaiting to wait for a grace period. Contains
+ * struct gp_waiters_thread objects.
+ */
+static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
+
+static void mutex_lock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+#ifndef DISTRUST_SIGNALS_EXTREME
+ ret = pthread_mutex_lock(mutex);
+ if (ret)
+ urcu_die(ret);
+#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
+ while ((ret = pthread_mutex_trylock(mutex)) != 0) {
+ if (ret != EBUSY && ret != EINTR)
+ urcu_die(ret);
+ poll(NULL,0,10);
+ }
+#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
+}
+
+static void mutex_unlock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(mutex);
+ if (ret)
+ urcu_die(ret);
+}
+
+/*
+ * synchronize_rcu() waiting. Single thread.
+ */
+static void wait_gp(void)
+{
+ /* Read reader_gp before read futex */
+ cmm_smp_rmb();
+ if (uatomic_read(&rcu_gp.futex) != -1)
+ return;
+ while (futex_noasync(&rcu_gp.futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ return;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+}
+
+/*
+ * Always called with rcu_registry lock held. Releases this lock between
+ * iterations and grabs it again. Holds the lock when it returns.
+ */
+static void wait_for_readers(struct cds_list_head *input_readers,
+ struct cds_list_head *cur_snap_readers,
+ struct cds_list_head *qsreaders)
+{
+ unsigned int wait_loops = 0;
+ struct rcu_reader *index, *tmp;
+
+ /*
+ * Wait for each thread URCU_TLS(rcu_reader).ctr to either
+ * indicate quiescence (offline), or for them to observe the
+ * current rcu_gp.ctr value.
+ */
+ for (;;) {
+ if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
+ wait_loops++;
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ uatomic_set(&rcu_gp.futex, -1);
+ /*
+ * Write futex before write waiting (the other side
+ * reads them in the opposite order).
+ */
+ cmm_smp_wmb();
+ cds_list_for_each_entry(index, input_readers, node) {
+ _CMM_STORE_SHARED(index->waiting, 1);
+ }
+ /* Write futex before read reader_gp */
+ cmm_smp_mb();
+ }
+ cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
+ switch (rcu_reader_state(&index->ctr)) {
+ case RCU_READER_ACTIVE_CURRENT:
+ if (cur_snap_readers) {
+ cds_list_move(&index->node,
+ cur_snap_readers);
+ break;
+ }
+ /* Fall-through */
+ case RCU_READER_INACTIVE:
+ cds_list_move(&index->node, qsreaders);
+ break;
+ case RCU_READER_ACTIVE_OLD:
+ /*
+ * Old snapshot. Leaving node in
+ * input_readers will make us busy-loop
+ * until the snapshot becomes current or
+ * the reader becomes inactive.
+ */
+ break;
+ }
+ }
+
+ if (cds_list_empty(input_readers)) {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* Read reader_gp before write futex */
+ cmm_smp_mb();
+ uatomic_set(&rcu_gp.futex, 0);
+ }
+ break;
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ wait_gp();
+ } else {
+#ifndef HAS_INCOHERENT_CACHES
+ caa_cpu_relax();
+#else /* #ifndef HAS_INCOHERENT_CACHES */
+ cmm_smp_mb();
+#endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+ }
+ /* Re-lock the registry lock before the next loop. */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+}
+
+/*
+ * Using a two-subphases algorithm for architectures with smaller than 64-bit
+ * long-size to ensure we do not encounter an overflow bug.
+ */
+
+#if (CAA_BITS_PER_LONG < 64)
+void synchronize_rcu(void)
+{
+ CDS_LIST_HEAD(cur_snap_readers);
+ CDS_LIST_HEAD(qsreaders);
+ unsigned long was_online;
+ DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
+ struct urcu_waiters waiters;
+
+ was_online = rcu_read_ongoing();
+
+ /* All threads should read qparity before accessing data structure
+ * where new ptr points to. In the "then" case, rcu_thread_offline
+ * includes a memory barrier.
+ *
+ * Mark the writer thread offline to make sure we don't wait for
+ * our own quiescent state. This allows using synchronize_rcu()
+ * in threads registered as readers.
+ */
+ if (was_online)
+ rcu_thread_offline();
+ else
+ cmm_smp_mb();
+
+ /*
+ * Add ourself to gp_waiters queue of threads awaiting to wait
+ * for a grace period. Proceed to perform the grace period only
+ * if we are the first thread added into the queue.
+ */
+ if (urcu_wait_add(&gp_waiters, &wait) != 0) {
+ /* Not first in queue: will be awakened by another thread. */
+ urcu_adaptative_busy_wait(&wait);
+ goto gp_end;
+ }
+ /* We won't need to wake ourself up */
+ urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
+
+ mutex_lock(&rcu_gp_lock);
+
+ /*
+ * Move all waiters into our local queue.
+ */
+ urcu_move_waiters(&waiters, &gp_waiters);
+
+ mutex_lock(&rcu_registry_lock);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /*
+ * Wait for readers to observe original parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
+
+ /*
+ * Must finish waiting for quiescent state for original parity
+ * before committing next rcu_gp.ctr update to memory. Failure
+ * to do so could result in the writer waiting forever while new
+ * readers are always accessing data (no progress). Enforce
+ * compiler-order of load URCU_TLS(rcu_reader).ctr before store
+ * to rcu_gp.ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /* Switch parity: 0 -> 1, 1 -> 0 */
+ CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR);
+
+ /*
+ * Must commit rcu_gp.ctr update to memory before waiting for
+ * quiescent state. Failure to do so could result in the writer
+ * waiting forever while new readers are always accessing data
+ * (no progress). Enforce compiler-order of store to rcu_gp.ctr
+ * before load URCU_TLS(rcu_reader).ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /*
+ * Wait for readers to observe new parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
+
+ /*
+ * Put quiescent reader list back into registry.
+ */
+ cds_list_splice(&qsreaders, ®istry);
+out:
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+ urcu_wake_all_waiters(&waiters);
+gp_end:
+ /*
+ * Finish waiting for reader threads before letting the old ptr being
+ * freed.
+ */
+ if (was_online)
+ rcu_thread_online();
+ else
+ cmm_smp_mb();
+}
+#else /* !(CAA_BITS_PER_LONG < 64) */
+void synchronize_rcu(void)
+{
+ CDS_LIST_HEAD(qsreaders);
+ unsigned long was_online;
+ DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
+ struct urcu_waiters waiters;
+
+ was_online = rcu_read_ongoing();
+
+ /*
+ * Mark the writer thread offline to make sure we don't wait for
+ * our own quiescent state. This allows using synchronize_rcu()
+ * in threads registered as readers.
+ */
+ if (was_online)
+ rcu_thread_offline();
+ else
+ cmm_smp_mb();
+
+ /*
+ * Add ourself to gp_waiters queue of threads awaiting to wait
+ * for a grace period. Proceed to perform the grace period only
+ * if we are the first thread added into the queue.
+ */
+ if (urcu_wait_add(&gp_waiters, &wait) != 0) {
+ /* Not first in queue: will be awakened by another thread. */
+ urcu_adaptative_busy_wait(&wait);
+ goto gp_end;
+ }
+ /* We won't need to wake ourself up */
+ urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
+
+ mutex_lock(&rcu_gp_lock);
+
+ /*
+ * Move all waiters into our local queue.
+ */
+ urcu_move_waiters(&waiters, &gp_waiters);
+
+ mutex_lock(&rcu_registry_lock);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /* Increment current G.P. */
+ CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr + RCU_GP_CTR);
+
+ /*
+ * Must commit rcu_gp.ctr update to memory before waiting for
+ * quiescent state. Failure to do so could result in the writer
+ * waiting forever while new readers are always accessing data
+ * (no progress). Enforce compiler-order of store to rcu_gp.ctr
+ * before load URCU_TLS(rcu_reader).ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /*
+ * Wait for readers to observe new count of be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(®istry, NULL, &qsreaders);
+
+ /*
+ * Put quiescent reader list back into registry.
+ */
+ cds_list_splice(&qsreaders, ®istry);
+out:
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+ urcu_wake_all_waiters(&waiters);
+gp_end:
+ if (was_online)
+ rcu_thread_online();
+ else
+ cmm_smp_mb();
+}
+#endif /* !(CAA_BITS_PER_LONG < 64) */
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void rcu_read_lock(void)
+{
+ _rcu_read_lock();
+}
+
+void rcu_read_unlock(void)
+{
+ _rcu_read_unlock();
+}
+
+int rcu_read_ongoing(void)
+{
+ return _rcu_read_ongoing();
+}
+
+void rcu_quiescent_state(void)
+{
+ _rcu_quiescent_state();
+}
+
+void rcu_thread_offline(void)
+{
+ _rcu_thread_offline();
+}
+
+void rcu_thread_online(void)
+{
+ _rcu_thread_online();
+}
+
+void rcu_register_thread(void)
+{
+ URCU_TLS(rcu_reader).tid = pthread_self();
+ assert(URCU_TLS(rcu_reader).ctr == 0);
+
+ mutex_lock(&rcu_registry_lock);
+ assert(!URCU_TLS(rcu_reader).registered);
+ URCU_TLS(rcu_reader).registered = 1;
+ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry);
+ mutex_unlock(&rcu_registry_lock);
+ _rcu_thread_online();
+}
+
+void rcu_unregister_thread(void)
+{
+ /*
+ * We have to make the thread offline otherwise we end up dealocking
+ * with a waiting writer.
+ */
+ _rcu_thread_offline();
+ assert(URCU_TLS(rcu_reader).registered);
+ URCU_TLS(rcu_reader).registered = 0;
+ mutex_lock(&rcu_registry_lock);
+ cds_list_del(&URCU_TLS(rcu_reader).node);
+ mutex_unlock(&rcu_registry_lock);
+}
+
+void rcu_exit(void)
+{
+ /*
+ * Assertion disabled because call_rcu threads are now rcu
+ * readers, and left running at exit.
+ * assert(cds_list_empty(®istry));
+ */
+}
+
+DEFINE_RCU_FLAVOR(rcu_flavor);
+
+#include "urcu-call-rcu-impl.h"
+#include "urcu-defer-impl.h"
--- /dev/null
+#ifndef _URCU_QSBR_H
+#define _URCU_QSBR_H
+
+/*
+ * urcu-qsbr.h
+ *
+ * Userspace RCU QSBR header.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+
+/*
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
+ */
+#include <urcu-pointer.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <urcu/map/urcu-qsbr.h>
+
+#ifdef RCU_DEBUG /* For backward compatibility */
+#define DEBUG_RCU
+#endif
+
+/*
+ * Important !
+ *
+ * Each thread containing read-side critical sections must be registered
+ * with rcu_register_thread() before calling rcu_read_lock().
+ * rcu_unregister_thread() should be called before the thread exits.
+ */
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/urcu-qsbr.h>
+
+/*
+ * Mappings for static use of the userspace RCU library.
+ * Should only be used in LGPL-compatible code.
+ */
+
+/*
+ * rcu_read_lock()
+ * rcu_read_unlock()
+ *
+ * Mark the beginning and end of a read-side critical section.
+ * DON'T FORGET TO USE rcu_register_thread/rcu_unregister_thread()
+ * FOR EACH THREAD WITH READ-SIDE CRITICAL SECTION.
+ */
+#define rcu_read_lock_qsbr _rcu_read_lock
+#define rcu_read_unlock_qsbr _rcu_read_unlock
+#define rcu_read_ongoing_qsbr _rcu_read_ongoing
+
+#define rcu_quiescent_state_qsbr _rcu_quiescent_state
+#define rcu_thread_offline_qsbr _rcu_thread_offline
+#define rcu_thread_online_qsbr _rcu_thread_online
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+/*
+ * QSBR read lock/unlock are guaranteed to be no-ops. Therefore, we expose them
+ * in the LGPL header for any code to use. However, the debug version is not
+ * nops and may contain sanity checks. To activate it, applications must be
+ * recompiled with -DDEBUG_RCU (even non-LGPL/GPL applications). This is the
+ * best trade-off between license/performance/code triviality and
+ * library debugging & tracing features we could come up with.
+ */
+
+#if (!defined(BUILD_QSBR_LIB) && !defined(DEBUG_RCU))
+
+static inline void rcu_read_lock(void)
+{
+}
+
+static inline void rcu_read_unlock(void)
+{
+}
+
+#else /* !DEBUG_RCU */
+
+extern void rcu_read_lock(void);
+extern void rcu_read_unlock(void);
+
+#endif /* !DEBUG_RCU */
+
+extern int rcu_read_ongoing(void);
+extern void rcu_quiescent_state(void);
+extern void rcu_thread_offline(void);
+extern void rcu_thread_online(void);
+
+#endif /* !_LGPL_SOURCE */
+
+extern void synchronize_rcu(void);
+
+/*
+ * Reader thread registration.
+ */
+extern void rcu_register_thread(void);
+extern void rcu_unregister_thread(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu-call-rcu.h>
+#include <urcu-defer.h>
+#include <urcu-flavor.h>
+
+#endif /* _URCU_QSBR_H */
--- /dev/null
+#ifndef _URCU_WAIT_H
+#define _URCU_WAIT_H
+
+/*
+ * urcu-wait.h
+ *
+ * Userspace RCU library wait/wakeup management
+ *
+ * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <urcu/uatomic.h>
+#include <urcu/wfstack.h>
+#include "urcu-die.h"
+
+/*
+ * Number of busy-loop attempts before waiting on futex for grace period
+ * batching.
+ */
+#define URCU_WAIT_ATTEMPTS 1000
+
+enum urcu_wait_state {
+ /* URCU_WAIT_WAITING is compared directly (futex compares it). */
+ URCU_WAIT_WAITING = 0,
+ /* non-zero are used as masks. */
+ URCU_WAIT_WAKEUP = (1 << 0),
+ URCU_WAIT_RUNNING = (1 << 1),
+ URCU_WAIT_TEARDOWN = (1 << 2),
+};
+
+struct urcu_wait_node {
+ struct cds_wfs_node node;
+ int32_t state; /* enum urcu_wait_state */
+};
+
+#define URCU_WAIT_NODE_INIT(name, _state) \
+ { .state = _state }
+
+#define DEFINE_URCU_WAIT_NODE(name, state) \
+ struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)
+
+#define DECLARE_URCU_WAIT_NODE(name) \
+ struct urcu_wait_node name
+
+struct urcu_wait_queue {
+ struct cds_wfs_stack stack;
+};
+
+#define URCU_WAIT_QUEUE_HEAD_INIT(name) \
+ { .stack.head = CDS_WFS_END, .stack.lock = PTHREAD_MUTEX_INITIALIZER }
+
+#define DECLARE_URCU_WAIT_QUEUE(name) \
+ struct urcu_wait_queue name
+
+#define DEFINE_URCU_WAIT_QUEUE(name) \
+ struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)
+
+struct urcu_waiters {
+ struct cds_wfs_head *head;
+};
+
+/*
+ * Add ourself atomically to a wait queue. Return 0 if queue was
+ * previously empty, else return 1.
+ * A full memory barrier is issued before being added to the wait queue.
+ */
+static inline
+bool urcu_wait_add(struct urcu_wait_queue *queue,
+ struct urcu_wait_node *node)
+{
+ return cds_wfs_push(&queue->stack, &node->node);
+}
+
+/*
+ * Atomically move all waiters from wait queue into our local struct
+ * urcu_waiters.
+ */
+static inline
+void urcu_move_waiters(struct urcu_waiters *waiters,
+ struct urcu_wait_queue *queue)
+{
+ waiters->head = __cds_wfs_pop_all(&queue->stack);
+}
+
+static inline
+void urcu_wait_set_state(struct urcu_wait_node *node,
+ enum urcu_wait_state state)
+{
+ node->state = state;
+}
+
+static inline
+void urcu_wait_node_init(struct urcu_wait_node *node,
+ enum urcu_wait_state state)
+{
+ urcu_wait_set_state(node, state);
+ cds_wfs_node_init(&node->node);
+}
+
+/*
+ * Note: urcu_adaptative_wake_up needs "value" to stay allocated
+ * throughout its execution. In this scheme, the waiter owns the node
+ * memory, and we only allow it to free this memory when it receives the
+ * URCU_WAIT_TEARDOWN flag.
+ */
+static inline
+void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
+{
+ cmm_smp_mb();
+ assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
+ uatomic_set(&wait->state, URCU_WAIT_WAKEUP);
+ if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
+ if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
+ NULL, NULL, 0) < 0)
+ urcu_die(errno);
+ }
+ /* Allow teardown of struct urcu_wait memory. */
+ uatomic_or(&wait->state, URCU_WAIT_TEARDOWN);
+}
+
+/*
+ * Caller must initialize "value" to URCU_WAIT_WAITING before passing its
+ * memory to waker thread.
+ */
+static inline
+void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
+{
+ unsigned int i;
+
+ /* Load and test condition before read state */
+ cmm_smp_rmb();
+ for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
+ if (uatomic_read(&wait->state) != URCU_WAIT_WAITING)
+ goto skip_futex_wait;
+ caa_cpu_relax();
+ }
+ while (futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ goto skip_futex_wait;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+skip_futex_wait:
+
+ /* Tell waker thread than we are running. */
+ uatomic_or(&wait->state, URCU_WAIT_RUNNING);
+
+ /*
+ * Wait until waker thread lets us know it's ok to tear down
+ * memory allocated for struct urcu_wait.
+ */
+ for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
+ if (uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN)
+ break;
+ caa_cpu_relax();
+ }
+ while (!(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN))
+ poll(NULL, 0, 10);
+ assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
+}
+
+static inline
+void urcu_wake_all_waiters(struct urcu_waiters *waiters)
+{
+ struct cds_wfs_node *iter, *iter_n;
+
+ /* Wake all waiters in our stack head */
+ cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
+ struct urcu_wait_node *wait_node =
+ caa_container_of(iter, struct urcu_wait_node, node);
+
+ /* Don't wake already running threads */
+ if (wait_node->state & URCU_WAIT_RUNNING)
+ continue;
+ urcu_adaptative_wake_up(wait_node);
+ }
+}
+
+#endif /* _URCU_WAIT_H */
--- /dev/null
+/*
+ * urcu.c
+ *
+ * Userspace RCU library
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#define _BSD_SOURCE
+#define _LGPL_SOURCE
+#define _DEFAULT_SOURCE
+#include <stdio.h>
+#include <pthread.h>
+#include <signal.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <poll.h>
+
+#include "urcu/arch.h"
+#include "urcu/wfcqueue.h"
+#include "urcu/map/urcu.h"
+#include "urcu/static/urcu.h"
+#include "urcu-pointer.h"
+#include "urcu/tls-compat.h"
+
+#include "urcu-die.h"
+#include "urcu-wait.h"
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#undef _LGPL_SOURCE
+#include "urcu.h"
+#define _LGPL_SOURCE
+
+/*
+ * If a reader is really non-cooperative and refuses to commit its
+ * rcu_active_readers count to memory (there is no barrier in the reader
+ * per-se), kick it after 10 loops waiting for it.
+ */
+#define KICK_READER_LOOPS 10
+
+/*
+ * Active attempts to check for reader Q.S. before calling futex().
+ */
+#define RCU_QS_ACTIVE_ATTEMPTS 100
+
+/* If the headers do not support membarrier system call, fall back on RCU_MB */
+#ifdef __NR_membarrier
+# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
+#else
+# define membarrier(...) -ENOSYS
+#endif
+
+enum membarrier_cmd {
+ MEMBARRIER_CMD_QUERY = 0,
+ MEMBARRIER_CMD_SHARED = (1 << 0),
+};
+
+#ifdef RCU_MEMBARRIER
+static int init_done;
+int rcu_has_sys_membarrier;
+
+void __attribute__((constructor)) rcu_init(void);
+#endif
+
+#ifdef RCU_MB
+void rcu_init(void)
+{
+}
+#endif
+
+#ifdef RCU_SIGNAL
+static int init_done;
+
+void __attribute__((constructor)) rcu_init(void);
+void __attribute__((destructor)) rcu_exit(void);
+#endif
+
+/*
+ * rcu_gp_lock ensures mutual exclusion between threads calling
+ * synchronize_rcu().
+ */
+static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
+/*
+ * rcu_registry_lock ensures mutual exclusion between threads
+ * registering and unregistering themselves to/from the registry, and
+ * with threads reading that registry from synchronize_rcu(). However,
+ * this lock is not held all the way through the completion of awaiting
+ * for the grace period. It is sporadically released between iterations
+ * on the registry.
+ * rcu_registry_lock may nest inside rcu_gp_lock.
+ */
+static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
+struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
+
+/*
+ * Written to only by each individual reader. Read by both the reader and the
+ * writers.
+ */
+DEFINE_URCU_TLS(struct rcu_reader, rcu_reader);
+
+static CDS_LIST_HEAD(registry);
+
+/*
+ * Queue keeping threads awaiting to wait for a grace period. Contains
+ * struct gp_waiters_thread objects.
+ */
+static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
+
+static void mutex_lock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+#ifndef DISTRUST_SIGNALS_EXTREME
+ ret = pthread_mutex_lock(mutex);
+ if (ret)
+ urcu_die(ret);
+#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
+ while ((ret = pthread_mutex_trylock(mutex)) != 0) {
+ if (ret != EBUSY && ret != EINTR)
+ urcu_die(ret);
+ if (CMM_LOAD_SHARED(URCU_TLS(rcu_reader).need_mb)) {
+ cmm_smp_mb();
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
+ cmm_smp_mb();
+ }
+ (void) poll(NULL, 0, 10);
+ }
+#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
+}
+
+static void mutex_unlock(pthread_mutex_t *mutex)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(mutex);
+ if (ret)
+ urcu_die(ret);
+}
+
+#ifdef RCU_MEMBARRIER
+static void smp_mb_master(void)
+{
+ if (caa_likely(rcu_has_sys_membarrier))
+ (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
+ else
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_MB
+static void smp_mb_master(void)
+{
+ cmm_smp_mb();
+}
+#endif
+
+#ifdef RCU_SIGNAL
+static void force_mb_all_readers(void)
+{
+ struct rcu_reader *index;
+
+ /*
+ * Ask for each threads to execute a cmm_smp_mb() so we can consider the
+ * compiler barriers around rcu read lock as real memory barriers.
+ */
+ if (cds_list_empty(®istry))
+ return;
+ /*
+ * pthread_kill has a cmm_smp_mb(). But beware, we assume it performs
+ * a cache flush on architectures with non-coherent cache. Let's play
+ * safe and don't assume anything : we use cmm_smp_mc() to make sure the
+ * cache flush is enforced.
+ */
+ cds_list_for_each_entry(index, ®istry, node) {
+ CMM_STORE_SHARED(index->need_mb, 1);
+ pthread_kill(index->tid, SIGRCU);
+ }
+ /*
+ * Wait for sighandler (and thus mb()) to execute on every thread.
+ *
+ * Note that the pthread_kill() will never be executed on systems
+ * that correctly deliver signals in a timely manner. However, it
+ * is not uncommon for kernels to have bugs that can result in
+ * lost or unduly delayed signals.
+ *
+ * If you are seeing the below pthread_kill() executing much at
+ * all, we suggest testing the underlying kernel and filing the
+ * relevant bug report. For Linux kernels, we recommend getting
+ * the Linux Test Project (LTP).
+ */
+ cds_list_for_each_entry(index, ®istry, node) {
+ while (CMM_LOAD_SHARED(index->need_mb)) {
+ pthread_kill(index->tid, SIGRCU);
+ (void) poll(NULL, 0, 1);
+ }
+ }
+ cmm_smp_mb(); /* read ->need_mb before ending the barrier */
+}
+
+static void smp_mb_master(void)
+{
+ force_mb_all_readers();
+}
+#endif /* #ifdef RCU_SIGNAL */
+
+/*
+ * synchronize_rcu() waiting. Single thread.
+ * Always called with rcu_registry lock held. Releases this lock and
+ * grabs it again. Holds the lock when it returns.
+ */
+static void wait_gp(void)
+{
+ /*
+ * Read reader_gp before read futex. smp_mb_master() needs to
+ * be called with the rcu registry lock held in RCU_SIGNAL
+ * flavor.
+ */
+ smp_mb_master();
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ if (uatomic_read(&rcu_gp.futex) != -1)
+ goto end;
+ while (futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
+ NULL, NULL, 0)) {
+ switch (errno) {
+ case EWOULDBLOCK:
+ /* Value already changed. */
+ goto end;
+ case EINTR:
+ /* Retry if interrupted by signal. */
+ break; /* Get out of switch. */
+ default:
+ /* Unexpected error. */
+ urcu_die(errno);
+ }
+ }
+end:
+ /*
+ * Re-lock the registry lock before the next loop.
+ */
+ mutex_lock(&rcu_registry_lock);
+}
+
+/*
+ * Always called with rcu_registry lock held. Releases this lock between
+ * iterations and grabs it again. Holds the lock when it returns.
+ */
+static void wait_for_readers(struct cds_list_head *input_readers,
+ struct cds_list_head *cur_snap_readers,
+ struct cds_list_head *qsreaders)
+{
+ unsigned int wait_loops = 0;
+ struct rcu_reader *index, *tmp;
+#ifdef HAS_INCOHERENT_CACHES
+ unsigned int wait_gp_loops = 0;
+#endif /* HAS_INCOHERENT_CACHES */
+
+ /*
+ * Wait for each thread URCU_TLS(rcu_reader).ctr to either
+ * indicate quiescence (not nested), or observe the current
+ * rcu_gp.ctr value.
+ */
+ for (;;) {
+ if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
+ wait_loops++;
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ uatomic_dec(&rcu_gp.futex);
+ /* Write futex before read reader_gp */
+ smp_mb_master();
+ }
+
+ cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
+ switch (rcu_reader_state(&index->ctr)) {
+ case RCU_READER_ACTIVE_CURRENT:
+ if (cur_snap_readers) {
+ cds_list_move(&index->node,
+ cur_snap_readers);
+ break;
+ }
+ /* Fall-through */
+ case RCU_READER_INACTIVE:
+ cds_list_move(&index->node, qsreaders);
+ break;
+ case RCU_READER_ACTIVE_OLD:
+ /*
+ * Old snapshot. Leaving node in
+ * input_readers will make us busy-loop
+ * until the snapshot becomes current or
+ * the reader becomes inactive.
+ */
+ break;
+ }
+ }
+
+#ifndef HAS_INCOHERENT_CACHES
+ if (cds_list_empty(input_readers)) {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* Read reader_gp before write futex */
+ smp_mb_master();
+ uatomic_set(&rcu_gp.futex, 0);
+ }
+ break;
+ } else {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* wait_gp unlocks/locks registry lock. */
+ wait_gp();
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ caa_cpu_relax();
+ /*
+ * Re-lock the registry lock before the
+ * next loop.
+ */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+#else /* #ifndef HAS_INCOHERENT_CACHES */
+ /*
+ * BUSY-LOOP. Force the reader thread to commit its
+ * URCU_TLS(rcu_reader).ctr update to memory if we wait
+ * for too long.
+ */
+ if (cds_list_empty(input_readers)) {
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* Read reader_gp before write futex */
+ smp_mb_master();
+ uatomic_set(&rcu_gp.futex, 0);
+ }
+ break;
+ } else {
+ if (wait_gp_loops == KICK_READER_LOOPS) {
+ smp_mb_master();
+ wait_gp_loops = 0;
+ }
+ if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
+ /* wait_gp unlocks/locks registry lock. */
+ wait_gp();
+ wait_gp_loops++;
+ } else {
+ /* Temporarily unlock the registry lock. */
+ mutex_unlock(&rcu_registry_lock);
+ caa_cpu_relax();
+ /*
+ * Re-lock the registry lock before the
+ * next loop.
+ */
+ mutex_lock(&rcu_registry_lock);
+ }
+ }
+#endif /* #else #ifndef HAS_INCOHERENT_CACHES */
+ }
+}
+
+void synchronize_rcu(void)
+{
+ CDS_LIST_HEAD(cur_snap_readers);
+ CDS_LIST_HEAD(qsreaders);
+ DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
+ struct urcu_waiters waiters;
+
+ /*
+ * Add ourself to gp_waiters queue of threads awaiting to wait
+ * for a grace period. Proceed to perform the grace period only
+ * if we are the first thread added into the queue.
+ * The implicit memory barrier before urcu_wait_add()
+ * orders prior memory accesses of threads put into the wait
+ * queue before their insertion into the wait queue.
+ */
+ if (urcu_wait_add(&gp_waiters, &wait) != 0) {
+ /* Not first in queue: will be awakened by another thread. */
+ urcu_adaptative_busy_wait(&wait);
+ /* Order following memory accesses after grace period. */
+ cmm_smp_mb();
+ return;
+ }
+ /* We won't need to wake ourself up */
+ urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
+
+ mutex_lock(&rcu_gp_lock);
+
+ /*
+ * Move all waiters into our local queue.
+ */
+ urcu_move_waiters(&waiters, &gp_waiters);
+
+ mutex_lock(&rcu_registry_lock);
+
+ if (cds_list_empty(®istry))
+ goto out;
+
+ /*
+ * All threads should read qparity before accessing data structure
+ * where new ptr points to. Must be done within rcu_registry_lock
+ * because it iterates on reader threads.
+ */
+ /* Write new ptr before changing the qparity */
+ smp_mb_master();
+
+ /*
+ * Wait for readers to observe original parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
+
+ /*
+ * Must finish waiting for quiescent state for original parity before
+ * committing next rcu_gp.ctr update to memory. Failure to do so could
+ * result in the writer waiting forever while new readers are always
+ * accessing data (no progress). Enforce compiler-order of load
+ * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
+ */
+ cmm_barrier();
+
+ /*
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /* Switch parity: 0 -> 1, 1 -> 0 */
+ CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
+
+ /*
+ * Must commit rcu_gp.ctr update to memory before waiting for quiescent
+ * state. Failure to do so could result in the writer waiting forever
+ * while new readers are always accessing data (no progress). Enforce
+ * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
+ */
+ cmm_barrier();
+
+ /*
+ *
+ * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
+ * model easier to understand. It does not have a big performance impact
+ * anyway, given this is the write-side.
+ */
+ cmm_smp_mb();
+
+ /*
+ * Wait for readers to observe new parity or be quiescent.
+ * wait_for_readers() can release and grab again rcu_registry_lock
+ * interally.
+ */
+ wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
+
+ /*
+ * Put quiescent reader list back into registry.
+ */
+ cds_list_splice(&qsreaders, ®istry);
+
+ /*
+ * Finish waiting for reader threads before letting the old ptr
+ * being freed. Must be done within rcu_registry_lock because it
+ * iterates on reader threads.
+ */
+ smp_mb_master();
+out:
+ mutex_unlock(&rcu_registry_lock);
+ mutex_unlock(&rcu_gp_lock);
+
+ /*
+ * Wakeup waiters only after we have completed the grace period
+ * and have ensured the memory barriers at the end of the grace
+ * period have been issued.
+ */
+ urcu_wake_all_waiters(&waiters);
+}
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void rcu_read_lock(void)
+{
+ _rcu_read_lock();
+}
+
+void rcu_read_unlock(void)
+{
+ _rcu_read_unlock();
+}
+
+int rcu_read_ongoing(void)
+{
+ return _rcu_read_ongoing();
+}
+
+void rcu_register_thread(void)
+{
+ URCU_TLS(rcu_reader).tid = pthread_self();
+ assert(URCU_TLS(rcu_reader).need_mb == 0);
+ assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK));
+
+ mutex_lock(&rcu_registry_lock);
+ assert(!URCU_TLS(rcu_reader).registered);
+ URCU_TLS(rcu_reader).registered = 1;
+ rcu_init(); /* In case gcc does not support constructor attribute */
+ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry);
+ mutex_unlock(&rcu_registry_lock);
+}
+
+void rcu_unregister_thread(void)
+{
+ mutex_lock(&rcu_registry_lock);
+ assert(URCU_TLS(rcu_reader).registered);
+ URCU_TLS(rcu_reader).registered = 0;
+ cds_list_del(&URCU_TLS(rcu_reader).node);
+ mutex_unlock(&rcu_registry_lock);
+}
+
+#ifdef RCU_MEMBARRIER
+void rcu_init(void)
+{
+ int ret;
+
+ if (init_done)
+ return;
+ init_done = 1;
+ ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
+ if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
+ rcu_has_sys_membarrier = 1;
+ }
+}
+#endif
+
+#ifdef RCU_SIGNAL
+static void sigrcu_handler(int signo, siginfo_t *siginfo, void *context)
+{
+ /*
+ * Executing this cmm_smp_mb() is the only purpose of this signal handler.
+ * It punctually promotes cmm_barrier() into cmm_smp_mb() on every thread it is
+ * executed on.
+ */
+ cmm_smp_mb();
+ _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
+ cmm_smp_mb();
+}
+
+/*
+ * rcu_init constructor. Called when the library is linked, but also when
+ * reader threads are calling rcu_register_thread().
+ * Should only be called by a single thread at a given time. This is ensured by
+ * holing the rcu_registry_lock from rcu_register_thread() or by running
+ * at library load time, which should not be executed by multiple
+ * threads nor concurrently with rcu_register_thread() anyway.
+ */
+void rcu_init(void)
+{
+ struct sigaction act;
+ int ret;
+
+ if (init_done)
+ return;
+ init_done = 1;
+
+ act.sa_sigaction = sigrcu_handler;
+ act.sa_flags = SA_SIGINFO | SA_RESTART;
+ sigemptyset(&act.sa_mask);
+ ret = sigaction(SIGRCU, &act, NULL);
+ if (ret)
+ urcu_die(errno);
+}
+
+void rcu_exit(void)
+{
+ /*
+ * Don't unregister the SIGRCU signal handler anymore, because
+ * call_rcu threads could still be using it shortly before the
+ * application exits.
+ * Assertion disabled because call_rcu threads are now rcu
+ * readers, and left running at exit.
+ * assert(cds_list_empty(®istry));
+ */
+}
+
+#endif /* #ifdef RCU_SIGNAL */
+
+DEFINE_RCU_FLAVOR(rcu_flavor);
+
+#include "urcu-call-rcu-impl.h"
+#include "urcu-defer-impl.h"
--- /dev/null
+#ifndef _URCU_H
+#define _URCU_H
+
+/*
+ * urcu.h
+ *
+ * Userspace RCU header
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ *
+ * LGPL-compatible code should include this header with :
+ *
+ * #define _LGPL_SOURCE
+ * #include <urcu.h>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include <stdlib.h>
+#include <pthread.h>
+
+/*
+ * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
+ * publication headers.
+ */
+#include <urcu-pointer.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <urcu/map/urcu.h>
+
+/*
+ * Important !
+ *
+ * Each thread containing read-side critical sections must be registered
+ * with rcu_register_thread_mb() before calling rcu_read_lock_mb().
+ * rcu_unregister_thread_mb() should be called before the thread exits.
+ */
+
+#ifdef _LGPL_SOURCE
+
+#include <urcu/static/urcu.h>
+
+/*
+ * Mappings for static use of the userspace RCU library.
+ * Should only be used in LGPL-compatible code.
+ */
+
+/*
+ * rcu_read_lock()
+ * rcu_read_unlock()
+ *
+ * Mark the beginning and end of a read-side critical section.
+ * DON'T FORGET TO USE RCU_REGISTER/UNREGISTER_THREAD() FOR EACH THREAD WITH
+ * READ-SIDE CRITICAL SECTION.
+ */
+#ifdef RCU_MEMBARRIER
+#define rcu_read_lock_memb _rcu_read_lock
+#define rcu_read_unlock_memb _rcu_read_unlock
+#define rcu_read_ongoing_memb _rcu_read_ongoing
+#elif defined(RCU_SIGNAL)
+#define rcu_read_lock_sig _rcu_read_lock
+#define rcu_read_unlock_sig _rcu_read_unlock
+#define rcu_read_ongoing_sig _rcu_read_ongoing
+#elif defined(RCU_MB)
+#define rcu_read_lock_mb _rcu_read_lock
+#define rcu_read_unlock_mb _rcu_read_unlock
+#define rcu_read_ongoing_mb _rcu_read_ongoing
+#endif
+
+#else /* !_LGPL_SOURCE */
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ * See LGPL-only urcu/static/urcu-pointer.h for documentation.
+ */
+
+extern void rcu_read_lock(void);
+extern void rcu_read_unlock(void);
+extern int rcu_read_ongoing(void);
+
+#endif /* !_LGPL_SOURCE */
+
+extern void synchronize_rcu(void);
+
+/*
+ * Reader thread registration.
+ */
+extern void rcu_register_thread(void);
+extern void rcu_unregister_thread(void);
+
+/*
+ * Explicit rcu initialization, for "early" use within library constructors.
+ */
+extern void rcu_init(void);
+
+/*
+ * Q.S. reporting are no-ops for these URCU flavors.
+ */
+static inline void rcu_quiescent_state(void)
+{
+}
+
+static inline void rcu_thread_offline(void)
+{
+}
+
+static inline void rcu_thread_online(void)
+{
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#include <urcu-call-rcu.h>
+#include <urcu-defer.h>
+#include <urcu-flavor.h>
+
+#endif /* _URCU_H */
--- /dev/null
+/*
+ * wfcqueue.c
+ *
+ * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#include "urcu/wfcqueue.h"
+#include "urcu/static/wfcqueue.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void cds_wfcq_node_init(struct cds_wfcq_node *node)
+{
+ _cds_wfcq_node_init(node);
+}
+
+void cds_wfcq_init(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ _cds_wfcq_init(head, tail);
+}
+
+void cds_wfcq_destroy(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ _cds_wfcq_destroy(head, tail);
+}
+
+void __cds_wfcq_init(struct __cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ ___cds_wfcq_init(head, tail);
+}
+
+bool cds_wfcq_empty(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+
+{
+ return _cds_wfcq_empty(head, tail);
+}
+
+bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node)
+{
+ return _cds_wfcq_enqueue(head, tail, node);
+}
+
+void cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ _cds_wfcq_dequeue_lock(head, tail);
+}
+
+void cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ _cds_wfcq_dequeue_unlock(head, tail);
+}
+
+struct cds_wfcq_node *cds_wfcq_dequeue_blocking(
+ struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail)
+{
+ return _cds_wfcq_dequeue_blocking(head, tail);
+}
+
+struct cds_wfcq_node *cds_wfcq_dequeue_with_state_blocking(
+ struct cds_wfcq_head *head,
+ struct cds_wfcq_tail *tail,
+ int *state)
+{
+ return _cds_wfcq_dequeue_with_state_blocking(head, tail, state);
+}
+
+enum cds_wfcq_ret cds_wfcq_splice_blocking(
+ struct cds_wfcq_head *dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ struct cds_wfcq_head *src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ return _cds_wfcq_splice_blocking(dest_q_head, dest_q_tail,
+ src_q_head, src_q_tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_dequeue_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_dequeue_blocking(head, tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ int *state)
+{
+ return ___cds_wfcq_dequeue_with_state_blocking(head, tail, state);
+}
+
+struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_dequeue_nonblocking(head, tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ int *state)
+{
+ return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, state);
+}
+
+enum cds_wfcq_ret __cds_wfcq_splice_blocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ return ___cds_wfcq_splice_blocking(dest_q_head, dest_q_tail,
+ src_q_head, src_q_tail);
+}
+
+enum cds_wfcq_ret __cds_wfcq_splice_nonblocking(
+ cds_wfcq_head_ptr_t dest_q_head,
+ struct cds_wfcq_tail *dest_q_tail,
+ cds_wfcq_head_ptr_t src_q_head,
+ struct cds_wfcq_tail *src_q_tail)
+{
+ return ___cds_wfcq_splice_nonblocking(dest_q_head, dest_q_tail,
+ src_q_head, src_q_tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_first_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_first_blocking(head, tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_first_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail)
+{
+ return ___cds_wfcq_first_nonblocking(head, tail);
+}
+
+struct cds_wfcq_node *__cds_wfcq_next_blocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node)
+{
+ return ___cds_wfcq_next_blocking(head, tail, node);
+}
+
+struct cds_wfcq_node *__cds_wfcq_next_nonblocking(
+ cds_wfcq_head_ptr_t head,
+ struct cds_wfcq_tail *tail,
+ struct cds_wfcq_node *node)
+{
+ return ___cds_wfcq_next_nonblocking(head, tail, node);
+}
--- /dev/null
+/*
+ * wfqueue.c
+ *
+ * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
+ *
+ * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Remove deprecation warnings from LGPL wrapper build. */
+#define CDS_WFQ_DEPRECATED
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#include "urcu/wfqueue.h"
+#include "urcu/static/wfqueue.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void cds_wfq_node_init(struct cds_wfq_node *node)
+{
+ _cds_wfq_node_init(node);
+}
+
+void cds_wfq_init(struct cds_wfq_queue *q)
+{
+ _cds_wfq_init(q);
+}
+
+void cds_wfq_destroy(struct cds_wfq_queue *q)
+{
+ _cds_wfq_destroy(q);
+}
+
+void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node)
+{
+ _cds_wfq_enqueue(q, node);
+}
+
+struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ return ___cds_wfq_dequeue_blocking(q);
+}
+
+struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
+{
+ return _cds_wfq_dequeue_blocking(q);
+}
--- /dev/null
+/*
+ * wfstack.c
+ *
+ * Userspace RCU library - Stack with wait-free push, blocking traversal.
+ *
+ * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
+#include "urcu/wfstack.h"
+#include "urcu/static/wfstack.h"
+
+/*
+ * library wrappers to be used by non-LGPL compatible source code.
+ */
+
+void cds_wfs_node_init(struct cds_wfs_node *node)
+{
+ _cds_wfs_node_init(node);
+}
+
+void cds_wfs_init(struct cds_wfs_stack *s)
+{
+ _cds_wfs_init(s);
+}
+
+void cds_wfs_destroy(struct cds_wfs_stack *s)
+{
+ _cds_wfs_destroy(s);
+}
+
+void __cds_wfs_init(struct __cds_wfs_stack *s)
+{
+ ___cds_wfs_init(s);
+}
+
+bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
+{
+ return _cds_wfs_empty(u_stack);
+}
+
+int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
+{
+ return _cds_wfs_push(u_stack, node);
+}
+
+struct cds_wfs_node *cds_wfs_pop_blocking(struct cds_wfs_stack *s)
+{
+ return _cds_wfs_pop_blocking(s);
+}
+
+struct cds_wfs_node *
+ cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
+{
+ return _cds_wfs_pop_with_state_blocking(s, state);
+}
+
+struct cds_wfs_head *cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
+{
+ return _cds_wfs_pop_all_blocking(s);
+}
+
+struct cds_wfs_node *cds_wfs_first(struct cds_wfs_head *head)
+{
+ return _cds_wfs_first(head);
+}
+
+struct cds_wfs_node *cds_wfs_next_blocking(struct cds_wfs_node *node)
+{
+ return _cds_wfs_next_blocking(node);
+}
+
+struct cds_wfs_node *cds_wfs_next_nonblocking(struct cds_wfs_node *node)
+{
+ return _cds_wfs_next_nonblocking(node);
+}
+
+void cds_wfs_pop_lock(struct cds_wfs_stack *s)
+{
+ _cds_wfs_pop_lock(s);
+}
+
+void cds_wfs_pop_unlock(struct cds_wfs_stack *s)
+{
+ _cds_wfs_pop_unlock(s);
+}
+
+struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
+{
+ return ___cds_wfs_pop_blocking(u_stack);
+}
+
+struct cds_wfs_node *
+ __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack,
+ int *state)
+{
+ return ___cds_wfs_pop_with_state_blocking(u_stack, state);
+}
+
+struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
+{
+ return ___cds_wfs_pop_nonblocking(u_stack);
+}
+
+struct cds_wfs_node *
+ __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack,
+ int *state)
+{
+ return ___cds_wfs_pop_with_state_nonblocking(u_stack, state);
+}
+
+struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
+{
+ return ___cds_wfs_pop_all(u_stack);
+}
-AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src -I$(top_srcdir)/tests/common -g
SCRIPT_LIST = common.sh \
run.sh \
test_urcu_lfq_dynlink test_urcu_lfs_dynlink test_urcu_hash \
test_urcu_lfs_rcu_dynlink
-URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la
-URCU_LIB=$(top_builddir)/liburcu.la
-URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la
-URCU_MB_LIB=$(top_builddir)/liburcu-mb.la
-URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la
-URCU_BP_LIB=$(top_builddir)/liburcu-bp.la
-URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la
+URCU_COMMON_LIB=$(top_builddir)/src/liburcu-common.la
+URCU_LIB=$(top_builddir)/src/liburcu.la
+URCU_QSBR_LIB=$(top_builddir)/src/liburcu-qsbr.la
+URCU_MB_LIB=$(top_builddir)/src/liburcu-mb.la
+URCU_SIGNAL_LIB=$(top_builddir)/src/liburcu-signal.la
+URCU_BP_LIB=$(top_builddir)/src/liburcu-bp.la
+URCU_CDS_LIB=$(top_builddir)/src/liburcu-cds.la
DEBUG_YIELD_LIB=$(builddir)/../common/libdebug-yield.la
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
#include "test_urcu_hash.h"
enum test_hash {
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
#include "test_urcu_hash.h"
enum urcu_hash_addremove {
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
#include "test_urcu_hash.h"
enum urcu_hash_addremove {
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
-AM_CPPFLAGS = -I$(top_srcdir) -I$(top_builddir)
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src
noinst_HEADERS = cpuset.h thread-id.h
#ifndef _INCLUDE_API_H
#define _INCLUDE_API_H
-#define _GNU_SOURCE
-#include "config.h"
-
/*
* common.h: Common Linux kernel-isms.
*
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#include "config.h"
-
#if defined(HAVE_SCHED_SETAFFINITY) || defined(HAVE_CPU_SET_T) \
|| defined(HAVE_CPU_ZERO) || defined(HAVE_CPU_SET)
# include <sched.h>
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*/
-#include <config.h>
#ifdef __linux__
# include <urcu/syscall-compat.h>
-AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/utils -I$(top_srcdir)/tests/common -g
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src -I$(top_srcdir)/tests/utils -I$(top_srcdir)/tests/common -g
SCRIPT_LIST = run.sh
noinst_HEADERS = rcutorture.h
-URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la
-URCU_LIB=$(top_builddir)/liburcu.la
-URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la
-URCU_MB_LIB=$(top_builddir)/liburcu-mb.la
-URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la
-URCU_BP_LIB=$(top_builddir)/liburcu-bp.la
-URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la
+URCU_COMMON_LIB=$(top_builddir)/src/liburcu-common.la
+URCU_LIB=$(top_builddir)/src/liburcu.la
+URCU_QSBR_LIB=$(top_builddir)/src/liburcu-qsbr.la
+URCU_MB_LIB=$(top_builddir)/src/liburcu-mb.la
+URCU_SIGNAL_LIB=$(top_builddir)/src/liburcu-signal.la
+URCU_BP_LIB=$(top_builddir)/src/liburcu-bp.la
+URCU_CDS_LIB=$(top_builddir)/src/liburcu-cds.la
TAP_LIB=$(top_builddir)/tests/utils/libtap.a
test_urcu_fork_SOURCES = test_urcu_fork.c
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
-#define _GNU_SOURCE
-#include "config.h"
#include <stdio.h>
#include <pthread.h>
#include <stdlib.h>
-#define _GNU_SOURCE
#include <string.h>
#include <sys/time.h>
#include <poll.h>
-AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/utils -I$(top_srcdir)/tests/common -g
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src -I$(top_srcdir)/tests/utils -I$(top_srcdir)/tests/common -g
SCRIPT_LIST = test_loop run.sh unit_tests
noinst_HEADERS = test_urcu_multiflavor.h
-URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la
-URCU_LIB=$(top_builddir)/liburcu.la
-URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la
-URCU_MB_LIB=$(top_builddir)/liburcu-mb.la
-URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la
-URCU_BP_LIB=$(top_builddir)/liburcu-bp.la
-URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la
+URCU_COMMON_LIB=$(top_builddir)/src/liburcu-common.la
+URCU_LIB=$(top_builddir)/src/liburcu.la
+URCU_QSBR_LIB=$(top_builddir)/src/liburcu-qsbr.la
+URCU_MB_LIB=$(top_builddir)/src/liburcu-mb.la
+URCU_SIGNAL_LIB=$(top_builddir)/src/liburcu-signal.la
+URCU_BP_LIB=$(top_builddir)/src/liburcu-bp.la
+URCU_CDS_LIB=$(top_builddir)/src/liburcu-cds.la
TAP_LIB=$(top_builddir)/tests/utils/libtap.a
test_uatomic_SOURCES = test_uatomic.c
+AM_CFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include -I$(top_srcdir)/src
+
noinst_LIBRARIES = libtap.a
libtap_a_SOURCES = tap.c tap.h
* SUCH DAMAGE.
*/
-#define _GNU_SOURCE
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
+++ /dev/null
-/*
- * urcu-bp.c
- *
- * Userspace RCU library, "bulletproof" version.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#define _GNU_SOURCE
-#define _LGPL_SOURCE
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <poll.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-#include "urcu/arch.h"
-#include "urcu/wfcqueue.h"
-#include "urcu/map/urcu-bp.h"
-#include "urcu/static/urcu-bp.h"
-#include "urcu-pointer.h"
-#include "urcu/tls-compat.h"
-
-#include "urcu-die.h"
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu-bp.h"
-#define _LGPL_SOURCE
-
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-#ifdef __linux__
-static
-void *mremap_wrapper(void *old_address, size_t old_size,
- size_t new_size, int flags)
-{
- return mremap(old_address, old_size, new_size, flags);
-}
-#else
-
-#define MREMAP_MAYMOVE 1
-#define MREMAP_FIXED 2
-
-/*
- * mremap wrapper for non-Linux systems not allowing MAYMOVE.
- * This is not generic.
-*/
-static
-void *mremap_wrapper(void *old_address, size_t old_size,
- size_t new_size, int flags)
-{
- assert(!(flags & MREMAP_MAYMOVE));
-
- return MAP_FAILED;
-}
-#endif
-
-/* Sleep delay in ms */
-#define RCU_SLEEP_DELAY_MS 10
-#define INIT_NR_THREADS 8
-#define ARENA_INIT_ALLOC \
- sizeof(struct registry_chunk) \
- + INIT_NR_THREADS * sizeof(struct rcu_reader)
-
-/*
- * Active attempts to check for reader Q.S. before calling sleep().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-static
-int rcu_bp_refcount;
-
-/* If the headers do not support membarrier system call, fall back smp_mb. */
-#ifdef __NR_membarrier
-# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
-#else
-# define membarrier(...) -ENOSYS
-#endif
-
-enum membarrier_cmd {
- MEMBARRIER_CMD_QUERY = 0,
- MEMBARRIER_CMD_SHARED = (1 << 0),
-};
-
-static
-void __attribute__((constructor)) rcu_bp_init(void);
-static
-void __attribute__((destructor)) rcu_bp_exit(void);
-
-int urcu_bp_has_sys_membarrier;
-
-/*
- * rcu_gp_lock ensures mutual exclusion between threads calling
- * synchronize_rcu().
- */
-static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-/*
- * rcu_registry_lock ensures mutual exclusion between threads
- * registering and unregistering themselves to/from the registry, and
- * with threads reading that registry from synchronize_rcu(). However,
- * this lock is not held all the way through the completion of awaiting
- * for the grace period. It is sporadically released between iterations
- * on the registry.
- * rcu_registry_lock may nest inside rcu_gp_lock.
- */
-static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
-
-static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
-static int initialized;
-
-static pthread_key_t urcu_bp_key;
-
-struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
-
-/*
- * Pointer to registry elements. Written to only by each individual reader. Read
- * by both the reader and the writers.
- */
-DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
-
-static CDS_LIST_HEAD(registry);
-
-struct registry_chunk {
- size_t data_len; /* data length */
- size_t used; /* amount of data used */
- struct cds_list_head node; /* chunk_list node */
- char data[];
-};
-
-struct registry_arena {
- struct cds_list_head chunk_list;
-};
-
-static struct registry_arena registry_arena = {
- .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list),
-};
-
-/* Saved fork signal mask, protected by rcu_gp_lock */
-static sigset_t saved_fork_signal_mask;
-
-static void mutex_lock(pthread_mutex_t *mutex)
-{
- int ret;
-
-#ifndef DISTRUST_SIGNALS_EXTREME
- ret = pthread_mutex_lock(mutex);
- if (ret)
- urcu_die(ret);
-#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
- while ((ret = pthread_mutex_trylock(mutex)) != 0) {
- if (ret != EBUSY && ret != EINTR)
- urcu_die(ret);
- poll(NULL,0,10);
- }
-#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
-}
-
-static void mutex_unlock(pthread_mutex_t *mutex)
-{
- int ret;
-
- ret = pthread_mutex_unlock(mutex);
- if (ret)
- urcu_die(ret);
-}
-
-static void smp_mb_master(void)
-{
- if (caa_likely(urcu_bp_has_sys_membarrier))
- (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
- else
- cmm_smp_mb();
-}
-
-/*
- * Always called with rcu_registry lock held. Releases this lock between
- * iterations and grabs it again. Holds the lock when it returns.
- */
-static void wait_for_readers(struct cds_list_head *input_readers,
- struct cds_list_head *cur_snap_readers,
- struct cds_list_head *qsreaders)
-{
- unsigned int wait_loops = 0;
- struct rcu_reader *index, *tmp;
-
- /*
- * Wait for each thread URCU_TLS(rcu_reader).ctr to either
- * indicate quiescence (not nested), or observe the current
- * rcu_gp.ctr value.
- */
- for (;;) {
- if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
- wait_loops++;
-
- cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
- switch (rcu_reader_state(&index->ctr)) {
- case RCU_READER_ACTIVE_CURRENT:
- if (cur_snap_readers) {
- cds_list_move(&index->node,
- cur_snap_readers);
- break;
- }
- /* Fall-through */
- case RCU_READER_INACTIVE:
- cds_list_move(&index->node, qsreaders);
- break;
- case RCU_READER_ACTIVE_OLD:
- /*
- * Old snapshot. Leaving node in
- * input_readers will make us busy-loop
- * until the snapshot becomes current or
- * the reader becomes inactive.
- */
- break;
- }
- }
-
- if (cds_list_empty(input_readers)) {
- break;
- } else {
- /* Temporarily unlock the registry lock. */
- mutex_unlock(&rcu_registry_lock);
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS)
- (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS);
- else
- caa_cpu_relax();
- /* Re-lock the registry lock before the next loop. */
- mutex_lock(&rcu_registry_lock);
- }
- }
-}
-
-void synchronize_rcu(void)
-{
- CDS_LIST_HEAD(cur_snap_readers);
- CDS_LIST_HEAD(qsreaders);
- sigset_t newmask, oldmask;
- int ret;
-
- ret = sigfillset(&newmask);
- assert(!ret);
- ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
- assert(!ret);
-
- mutex_lock(&rcu_gp_lock);
-
- mutex_lock(&rcu_registry_lock);
-
- if (cds_list_empty(®istry))
- goto out;
-
- /* All threads should read qparity before accessing data structure
- * where new ptr points to. */
- /* Write new ptr before changing the qparity */
- smp_mb_master();
-
- /*
- * Wait for readers to observe original parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /* Switch parity: 0 -> 1, 1 -> 0 */
- CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
-
- /*
- * Must commit qparity update to memory before waiting for other parity
- * quiescent state. Failure to do so could result in the writer waiting
- * forever while new readers are always accessing data (no progress).
- * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
- */
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /*
- * Wait for readers to observe new parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
-
- /*
- * Put quiescent reader list back into registry.
- */
- cds_list_splice(&qsreaders, ®istry);
-
- /*
- * Finish waiting for reader threads before letting the old ptr being
- * freed.
- */
- smp_mb_master();
-out:
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
- ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
- assert(!ret);
-}
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void rcu_read_lock(void)
-{
- _rcu_read_lock();
-}
-
-void rcu_read_unlock(void)
-{
- _rcu_read_unlock();
-}
-
-int rcu_read_ongoing(void)
-{
- return _rcu_read_ongoing();
-}
-
-/*
- * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk.
- * Else, try expanding the last chunk. If this fails, allocate a new
- * chunk twice as big as the last chunk.
- * Memory used by chunks _never_ moves. A chunk could theoretically be
- * freed when all "used" slots are released, but we don't do it at this
- * point.
- */
-static
-void expand_arena(struct registry_arena *arena)
-{
- struct registry_chunk *new_chunk, *last_chunk;
- size_t old_chunk_len, new_chunk_len;
-
- /* No chunk. */
- if (cds_list_empty(&arena->chunk_list)) {
- assert(ARENA_INIT_ALLOC >=
- sizeof(struct registry_chunk)
- + sizeof(struct rcu_reader));
- new_chunk_len = ARENA_INIT_ALLOC;
- new_chunk = mmap(NULL, new_chunk_len,
- PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE,
- -1, 0);
- if (new_chunk == MAP_FAILED)
- abort();
- memset(new_chunk, 0, new_chunk_len);
- new_chunk->data_len =
- new_chunk_len - sizeof(struct registry_chunk);
- cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
- return; /* We're done. */
- }
-
- /* Try expanding last chunk. */
- last_chunk = cds_list_entry(arena->chunk_list.prev,
- struct registry_chunk, node);
- old_chunk_len =
- last_chunk->data_len + sizeof(struct registry_chunk);
- new_chunk_len = old_chunk_len << 1;
-
- /* Don't allow memory mapping to move, just expand. */
- new_chunk = mremap_wrapper(last_chunk, old_chunk_len,
- new_chunk_len, 0);
- if (new_chunk != MAP_FAILED) {
- /* Should not have moved. */
- assert(new_chunk == last_chunk);
- memset((char *) last_chunk + old_chunk_len, 0,
- new_chunk_len - old_chunk_len);
- last_chunk->data_len =
- new_chunk_len - sizeof(struct registry_chunk);
- return; /* We're done. */
- }
-
- /* Remap did not succeed, we need to add a new chunk. */
- new_chunk = mmap(NULL, new_chunk_len,
- PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE,
- -1, 0);
- if (new_chunk == MAP_FAILED)
- abort();
- memset(new_chunk, 0, new_chunk_len);
- new_chunk->data_len =
- new_chunk_len - sizeof(struct registry_chunk);
- cds_list_add_tail(&new_chunk->node, &arena->chunk_list);
-}
-
-static
-struct rcu_reader *arena_alloc(struct registry_arena *arena)
-{
- struct registry_chunk *chunk;
- struct rcu_reader *rcu_reader_reg;
- int expand_done = 0; /* Only allow to expand once per alloc */
- size_t len = sizeof(struct rcu_reader);
-
-retry:
- cds_list_for_each_entry(chunk, &arena->chunk_list, node) {
- if (chunk->data_len - chunk->used < len)
- continue;
- /* Find spot */
- for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
- rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
- rcu_reader_reg++) {
- if (!rcu_reader_reg->alloc) {
- rcu_reader_reg->alloc = 1;
- chunk->used += len;
- return rcu_reader_reg;
- }
- }
- }
-
- if (!expand_done) {
- expand_arena(arena);
- expand_done = 1;
- goto retry;
- }
-
- return NULL;
-}
-
-/* Called with signals off and mutex locked */
-static
-void add_thread(void)
-{
- struct rcu_reader *rcu_reader_reg;
- int ret;
-
- rcu_reader_reg = arena_alloc(®istry_arena);
- if (!rcu_reader_reg)
- abort();
- ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg);
- if (ret)
- abort();
-
- /* Add to registry */
- rcu_reader_reg->tid = pthread_self();
- assert(rcu_reader_reg->ctr == 0);
- cds_list_add(&rcu_reader_reg->node, ®istry);
- /*
- * Reader threads are pointing to the reader registry. This is
- * why its memory should never be relocated.
- */
- URCU_TLS(rcu_reader) = rcu_reader_reg;
-}
-
-/* Called with mutex locked */
-static
-void cleanup_thread(struct registry_chunk *chunk,
- struct rcu_reader *rcu_reader_reg)
-{
- rcu_reader_reg->ctr = 0;
- cds_list_del(&rcu_reader_reg->node);
- rcu_reader_reg->tid = 0;
- rcu_reader_reg->alloc = 0;
- chunk->used -= sizeof(struct rcu_reader);
-}
-
-static
-struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg)
-{
- struct registry_chunk *chunk;
-
- cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) {
- if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0])
- continue;
- if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len])
- continue;
- return chunk;
- }
- return NULL;
-}
-
-/* Called with signals off and mutex locked */
-static
-void remove_thread(struct rcu_reader *rcu_reader_reg)
-{
- cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg);
- URCU_TLS(rcu_reader) = NULL;
-}
-
-/* Disable signals, take mutex, add to registry */
-void rcu_bp_register(void)
-{
- sigset_t newmask, oldmask;
- int ret;
-
- ret = sigfillset(&newmask);
- if (ret)
- abort();
- ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
- if (ret)
- abort();
-
- /*
- * Check if a signal concurrently registered our thread since
- * the check in rcu_read_lock().
- */
- if (URCU_TLS(rcu_reader))
- goto end;
-
- /*
- * Take care of early registration before urcu_bp constructor.
- */
- rcu_bp_init();
-
- mutex_lock(&rcu_registry_lock);
- add_thread();
- mutex_unlock(&rcu_registry_lock);
-end:
- ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
- if (ret)
- abort();
-}
-
-/* Disable signals, take mutex, remove from registry */
-static
-void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg)
-{
- sigset_t newmask, oldmask;
- int ret;
-
- ret = sigfillset(&newmask);
- if (ret)
- abort();
- ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
- if (ret)
- abort();
-
- mutex_lock(&rcu_registry_lock);
- remove_thread(rcu_reader_reg);
- mutex_unlock(&rcu_registry_lock);
- ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
- if (ret)
- abort();
- rcu_bp_exit();
-}
-
-/*
- * Remove thread from the registry when it exits, and flag it as
- * destroyed so garbage collection can take care of it.
- */
-static
-void urcu_bp_thread_exit_notifier(void *rcu_key)
-{
- rcu_bp_unregister(rcu_key);
-}
-
-static
-void rcu_bp_init(void)
-{
- mutex_lock(&init_lock);
- if (!rcu_bp_refcount++) {
- int ret;
-
- ret = pthread_key_create(&urcu_bp_key,
- urcu_bp_thread_exit_notifier);
- if (ret)
- abort();
- ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
- if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
- urcu_bp_has_sys_membarrier = 1;
- }
- initialized = 1;
- }
- mutex_unlock(&init_lock);
-}
-
-static
-void rcu_bp_exit(void)
-{
- mutex_lock(&init_lock);
- if (!--rcu_bp_refcount) {
- struct registry_chunk *chunk, *tmp;
- int ret;
-
- cds_list_for_each_entry_safe(chunk, tmp,
- ®istry_arena.chunk_list, node) {
- munmap(chunk, chunk->data_len
- + sizeof(struct registry_chunk));
- }
- CDS_INIT_LIST_HEAD(®istry_arena.chunk_list);
- ret = pthread_key_delete(urcu_bp_key);
- if (ret)
- abort();
- }
- mutex_unlock(&init_lock);
-}
-
-/*
- * Holding the rcu_gp_lock and rcu_registry_lock across fork will make
- * sure we fork() don't race with a concurrent thread executing with
- * any of those locks held. This ensures that the registry and data
- * protected by rcu_gp_lock are in a coherent state in the child.
- */
-void rcu_bp_before_fork(void)
-{
- sigset_t newmask, oldmask;
- int ret;
-
- ret = sigfillset(&newmask);
- assert(!ret);
- ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
- assert(!ret);
- mutex_lock(&rcu_gp_lock);
- mutex_lock(&rcu_registry_lock);
- saved_fork_signal_mask = oldmask;
-}
-
-void rcu_bp_after_fork_parent(void)
-{
- sigset_t oldmask;
- int ret;
-
- oldmask = saved_fork_signal_mask;
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
- ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
- assert(!ret);
-}
-
-/*
- * Prune all entries from registry except our own thread. Fits the Linux
- * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held.
- */
-static
-void urcu_bp_prune_registry(void)
-{
- struct registry_chunk *chunk;
- struct rcu_reader *rcu_reader_reg;
-
- cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) {
- for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0];
- rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len];
- rcu_reader_reg++) {
- if (!rcu_reader_reg->alloc)
- continue;
- if (rcu_reader_reg->tid == pthread_self())
- continue;
- cleanup_thread(chunk, rcu_reader_reg);
- }
- }
-}
-
-void rcu_bp_after_fork_child(void)
-{
- sigset_t oldmask;
- int ret;
-
- urcu_bp_prune_registry();
- oldmask = saved_fork_signal_mask;
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
- ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
- assert(!ret);
-}
-
-void *rcu_dereference_sym_bp(void *p)
-{
- return _rcu_dereference(p);
-}
-
-void *rcu_set_pointer_sym_bp(void **p, void *v)
-{
- cmm_wmb();
- uatomic_set(p, v);
- return v;
-}
-
-void *rcu_xchg_pointer_sym_bp(void **p, void *v)
-{
- cmm_wmb();
- return uatomic_xchg(p, v);
-}
-
-void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
-{
- cmm_wmb();
- return uatomic_cmpxchg(p, old, _new);
-}
-
-DEFINE_RCU_FLAVOR(rcu_flavor);
-
-#include "urcu-call-rcu-impl.h"
-#include "urcu-defer-impl.h"
+++ /dev/null
-#ifndef _URCU_BP_H
-#define _URCU_BP_H
-
-/*
- * urcu-bp.h
- *
- * Userspace RCU header, "bulletproof" version.
- *
- * Slower RCU read-side adapted for tracing library. Does not require thread
- * registration nor unregistration. Also signal-safe.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <urcu/map/urcu-bp.h>
-
-/*
- * Important !
- *
- * Each thread containing read-side critical sections must be registered
- * with rcu_register_thread() before calling rcu_read_lock().
- * rcu_unregister_thread() should be called before the thread exits.
- */
-
-/*
- * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
- * publication headers.
- */
-#include <urcu-pointer.h>
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/urcu-bp.h>
-
-/*
- * Mappings for static use of the userspace RCU library.
- * Should only be used in LGPL-compatible code.
- */
-
-/*
- * rcu_read_lock()
- * rcu_read_unlock()
- *
- * Mark the beginning and end of a read-side critical section.
- */
-#define rcu_read_lock_bp _rcu_read_lock
-#define rcu_read_unlock_bp _rcu_read_unlock
-#define rcu_read_ongoing_bp _rcu_read_ongoing
-
-#define rcu_dereference_bp rcu_dereference
-#define rcu_cmpxchg_pointer_bp rcu_cmpxchg_pointer
-#define rcu_xchg_pointer_bp rcu_xchg_pointer
-#define rcu_set_pointer_bp rcu_set_pointer
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- * See LGPL-only urcu/static/urcu-pointer.h for documentation.
- */
-
-extern void rcu_read_lock(void);
-extern void rcu_read_unlock(void);
-extern int rcu_read_ongoing(void);
-
-extern void *rcu_dereference_sym_bp(void *p);
-#define rcu_dereference_bp(p) \
- __extension__ \
- ({ \
- __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \
- rcu_dereference_sym_bp(URCU_FORCE_CAST(void *, p))); \
- (_________p1); \
- })
-
-extern void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new);
-#define rcu_cmpxchg_pointer_bp(p, old, _new) \
- __extension__ \
- ({ \
- __typeof__(*(p)) _________pold = (old); \
- __typeof__(*(p)) _________pnew = (_new); \
- __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
- rcu_cmpxchg_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
- _________pold, \
- _________pnew)); \
- (_________p1); \
- })
-
-extern void *rcu_xchg_pointer_sym_bp(void **p, void *v);
-#define rcu_xchg_pointer_bp(p, v) \
- __extension__ \
- ({ \
- __typeof__(*(p)) _________pv = (v); \
- __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)),\
- rcu_xchg_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
- _________pv)); \
- (_________p1); \
- })
-
-extern void *rcu_set_pointer_sym_bp(void **p, void *v);
-#define rcu_set_pointer_bp(p, v) \
- __extension__ \
- ({ \
- __typeof__(*(p)) _________pv = (v); \
- __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
- rcu_set_pointer_sym_bp(URCU_FORCE_CAST(void **, p), \
- _________pv)); \
- (_________p1); \
- })
-
-#endif /* !_LGPL_SOURCE */
-
-extern void synchronize_rcu(void);
-
-/*
- * rcu_bp_before_fork, rcu_bp_after_fork_parent and rcu_bp_after_fork_child
- * should be called around fork() system calls when the child process is not
- * expected to immediately perform an exec(). For pthread users, see
- * pthread_atfork(3).
- */
-extern void rcu_bp_before_fork(void);
-extern void rcu_bp_after_fork_parent(void);
-extern void rcu_bp_after_fork_child(void);
-
-/*
- * In the bulletproof version, the following functions are no-ops.
- */
-static inline void rcu_register_thread(void)
-{
-}
-
-static inline void rcu_unregister_thread(void)
-{
-}
-
-static inline void rcu_init(void)
-{
-}
-
-/*
- * Q.S. reporting are no-ops for these URCU flavors.
- */
-static inline void rcu_quiescent_state(void)
-{
-}
-
-static inline void rcu_thread_offline(void)
-{
-}
-
-static inline void rcu_thread_online(void)
-{
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu-call-rcu.h>
-#include <urcu-defer.h>
-#include <urcu-flavor.h>
-
-#endif /* _URCU_BP_H */
+++ /dev/null
-/*
- * urcu-call-rcu.c
- *
- * Userspace RCU library - batch memory reclamation with kernel API
- *
- * Copyright (c) 2010 Paul E. McKenney <paulmck@linux.vnet.ibm.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#define _GNU_SOURCE
-#define _LGPL_SOURCE
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-#include <errno.h>
-#include <poll.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <sched.h>
-
-#include "config.h"
-#include "compat-getcpu.h"
-#include "urcu/wfcqueue.h"
-#include "urcu-call-rcu.h"
-#include "urcu-pointer.h"
-#include "urcu/list.h"
-#include "urcu/futex.h"
-#include "urcu/tls-compat.h"
-#include "urcu/ref.h"
-#include "urcu-die.h"
-
-#define SET_AFFINITY_CHECK_PERIOD (1U << 8) /* 256 */
-#define SET_AFFINITY_CHECK_PERIOD_MASK (SET_AFFINITY_CHECK_PERIOD - 1)
-
-/* Data structure that identifies a call_rcu thread. */
-
-struct call_rcu_data {
- /*
- * We do not align head on a different cache-line than tail
- * mainly because call_rcu callback-invocation threads use
- * batching ("splice") to get an entire list of callbacks, which
- * effectively empties the queue, and requires to touch the tail
- * anyway.
- */
- struct cds_wfcq_tail cbs_tail;
- struct cds_wfcq_head cbs_head;
- unsigned long flags;
- int32_t futex;
- unsigned long qlen; /* maintained for debugging. */
- pthread_t tid;
- int cpu_affinity;
- unsigned long gp_count;
- struct cds_list_head list;
-} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
-
-struct call_rcu_completion {
- int barrier_count;
- int32_t futex;
- struct urcu_ref ref;
-};
-
-struct call_rcu_completion_work {
- struct rcu_head head;
- struct call_rcu_completion *completion;
-};
-
-/*
- * List of all call_rcu_data structures to keep valgrind happy.
- * Protected by call_rcu_mutex.
- */
-
-static CDS_LIST_HEAD(call_rcu_data_list);
-
-/* Link a thread using call_rcu() to its call_rcu thread. */
-
-static DEFINE_URCU_TLS(struct call_rcu_data *, thread_call_rcu_data);
-
-/*
- * Guard call_rcu thread creation and atfork handlers.
- */
-static pthread_mutex_t call_rcu_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-/* If a given thread does not have its own call_rcu thread, this is default. */
-
-static struct call_rcu_data *default_call_rcu_data;
-
-/*
- * If the sched_getcpu() and sysconf(_SC_NPROCESSORS_CONF) calls are
- * available, then we can have call_rcu threads assigned to individual
- * CPUs rather than only to specific threads.
- */
-
-#if defined(HAVE_SYSCONF) && (defined(HAVE_SCHED_GETCPU) || defined(HAVE_GETCPUID))
-
-/*
- * Pointer to array of pointers to per-CPU call_rcu_data structures
- * and # CPUs. per_cpu_call_rcu_data is a RCU-protected pointer to an
- * array of RCU-protected pointers to call_rcu_data. call_rcu acts as a
- * RCU read-side and reads per_cpu_call_rcu_data and the per-cpu pointer
- * without mutex. The call_rcu_mutex protects updates.
- */
-
-static struct call_rcu_data **per_cpu_call_rcu_data;
-static long maxcpus;
-
-static void maxcpus_reset(void)
-{
- maxcpus = 0;
-}
-
-/* Allocate the array if it has not already been allocated. */
-
-static void alloc_cpu_call_rcu_data(void)
-{
- struct call_rcu_data **p;
- static int warned = 0;
-
- if (maxcpus != 0)
- return;
- maxcpus = sysconf(_SC_NPROCESSORS_CONF);
- if (maxcpus <= 0) {
- return;
- }
- p = malloc(maxcpus * sizeof(*per_cpu_call_rcu_data));
- if (p != NULL) {
- memset(p, '\0', maxcpus * sizeof(*per_cpu_call_rcu_data));
- rcu_set_pointer(&per_cpu_call_rcu_data, p);
- } else {
- if (!warned) {
- fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n");
- }
- warned = 1;
- }
-}
-
-#else /* #if defined(HAVE_SYSCONF) && defined(HAVE_SCHED_GETCPU) */
-
-/*
- * per_cpu_call_rcu_data should be constant, but some functions below, used both
- * for cases where cpu number is available and not available, assume it it not
- * constant.
- */
-static struct call_rcu_data **per_cpu_call_rcu_data = NULL;
-static const long maxcpus = -1;
-
-static void maxcpus_reset(void)
-{
-}
-
-static void alloc_cpu_call_rcu_data(void)
-{
-}
-
-#endif /* #else #if defined(HAVE_SYSCONF) && defined(HAVE_SCHED_GETCPU) */
-
-/* Acquire the specified pthread mutex. */
-
-static void call_rcu_lock(pthread_mutex_t *pmp)
-{
- int ret;
-
- ret = pthread_mutex_lock(pmp);
- if (ret)
- urcu_die(ret);
-}
-
-/* Release the specified pthread mutex. */
-
-static void call_rcu_unlock(pthread_mutex_t *pmp)
-{
- int ret;
-
- ret = pthread_mutex_unlock(pmp);
- if (ret)
- urcu_die(ret);
-}
-
-/*
- * Periodically retry setting CPU affinity if we migrate.
- * Losing affinity can be caused by CPU hotunplug/hotplug, or by
- * cpuset(7).
- */
-#if HAVE_SCHED_SETAFFINITY
-static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
-{
- cpu_set_t mask;
- int ret;
-
- if (crdp->cpu_affinity < 0)
- return 0;
- if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
- return 0;
- if (urcu_sched_getcpu() == crdp->cpu_affinity)
- return 0;
-
- CPU_ZERO(&mask);
- CPU_SET(crdp->cpu_affinity, &mask);
-#if SCHED_SETAFFINITY_ARGS == 2
- ret = sched_setaffinity(0, &mask);
-#else
- ret = sched_setaffinity(0, sizeof(mask), &mask);
-#endif
- /*
- * EINVAL is fine: can be caused by hotunplugged CPUs, or by
- * cpuset(7). This is why we should always retry if we detect
- * migration.
- */
- if (ret && errno == EINVAL) {
- ret = 0;
- errno = 0;
- }
- return ret;
-}
-#else
-static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
-{
- return 0;
-}
-#endif
-
-static void call_rcu_wait(struct call_rcu_data *crdp)
-{
- /* Read call_rcu list before read futex */
- cmm_smp_mb();
- if (uatomic_read(&crdp->futex) != -1)
- return;
- while (futex_async(&crdp->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- return;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
-}
-
-static void call_rcu_wake_up(struct call_rcu_data *crdp)
-{
- /* Write to call_rcu list before reading/writing futex */
- cmm_smp_mb();
- if (caa_unlikely(uatomic_read(&crdp->futex) == -1)) {
- uatomic_set(&crdp->futex, 0);
- if (futex_async(&crdp->futex, FUTEX_WAKE, 1,
- NULL, NULL, 0) < 0)
- urcu_die(errno);
- }
-}
-
-static void call_rcu_completion_wait(struct call_rcu_completion *completion)
-{
- /* Read completion barrier count before read futex */
- cmm_smp_mb();
- if (uatomic_read(&completion->futex) != -1)
- return;
- while (futex_async(&completion->futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- return;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
-}
-
-static void call_rcu_completion_wake_up(struct call_rcu_completion *completion)
-{
- /* Write to completion barrier count before reading/writing futex */
- cmm_smp_mb();
- if (caa_unlikely(uatomic_read(&completion->futex) == -1)) {
- uatomic_set(&completion->futex, 0);
- if (futex_async(&completion->futex, FUTEX_WAKE, 1,
- NULL, NULL, 0) < 0)
- urcu_die(errno);
- }
-}
-
-/* This is the code run by each call_rcu thread. */
-
-static void *call_rcu_thread(void *arg)
-{
- unsigned long cbcount;
- struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
- int rt = !!(uatomic_read(&crdp->flags) & URCU_CALL_RCU_RT);
-
- if (set_thread_cpu_affinity(crdp))
- urcu_die(errno);
-
- /*
- * If callbacks take a read-side lock, we need to be registered.
- */
- rcu_register_thread();
-
- URCU_TLS(thread_call_rcu_data) = crdp;
- if (!rt) {
- uatomic_dec(&crdp->futex);
- /* Decrement futex before reading call_rcu list */
- cmm_smp_mb();
- }
- for (;;) {
- struct cds_wfcq_head cbs_tmp_head;
- struct cds_wfcq_tail cbs_tmp_tail;
- struct cds_wfcq_node *cbs, *cbs_tmp_n;
- enum cds_wfcq_ret splice_ret;
-
- if (set_thread_cpu_affinity(crdp))
- urcu_die(errno);
-
- if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
- /*
- * Pause requested. Become quiescent: remove
- * ourself from all global lists, and don't
- * process any callback. The callback lists may
- * still be non-empty though.
- */
- rcu_unregister_thread();
- cmm_smp_mb__before_uatomic_or();
- uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSED);
- while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) != 0)
- (void) poll(NULL, 0, 1);
- uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSED);
- cmm_smp_mb__after_uatomic_and();
- rcu_register_thread();
- }
-
- cds_wfcq_init(&cbs_tmp_head, &cbs_tmp_tail);
- splice_ret = __cds_wfcq_splice_blocking(&cbs_tmp_head,
- &cbs_tmp_tail, &crdp->cbs_head, &crdp->cbs_tail);
- assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
- assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
- if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
- synchronize_rcu();
- cbcount = 0;
- __cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
- &cbs_tmp_tail, cbs, cbs_tmp_n) {
- struct rcu_head *rhp;
-
- rhp = caa_container_of(cbs,
- struct rcu_head, next);
- rhp->func(rhp);
- cbcount++;
- }
- uatomic_sub(&crdp->qlen, cbcount);
- }
- if (uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOP)
- break;
- rcu_thread_offline();
- if (!rt) {
- if (cds_wfcq_empty(&crdp->cbs_head,
- &crdp->cbs_tail)) {
- call_rcu_wait(crdp);
- (void) poll(NULL, 0, 10);
- uatomic_dec(&crdp->futex);
- /*
- * Decrement futex before reading
- * call_rcu list.
- */
- cmm_smp_mb();
- } else {
- (void) poll(NULL, 0, 10);
- }
- } else {
- (void) poll(NULL, 0, 10);
- }
- rcu_thread_online();
- }
- if (!rt) {
- /*
- * Read call_rcu list before write futex.
- */
- cmm_smp_mb();
- uatomic_set(&crdp->futex, 0);
- }
- uatomic_or(&crdp->flags, URCU_CALL_RCU_STOPPED);
- rcu_unregister_thread();
- return NULL;
-}
-
-/*
- * Create both a call_rcu thread and the corresponding call_rcu_data
- * structure, linking the structure in as specified. Caller must hold
- * call_rcu_mutex.
- */
-
-static void call_rcu_data_init(struct call_rcu_data **crdpp,
- unsigned long flags,
- int cpu_affinity)
-{
- struct call_rcu_data *crdp;
- int ret;
-
- crdp = malloc(sizeof(*crdp));
- if (crdp == NULL)
- urcu_die(errno);
- memset(crdp, '\0', sizeof(*crdp));
- cds_wfcq_init(&crdp->cbs_head, &crdp->cbs_tail);
- crdp->qlen = 0;
- crdp->futex = 0;
- crdp->flags = flags;
- cds_list_add(&crdp->list, &call_rcu_data_list);
- crdp->cpu_affinity = cpu_affinity;
- crdp->gp_count = 0;
- cmm_smp_mb(); /* Structure initialized before pointer is planted. */
- *crdpp = crdp;
- ret = pthread_create(&crdp->tid, NULL, call_rcu_thread, crdp);
- if (ret)
- urcu_die(ret);
-}
-
-/*
- * Return a pointer to the call_rcu_data structure for the specified
- * CPU, returning NULL if there is none. We cannot automatically
- * created it because the platform we are running on might not define
- * urcu_sched_getcpu().
- *
- * The call to this function and use of the returned call_rcu_data
- * should be protected by RCU read-side lock.
- */
-
-struct call_rcu_data *get_cpu_call_rcu_data(int cpu)
-{
- static int warned = 0;
- struct call_rcu_data **pcpu_crdp;
-
- pcpu_crdp = rcu_dereference(per_cpu_call_rcu_data);
- if (pcpu_crdp == NULL)
- return NULL;
- if (!warned && maxcpus > 0 && (cpu < 0 || maxcpus <= cpu)) {
- fprintf(stderr, "[error] liburcu: get CPU # out of range\n");
- warned = 1;
- }
- if (cpu < 0 || maxcpus <= cpu)
- return NULL;
- return rcu_dereference(pcpu_crdp[cpu]);
-}
-
-/*
- * Return the tid corresponding to the call_rcu thread whose
- * call_rcu_data structure is specified.
- */
-
-pthread_t get_call_rcu_thread(struct call_rcu_data *crdp)
-{
- return crdp->tid;
-}
-
-/*
- * Create a call_rcu_data structure (with thread) and return a pointer.
- */
-
-static struct call_rcu_data *__create_call_rcu_data(unsigned long flags,
- int cpu_affinity)
-{
- struct call_rcu_data *crdp;
-
- call_rcu_data_init(&crdp, flags, cpu_affinity);
- return crdp;
-}
-
-struct call_rcu_data *create_call_rcu_data(unsigned long flags,
- int cpu_affinity)
-{
- struct call_rcu_data *crdp;
-
- call_rcu_lock(&call_rcu_mutex);
- crdp = __create_call_rcu_data(flags, cpu_affinity);
- call_rcu_unlock(&call_rcu_mutex);
- return crdp;
-}
-
-/*
- * Set the specified CPU to use the specified call_rcu_data structure.
- *
- * Use NULL to remove a CPU's call_rcu_data structure, but it is
- * the caller's responsibility to dispose of the removed structure.
- * Use get_cpu_call_rcu_data() to obtain a pointer to the old structure
- * (prior to NULLing it out, of course).
- *
- * The caller must wait for a grace-period to pass between return from
- * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
- * previous call rcu data as argument.
- */
-
-int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp)
-{
- static int warned = 0;
-
- call_rcu_lock(&call_rcu_mutex);
- alloc_cpu_call_rcu_data();
- if (cpu < 0 || maxcpus <= cpu) {
- if (!warned) {
- fprintf(stderr, "[error] liburcu: set CPU # out of range\n");
- warned = 1;
- }
- call_rcu_unlock(&call_rcu_mutex);
- errno = EINVAL;
- return -EINVAL;
- }
-
- if (per_cpu_call_rcu_data == NULL) {
- call_rcu_unlock(&call_rcu_mutex);
- errno = ENOMEM;
- return -ENOMEM;
- }
-
- if (per_cpu_call_rcu_data[cpu] != NULL && crdp != NULL) {
- call_rcu_unlock(&call_rcu_mutex);
- errno = EEXIST;
- return -EEXIST;
- }
-
- rcu_set_pointer(&per_cpu_call_rcu_data[cpu], crdp);
- call_rcu_unlock(&call_rcu_mutex);
- return 0;
-}
-
-/*
- * Return a pointer to the default call_rcu_data structure, creating
- * one if need be. Because we never free call_rcu_data structures,
- * we don't need to be in an RCU read-side critical section.
- */
-
-struct call_rcu_data *get_default_call_rcu_data(void)
-{
- if (default_call_rcu_data != NULL)
- return rcu_dereference(default_call_rcu_data);
- call_rcu_lock(&call_rcu_mutex);
- if (default_call_rcu_data != NULL) {
- call_rcu_unlock(&call_rcu_mutex);
- return default_call_rcu_data;
- }
- call_rcu_data_init(&default_call_rcu_data, 0, -1);
- call_rcu_unlock(&call_rcu_mutex);
- return default_call_rcu_data;
-}
-
-/*
- * Return the call_rcu_data structure that applies to the currently
- * running thread. Any call_rcu_data structure assigned specifically
- * to this thread has first priority, followed by any call_rcu_data
- * structure assigned to the CPU on which the thread is running,
- * followed by the default call_rcu_data structure. If there is not
- * yet a default call_rcu_data structure, one will be created.
- *
- * Calls to this function and use of the returned call_rcu_data should
- * be protected by RCU read-side lock.
- */
-struct call_rcu_data *get_call_rcu_data(void)
-{
- struct call_rcu_data *crd;
-
- if (URCU_TLS(thread_call_rcu_data) != NULL)
- return URCU_TLS(thread_call_rcu_data);
-
- if (maxcpus > 0) {
- crd = get_cpu_call_rcu_data(urcu_sched_getcpu());
- if (crd)
- return crd;
- }
-
- return get_default_call_rcu_data();
-}
-
-/*
- * Return a pointer to this task's call_rcu_data if there is one.
- */
-
-struct call_rcu_data *get_thread_call_rcu_data(void)
-{
- return URCU_TLS(thread_call_rcu_data);
-}
-
-/*
- * Set this task's call_rcu_data structure as specified, regardless
- * of whether or not this task already had one. (This allows switching
- * to and from real-time call_rcu threads, for example.)
- *
- * Use NULL to remove a thread's call_rcu_data structure, but it is
- * the caller's responsibility to dispose of the removed structure.
- * Use get_thread_call_rcu_data() to obtain a pointer to the old structure
- * (prior to NULLing it out, of course).
- */
-
-void set_thread_call_rcu_data(struct call_rcu_data *crdp)
-{
- URCU_TLS(thread_call_rcu_data) = crdp;
-}
-
-/*
- * Create a separate call_rcu thread for each CPU. This does not
- * replace a pre-existing call_rcu thread -- use the set_cpu_call_rcu_data()
- * function if you want that behavior. Should be paired with
- * free_all_cpu_call_rcu_data() to teardown these call_rcu worker
- * threads.
- */
-
-int create_all_cpu_call_rcu_data(unsigned long flags)
-{
- int i;
- struct call_rcu_data *crdp;
- int ret;
-
- call_rcu_lock(&call_rcu_mutex);
- alloc_cpu_call_rcu_data();
- call_rcu_unlock(&call_rcu_mutex);
- if (maxcpus <= 0) {
- errno = EINVAL;
- return -EINVAL;
- }
- if (per_cpu_call_rcu_data == NULL) {
- errno = ENOMEM;
- return -ENOMEM;
- }
- for (i = 0; i < maxcpus; i++) {
- call_rcu_lock(&call_rcu_mutex);
- if (get_cpu_call_rcu_data(i)) {
- call_rcu_unlock(&call_rcu_mutex);
- continue;
- }
- crdp = __create_call_rcu_data(flags, i);
- if (crdp == NULL) {
- call_rcu_unlock(&call_rcu_mutex);
- errno = ENOMEM;
- return -ENOMEM;
- }
- call_rcu_unlock(&call_rcu_mutex);
- if ((ret = set_cpu_call_rcu_data(i, crdp)) != 0) {
- call_rcu_data_free(crdp);
-
- /* it has been created by other thread */
- if (ret == -EEXIST)
- continue;
-
- return ret;
- }
- }
- return 0;
-}
-
-/*
- * Wake up the call_rcu thread corresponding to the specified
- * call_rcu_data structure.
- */
-static void wake_call_rcu_thread(struct call_rcu_data *crdp)
-{
- if (!(_CMM_LOAD_SHARED(crdp->flags) & URCU_CALL_RCU_RT))
- call_rcu_wake_up(crdp);
-}
-
-static void _call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head),
- struct call_rcu_data *crdp)
-{
- cds_wfcq_node_init(&head->next);
- head->func = func;
- cds_wfcq_enqueue(&crdp->cbs_head, &crdp->cbs_tail, &head->next);
- uatomic_inc(&crdp->qlen);
- wake_call_rcu_thread(crdp);
-}
-
-/*
- * Schedule a function to be invoked after a following grace period.
- * This is the only function that must be called -- the others are
- * only present to allow applications to tune their use of RCU for
- * maximum performance.
- *
- * Note that unless a call_rcu thread has not already been created,
- * the first invocation of call_rcu() will create one. So, if you
- * need the first invocation of call_rcu() to be fast, make sure
- * to create a call_rcu thread first. One way to accomplish this is
- * "get_call_rcu_data();", and another is create_all_cpu_call_rcu_data().
- *
- * call_rcu must be called by registered RCU read-side threads.
- */
-void call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head))
-{
- struct call_rcu_data *crdp;
-
- /* Holding rcu read-side lock across use of per-cpu crdp */
- _rcu_read_lock();
- crdp = get_call_rcu_data();
- _call_rcu(head, func, crdp);
- _rcu_read_unlock();
-}
-
-/*
- * Free up the specified call_rcu_data structure, terminating the
- * associated call_rcu thread. The caller must have previously
- * removed the call_rcu_data structure from per-thread or per-CPU
- * usage. For example, set_cpu_call_rcu_data(cpu, NULL) for per-CPU
- * call_rcu_data structures or set_thread_call_rcu_data(NULL) for
- * per-thread call_rcu_data structures.
- *
- * We silently refuse to free up the default call_rcu_data structure
- * because that is where we put any leftover callbacks. Note that
- * the possibility of self-spawning callbacks makes it impossible
- * to execute all the callbacks in finite time without putting any
- * newly spawned callbacks somewhere else. The "somewhere else" of
- * last resort is the default call_rcu_data structure.
- *
- * We also silently refuse to free NULL pointers. This simplifies
- * the calling code.
- *
- * The caller must wait for a grace-period to pass between return from
- * set_cpu_call_rcu_data() and call to call_rcu_data_free() passing the
- * previous call rcu data as argument.
- *
- * Note: introducing __cds_wfcq_splice_blocking() in this function fixed
- * a list corruption bug in the 0.7.x series. The equivalent fix
- * appeared in 0.6.8 for the stable-0.6 branch.
- */
-void call_rcu_data_free(struct call_rcu_data *crdp)
-{
- if (crdp == NULL || crdp == default_call_rcu_data) {
- return;
- }
- if ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0) {
- uatomic_or(&crdp->flags, URCU_CALL_RCU_STOP);
- wake_call_rcu_thread(crdp);
- while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_STOPPED) == 0)
- (void) poll(NULL, 0, 1);
- }
- if (!cds_wfcq_empty(&crdp->cbs_head, &crdp->cbs_tail)) {
- /* Create default call rcu data if need be */
- (void) get_default_call_rcu_data();
- __cds_wfcq_splice_blocking(&default_call_rcu_data->cbs_head,
- &default_call_rcu_data->cbs_tail,
- &crdp->cbs_head, &crdp->cbs_tail);
- uatomic_add(&default_call_rcu_data->qlen,
- uatomic_read(&crdp->qlen));
- wake_call_rcu_thread(default_call_rcu_data);
- }
-
- call_rcu_lock(&call_rcu_mutex);
- cds_list_del(&crdp->list);
- call_rcu_unlock(&call_rcu_mutex);
-
- free(crdp);
-}
-
-/*
- * Clean up all the per-CPU call_rcu threads.
- */
-void free_all_cpu_call_rcu_data(void)
-{
- int cpu;
- struct call_rcu_data **crdp;
- static int warned = 0;
-
- if (maxcpus <= 0)
- return;
-
- crdp = malloc(sizeof(*crdp) * maxcpus);
- if (!crdp) {
- if (!warned) {
- fprintf(stderr, "[error] liburcu: unable to allocate per-CPU pointer array\n");
- }
- warned = 1;
- return;
- }
-
- for (cpu = 0; cpu < maxcpus; cpu++) {
- crdp[cpu] = get_cpu_call_rcu_data(cpu);
- if (crdp[cpu] == NULL)
- continue;
- set_cpu_call_rcu_data(cpu, NULL);
- }
- /*
- * Wait for call_rcu sites acting as RCU readers of the
- * call_rcu_data to become quiescent.
- */
- synchronize_rcu();
- for (cpu = 0; cpu < maxcpus; cpu++) {
- if (crdp[cpu] == NULL)
- continue;
- call_rcu_data_free(crdp[cpu]);
- }
- free(crdp);
-}
-
-static
-void free_completion(struct urcu_ref *ref)
-{
- struct call_rcu_completion *completion;
-
- completion = caa_container_of(ref, struct call_rcu_completion, ref);
- free(completion);
-}
-
-static
-void _rcu_barrier_complete(struct rcu_head *head)
-{
- struct call_rcu_completion_work *work;
- struct call_rcu_completion *completion;
-
- work = caa_container_of(head, struct call_rcu_completion_work, head);
- completion = work->completion;
- if (!uatomic_sub_return(&completion->barrier_count, 1))
- call_rcu_completion_wake_up(completion);
- urcu_ref_put(&completion->ref, free_completion);
- free(work);
-}
-
-/*
- * Wait for all in-flight call_rcu callbacks to complete execution.
- */
-void rcu_barrier(void)
-{
- struct call_rcu_data *crdp;
- struct call_rcu_completion *completion;
- int count = 0;
- int was_online;
-
- /* Put in offline state in QSBR. */
- was_online = _rcu_read_ongoing();
- if (was_online)
- rcu_thread_offline();
- /*
- * Calling a rcu_barrier() within a RCU read-side critical
- * section is an error.
- */
- if (_rcu_read_ongoing()) {
- static int warned = 0;
-
- if (!warned) {
- fprintf(stderr, "[error] liburcu: rcu_barrier() called from within RCU read-side critical section.\n");
- }
- warned = 1;
- goto online;
- }
-
- completion = calloc(sizeof(*completion), 1);
- if (!completion)
- urcu_die(errno);
-
- call_rcu_lock(&call_rcu_mutex);
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
- count++;
-
- /* Referenced by rcu_barrier() and each call_rcu thread. */
- urcu_ref_set(&completion->ref, count + 1);
- completion->barrier_count = count;
-
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
- struct call_rcu_completion_work *work;
-
- work = calloc(sizeof(*work), 1);
- if (!work)
- urcu_die(errno);
- work->completion = completion;
- _call_rcu(&work->head, _rcu_barrier_complete, crdp);
- }
- call_rcu_unlock(&call_rcu_mutex);
-
- /* Wait for them */
- for (;;) {
- uatomic_dec(&completion->futex);
- /* Decrement futex before reading barrier_count */
- cmm_smp_mb();
- if (!uatomic_read(&completion->barrier_count))
- break;
- call_rcu_completion_wait(completion);
- }
-
- urcu_ref_put(&completion->ref, free_completion);
-
-online:
- if (was_online)
- rcu_thread_online();
-}
-
-/*
- * Acquire the call_rcu_mutex in order to ensure that the child sees
- * all of the call_rcu() data structures in a consistent state. Ensure
- * that all call_rcu threads are in a quiescent state across fork.
- * Suitable for pthread_atfork() and friends.
- */
-void call_rcu_before_fork(void)
-{
- struct call_rcu_data *crdp;
-
- call_rcu_lock(&call_rcu_mutex);
-
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
- uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSE);
- cmm_smp_mb__after_uatomic_or();
- wake_call_rcu_thread(crdp);
- }
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
- while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) == 0)
- (void) poll(NULL, 0, 1);
- }
-}
-
-/*
- * Clean up call_rcu data structures in the parent of a successful fork()
- * that is not followed by exec() in the child. Suitable for
- * pthread_atfork() and friends.
- */
-void call_rcu_after_fork_parent(void)
-{
- struct call_rcu_data *crdp;
-
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list)
- uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSE);
- cds_list_for_each_entry(crdp, &call_rcu_data_list, list) {
- while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) != 0)
- (void) poll(NULL, 0, 1);
- }
- call_rcu_unlock(&call_rcu_mutex);
-}
-
-/*
- * Clean up call_rcu data structures in the child of a successful fork()
- * that is not followed by exec(). Suitable for pthread_atfork() and
- * friends.
- */
-void call_rcu_after_fork_child(void)
-{
- struct call_rcu_data *crdp, *next;
-
- /* Release the mutex. */
- call_rcu_unlock(&call_rcu_mutex);
-
- /* Do nothing when call_rcu() has not been used */
- if (cds_list_empty(&call_rcu_data_list))
- return;
-
- /*
- * Allocate a new default call_rcu_data structure in order
- * to get a working call_rcu thread to go with it.
- */
- default_call_rcu_data = NULL;
- (void)get_default_call_rcu_data();
-
- /* Cleanup call_rcu_data pointers before use */
- maxcpus_reset();
- free(per_cpu_call_rcu_data);
- rcu_set_pointer(&per_cpu_call_rcu_data, NULL);
- URCU_TLS(thread_call_rcu_data) = NULL;
-
- /*
- * Dispose of all of the rest of the call_rcu_data structures.
- * Leftover call_rcu callbacks will be merged into the new
- * default call_rcu thread queue.
- */
- cds_list_for_each_entry_safe(crdp, next, &call_rcu_data_list, list) {
- if (crdp == default_call_rcu_data)
- continue;
- uatomic_set(&crdp->flags, URCU_CALL_RCU_STOPPED);
- call_rcu_data_free(crdp);
- }
-}
+++ /dev/null
-#ifndef _URCU_CALL_RCU_H
-#define _URCU_CALL_RCU_H
-
-/*
- * urcu-call-rcu.h
- *
- * Userspace RCU header - deferred execution
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu-defer.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-
-#include <urcu/wfcqueue.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Note that struct call_rcu_data is opaque to callers. */
-
-struct call_rcu_data;
-
-/* Flag values. */
-
-#define URCU_CALL_RCU_RT (1U << 0)
-#define URCU_CALL_RCU_RUNNING (1U << 1)
-#define URCU_CALL_RCU_STOP (1U << 2)
-#define URCU_CALL_RCU_STOPPED (1U << 3)
-#define URCU_CALL_RCU_PAUSE (1U << 4)
-#define URCU_CALL_RCU_PAUSED (1U << 5)
-
-/*
- * The rcu_head data structure is placed in the structure to be freed
- * via call_rcu().
- */
-
-struct rcu_head {
- struct cds_wfcq_node next;
- void (*func)(struct rcu_head *head);
-};
-
-/*
- * Exported functions
- *
- * Important: see rcu-api.md in userspace-rcu documentation for
- * call_rcu family of functions usage detail, including the surrounding
- * RCU usage required when using these primitives.
- */
-
-void call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head));
-
-struct call_rcu_data *create_call_rcu_data(unsigned long flags,
- int cpu_affinity);
-void call_rcu_data_free(struct call_rcu_data *crdp);
-
-struct call_rcu_data *get_default_call_rcu_data(void);
-struct call_rcu_data *get_cpu_call_rcu_data(int cpu);
-struct call_rcu_data *get_thread_call_rcu_data(void);
-struct call_rcu_data *get_call_rcu_data(void);
-pthread_t get_call_rcu_thread(struct call_rcu_data *crdp);
-
-void set_thread_call_rcu_data(struct call_rcu_data *crdp);
-int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp);
-
-int create_all_cpu_call_rcu_data(unsigned long flags);
-void free_all_cpu_call_rcu_data(void);
-
-void call_rcu_before_fork(void);
-void call_rcu_after_fork_parent(void);
-void call_rcu_after_fork_child(void);
-
-void rcu_barrier(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_CALL_RCU_H */
+++ /dev/null
-#ifndef _URCU_DEFER_IMPL_H
-#define _URCU_DEFER_IMPL_H
-
-/*
- * urcu-defer-impl.h
- *
- * Userspace RCU header - memory reclamation.
- *
- * TO BE INCLUDED ONLY FROM URCU LIBRARY CODE. See urcu-defer.h for linking
- * dynamically with the userspace rcu reclamation library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <signal.h>
-#include <assert.h>
-#include <string.h>
-#include <errno.h>
-#include <poll.h>
-#include <sys/time.h>
-#include <unistd.h>
-#include <stdint.h>
-
-#include "urcu/futex.h"
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/uatomic.h>
-#include <urcu/list.h>
-#include <urcu/system.h>
-#include <urcu/tls-compat.h>
-#include "urcu-die.h"
-
-/*
- * Number of entries in the per-thread defer queue. Must be power of 2.
- */
-#define DEFER_QUEUE_SIZE (1 << 12)
-#define DEFER_QUEUE_MASK (DEFER_QUEUE_SIZE - 1)
-
-/*
- * Typically, data is aligned at least on the architecture size.
- * Use lowest bit to indicate that the current callback is changing.
- * Assumes that (void *)-2L is not used often. Used to encode non-aligned
- * functions and non-aligned data using extra space.
- * We encode the (void *)-2L fct as: -2L, fct, data.
- * We encode the (void *)-2L data as either:
- * fct | DQ_FCT_BIT, data (if fct is aligned), or
- * -2L, fct, data (if fct is not aligned).
- * Here, DQ_FCT_MARK == ~DQ_FCT_BIT. Required for the test order.
- */
-#define DQ_FCT_BIT (1 << 0)
-#define DQ_IS_FCT_BIT(x) ((unsigned long)(x) & DQ_FCT_BIT)
-#define DQ_SET_FCT_BIT(x) \
- (x = (void *)((unsigned long)(x) | DQ_FCT_BIT))
-#define DQ_CLEAR_FCT_BIT(x) \
- (x = (void *)((unsigned long)(x) & ~DQ_FCT_BIT))
-#define DQ_FCT_MARK ((void *)(~DQ_FCT_BIT))
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-/*
- * defer queue.
- * Contains pointers. Encoded to save space when same callback is often used.
- * When looking up the next item:
- * - if DQ_FCT_BIT is set, set the current callback to DQ_CLEAR_FCT_BIT(ptr)
- * - next element contains pointer to data.
- * - else if item == DQ_FCT_MARK
- * - set the current callback to next element ptr
- * - following next element contains pointer to data.
- * - else current element contains data
- */
-struct defer_queue {
- unsigned long head; /* add element at head */
- void *last_fct_in; /* last fct pointer encoded */
- unsigned long tail; /* next element to remove at tail */
- void *last_fct_out; /* last fct pointer encoded */
- void **q;
- /* registry information */
- unsigned long last_head;
- struct cds_list_head list; /* list of thread queues */
-};
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#include "urcu-defer.h"
-
-void __attribute__((destructor)) rcu_defer_exit(void);
-
-extern void synchronize_rcu(void);
-
-/*
- * rcu_defer_mutex nests inside defer_thread_mutex.
- */
-static pthread_mutex_t rcu_defer_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_mutex_t defer_thread_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-static int32_t defer_thread_futex;
-static int32_t defer_thread_stop;
-
-/*
- * Written to only by each individual deferer. Read by both the deferer and
- * the reclamation tread.
- */
-static DEFINE_URCU_TLS(struct defer_queue, defer_queue);
-static CDS_LIST_HEAD(registry_defer);
-static pthread_t tid_defer;
-
-static void mutex_lock_defer(pthread_mutex_t *mutex)
-{
- int ret;
-
-#ifndef DISTRUST_SIGNALS_EXTREME
- ret = pthread_mutex_lock(mutex);
- if (ret)
- urcu_die(ret);
-#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
- while ((ret = pthread_mutex_trylock(mutex)) != 0) {
- if (ret != EBUSY && ret != EINTR)
- urcu_die(ret);
- (void) poll(NULL,0,10);
- }
-#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
-}
-
-/*
- * Wake-up any waiting defer thread. Called from many concurrent threads.
- */
-static void wake_up_defer(void)
-{
- if (caa_unlikely(uatomic_read(&defer_thread_futex) == -1)) {
- uatomic_set(&defer_thread_futex, 0);
- if (futex_noasync(&defer_thread_futex, FUTEX_WAKE, 1,
- NULL, NULL, 0) < 0)
- urcu_die(errno);
- }
-}
-
-static unsigned long rcu_defer_num_callbacks(void)
-{
- unsigned long num_items = 0, head;
- struct defer_queue *index;
-
- mutex_lock_defer(&rcu_defer_mutex);
- cds_list_for_each_entry(index, ®istry_defer, list) {
- head = CMM_LOAD_SHARED(index->head);
- num_items += head - index->tail;
- }
- mutex_unlock(&rcu_defer_mutex);
- return num_items;
-}
-
-/*
- * Defer thread waiting. Single thread.
- */
-static void wait_defer(void)
-{
- uatomic_dec(&defer_thread_futex);
- /* Write futex before read queue */
- /* Write futex before read defer_thread_stop */
- cmm_smp_mb();
- if (_CMM_LOAD_SHARED(defer_thread_stop)) {
- uatomic_set(&defer_thread_futex, 0);
- pthread_exit(0);
- }
- if (rcu_defer_num_callbacks()) {
- cmm_smp_mb(); /* Read queue before write futex */
- /* Callbacks are queued, don't wait. */
- uatomic_set(&defer_thread_futex, 0);
- } else {
- cmm_smp_rmb(); /* Read queue before read futex */
- if (uatomic_read(&defer_thread_futex) != -1)
- return;
- while (futex_noasync(&defer_thread_futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- return;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
- }
-}
-
-/*
- * Must be called after Q.S. is reached.
- */
-static void rcu_defer_barrier_queue(struct defer_queue *queue,
- unsigned long head)
-{
- unsigned long i;
- void (*fct)(void *p);
- void *p;
-
- /*
- * Tail is only modified when lock is held.
- * Head is only modified by owner thread.
- */
-
- for (i = queue->tail; i != head;) {
- cmm_smp_rmb(); /* read head before q[]. */
- p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
- if (caa_unlikely(DQ_IS_FCT_BIT(p))) {
- DQ_CLEAR_FCT_BIT(p);
- queue->last_fct_out = p;
- p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
- } else if (caa_unlikely(p == DQ_FCT_MARK)) {
- p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
- queue->last_fct_out = p;
- p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]);
- }
- fct = queue->last_fct_out;
- fct(p);
- }
- cmm_smp_mb(); /* push tail after having used q[] */
- CMM_STORE_SHARED(queue->tail, i);
-}
-
-static void _rcu_defer_barrier_thread(void)
-{
- unsigned long head, num_items;
-
- head = URCU_TLS(defer_queue).head;
- num_items = head - URCU_TLS(defer_queue).tail;
- if (caa_unlikely(!num_items))
- return;
- synchronize_rcu();
- rcu_defer_barrier_queue(&URCU_TLS(defer_queue), head);
-}
-
-void rcu_defer_barrier_thread(void)
-{
- mutex_lock_defer(&rcu_defer_mutex);
- _rcu_defer_barrier_thread();
- mutex_unlock(&rcu_defer_mutex);
-}
-
-/*
- * rcu_defer_barrier - Execute all queued rcu callbacks.
- *
- * Execute all RCU callbacks queued before rcu_defer_barrier() execution.
- * All callbacks queued on the local thread prior to a rcu_defer_barrier() call
- * are guaranteed to be executed.
- * Callbacks queued by other threads concurrently with rcu_defer_barrier()
- * execution are not guaranteed to be executed in the current batch (could
- * be left for the next batch). These callbacks queued by other threads are only
- * guaranteed to be executed if there is explicit synchronization between
- * the thread adding to the queue and the thread issuing the defer_barrier call.
- */
-
-void rcu_defer_barrier(void)
-{
- struct defer_queue *index;
- unsigned long num_items = 0;
-
- if (cds_list_empty(®istry_defer))
- return;
-
- mutex_lock_defer(&rcu_defer_mutex);
- cds_list_for_each_entry(index, ®istry_defer, list) {
- index->last_head = CMM_LOAD_SHARED(index->head);
- num_items += index->last_head - index->tail;
- }
- if (caa_likely(!num_items)) {
- /*
- * We skip the grace period because there are no queued
- * callbacks to execute.
- */
- goto end;
- }
- synchronize_rcu();
- cds_list_for_each_entry(index, ®istry_defer, list)
- rcu_defer_barrier_queue(index, index->last_head);
-end:
- mutex_unlock(&rcu_defer_mutex);
-}
-
-/*
- * _defer_rcu - Queue a RCU callback.
- */
-static void _defer_rcu(void (*fct)(void *p), void *p)
-{
- unsigned long head, tail;
-
- /*
- * Head is only modified by ourself. Tail can be modified by reclamation
- * thread.
- */
- head = URCU_TLS(defer_queue).head;
- tail = CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail);
-
- /*
- * If queue is full, or reached threshold. Empty queue ourself.
- * Worse-case: must allow 2 supplementary entries for fct pointer.
- */
- if (caa_unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) {
- assert(head - tail <= DEFER_QUEUE_SIZE);
- rcu_defer_barrier_thread();
- assert(head - CMM_LOAD_SHARED(URCU_TLS(defer_queue).tail) == 0);
- }
-
- /*
- * Encode:
- * if the function is not changed and the data is aligned and it is
- * not the marker:
- * store the data
- * otherwise if the function is aligned and its not the marker:
- * store the function with DQ_FCT_BIT
- * store the data
- * otherwise:
- * store the marker (DQ_FCT_MARK)
- * store the function
- * store the data
- *
- * Decode: see the comments before 'struct defer_queue'
- * or the code in rcu_defer_barrier_queue().
- */
- if (caa_unlikely(URCU_TLS(defer_queue).last_fct_in != fct
- || DQ_IS_FCT_BIT(p)
- || p == DQ_FCT_MARK)) {
- URCU_TLS(defer_queue).last_fct_in = fct;
- if (caa_unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) {
- _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
- DQ_FCT_MARK);
- _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
- fct);
- } else {
- DQ_SET_FCT_BIT(fct);
- _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK],
- fct);
- }
- }
- _CMM_STORE_SHARED(URCU_TLS(defer_queue).q[head++ & DEFER_QUEUE_MASK], p);
- cmm_smp_wmb(); /* Publish new pointer before head */
- /* Write q[] before head. */
- CMM_STORE_SHARED(URCU_TLS(defer_queue).head, head);
- cmm_smp_mb(); /* Write queue head before read futex */
- /*
- * Wake-up any waiting defer thread.
- */
- wake_up_defer();
-}
-
-static void *thr_defer(void *args)
-{
- for (;;) {
- /*
- * "Be green". Don't wake up the CPU if there is no RCU work
- * to perform whatsoever. Aims at saving laptop battery life by
- * leaving the processor in sleep state when idle.
- */
- wait_defer();
- /* Sleeping after wait_defer to let many callbacks enqueue */
- (void) poll(NULL,0,100); /* wait for 100ms */
- rcu_defer_barrier();
- }
-
- return NULL;
-}
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void defer_rcu(void (*fct)(void *p), void *p)
-{
- _defer_rcu(fct, p);
-}
-
-static void start_defer_thread(void)
-{
- int ret;
-
- ret = pthread_create(&tid_defer, NULL, thr_defer, NULL);
- assert(!ret);
-}
-
-static void stop_defer_thread(void)
-{
- int ret;
- void *tret;
-
- _CMM_STORE_SHARED(defer_thread_stop, 1);
- /* Store defer_thread_stop before testing futex */
- cmm_smp_mb();
- wake_up_defer();
-
- ret = pthread_join(tid_defer, &tret);
- assert(!ret);
-
- CMM_STORE_SHARED(defer_thread_stop, 0);
- /* defer thread should always exit when futex value is 0 */
- assert(uatomic_read(&defer_thread_futex) == 0);
-}
-
-int rcu_defer_register_thread(void)
-{
- int was_empty;
-
- assert(URCU_TLS(defer_queue).last_head == 0);
- assert(URCU_TLS(defer_queue).q == NULL);
- URCU_TLS(defer_queue).q = malloc(sizeof(void *) * DEFER_QUEUE_SIZE);
- if (!URCU_TLS(defer_queue).q)
- return -ENOMEM;
-
- mutex_lock_defer(&defer_thread_mutex);
- mutex_lock_defer(&rcu_defer_mutex);
- was_empty = cds_list_empty(®istry_defer);
- cds_list_add(&URCU_TLS(defer_queue).list, ®istry_defer);
- mutex_unlock(&rcu_defer_mutex);
-
- if (was_empty)
- start_defer_thread();
- mutex_unlock(&defer_thread_mutex);
- return 0;
-}
-
-void rcu_defer_unregister_thread(void)
-{
- int is_empty;
-
- mutex_lock_defer(&defer_thread_mutex);
- mutex_lock_defer(&rcu_defer_mutex);
- cds_list_del(&URCU_TLS(defer_queue).list);
- _rcu_defer_barrier_thread();
- free(URCU_TLS(defer_queue).q);
- URCU_TLS(defer_queue).q = NULL;
- is_empty = cds_list_empty(®istry_defer);
- mutex_unlock(&rcu_defer_mutex);
-
- if (is_empty)
- stop_defer_thread();
- mutex_unlock(&defer_thread_mutex);
-}
-
-void rcu_defer_exit(void)
-{
- assert(cds_list_empty(®istry_defer));
-}
-
-#endif /* _URCU_DEFER_IMPL_H */
+++ /dev/null
-#ifndef _URCU_BATCH_H
-#define _URCU_BATCH_H
-
-/*
- * urcu-defer.h
- *
- * Userspace RCU header - deferred execution
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu-defer.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Note: the defer_rcu() API is currently EXPERIMENTAL. It may change in the
- * future.
- *
- * Important !
- *
- * Each thread queuing memory reclamation must be registered with
- * rcu_defer_register_thread(). rcu_defer_unregister_thread() should be
- * called before the thread exits.
- *
- * *NEVER* use defer_rcu() within a RCU read-side critical section, because this
- * primitive need to call synchronize_rcu() if the thread queue is full.
- */
-
-extern void defer_rcu(void (*fct)(void *p), void *p);
-
-/*
- * Thread registration for reclamation.
- */
-extern int rcu_defer_register_thread(void);
-extern void rcu_defer_unregister_thread(void);
-extern void rcu_defer_barrier(void);
-extern void rcu_defer_barrier_thread(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_BATCH_H */
+++ /dev/null
-#ifndef _URCU_DIE_H
-#define _URCU_DIE_H
-
-/*
- * urcu-die.h
- *
- * Userspace RCU library unrecoverable error handling
- *
- * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-#define urcu_die(cause) \
-do { \
- fprintf(stderr, "(" __FILE__ ":%s@%u) Unrecoverable error: %s\n", \
- __func__, __LINE__, strerror(cause)); \
- abort(); \
-} while (0)
-
-#endif /* _URCU_DIE_H */
+++ /dev/null
-#ifndef _URCU_FLAVOR_H
-#define _URCU_FLAVOR_H
-
-/*
- * urcu-flavor.h
- *
- * Userspace RCU header - rcu flavor declarations
- *
- * Copyright (c) 2011 Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct rcu_flavor_struct {
- void (*read_lock)(void);
- void (*read_unlock)(void);
- int (*read_ongoing)(void);
- void (*read_quiescent_state)(void);
- void (*update_call_rcu)(struct rcu_head *head,
- void (*func)(struct rcu_head *head));
- void (*update_synchronize_rcu)(void);
- void (*update_defer_rcu)(void (*fct)(void *p), void *p);
-
- void (*thread_offline)(void);
- void (*thread_online)(void);
- void (*register_thread)(void);
- void (*unregister_thread)(void);
-
- void (*barrier)(void);
-};
-
-#define DEFINE_RCU_FLAVOR(x) \
-const struct rcu_flavor_struct x = { \
- .read_lock = rcu_read_lock, \
- .read_unlock = rcu_read_unlock, \
- .read_ongoing = rcu_read_ongoing, \
- .read_quiescent_state = rcu_quiescent_state, \
- .update_call_rcu = call_rcu, \
- .update_synchronize_rcu = synchronize_rcu, \
- .update_defer_rcu = defer_rcu, \
- .thread_offline = rcu_thread_offline, \
- .thread_online = rcu_thread_online, \
- .register_thread = rcu_register_thread, \
- .unregister_thread = rcu_unregister_thread,\
- .barrier = rcu_barrier, \
-}
-
-extern const struct rcu_flavor_struct rcu_flavor;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_FLAVOR_H */
+++ /dev/null
-
-/*
- * urcu-pointer.c
- *
- * library wrappers to be used by non-LGPL compatible source code.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <urcu/uatomic.h>
-
-#include "urcu/static/urcu-pointer.h"
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#include "urcu-pointer.h"
-
-extern void synchronize_rcu(void);
-
-void *rcu_dereference_sym(void *p)
-{
- return _rcu_dereference(p);
-}
-
-void *rcu_set_pointer_sym(void **p, void *v)
-{
- cmm_wmb();
- uatomic_set(p, v);
- return v;
-}
-
-void *rcu_xchg_pointer_sym(void **p, void *v)
-{
- cmm_wmb();
- return uatomic_xchg(p, v);
-}
-
-void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new)
-{
- cmm_wmb();
- return uatomic_cmpxchg(p, old, _new);
-}
+++ /dev/null
-#ifndef _URCU_POINTER_H
-#define _URCU_POINTER_H
-
-/*
- * urcu-pointer.h
- *
- * Userspace RCU header. Operations on pointers.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/uatomic.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)
-
-#include <urcu/static/urcu-pointer.h>
-
-/*
- * rcu_dereference(ptr)
- *
- * Fetch a RCU-protected pointer. Typically used to copy the variable ptr to a
- * local variable.
- */
-#define rcu_dereference _rcu_dereference
-
-/*
- * type *rcu_cmpxchg_pointer(type **ptr, type *new, type *old)
- * type *rcu_xchg_pointer(type **ptr, type *new)
- * void rcu_set_pointer(type **ptr, type *new)
- *
- * RCU pointer updates.
- * @ptr: address of the pointer to modify
- * @new: new pointer value
- * @old: old pointer value (expected)
- *
- * return: old pointer value
- */
-#define rcu_cmpxchg_pointer _rcu_cmpxchg_pointer
-#define rcu_xchg_pointer _rcu_xchg_pointer
-#define rcu_set_pointer _rcu_set_pointer
-
-#else /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */
-
-extern void *rcu_dereference_sym(void *p);
-#define rcu_dereference(p) \
- __extension__ \
- ({ \
- __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \
- rcu_dereference_sym(URCU_FORCE_CAST(void *, p))); \
- (_________p1); \
- })
-
-extern void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new);
-#define rcu_cmpxchg_pointer(p, old, _new) \
- __extension__ \
- ({ \
- __typeof__(*(p)) _________pold = (old); \
- __typeof__(*(p)) _________pnew = (_new); \
- __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
- rcu_cmpxchg_pointer_sym(URCU_FORCE_CAST(void **, p), \
- _________pold, \
- _________pnew)); \
- (_________p1); \
- })
-
-extern void *rcu_xchg_pointer_sym(void **p, void *v);
-#define rcu_xchg_pointer(p, v) \
- __extension__ \
- ({ \
- __typeof__(*(p)) _________pv = (v); \
- __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \
- rcu_xchg_pointer_sym(URCU_FORCE_CAST(void **, p), \
- _________pv)); \
- (_________p1); \
- })
-
-/*
- * Note: rcu_set_pointer_sym returns @v because we don't want to break
- * the ABI. At the API level, rcu_set_pointer() now returns void. Use of
- * the return value is therefore deprecated, and will cause a build
- * error.
- */
-extern void *rcu_set_pointer_sym(void **p, void *v);
-#define rcu_set_pointer(p, v) \
- do { \
- __typeof__(*(p)) _________pv = (v); \
- (void) rcu_set_pointer_sym(URCU_FORCE_CAST(void **, p), \
- _________pv); \
- } while (0)
-
-#endif /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */
-
-/*
- * void rcu_assign_pointer(type *ptr, type *new)
- *
- * Same as rcu_set_pointer, but takes the pointer to assign to rather than its
- * address as first parameter. Provided for compatibility with the Linux kernel
- * RCU semantic.
- */
-#define rcu_assign_pointer(p, v) rcu_set_pointer((&p), (v))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_POINTER_H */
+++ /dev/null
-/*
- * urcu-qsbr.c
- *
- * Userspace RCU QSBR library
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#define _GNU_SOURCE
-#define _LGPL_SOURCE
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-#include <errno.h>
-#include <poll.h>
-
-#include "urcu/wfcqueue.h"
-#include "urcu/map/urcu-qsbr.h"
-#define BUILD_QSBR_LIB
-#include "urcu/static/urcu-qsbr.h"
-#include "urcu-pointer.h"
-#include "urcu/tls-compat.h"
-
-#include "urcu-die.h"
-#include "urcu-wait.h"
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu-qsbr.h"
-#define _LGPL_SOURCE
-
-void __attribute__((destructor)) rcu_exit(void);
-
-/*
- * rcu_gp_lock ensures mutual exclusion between threads calling
- * synchronize_rcu().
- */
-static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-/*
- * rcu_registry_lock ensures mutual exclusion between threads
- * registering and unregistering themselves to/from the registry, and
- * with threads reading that registry from synchronize_rcu(). However,
- * this lock is not held all the way through the completion of awaiting
- * for the grace period. It is sporadically released between iterations
- * on the registry.
- * rcu_registry_lock may nest inside rcu_gp_lock.
- */
-static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
-struct rcu_gp rcu_gp = { .ctr = RCU_GP_ONLINE };
-
-/*
- * Active attempts to check for reader Q.S. before calling futex().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-/*
- * Written to only by each individual reader. Read by both the reader and the
- * writers.
- */
-DEFINE_URCU_TLS(struct rcu_reader, rcu_reader);
-
-static CDS_LIST_HEAD(registry);
-
-/*
- * Queue keeping threads awaiting to wait for a grace period. Contains
- * struct gp_waiters_thread objects.
- */
-static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
-
-static void mutex_lock(pthread_mutex_t *mutex)
-{
- int ret;
-
-#ifndef DISTRUST_SIGNALS_EXTREME
- ret = pthread_mutex_lock(mutex);
- if (ret)
- urcu_die(ret);
-#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
- while ((ret = pthread_mutex_trylock(mutex)) != 0) {
- if (ret != EBUSY && ret != EINTR)
- urcu_die(ret);
- poll(NULL,0,10);
- }
-#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
-}
-
-static void mutex_unlock(pthread_mutex_t *mutex)
-{
- int ret;
-
- ret = pthread_mutex_unlock(mutex);
- if (ret)
- urcu_die(ret);
-}
-
-/*
- * synchronize_rcu() waiting. Single thread.
- */
-static void wait_gp(void)
-{
- /* Read reader_gp before read futex */
- cmm_smp_rmb();
- if (uatomic_read(&rcu_gp.futex) != -1)
- return;
- while (futex_noasync(&rcu_gp.futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- return;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
-}
-
-/*
- * Always called with rcu_registry lock held. Releases this lock between
- * iterations and grabs it again. Holds the lock when it returns.
- */
-static void wait_for_readers(struct cds_list_head *input_readers,
- struct cds_list_head *cur_snap_readers,
- struct cds_list_head *qsreaders)
-{
- unsigned int wait_loops = 0;
- struct rcu_reader *index, *tmp;
-
- /*
- * Wait for each thread URCU_TLS(rcu_reader).ctr to either
- * indicate quiescence (offline), or for them to observe the
- * current rcu_gp.ctr value.
- */
- for (;;) {
- if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
- wait_loops++;
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- uatomic_set(&rcu_gp.futex, -1);
- /*
- * Write futex before write waiting (the other side
- * reads them in the opposite order).
- */
- cmm_smp_wmb();
- cds_list_for_each_entry(index, input_readers, node) {
- _CMM_STORE_SHARED(index->waiting, 1);
- }
- /* Write futex before read reader_gp */
- cmm_smp_mb();
- }
- cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
- switch (rcu_reader_state(&index->ctr)) {
- case RCU_READER_ACTIVE_CURRENT:
- if (cur_snap_readers) {
- cds_list_move(&index->node,
- cur_snap_readers);
- break;
- }
- /* Fall-through */
- case RCU_READER_INACTIVE:
- cds_list_move(&index->node, qsreaders);
- break;
- case RCU_READER_ACTIVE_OLD:
- /*
- * Old snapshot. Leaving node in
- * input_readers will make us busy-loop
- * until the snapshot becomes current or
- * the reader becomes inactive.
- */
- break;
- }
- }
-
- if (cds_list_empty(input_readers)) {
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- /* Read reader_gp before write futex */
- cmm_smp_mb();
- uatomic_set(&rcu_gp.futex, 0);
- }
- break;
- } else {
- /* Temporarily unlock the registry lock. */
- mutex_unlock(&rcu_registry_lock);
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- wait_gp();
- } else {
-#ifndef HAS_INCOHERENT_CACHES
- caa_cpu_relax();
-#else /* #ifndef HAS_INCOHERENT_CACHES */
- cmm_smp_mb();
-#endif /* #else #ifndef HAS_INCOHERENT_CACHES */
- }
- /* Re-lock the registry lock before the next loop. */
- mutex_lock(&rcu_registry_lock);
- }
- }
-}
-
-/*
- * Using a two-subphases algorithm for architectures with smaller than 64-bit
- * long-size to ensure we do not encounter an overflow bug.
- */
-
-#if (CAA_BITS_PER_LONG < 64)
-void synchronize_rcu(void)
-{
- CDS_LIST_HEAD(cur_snap_readers);
- CDS_LIST_HEAD(qsreaders);
- unsigned long was_online;
- DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
- struct urcu_waiters waiters;
-
- was_online = rcu_read_ongoing();
-
- /* All threads should read qparity before accessing data structure
- * where new ptr points to. In the "then" case, rcu_thread_offline
- * includes a memory barrier.
- *
- * Mark the writer thread offline to make sure we don't wait for
- * our own quiescent state. This allows using synchronize_rcu()
- * in threads registered as readers.
- */
- if (was_online)
- rcu_thread_offline();
- else
- cmm_smp_mb();
-
- /*
- * Add ourself to gp_waiters queue of threads awaiting to wait
- * for a grace period. Proceed to perform the grace period only
- * if we are the first thread added into the queue.
- */
- if (urcu_wait_add(&gp_waiters, &wait) != 0) {
- /* Not first in queue: will be awakened by another thread. */
- urcu_adaptative_busy_wait(&wait);
- goto gp_end;
- }
- /* We won't need to wake ourself up */
- urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
-
- mutex_lock(&rcu_gp_lock);
-
- /*
- * Move all waiters into our local queue.
- */
- urcu_move_waiters(&waiters, &gp_waiters);
-
- mutex_lock(&rcu_registry_lock);
-
- if (cds_list_empty(®istry))
- goto out;
-
- /*
- * Wait for readers to observe original parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
-
- /*
- * Must finish waiting for quiescent state for original parity
- * before committing next rcu_gp.ctr update to memory. Failure
- * to do so could result in the writer waiting forever while new
- * readers are always accessing data (no progress). Enforce
- * compiler-order of load URCU_TLS(rcu_reader).ctr before store
- * to rcu_gp.ctr.
- */
- cmm_barrier();
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /* Switch parity: 0 -> 1, 1 -> 0 */
- CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR);
-
- /*
- * Must commit rcu_gp.ctr update to memory before waiting for
- * quiescent state. Failure to do so could result in the writer
- * waiting forever while new readers are always accessing data
- * (no progress). Enforce compiler-order of store to rcu_gp.ctr
- * before load URCU_TLS(rcu_reader).ctr.
- */
- cmm_barrier();
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /*
- * Wait for readers to observe new parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
-
- /*
- * Put quiescent reader list back into registry.
- */
- cds_list_splice(&qsreaders, ®istry);
-out:
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
- urcu_wake_all_waiters(&waiters);
-gp_end:
- /*
- * Finish waiting for reader threads before letting the old ptr being
- * freed.
- */
- if (was_online)
- rcu_thread_online();
- else
- cmm_smp_mb();
-}
-#else /* !(CAA_BITS_PER_LONG < 64) */
-void synchronize_rcu(void)
-{
- CDS_LIST_HEAD(qsreaders);
- unsigned long was_online;
- DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
- struct urcu_waiters waiters;
-
- was_online = rcu_read_ongoing();
-
- /*
- * Mark the writer thread offline to make sure we don't wait for
- * our own quiescent state. This allows using synchronize_rcu()
- * in threads registered as readers.
- */
- if (was_online)
- rcu_thread_offline();
- else
- cmm_smp_mb();
-
- /*
- * Add ourself to gp_waiters queue of threads awaiting to wait
- * for a grace period. Proceed to perform the grace period only
- * if we are the first thread added into the queue.
- */
- if (urcu_wait_add(&gp_waiters, &wait) != 0) {
- /* Not first in queue: will be awakened by another thread. */
- urcu_adaptative_busy_wait(&wait);
- goto gp_end;
- }
- /* We won't need to wake ourself up */
- urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
-
- mutex_lock(&rcu_gp_lock);
-
- /*
- * Move all waiters into our local queue.
- */
- urcu_move_waiters(&waiters, &gp_waiters);
-
- mutex_lock(&rcu_registry_lock);
-
- if (cds_list_empty(®istry))
- goto out;
-
- /* Increment current G.P. */
- CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr + RCU_GP_CTR);
-
- /*
- * Must commit rcu_gp.ctr update to memory before waiting for
- * quiescent state. Failure to do so could result in the writer
- * waiting forever while new readers are always accessing data
- * (no progress). Enforce compiler-order of store to rcu_gp.ctr
- * before load URCU_TLS(rcu_reader).ctr.
- */
- cmm_barrier();
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /*
- * Wait for readers to observe new count of be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(®istry, NULL, &qsreaders);
-
- /*
- * Put quiescent reader list back into registry.
- */
- cds_list_splice(&qsreaders, ®istry);
-out:
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
- urcu_wake_all_waiters(&waiters);
-gp_end:
- if (was_online)
- rcu_thread_online();
- else
- cmm_smp_mb();
-}
-#endif /* !(CAA_BITS_PER_LONG < 64) */
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void rcu_read_lock(void)
-{
- _rcu_read_lock();
-}
-
-void rcu_read_unlock(void)
-{
- _rcu_read_unlock();
-}
-
-int rcu_read_ongoing(void)
-{
- return _rcu_read_ongoing();
-}
-
-void rcu_quiescent_state(void)
-{
- _rcu_quiescent_state();
-}
-
-void rcu_thread_offline(void)
-{
- _rcu_thread_offline();
-}
-
-void rcu_thread_online(void)
-{
- _rcu_thread_online();
-}
-
-void rcu_register_thread(void)
-{
- URCU_TLS(rcu_reader).tid = pthread_self();
- assert(URCU_TLS(rcu_reader).ctr == 0);
-
- mutex_lock(&rcu_registry_lock);
- assert(!URCU_TLS(rcu_reader).registered);
- URCU_TLS(rcu_reader).registered = 1;
- cds_list_add(&URCU_TLS(rcu_reader).node, ®istry);
- mutex_unlock(&rcu_registry_lock);
- _rcu_thread_online();
-}
-
-void rcu_unregister_thread(void)
-{
- /*
- * We have to make the thread offline otherwise we end up dealocking
- * with a waiting writer.
- */
- _rcu_thread_offline();
- assert(URCU_TLS(rcu_reader).registered);
- URCU_TLS(rcu_reader).registered = 0;
- mutex_lock(&rcu_registry_lock);
- cds_list_del(&URCU_TLS(rcu_reader).node);
- mutex_unlock(&rcu_registry_lock);
-}
-
-void rcu_exit(void)
-{
- /*
- * Assertion disabled because call_rcu threads are now rcu
- * readers, and left running at exit.
- * assert(cds_list_empty(®istry));
- */
-}
-
-DEFINE_RCU_FLAVOR(rcu_flavor);
-
-#include "urcu-call-rcu-impl.h"
-#include "urcu-defer-impl.h"
+++ /dev/null
-#ifndef _URCU_QSBR_H
-#define _URCU_QSBR_H
-
-/*
- * urcu-qsbr.h
- *
- * Userspace RCU QSBR header.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-
-/*
- * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
- * publication headers.
- */
-#include <urcu-pointer.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <urcu/map/urcu-qsbr.h>
-
-#ifdef RCU_DEBUG /* For backward compatibility */
-#define DEBUG_RCU
-#endif
-
-/*
- * Important !
- *
- * Each thread containing read-side critical sections must be registered
- * with rcu_register_thread() before calling rcu_read_lock().
- * rcu_unregister_thread() should be called before the thread exits.
- */
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/urcu-qsbr.h>
-
-/*
- * Mappings for static use of the userspace RCU library.
- * Should only be used in LGPL-compatible code.
- */
-
-/*
- * rcu_read_lock()
- * rcu_read_unlock()
- *
- * Mark the beginning and end of a read-side critical section.
- * DON'T FORGET TO USE rcu_register_thread/rcu_unregister_thread()
- * FOR EACH THREAD WITH READ-SIDE CRITICAL SECTION.
- */
-#define rcu_read_lock_qsbr _rcu_read_lock
-#define rcu_read_unlock_qsbr _rcu_read_unlock
-#define rcu_read_ongoing_qsbr _rcu_read_ongoing
-
-#define rcu_quiescent_state_qsbr _rcu_quiescent_state
-#define rcu_thread_offline_qsbr _rcu_thread_offline
-#define rcu_thread_online_qsbr _rcu_thread_online
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-/*
- * QSBR read lock/unlock are guaranteed to be no-ops. Therefore, we expose them
- * in the LGPL header for any code to use. However, the debug version is not
- * nops and may contain sanity checks. To activate it, applications must be
- * recompiled with -DDEBUG_RCU (even non-LGPL/GPL applications). This is the
- * best trade-off between license/performance/code triviality and
- * library debugging & tracing features we could come up with.
- */
-
-#if (!defined(BUILD_QSBR_LIB) && !defined(DEBUG_RCU))
-
-static inline void rcu_read_lock(void)
-{
-}
-
-static inline void rcu_read_unlock(void)
-{
-}
-
-#else /* !DEBUG_RCU */
-
-extern void rcu_read_lock(void);
-extern void rcu_read_unlock(void);
-
-#endif /* !DEBUG_RCU */
-
-extern int rcu_read_ongoing(void);
-extern void rcu_quiescent_state(void);
-extern void rcu_thread_offline(void);
-extern void rcu_thread_online(void);
-
-#endif /* !_LGPL_SOURCE */
-
-extern void synchronize_rcu(void);
-
-/*
- * Reader thread registration.
- */
-extern void rcu_register_thread(void);
-extern void rcu_unregister_thread(void);
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu-call-rcu.h>
-#include <urcu-defer.h>
-#include <urcu-flavor.h>
-
-#endif /* _URCU_QSBR_H */
+++ /dev/null
-#ifndef _URCU_WAIT_H
-#define _URCU_WAIT_H
-
-/*
- * urcu-wait.h
- *
- * Userspace RCU library wait/wakeup management
- *
- * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/uatomic.h>
-#include <urcu/wfstack.h>
-#include "urcu-die.h"
-
-/*
- * Number of busy-loop attempts before waiting on futex for grace period
- * batching.
- */
-#define URCU_WAIT_ATTEMPTS 1000
-
-enum urcu_wait_state {
- /* URCU_WAIT_WAITING is compared directly (futex compares it). */
- URCU_WAIT_WAITING = 0,
- /* non-zero are used as masks. */
- URCU_WAIT_WAKEUP = (1 << 0),
- URCU_WAIT_RUNNING = (1 << 1),
- URCU_WAIT_TEARDOWN = (1 << 2),
-};
-
-struct urcu_wait_node {
- struct cds_wfs_node node;
- int32_t state; /* enum urcu_wait_state */
-};
-
-#define URCU_WAIT_NODE_INIT(name, _state) \
- { .state = _state }
-
-#define DEFINE_URCU_WAIT_NODE(name, state) \
- struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)
-
-#define DECLARE_URCU_WAIT_NODE(name) \
- struct urcu_wait_node name
-
-struct urcu_wait_queue {
- struct cds_wfs_stack stack;
-};
-
-#define URCU_WAIT_QUEUE_HEAD_INIT(name) \
- { .stack.head = CDS_WFS_END, .stack.lock = PTHREAD_MUTEX_INITIALIZER }
-
-#define DECLARE_URCU_WAIT_QUEUE(name) \
- struct urcu_wait_queue name
-
-#define DEFINE_URCU_WAIT_QUEUE(name) \
- struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)
-
-struct urcu_waiters {
- struct cds_wfs_head *head;
-};
-
-/*
- * Add ourself atomically to a wait queue. Return 0 if queue was
- * previously empty, else return 1.
- * A full memory barrier is issued before being added to the wait queue.
- */
-static inline
-bool urcu_wait_add(struct urcu_wait_queue *queue,
- struct urcu_wait_node *node)
-{
- return cds_wfs_push(&queue->stack, &node->node);
-}
-
-/*
- * Atomically move all waiters from wait queue into our local struct
- * urcu_waiters.
- */
-static inline
-void urcu_move_waiters(struct urcu_waiters *waiters,
- struct urcu_wait_queue *queue)
-{
- waiters->head = __cds_wfs_pop_all(&queue->stack);
-}
-
-static inline
-void urcu_wait_set_state(struct urcu_wait_node *node,
- enum urcu_wait_state state)
-{
- node->state = state;
-}
-
-static inline
-void urcu_wait_node_init(struct urcu_wait_node *node,
- enum urcu_wait_state state)
-{
- urcu_wait_set_state(node, state);
- cds_wfs_node_init(&node->node);
-}
-
-/*
- * Note: urcu_adaptative_wake_up needs "value" to stay allocated
- * throughout its execution. In this scheme, the waiter owns the node
- * memory, and we only allow it to free this memory when it receives the
- * URCU_WAIT_TEARDOWN flag.
- */
-static inline
-void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
-{
- cmm_smp_mb();
- assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
- uatomic_set(&wait->state, URCU_WAIT_WAKEUP);
- if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
- if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
- NULL, NULL, 0) < 0)
- urcu_die(errno);
- }
- /* Allow teardown of struct urcu_wait memory. */
- uatomic_or(&wait->state, URCU_WAIT_TEARDOWN);
-}
-
-/*
- * Caller must initialize "value" to URCU_WAIT_WAITING before passing its
- * memory to waker thread.
- */
-static inline
-void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
-{
- unsigned int i;
-
- /* Load and test condition before read state */
- cmm_smp_rmb();
- for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
- if (uatomic_read(&wait->state) != URCU_WAIT_WAITING)
- goto skip_futex_wait;
- caa_cpu_relax();
- }
- while (futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- goto skip_futex_wait;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
-skip_futex_wait:
-
- /* Tell waker thread than we are running. */
- uatomic_or(&wait->state, URCU_WAIT_RUNNING);
-
- /*
- * Wait until waker thread lets us know it's ok to tear down
- * memory allocated for struct urcu_wait.
- */
- for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
- if (uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN)
- break;
- caa_cpu_relax();
- }
- while (!(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN))
- poll(NULL, 0, 10);
- assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
-}
-
-static inline
-void urcu_wake_all_waiters(struct urcu_waiters *waiters)
-{
- struct cds_wfs_node *iter, *iter_n;
-
- /* Wake all waiters in our stack head */
- cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
- struct urcu_wait_node *wait_node =
- caa_container_of(iter, struct urcu_wait_node, node);
-
- /* Don't wake already running threads */
- if (wait_node->state & URCU_WAIT_RUNNING)
- continue;
- urcu_adaptative_wake_up(wait_node);
- }
-}
-
-#endif /* _URCU_WAIT_H */
+++ /dev/null
-/*
- * urcu.c
- *
- * Userspace RCU library
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#define _BSD_SOURCE
-#define _GNU_SOURCE
-#define _LGPL_SOURCE
-#define _DEFAULT_SOURCE
-#include <stdio.h>
-#include <pthread.h>
-#include <signal.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <string.h>
-#include <errno.h>
-#include <poll.h>
-
-#include "urcu/arch.h"
-#include "urcu/wfcqueue.h"
-#include "urcu/map/urcu.h"
-#include "urcu/static/urcu.h"
-#include "urcu-pointer.h"
-#include "urcu/tls-compat.h"
-
-#include "urcu-die.h"
-#include "urcu-wait.h"
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#undef _LGPL_SOURCE
-#include "urcu.h"
-#define _LGPL_SOURCE
-
-/*
- * If a reader is really non-cooperative and refuses to commit its
- * rcu_active_readers count to memory (there is no barrier in the reader
- * per-se), kick it after 10 loops waiting for it.
- */
-#define KICK_READER_LOOPS 10
-
-/*
- * Active attempts to check for reader Q.S. before calling futex().
- */
-#define RCU_QS_ACTIVE_ATTEMPTS 100
-
-/* If the headers do not support membarrier system call, fall back on RCU_MB */
-#ifdef __NR_membarrier
-# define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__)
-#else
-# define membarrier(...) -ENOSYS
-#endif
-
-enum membarrier_cmd {
- MEMBARRIER_CMD_QUERY = 0,
- MEMBARRIER_CMD_SHARED = (1 << 0),
-};
-
-#ifdef RCU_MEMBARRIER
-static int init_done;
-int rcu_has_sys_membarrier;
-
-void __attribute__((constructor)) rcu_init(void);
-#endif
-
-#ifdef RCU_MB
-void rcu_init(void)
-{
-}
-#endif
-
-#ifdef RCU_SIGNAL
-static int init_done;
-
-void __attribute__((constructor)) rcu_init(void);
-void __attribute__((destructor)) rcu_exit(void);
-#endif
-
-/*
- * rcu_gp_lock ensures mutual exclusion between threads calling
- * synchronize_rcu().
- */
-static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
-/*
- * rcu_registry_lock ensures mutual exclusion between threads
- * registering and unregistering themselves to/from the registry, and
- * with threads reading that registry from synchronize_rcu(). However,
- * this lock is not held all the way through the completion of awaiting
- * for the grace period. It is sporadically released between iterations
- * on the registry.
- * rcu_registry_lock may nest inside rcu_gp_lock.
- */
-static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER;
-struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
-
-/*
- * Written to only by each individual reader. Read by both the reader and the
- * writers.
- */
-DEFINE_URCU_TLS(struct rcu_reader, rcu_reader);
-
-static CDS_LIST_HEAD(registry);
-
-/*
- * Queue keeping threads awaiting to wait for a grace period. Contains
- * struct gp_waiters_thread objects.
- */
-static DEFINE_URCU_WAIT_QUEUE(gp_waiters);
-
-static void mutex_lock(pthread_mutex_t *mutex)
-{
- int ret;
-
-#ifndef DISTRUST_SIGNALS_EXTREME
- ret = pthread_mutex_lock(mutex);
- if (ret)
- urcu_die(ret);
-#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
- while ((ret = pthread_mutex_trylock(mutex)) != 0) {
- if (ret != EBUSY && ret != EINTR)
- urcu_die(ret);
- if (CMM_LOAD_SHARED(URCU_TLS(rcu_reader).need_mb)) {
- cmm_smp_mb();
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
- cmm_smp_mb();
- }
- (void) poll(NULL, 0, 10);
- }
-#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
-}
-
-static void mutex_unlock(pthread_mutex_t *mutex)
-{
- int ret;
-
- ret = pthread_mutex_unlock(mutex);
- if (ret)
- urcu_die(ret);
-}
-
-#ifdef RCU_MEMBARRIER
-static void smp_mb_master(void)
-{
- if (caa_likely(rcu_has_sys_membarrier))
- (void) membarrier(MEMBARRIER_CMD_SHARED, 0);
- else
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_MB
-static void smp_mb_master(void)
-{
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_SIGNAL
-static void force_mb_all_readers(void)
-{
- struct rcu_reader *index;
-
- /*
- * Ask for each threads to execute a cmm_smp_mb() so we can consider the
- * compiler barriers around rcu read lock as real memory barriers.
- */
- if (cds_list_empty(®istry))
- return;
- /*
- * pthread_kill has a cmm_smp_mb(). But beware, we assume it performs
- * a cache flush on architectures with non-coherent cache. Let's play
- * safe and don't assume anything : we use cmm_smp_mc() to make sure the
- * cache flush is enforced.
- */
- cds_list_for_each_entry(index, ®istry, node) {
- CMM_STORE_SHARED(index->need_mb, 1);
- pthread_kill(index->tid, SIGRCU);
- }
- /*
- * Wait for sighandler (and thus mb()) to execute on every thread.
- *
- * Note that the pthread_kill() will never be executed on systems
- * that correctly deliver signals in a timely manner. However, it
- * is not uncommon for kernels to have bugs that can result in
- * lost or unduly delayed signals.
- *
- * If you are seeing the below pthread_kill() executing much at
- * all, we suggest testing the underlying kernel and filing the
- * relevant bug report. For Linux kernels, we recommend getting
- * the Linux Test Project (LTP).
- */
- cds_list_for_each_entry(index, ®istry, node) {
- while (CMM_LOAD_SHARED(index->need_mb)) {
- pthread_kill(index->tid, SIGRCU);
- (void) poll(NULL, 0, 1);
- }
- }
- cmm_smp_mb(); /* read ->need_mb before ending the barrier */
-}
-
-static void smp_mb_master(void)
-{
- force_mb_all_readers();
-}
-#endif /* #ifdef RCU_SIGNAL */
-
-/*
- * synchronize_rcu() waiting. Single thread.
- * Always called with rcu_registry lock held. Releases this lock and
- * grabs it again. Holds the lock when it returns.
- */
-static void wait_gp(void)
-{
- /*
- * Read reader_gp before read futex. smp_mb_master() needs to
- * be called with the rcu registry lock held in RCU_SIGNAL
- * flavor.
- */
- smp_mb_master();
- /* Temporarily unlock the registry lock. */
- mutex_unlock(&rcu_registry_lock);
- if (uatomic_read(&rcu_gp.futex) != -1)
- goto end;
- while (futex_async(&rcu_gp.futex, FUTEX_WAIT, -1,
- NULL, NULL, 0)) {
- switch (errno) {
- case EWOULDBLOCK:
- /* Value already changed. */
- goto end;
- case EINTR:
- /* Retry if interrupted by signal. */
- break; /* Get out of switch. */
- default:
- /* Unexpected error. */
- urcu_die(errno);
- }
- }
-end:
- /*
- * Re-lock the registry lock before the next loop.
- */
- mutex_lock(&rcu_registry_lock);
-}
-
-/*
- * Always called with rcu_registry lock held. Releases this lock between
- * iterations and grabs it again. Holds the lock when it returns.
- */
-static void wait_for_readers(struct cds_list_head *input_readers,
- struct cds_list_head *cur_snap_readers,
- struct cds_list_head *qsreaders)
-{
- unsigned int wait_loops = 0;
- struct rcu_reader *index, *tmp;
-#ifdef HAS_INCOHERENT_CACHES
- unsigned int wait_gp_loops = 0;
-#endif /* HAS_INCOHERENT_CACHES */
-
- /*
- * Wait for each thread URCU_TLS(rcu_reader).ctr to either
- * indicate quiescence (not nested), or observe the current
- * rcu_gp.ctr value.
- */
- for (;;) {
- if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
- wait_loops++;
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- uatomic_dec(&rcu_gp.futex);
- /* Write futex before read reader_gp */
- smp_mb_master();
- }
-
- cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
- switch (rcu_reader_state(&index->ctr)) {
- case RCU_READER_ACTIVE_CURRENT:
- if (cur_snap_readers) {
- cds_list_move(&index->node,
- cur_snap_readers);
- break;
- }
- /* Fall-through */
- case RCU_READER_INACTIVE:
- cds_list_move(&index->node, qsreaders);
- break;
- case RCU_READER_ACTIVE_OLD:
- /*
- * Old snapshot. Leaving node in
- * input_readers will make us busy-loop
- * until the snapshot becomes current or
- * the reader becomes inactive.
- */
- break;
- }
- }
-
-#ifndef HAS_INCOHERENT_CACHES
- if (cds_list_empty(input_readers)) {
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- /* Read reader_gp before write futex */
- smp_mb_master();
- uatomic_set(&rcu_gp.futex, 0);
- }
- break;
- } else {
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- /* wait_gp unlocks/locks registry lock. */
- wait_gp();
- } else {
- /* Temporarily unlock the registry lock. */
- mutex_unlock(&rcu_registry_lock);
- caa_cpu_relax();
- /*
- * Re-lock the registry lock before the
- * next loop.
- */
- mutex_lock(&rcu_registry_lock);
- }
- }
-#else /* #ifndef HAS_INCOHERENT_CACHES */
- /*
- * BUSY-LOOP. Force the reader thread to commit its
- * URCU_TLS(rcu_reader).ctr update to memory if we wait
- * for too long.
- */
- if (cds_list_empty(input_readers)) {
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- /* Read reader_gp before write futex */
- smp_mb_master();
- uatomic_set(&rcu_gp.futex, 0);
- }
- break;
- } else {
- if (wait_gp_loops == KICK_READER_LOOPS) {
- smp_mb_master();
- wait_gp_loops = 0;
- }
- if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
- /* wait_gp unlocks/locks registry lock. */
- wait_gp();
- wait_gp_loops++;
- } else {
- /* Temporarily unlock the registry lock. */
- mutex_unlock(&rcu_registry_lock);
- caa_cpu_relax();
- /*
- * Re-lock the registry lock before the
- * next loop.
- */
- mutex_lock(&rcu_registry_lock);
- }
- }
-#endif /* #else #ifndef HAS_INCOHERENT_CACHES */
- }
-}
-
-void synchronize_rcu(void)
-{
- CDS_LIST_HEAD(cur_snap_readers);
- CDS_LIST_HEAD(qsreaders);
- DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING);
- struct urcu_waiters waiters;
-
- /*
- * Add ourself to gp_waiters queue of threads awaiting to wait
- * for a grace period. Proceed to perform the grace period only
- * if we are the first thread added into the queue.
- * The implicit memory barrier before urcu_wait_add()
- * orders prior memory accesses of threads put into the wait
- * queue before their insertion into the wait queue.
- */
- if (urcu_wait_add(&gp_waiters, &wait) != 0) {
- /* Not first in queue: will be awakened by another thread. */
- urcu_adaptative_busy_wait(&wait);
- /* Order following memory accesses after grace period. */
- cmm_smp_mb();
- return;
- }
- /* We won't need to wake ourself up */
- urcu_wait_set_state(&wait, URCU_WAIT_RUNNING);
-
- mutex_lock(&rcu_gp_lock);
-
- /*
- * Move all waiters into our local queue.
- */
- urcu_move_waiters(&waiters, &gp_waiters);
-
- mutex_lock(&rcu_registry_lock);
-
- if (cds_list_empty(®istry))
- goto out;
-
- /*
- * All threads should read qparity before accessing data structure
- * where new ptr points to. Must be done within rcu_registry_lock
- * because it iterates on reader threads.
- */
- /* Write new ptr before changing the qparity */
- smp_mb_master();
-
- /*
- * Wait for readers to observe original parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(®istry, &cur_snap_readers, &qsreaders);
-
- /*
- * Must finish waiting for quiescent state for original parity before
- * committing next rcu_gp.ctr update to memory. Failure to do so could
- * result in the writer waiting forever while new readers are always
- * accessing data (no progress). Enforce compiler-order of load
- * URCU_TLS(rcu_reader).ctr before store to rcu_gp.ctr.
- */
- cmm_barrier();
-
- /*
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /* Switch parity: 0 -> 1, 1 -> 0 */
- CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
-
- /*
- * Must commit rcu_gp.ctr update to memory before waiting for quiescent
- * state. Failure to do so could result in the writer waiting forever
- * while new readers are always accessing data (no progress). Enforce
- * compiler-order of store to rcu_gp.ctr before load rcu_reader ctr.
- */
- cmm_barrier();
-
- /*
- *
- * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
- * model easier to understand. It does not have a big performance impact
- * anyway, given this is the write-side.
- */
- cmm_smp_mb();
-
- /*
- * Wait for readers to observe new parity or be quiescent.
- * wait_for_readers() can release and grab again rcu_registry_lock
- * interally.
- */
- wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
-
- /*
- * Put quiescent reader list back into registry.
- */
- cds_list_splice(&qsreaders, ®istry);
-
- /*
- * Finish waiting for reader threads before letting the old ptr
- * being freed. Must be done within rcu_registry_lock because it
- * iterates on reader threads.
- */
- smp_mb_master();
-out:
- mutex_unlock(&rcu_registry_lock);
- mutex_unlock(&rcu_gp_lock);
-
- /*
- * Wakeup waiters only after we have completed the grace period
- * and have ensured the memory barriers at the end of the grace
- * period have been issued.
- */
- urcu_wake_all_waiters(&waiters);
-}
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void rcu_read_lock(void)
-{
- _rcu_read_lock();
-}
-
-void rcu_read_unlock(void)
-{
- _rcu_read_unlock();
-}
-
-int rcu_read_ongoing(void)
-{
- return _rcu_read_ongoing();
-}
-
-void rcu_register_thread(void)
-{
- URCU_TLS(rcu_reader).tid = pthread_self();
- assert(URCU_TLS(rcu_reader).need_mb == 0);
- assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK));
-
- mutex_lock(&rcu_registry_lock);
- assert(!URCU_TLS(rcu_reader).registered);
- URCU_TLS(rcu_reader).registered = 1;
- rcu_init(); /* In case gcc does not support constructor attribute */
- cds_list_add(&URCU_TLS(rcu_reader).node, ®istry);
- mutex_unlock(&rcu_registry_lock);
-}
-
-void rcu_unregister_thread(void)
-{
- mutex_lock(&rcu_registry_lock);
- assert(URCU_TLS(rcu_reader).registered);
- URCU_TLS(rcu_reader).registered = 0;
- cds_list_del(&URCU_TLS(rcu_reader).node);
- mutex_unlock(&rcu_registry_lock);
-}
-
-#ifdef RCU_MEMBARRIER
-void rcu_init(void)
-{
- int ret;
-
- if (init_done)
- return;
- init_done = 1;
- ret = membarrier(MEMBARRIER_CMD_QUERY, 0);
- if (ret >= 0 && (ret & MEMBARRIER_CMD_SHARED)) {
- rcu_has_sys_membarrier = 1;
- }
-}
-#endif
-
-#ifdef RCU_SIGNAL
-static void sigrcu_handler(int signo, siginfo_t *siginfo, void *context)
-{
- /*
- * Executing this cmm_smp_mb() is the only purpose of this signal handler.
- * It punctually promotes cmm_barrier() into cmm_smp_mb() on every thread it is
- * executed on.
- */
- cmm_smp_mb();
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).need_mb, 0);
- cmm_smp_mb();
-}
-
-/*
- * rcu_init constructor. Called when the library is linked, but also when
- * reader threads are calling rcu_register_thread().
- * Should only be called by a single thread at a given time. This is ensured by
- * holing the rcu_registry_lock from rcu_register_thread() or by running
- * at library load time, which should not be executed by multiple
- * threads nor concurrently with rcu_register_thread() anyway.
- */
-void rcu_init(void)
-{
- struct sigaction act;
- int ret;
-
- if (init_done)
- return;
- init_done = 1;
-
- act.sa_sigaction = sigrcu_handler;
- act.sa_flags = SA_SIGINFO | SA_RESTART;
- sigemptyset(&act.sa_mask);
- ret = sigaction(SIGRCU, &act, NULL);
- if (ret)
- urcu_die(errno);
-}
-
-void rcu_exit(void)
-{
- /*
- * Don't unregister the SIGRCU signal handler anymore, because
- * call_rcu threads could still be using it shortly before the
- * application exits.
- * Assertion disabled because call_rcu threads are now rcu
- * readers, and left running at exit.
- * assert(cds_list_empty(®istry));
- */
-}
-
-#endif /* #ifdef RCU_SIGNAL */
-
-DEFINE_RCU_FLAVOR(rcu_flavor);
-
-#include "urcu-call-rcu-impl.h"
-#include "urcu-defer-impl.h"
+++ /dev/null
-#ifndef _URCU_H
-#define _URCU_H
-
-/*
- * urcu.h
- *
- * Userspace RCU header
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-
-/*
- * See urcu-pointer.h and urcu/static/urcu-pointer.h for pointer
- * publication headers.
- */
-#include <urcu-pointer.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <urcu/map/urcu.h>
-
-/*
- * Important !
- *
- * Each thread containing read-side critical sections must be registered
- * with rcu_register_thread_mb() before calling rcu_read_lock_mb().
- * rcu_unregister_thread_mb() should be called before the thread exits.
- */
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/urcu.h>
-
-/*
- * Mappings for static use of the userspace RCU library.
- * Should only be used in LGPL-compatible code.
- */
-
-/*
- * rcu_read_lock()
- * rcu_read_unlock()
- *
- * Mark the beginning and end of a read-side critical section.
- * DON'T FORGET TO USE RCU_REGISTER/UNREGISTER_THREAD() FOR EACH THREAD WITH
- * READ-SIDE CRITICAL SECTION.
- */
-#ifdef RCU_MEMBARRIER
-#define rcu_read_lock_memb _rcu_read_lock
-#define rcu_read_unlock_memb _rcu_read_unlock
-#define rcu_read_ongoing_memb _rcu_read_ongoing
-#elif defined(RCU_SIGNAL)
-#define rcu_read_lock_sig _rcu_read_lock
-#define rcu_read_unlock_sig _rcu_read_unlock
-#define rcu_read_ongoing_sig _rcu_read_ongoing
-#elif defined(RCU_MB)
-#define rcu_read_lock_mb _rcu_read_lock
-#define rcu_read_unlock_mb _rcu_read_unlock
-#define rcu_read_ongoing_mb _rcu_read_ongoing
-#endif
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- * See LGPL-only urcu/static/urcu-pointer.h for documentation.
- */
-
-extern void rcu_read_lock(void);
-extern void rcu_read_unlock(void);
-extern int rcu_read_ongoing(void);
-
-#endif /* !_LGPL_SOURCE */
-
-extern void synchronize_rcu(void);
-
-/*
- * Reader thread registration.
- */
-extern void rcu_register_thread(void);
-extern void rcu_unregister_thread(void);
-
-/*
- * Explicit rcu initialization, for "early" use within library constructors.
- */
-extern void rcu_init(void);
-
-/*
- * Q.S. reporting are no-ops for these URCU flavors.
- */
-static inline void rcu_quiescent_state(void)
-{
-}
-
-static inline void rcu_thread_offline(void)
-{
-}
-
-static inline void rcu_thread_online(void)
-{
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu-call-rcu.h>
-#include <urcu-defer.h>
-#include <urcu-flavor.h>
-
-#endif /* _URCU_H */
+++ /dev/null
-#ifndef _URCU_ARCH_AARCH64_H
-#define _URCU_ARCH_AARCH64_H
-
-/*
- * arch/aarch64.h: definitions for aarch64 architecture
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers. aarch64 implements asm-generic/unistd.h system call
- * numbers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 283
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_AARCH64_H */
+++ /dev/null
-#ifndef _URCU_ARCH_ALPHA_H
-#define _URCU_ARCH_ALPHA_H
-
-/*
- * arch_alpha.h: trivial definitions for the Alpha architecture.
- *
- * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define cmm_mb() __asm__ __volatile__ ("mb":::"memory")
-#define cmm_wmb() __asm__ __volatile__ ("wmb":::"memory")
-#define cmm_read_barrier_depends() __asm__ __volatile__ ("mb":::"memory")
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_ALPHA_H */
+++ /dev/null
-#ifndef _URCU_ARCH_ARM_H
-#define _URCU_ARCH_ARM_H
-
-/*
- * arch_arm.h: trivial definitions for the ARM architecture.
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef CONFIG_RCU_ARM_HAVE_DMB
-#define cmm_mb() __asm__ __volatile__ ("dmb":::"memory")
-#define cmm_rmb() __asm__ __volatile__ ("dmb":::"memory")
-#define cmm_wmb() __asm__ __volatile__ ("dmb":::"memory")
-#endif /* CONFIG_RCU_ARM_HAVE_DMB */
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 389
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_ARM_H */
+++ /dev/null
-#ifndef _URCU_ARCH_GCC_H
-#define _URCU_ARCH_GCC_H
-
-/*
- * arch_gcc.h: trivial definitions for architectures using gcc __sync_
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_GCC_H */
+++ /dev/null
-#ifndef _URCU_ARCH_GENERIC_H
-#define _URCU_ARCH_GENERIC_H
-
-/*
- * arch_generic.h: common definitions for multiple architectures.
- *
- * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef CAA_CACHE_LINE_SIZE
-#define CAA_CACHE_LINE_SIZE 64
-#endif
-
-#if !defined(cmm_mc) && !defined(cmm_rmc) && !defined(cmm_wmc)
-#define CONFIG_HAVE_MEM_COHERENCY
-/*
- * Architectures with cache coherency must _not_ define cmm_mc/cmm_rmc/cmm_wmc.
- *
- * For them, cmm_mc/cmm_rmc/cmm_wmc are implemented with a simple
- * compiler barrier; in addition, we provide defaults for cmm_mb (using
- * GCC builtins) as well as cmm_rmb and cmm_wmb (defaulting to cmm_mb).
- */
-
-#ifndef cmm_mb
-#define cmm_mb() __sync_synchronize()
-#endif
-
-#ifndef cmm_rmb
-#define cmm_rmb() cmm_mb()
-#endif
-
-#ifndef cmm_wmb
-#define cmm_wmb() cmm_mb()
-#endif
-
-#define cmm_mc() cmm_barrier()
-#define cmm_rmc() cmm_barrier()
-#define cmm_wmc() cmm_barrier()
-#else
-/*
- * Architectures without cache coherency need something like the following:
- *
- * #define cmm_mc() arch_cache_flush()
- * #define cmm_rmc() arch_cache_flush_read()
- * #define cmm_wmc() arch_cache_flush_write()
- *
- * Of these, only cmm_mc is mandatory. cmm_rmc and cmm_wmc default to
- * cmm_mc. cmm_mb/cmm_rmb/cmm_wmb use these definitions by default:
- *
- * #define cmm_mb() cmm_mc()
- * #define cmm_rmb() cmm_rmc()
- * #define cmm_wmb() cmm_wmc()
- */
-
-#ifndef cmm_mb
-#define cmm_mb() cmm_mc()
-#endif
-
-#ifndef cmm_rmb
-#define cmm_rmb() cmm_rmc()
-#endif
-
-#ifndef cmm_wmb
-#define cmm_wmb() cmm_wmc()
-#endif
-
-#ifndef cmm_rmc
-#define cmm_rmc() cmm_mc()
-#endif
-
-#ifndef cmm_wmc
-#define cmm_wmc() cmm_mc()
-#endif
-#endif
-
-/* Nop everywhere except on alpha. */
-#ifndef cmm_read_barrier_depends
-#define cmm_read_barrier_depends()
-#endif
-
-#ifdef CONFIG_RCU_SMP
-#ifndef cmm_smp_mb
-#define cmm_smp_mb() cmm_mb()
-#endif
-#ifndef cmm_smp_rmb
-#define cmm_smp_rmb() cmm_rmb()
-#endif
-#ifndef cmm_smp_wmb
-#define cmm_smp_wmb() cmm_wmb()
-#endif
-#ifndef cmm_smp_mc
-#define cmm_smp_mc() cmm_mc()
-#endif
-#ifndef cmm_smp_rmc
-#define cmm_smp_rmc() cmm_rmc()
-#endif
-#ifndef cmm_smp_wmc
-#define cmm_smp_wmc() cmm_wmc()
-#endif
-#ifndef cmm_smp_read_barrier_depends
-#define cmm_smp_read_barrier_depends() cmm_read_barrier_depends()
-#endif
-#else
-#ifndef cmm_smp_mb
-#define cmm_smp_mb() cmm_barrier()
-#endif
-#ifndef cmm_smp_rmb
-#define cmm_smp_rmb() cmm_barrier()
-#endif
-#ifndef cmm_smp_wmb
-#define cmm_smp_wmb() cmm_barrier()
-#endif
-#ifndef cmm_smp_mc
-#define cmm_smp_mc() cmm_barrier()
-#endif
-#ifndef cmm_smp_rmc
-#define cmm_smp_rmc() cmm_barrier()
-#endif
-#ifndef cmm_smp_wmc
-#define cmm_smp_wmc() cmm_barrier()
-#endif
-#ifndef cmm_smp_read_barrier_depends
-#define cmm_smp_read_barrier_depends()
-#endif
-#endif
-
-#ifndef caa_cpu_relax
-#define caa_cpu_relax() cmm_barrier()
-#endif
-
-#ifndef HAS_CAA_GET_CYCLES
-#define HAS_CAA_GET_CYCLES
-
-#ifdef CONFIG_RCU_HAVE_CLOCK_GETTIME
-
-#include <time.h>
-#include <stdint.h>
-
-typedef uint64_t caa_cycles_t;
-
-static inline caa_cycles_t caa_get_cycles (void)
-{
- struct timespec ts;
-
- if (caa_unlikely(clock_gettime(CLOCK_MONOTONIC, &ts)))
- return -1ULL;
- return ((uint64_t) ts.tv_sec * 1000000000ULL) + ts.tv_nsec;
-}
-
-#elif defined(__APPLE__)
-
-#include <mach/mach.h>
-#include <mach/clock.h>
-#include <mach/mach_time.h>
-#include <time.h>
-#include <stdint.h>
-
-typedef uint64_t caa_cycles_t;
-
-static inline caa_cycles_t caa_get_cycles (void)
-{
- mach_timespec_t ts = { 0, 0 };
- static clock_serv_t clock_service;
-
- if (caa_unlikely(!clock_service)) {
- if (host_get_clock_service(mach_host_self(),
- SYSTEM_CLOCK, &clock_service))
- return -1ULL;
- }
- if (caa_unlikely(clock_get_time(clock_service, &ts)))
- return -1ULL;
- return ((uint64_t) ts.tv_sec * 1000000000ULL) + ts.tv_nsec;
-}
-
-#else
-
-#error caa_get_cycles() not implemented for this platform.
-
-#endif
-
-#endif /* HAS_CAA_GET_CYCLES */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_ARCH_GENERIC_H */
+++ /dev/null
-#ifndef _URCU_ARCH_HPPA_H
-#define _URCU_ARCH_HPPA_H
-
-/*
- * arch/hppa.h: definitions for hppa architecture
- *
- * Copyright (c) 2014 Helge Deller <deller@gmx.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 343
-#endif
-
-#define HAS_CAA_GET_CYCLES
-typedef unsigned long caa_cycles_t;
-
-static inline caa_cycles_t caa_get_cycles(void)
-{
- caa_cycles_t cycles;
-
- asm volatile("mfctl 16, %0" : "=r" (cycles));
- return cycles;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_HPPA_H */
+++ /dev/null
-#ifndef _URCU_ARCH_IA64_H
-#define _URCU_ARCH_IA64_H
-
-/*
- * arch/ia64.h: definitions for ia64 architecture
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 1344
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_IA64_H */
+++ /dev/null
-#ifndef _URCU_ARCH_MIPS_H
-#define _URCU_ARCH_MIPS_H
-
-/*
- * arch_mips.h: trivial definitions for the MIPS architecture.
- *
- * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
- * Copyright (c) 2012 Ralf Baechle <ralf@linux-mips.org>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define cmm_mb() __asm__ __volatile__ ( \
- " .set mips2 \n" \
- " sync \n" \
- " .set mips0 \n" \
- :::"memory")
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_MIPS_H */
+++ /dev/null
-#ifndef _URCU_ARCH_NIOS2_H
-#define _URCU_ARCH_NIOS2_H
-
-/*
- * arch_nios2.h: trivial definitions for the NIOS2 architecture.
- *
- * Copyright (c) 2016 Marek Vasut <marex@denx.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define cmm_mb() cmm_barrier()
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_NIOS2_H */
+++ /dev/null
-#ifndef _URCU_ARCH_PPC_H
-#define _URCU_ARCH_PPC_H
-
-/*
- * arch_ppc.h: trivial definitions for the powerpc architecture.
- *
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Include size of POWER5+ L3 cache lines: 256 bytes */
-#define CAA_CACHE_LINE_SIZE 256
-
-#ifdef __NO_LWSYNC__
-#define LWSYNC_OPCODE "sync\n"
-#else
-#define LWSYNC_OPCODE "lwsync\n"
-#endif
-
-/*
- * Use sync for all cmm_mb/rmb/wmb barriers because lwsync does not
- * preserve ordering of cacheable vs. non-cacheable accesses, so it
- * should not be used to order with respect to MMIO operations. An
- * eieio+lwsync pair is also not enough for cmm_rmb, because it will
- * order cacheable and non-cacheable memory operations separately---i.e.
- * not the latter against the former.
- */
-#define cmm_mb() __asm__ __volatile__ ("sync":::"memory")
-
-/*
- * lwsync orders loads in cacheable memory with respect to other loads,
- * and stores in cacheable memory with respect to other stores.
- * Therefore, use it for barriers ordering accesses to cacheable memory
- * only.
- */
-#define cmm_smp_rmb() __asm__ __volatile__ (LWSYNC_OPCODE:::"memory")
-#define cmm_smp_wmb() __asm__ __volatile__ (LWSYNC_OPCODE:::"memory")
-
-#define mftbl() \
- __extension__ \
- ({ \
- unsigned long rval; \
- __asm__ __volatile__ ("mftbl %0" : "=r" (rval)); \
- rval; \
- })
-
-#define mftbu() \
- __extension__ \
- ({ \
- unsigned long rval; \
- __asm__ __volatile__ ("mftbu %0" : "=r" (rval)); \
- rval; \
- })
-
-#define mftb() \
- __extension__ \
- ({ \
- unsigned long long rval; \
- __asm__ __volatile__ ("mftb %0" : "=r" (rval)); \
- rval; \
- })
-
-#define HAS_CAA_GET_CYCLES
-
-typedef uint64_t caa_cycles_t;
-
-#ifdef __powerpc64__
-static inline caa_cycles_t caa_get_cycles(void)
-{
- return (caa_cycles_t) mftb();
-}
-#else
-static inline caa_cycles_t caa_get_cycles(void)
-{
- unsigned long h, l;
-
- for (;;) {
- h = mftbu();
- cmm_barrier();
- l = mftbl();
- cmm_barrier();
- if (mftbu() == h)
- return (((caa_cycles_t) h) << 32) + l;
- }
-}
-#endif
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 365
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_PPC_H */
+++ /dev/null
-#ifndef _URCU_ARCH_S390_H
-#define _URCU_ARCH_S390_H
-
-/*
- * Trivial definitions for the S390 architecture based on information from the
- * Principles of Operation "CPU Serialization" (5-91), "BRANCH ON CONDITION"
- * (7-25) and "STORE CLOCK" (7-169).
- *
- * Copyright (c) 2009 Novell, Inc.
- * Author: Jan Blunck <jblunck@suse.de>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define CAA_CACHE_LINE_SIZE 128
-
-#define cmm_mb() __asm__ __volatile__("bcr 15,0" : : : "memory")
-
-#define HAS_CAA_GET_CYCLES
-
-typedef uint64_t caa_cycles_t;
-
-static inline caa_cycles_t caa_get_cycles (void)
-{
- caa_cycles_t cycles;
-
- __asm__ __volatile__("stck %0" : "=m" (cycles) : : "cc", "memory" );
-
- return cycles;
-}
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 356
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_S390_H */
+++ /dev/null
-#ifndef _URCU_ARCH_SPARC64_H
-#define _URCU_ARCH_SPARC64_H
-
-/*
- * arch_sparc64.h: trivial definitions for the Sparc64 architecture.
- *
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 351
-#endif
-
-#define CAA_CACHE_LINE_SIZE 256
-
-/*
- * Inspired from the Linux kernel. Workaround Spitfire bug #51.
- */
-#define membar_safe(type) \
-__asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \
- "membar " type "\n" \
- "1:\n" \
- : : : "memory")
-
-#define cmm_mb() membar_safe("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
-#define cmm_rmb() membar_safe("#LoadLoad")
-#define cmm_wmb() membar_safe("#StoreStore")
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_SPARC64_H */
+++ /dev/null
-#ifndef _URCU_ARCH_TILE_H
-#define _URCU_ARCH_TILE_H
-
-/*
- * arch/tile.h: definitions for tile architecture
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009-2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <sys/time.h>
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers. tile implements asm-generic/unistd.h system call
- * numbers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#define __NR_membarrier 283
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_TILE_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UNKNOWN_H
-#define _URCU_ARCH_UNKNOWN_H
-
-/*
- * arch_unknown.h: #error to prevent build on unknown architectures.
- *
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* See configure.ac for the list of recognized architectures. */
-#error "Cannot build: unrecognized architecture detected."
-
-#endif /* _URCU_ARCH_UNKNOWN_H */
+++ /dev/null
-#ifndef _URCU_ARCH_X86_H
-#define _URCU_ARCH_X86_H
-
-/*
- * arch_x86.h: trivial definitions for the x86 architecture.
- *
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-#include <urcu/config.h>
-#include <urcu/syscall-compat.h>
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define CAA_CACHE_LINE_SIZE 128
-
-#ifdef CONFIG_RCU_HAVE_FENCE
-#define cmm_mb() __asm__ __volatile__ ("mfence":::"memory")
-
-/*
- * Define cmm_rmb/cmm_wmb to "strict" barriers that may be needed when
- * using SSE or working with I/O areas. cmm_smp_rmb/cmm_smp_wmb are
- * only compiler barriers, which is enough for general use.
- */
-#define cmm_rmb() __asm__ __volatile__ ("lfence":::"memory")
-#define cmm_wmb() __asm__ __volatile__ ("sfence"::: "memory")
-#define cmm_smp_rmb() cmm_barrier()
-#define cmm_smp_wmb() cmm_barrier()
-#else
-/*
- * We leave smp_rmb/smp_wmb as full barriers for processors that do not have
- * fence instructions.
- *
- * An empty cmm_smp_rmb() may not be enough on old PentiumPro multiprocessor
- * systems, due to an erratum. The Linux kernel says that "Even distro
- * kernels should think twice before enabling this", but for now let's
- * be conservative and leave the full barrier on 32-bit processors. Also,
- * IDT WinChip supports weak store ordering, and the kernel may enable it
- * under our feet; cmm_smp_wmb() ceases to be a nop for these processors.
- */
-#if (CAA_BITS_PER_LONG == 32)
-#define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
-#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
-#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory")
-#else
-#define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
-#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
-#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory")
-#endif
-#endif
-
-#define caa_cpu_relax() __asm__ __volatile__ ("rep; nop" : : : "memory")
-
-#define HAS_CAA_GET_CYCLES
-
-#define rdtscll(val) \
- do { \
- unsigned int __a, __d; \
- __asm__ __volatile__ ("rdtsc" : "=a" (__a), "=d" (__d)); \
- (val) = ((unsigned long long)__a) \
- | (((unsigned long long)__d) << 32); \
- } while(0)
-
-typedef uint64_t caa_cycles_t;
-
-static inline caa_cycles_t caa_get_cycles(void)
-{
- caa_cycles_t ret = 0;
-
- rdtscll(ret);
- return ret;
-}
-
-/*
- * On Linux, define the membarrier system call number if not yet available in
- * the system headers.
- */
-#if (defined(__linux__) && !defined(__NR_membarrier))
-#if (CAA_BITS_PER_LONG == 32)
-#define __NR_membarrier 375
-#else
-#define __NR_membarrier 324
-#endif
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/arch/generic.h>
-
-#endif /* _URCU_ARCH_X86_H */
+++ /dev/null
-#ifndef _URCU_CDS_H
-#define _URCU_CDS_H
-
-/*
- * urcu/cds.h
- *
- * Userspace RCU library - Concurrent Data Structures
- *
- * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/hlist.h>
-#include <urcu/list.h>
-#include <urcu/rcuhlist.h>
-#include <urcu/rculist.h>
-#include <urcu/rculfqueue.h>
-#include <urcu/rculfstack.h>
-#include <urcu/rculfhash.h>
-#include <urcu/wfqueue.h>
-#include <urcu/wfcqueue.h>
-#include <urcu/wfstack.h>
-#include <urcu/lfstack.h>
-
-#endif /* _URCU_CDS_H */
+++ /dev/null
-#ifndef _URCU_COMPILER_H
-#define _URCU_COMPILER_H
-
-/*
- * compiler.h
- *
- * Compiler definitions.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- */
-
-#include <stddef.h> /* for offsetof */
-
-#define caa_likely(x) __builtin_expect(!!(x), 1)
-#define caa_unlikely(x) __builtin_expect(!!(x), 0)
-
-#define cmm_barrier() __asm__ __volatile__ ("" : : : "memory")
-
-/*
- * Instruct the compiler to perform only a single access to a variable
- * (prohibits merging and refetching). The compiler is also forbidden to reorder
- * successive instances of CMM_ACCESS_ONCE(), but only when the compiler is aware of
- * particular ordering. Compiler ordering can be ensured, for example, by
- * putting two CMM_ACCESS_ONCE() in separate C statements.
- *
- * This macro does absolutely -nothing- to prevent the CPU from reordering,
- * merging, or refetching absolutely anything at any time. Its main intended
- * use is to mediate communication between process-level code and irq/NMI
- * handlers, all running on the same CPU.
- */
-#define CMM_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
-
-#ifndef caa_max
-#define caa_max(a,b) ((a)>(b)?(a):(b))
-#endif
-
-#ifndef caa_min
-#define caa_min(a,b) ((a)<(b)?(a):(b))
-#endif
-
-#if defined(__SIZEOF_LONG__)
-#define CAA_BITS_PER_LONG (__SIZEOF_LONG__ * 8)
-#elif defined(_LP64)
-#define CAA_BITS_PER_LONG 64
-#else
-#define CAA_BITS_PER_LONG 32
-#endif
-
-/*
- * caa_container_of - Get the address of an object containing a field.
- *
- * @ptr: pointer to the field.
- * @type: type of the object.
- * @member: name of the field within the object.
- */
-#define caa_container_of(ptr, type, member) \
- __extension__ \
- ({ \
- const __typeof__(((type *) NULL)->member) * __ptr = (ptr); \
- (type *)((char *)__ptr - offsetof(type, member)); \
- })
-
-#define CAA_BUILD_BUG_ON_ZERO(cond) (sizeof(struct { int:-!!(cond); }))
-#define CAA_BUILD_BUG_ON(cond) ((void)CAA_BUILD_BUG_ON_ZERO(cond))
-
-/*
- * __rcu is an annotation that documents RCU pointer accesses that need
- * to be protected by a read-side critical section. Eventually, a static
- * checker will be able to use this annotation to detect incorrect RCU
- * usage.
- */
-#define __rcu
-
-#ifdef __cplusplus
-#define URCU_FORCE_CAST(type, arg) (reinterpret_cast<type>(arg))
-#else
-#define URCU_FORCE_CAST(type, arg) ((type) (arg))
-#endif
-
-#define caa_is_signed_type(type) ((type) -1 < (type) 0)
-
-/*
- * Cast to unsigned long, sign-extending if @v is signed.
- * Note: casting to a larger type or to same type size keeps the sign of
- * the expression being cast (see C99 6.3.1.3).
- */
-#define caa_cast_long_keep_sign(v) ((unsigned long) (v))
-
-#if defined (__GNUC__) \
- && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 5) \
- || __GNUC__ >= 5)
-#define CDS_DEPRECATED(msg) \
- __attribute__((deprecated(msg)))
-#else
-#define CDS_DEPRECATED(msg) \
- __attribute__((deprecated))
-#endif
-
-#define CAA_ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
-
-/*
- * Don't allow compiling with buggy compiler.
- */
-
-#ifdef __GNUC__
-# define URCU_GCC_VERSION (__GNUC__ * 10000 \
- + __GNUC_MINOR__ * 100 \
- + __GNUC_PATCHLEVEL__)
-
-/*
- * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58854
- */
-# ifdef __ARMEL__
-# if URCU_GCC_VERSION >= 40800 && URCU_GCC_VERSION <= 40802
-# error Your gcc version produces clobbered frame accesses
-# endif
-# endif
-#endif
-
-#endif /* _URCU_COMPILER_H */
+++ /dev/null
-/* urcu/config.h.in. Manually generated for control over the contained defs. */
-
-/* Defined when on a system that has memory fence instructions. */
-#undef CONFIG_RCU_HAVE_FENCE
-
-/* Defined when on a system with futex support. */
-#undef CONFIG_RCU_HAVE_FUTEX
-
-/* Enable SMP support. With SMP support enabled, uniprocessors are also
- supported. With SMP support disabled, UP systems work fine, but the
- behavior of SMP systems is undefined. */
-#undef CONFIG_RCU_SMP
-
-/* Compatibility mode for i386 which lacks cmpxchg instruction. */
-#undef CONFIG_RCU_COMPAT_ARCH
-
-/* Use the dmb instruction is available for use on ARM. */
-#undef CONFIG_RCU_ARM_HAVE_DMB
-
-/* TLS provided by the compiler. */
-#undef CONFIG_RCU_TLS
-
-/* clock_gettime() is detected. */
-#undef CONFIG_RCU_HAVE_CLOCK_GETTIME
+++ /dev/null
-#ifndef _URCU_DEBUG_H
-#define _URCU_DEBUG_H
-
-/*
- * urcu/debug.h
- *
- * Userspace RCU debugging facilities.
- *
- * Copyright (c) 2015 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- */
-
-#include <assert.h>
-
-#ifdef DEBUG_RCU
-#define urcu_assert(...) assert(__VA_ARGS__)
-#else
-#define urcu_assert(...)
-#endif
-
-#endif /* _URCU_DEBUG_H */
+++ /dev/null
-#ifndef _URCU_FUTEX_H
-#define _URCU_FUTEX_H
-
-/*
- * urcu-futex.h
- *
- * Userspace RCU - sys_futex/compat_futex header.
- *
- * Copyright 2011-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/config.h>
-#include <stdint.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-
-/*
- * sys_futex compatibility header.
- * Use *only* *either of* futex_noasync OR futex_async on a given address.
- *
- * futex_noasync cannot be executed in signal handlers, but ensures that
- * it will be put in a wait queue even in compatibility mode.
- *
- * futex_async is signal-handler safe for the wakeup. It uses polling
- * on the wait-side in compatibility mode.
- *
- * BEWARE: sys_futex() FUTEX_WAIT may return early if interrupted
- * (returns EINTR).
- */
-
-extern int compat_futex_noasync(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3);
-extern int compat_futex_async(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3);
-
-#ifdef CONFIG_RCU_HAVE_FUTEX
-
-#include <unistd.h>
-#include <errno.h>
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-
-static inline int futex(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- return syscall(__NR_futex, uaddr, op, val, timeout,
- uaddr2, val3);
-}
-
-static inline int futex_noasync(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- int ret;
-
- ret = futex(uaddr, op, val, timeout, uaddr2, val3);
- if (caa_unlikely(ret < 0 && errno == ENOSYS)) {
- /*
- * The fallback on ENOSYS is the async-safe version of
- * the compat futex implementation, because the
- * async-safe compat implementation allows being used
- * concurrently with calls to futex(). Indeed, sys_futex
- * FUTEX_WAIT, on some architectures (mips and parisc),
- * within a given process, spuriously return ENOSYS due
- * to signal restart bugs on some kernel versions.
- */
- return compat_futex_async(uaddr, op, val, timeout,
- uaddr2, val3);
- }
- return ret;
-
-}
-
-static inline int futex_async(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- int ret;
-
- ret = futex(uaddr, op, val, timeout, uaddr2, val3);
- if (caa_unlikely(ret < 0 && errno == ENOSYS)) {
- return compat_futex_async(uaddr, op, val, timeout,
- uaddr2, val3);
- }
- return ret;
-}
-
-#else
-
-static inline int futex_noasync(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- return compat_futex_noasync(uaddr, op, val, timeout, uaddr2, val3);
-}
-
-static inline int futex_async(int32_t *uaddr, int op, int32_t val,
- const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
-{
- return compat_futex_async(uaddr, op, val, timeout, uaddr2, val3);
-}
-
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_FUTEX_H */
+++ /dev/null
-#ifndef _KCOMPAT_HLIST_H
-#define _KCOMPAT_HLIST_H
-
-/*
- * Kernel sourcecode compatible lightweight single pointer list head useful
- * for implementing hash tables
- *
- * Copyright (C) 2009 Novell Inc.
- *
- * Author: Jan Blunck <jblunck@suse.de>
- *
- * Copyright (C) 2010-2013 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License version 2.1 as
- * published by the Free Software Foundation.
- */
-
-#include <stddef.h>
-
-struct cds_hlist_head {
- struct cds_hlist_node *next;
-};
-
-struct cds_hlist_node {
- struct cds_hlist_node *next, *prev;
-};
-
-/* Initialize a new list head. */
-static inline
-void CDS_INIT_HLIST_HEAD(struct cds_hlist_head *ptr)
-{
- ptr->next = NULL;
-}
-
-#define CDS_HLIST_HEAD(name) \
- struct cds_hlist_head name = { NULL }
-
-#define CDS_HLIST_HEAD_INIT(name) \
- { .next = NULL }
-
-/* Get typed element from list at a given position. */
-#define cds_hlist_entry(ptr, type, member) \
- ((type *) ((char *) (ptr) - (unsigned long) (&((type *) 0)->member)))
-
-/* Add new element at the head of the list. */
-static inline
-void cds_hlist_add_head(struct cds_hlist_node *newp,
- struct cds_hlist_head *head)
-{
- if (head->next)
- head->next->prev = newp;
- newp->next = head->next;
- newp->prev = (struct cds_hlist_node *) head;
- head->next = newp;
-}
-
-/* Remove element from list. */
-static inline
-void cds_hlist_del(struct cds_hlist_node *elem)
-{
- if (elem->next)
- elem->next->prev = elem->prev;
- elem->prev->next = elem->next;
-}
-
-#define cds_hlist_for_each(pos, head) \
- for (pos = (head)->next; pos != NULL; pos = pos->next)
-
-#define cds_hlist_for_each_safe(pos, p, head) \
- for (pos = (head)->next; \
- (pos != NULL) && (p = pos->next, 1); \
- pos = p)
-
-/*
- * cds_hlist_for_each_entry and cds_hlist_for_each_entry_safe take
- * respectively 4 and 5 arguments, while the Linux kernel APIs take 3,
- * and 4. We implement cds_hlist_for_each_entry_2() and
- * cds_hlist_for_each_entry_safe_2() to follow the Linux kernel APIs.
- */
-#define cds_hlist_for_each_entry(entry, pos, head, member) \
- for (pos = (head)->next, \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
- pos != NULL; \
- pos = pos->next, \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member))
-
-#define cds_hlist_for_each_entry_safe(entry, pos, p, head, member) \
- for (pos = (head)->next, \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
- (pos != NULL) && (p = pos->next, 1); \
- pos = p, \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member))
-
-#define cds_hlist_for_each_entry_2(entry, head, member) \
- for (entry = ((head)->next == NULL ? NULL \
- : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \
- entry != NULL; \
- entry = (entry->member.next == NULL ? NULL \
- : cds_hlist_entry(entry->member.next, __typeof__(*entry), member)))
-
-#define cds_hlist_for_each_entry_safe_2(entry, e, head, member) \
- for (entry = ((head)->next == NULL ? NULL \
- : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \
- (entry != NULL) && (e = (entry->member.next == NULL ? NULL \
- : cds_hlist_entry(entry->member.next, \
- __typeof__(*entry), member)), 1); \
- entry = e)
-
-#endif /* _KCOMPAT_HLIST_H */
+++ /dev/null
-#ifndef _URCU_LFSTACK_H
-#define _URCU_LFSTACK_H
-
-/*
- * lfstack.h
- *
- * Userspace RCU library - Lock-Free Stack
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdbool.h>
-#include <pthread.h>
-
-/*
- * Lock-free stack.
- *
- * Stack implementing push, pop, pop_all operations, as well as iterator
- * on the stack head returned by pop_all.
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * cds_lfs_push __cds_lfs_pop __cds_lfs_pop_all
- * cds_lfs_push - - -
- * __cds_lfs_pop - X X
- * __cds_lfs_pop_all - X -
- *
- * cds_lfs_pop_blocking and cds_lfs_pop_all_blocking use an internal
- * mutex to provide synchronization.
- */
-
-/*
- * struct cds_lfs_node is returned by cds_lfs_pop, and also used as
- * iterator on stack. It is not safe to dereference the node next
- * pointer when returned by cds_lfs_pop.
- */
-struct cds_lfs_node {
- struct cds_lfs_node *next;
-};
-
-/*
- * struct cds_lfs_head is returned by __cds_lfs_pop_all, and can be used
- * to begin iteration on the stack. "node" needs to be the first field
- * of cds_lfs_head, so the end-of-stack pointer value can be used for
- * both types.
- */
-struct cds_lfs_head {
- struct cds_lfs_node node;
-};
-
-struct __cds_lfs_stack {
- struct cds_lfs_head *head;
-};
-
-struct cds_lfs_stack {
- struct cds_lfs_head *head;
- pthread_mutex_t lock;
-};
-
-/*
- * The transparent union allows calling functions that work on both
- * struct cds_lfs_stack and struct __cds_lfs_stack on any of those two
- * types.
- */
-typedef union {
- struct __cds_lfs_stack *_s;
- struct cds_lfs_stack *s;
-} __attribute__((__transparent_union__)) cds_lfs_stack_ptr_t;
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/lfstack.h>
-
-#define cds_lfs_node_init _cds_lfs_node_init
-#define cds_lfs_init _cds_lfs_init
-#define cds_lfs_destroy _cds_lfs_destroy
-#define __cds_lfs_init ___cds_lfs_init
-#define cds_lfs_empty _cds_lfs_empty
-#define cds_lfs_push _cds_lfs_push
-
-/* Locking performed internally */
-#define cds_lfs_pop_blocking _cds_lfs_pop_blocking
-#define cds_lfs_pop_all_blocking _cds_lfs_pop_all_blocking
-
-/* Synchronize pop with internal mutex */
-#define cds_lfs_pop_lock _cds_lfs_pop_lock
-#define cds_lfs_pop_unlock _cds_lfs_pop_unlock
-
-/* Synchronization ensured by the caller. See synchronization table. */
-#define __cds_lfs_pop ___cds_lfs_pop
-#define __cds_lfs_pop_all ___cds_lfs_pop_all
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * cds_lfs_node_init: initialize lock-free stack node.
- */
-extern void cds_lfs_node_init(struct cds_lfs_node *node);
-
-/*
- * cds_lfs_init: initialize lock-free stack (with locking). Pair with
- * cds_lfs_destroy().
- */
-extern void cds_lfs_init(struct cds_lfs_stack *s);
-
-/*
- * cds_lfs_destroy: destroy lock-free stack (with lock). Pair with
- * cds_lfs_init().
- */
-extern void cds_lfs_destroy(struct cds_lfs_stack *s);
-
-/*
- * __cds_lfs_init: initialize lock-free stack (without lock).
- * Don't pair with any destroy function.
- */
-extern void __cds_lfs_init(struct __cds_lfs_stack *s);
-
-/*
- * cds_lfs_empty: return whether lock-free stack is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- */
-extern bool cds_lfs_empty(cds_lfs_stack_ptr_t s);
-
-/*
- * cds_lfs_push: push a node into the stack.
- *
- * Does not require any synchronization with other push nor pop.
- *
- * Returns 0 if the stack was empty prior to adding the node.
- * Returns non-zero otherwise.
- */
-extern bool cds_lfs_push(cds_lfs_stack_ptr_t s,
- struct cds_lfs_node *node);
-
-/*
- * cds_lfs_pop_blocking: pop a node from the stack.
- *
- * Calls __cds_lfs_pop with an internal pop mutex held.
- */
-extern struct cds_lfs_node *cds_lfs_pop_blocking(struct cds_lfs_stack *s);
-
-/*
- * cds_lfs_pop_all_blocking: pop all nodes from a stack.
- *
- * Calls __cds_lfs_pop_all with an internal pop mutex held.
- */
-extern struct cds_lfs_head *cds_lfs_pop_all_blocking(struct cds_lfs_stack *s);
-
-/*
- * cds_lfs_pop_lock: lock stack pop-protection mutex.
- */
-extern void cds_lfs_pop_lock(struct cds_lfs_stack *s);
-
-/*
- * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
- */
-extern void cds_lfs_pop_unlock(struct cds_lfs_stack *s);
-
-/*
- * __cds_lfs_pop: pop a node from the stack.
- *
- * Returns NULL if stack is empty.
- *
- * __cds_lfs_pop needs to be synchronized using one of the following
- * techniques:
- *
- * 1) Calling __cds_lfs_pop under rcu read lock critical section.
- * Both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
- * grace period to pass before freeing the returned node or pushing
- * the node back into the stack. It is valid to overwrite the content
- * of cds_lfs_node immediately after __cds_lfs_pop and
- * __cds_lfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop
- * and __cds_lfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
- * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-extern struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s);
-
-/*
- * __cds_lfs_pop_all: pop all nodes from a stack.
- *
- * __cds_lfs_pop_all does not require any synchronization with other
- * push, nor with other __cds_lfs_pop_all, but requires synchronization
- * matching the technique used to synchronize __cds_lfs_pop:
- *
- * 1) If __cds_lfs_pop is called under rcu read lock critical section,
- * both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
- * grace period to pass before freeing the returned node or pushing
- * the node back into the stack. It is valid to overwrite the content
- * of cds_lfs_node immediately after __cds_lfs_pop and
- * __cds_lfs_pop_all. No RCU read-side critical section is needed
- * around __cds_lfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
- * __cds_lfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
- * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-extern struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s);
-
-#endif /* !_LGPL_SOURCE */
-
-/*
- * cds_lfs_for_each: Iterate over all nodes returned by
- * __cds_lfs_pop_all.
- * @__head: node returned by __cds_lfs_pop_all (struct cds_lfs_head pointer).
- * @__node: node to use as iterator (struct cds_lfs_node pointer).
- *
- * Content written into each node before push is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- */
-#define cds_lfs_for_each(__head, __node) \
- for (__node = &__head->node; \
- __node != NULL; \
- __node = __node->next)
-
-/*
- * cds_lfs_for_each_safe: Iterate over all nodes returned by
- * __cds_lfs_pop_all, safe against node deletion.
- * @__head: node returned by __cds_lfs_pop_all (struct cds_lfs_head pointer).
- * @__node: node to use as iterator (struct cds_lfs_node pointer).
- * @__n: struct cds_lfs_node pointer holding the next pointer (used
- * internally).
- *
- * Content written into each node before push is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- */
-#define cds_lfs_for_each_safe(__head, __node, __n) \
- for (__node = &__head->node, __n = (__node ? __node->next : NULL); \
- __node != NULL; \
- __node = __n, __n = (__node ? __node->next : NULL))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_LFSTACK_H */
+++ /dev/null
-/*
- * Copyright (C) 2002 Free Software Foundation, Inc.
- * (originally part of the GNU C Library)
- * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
- *
- * Copyright (C) 2009 Pierre-Marc Fournier
- * Conversion to RCU list.
- * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifndef _CDS_LIST_H
-#define _CDS_LIST_H 1
-
-/*
- * The definitions of this file are adopted from those which can be
- * found in the Linux kernel headers to enable people familiar with the
- * latter find their way in these sources as well.
- */
-
-/* Basic type for the double-link list. */
-struct cds_list_head {
- struct cds_list_head *next, *prev;
-};
-
-/* Define a variable with the head and tail of the list. */
-#define CDS_LIST_HEAD(name) \
- struct cds_list_head name = { &(name), &(name) }
-
-/* Initialize a new list head. */
-#define CDS_INIT_LIST_HEAD(ptr) \
- (ptr)->next = (ptr)->prev = (ptr)
-
-#define CDS_LIST_HEAD_INIT(name) { .prev = &(name), .next = &(name) }
-
-/* Add new element at the head of the list. */
-static inline
-void cds_list_add(struct cds_list_head *newp, struct cds_list_head *head)
-{
- head->next->prev = newp;
- newp->next = head->next;
- newp->prev = head;
- head->next = newp;
-}
-
-/* Add new element at the tail of the list. */
-static inline
-void cds_list_add_tail(struct cds_list_head *newp, struct cds_list_head *head)
-{
- head->prev->next = newp;
- newp->next = head;
- newp->prev = head->prev;
- head->prev = newp;
-}
-
-/* Remove element from list. */
-static inline
-void __cds_list_del(struct cds_list_head *prev, struct cds_list_head *next)
-{
- next->prev = prev;
- prev->next = next;
-}
-
-/* Remove element from list. */
-static inline
-void cds_list_del(struct cds_list_head *elem)
-{
- __cds_list_del(elem->prev, elem->next);
-}
-
-/* Remove element from list, initializing the element's list pointers. */
-static inline
-void cds_list_del_init(struct cds_list_head *elem)
-{
- cds_list_del(elem);
- CDS_INIT_LIST_HEAD(elem);
-}
-
-/* Delete from list, add to another list as head. */
-static inline
-void cds_list_move(struct cds_list_head *elem, struct cds_list_head *head)
-{
- __cds_list_del(elem->prev, elem->next);
- cds_list_add(elem, head);
-}
-
-/* Replace an old entry. */
-static inline
-void cds_list_replace(struct cds_list_head *old, struct cds_list_head *_new)
-{
- _new->next = old->next;
- _new->prev = old->prev;
- _new->prev->next = _new;
- _new->next->prev = _new;
-}
-
-/* Join two lists. */
-static inline
-void cds_list_splice(struct cds_list_head *add, struct cds_list_head *head)
-{
- /* Do nothing if the list which gets added is empty. */
- if (add != add->next) {
- add->next->prev = head;
- add->prev->next = head->next;
- head->next->prev = add->prev;
- head->next = add->next;
- }
-}
-
-/* Get typed element from list at a given position. */
-#define cds_list_entry(ptr, type, member) \
- ((type *) ((char *) (ptr) - (unsigned long) (&((type *) 0)->member)))
-
-
-/* Get first entry from a list. */
-#define cds_list_first_entry(ptr, type, member) \
- cds_list_entry((ptr)->next, type, member)
-
-/* Iterate forward over the elements of the list. */
-#define cds_list_for_each(pos, head) \
- for (pos = (head)->next; pos != (head); pos = pos->next)
-
-/*
- * Iterate forward over the elements list. The list elements can be
- * removed from the list while doing this.
- */
-#define cds_list_for_each_safe(pos, p, head) \
- for (pos = (head)->next, p = pos->next; \
- pos != (head); \
- pos = p, p = pos->next)
-
-/* Iterate backward over the elements of the list. */
-#define cds_list_for_each_prev(pos, head) \
- for (pos = (head)->prev; pos != (head); pos = pos->prev)
-
-/*
- * Iterate backwards over the elements list. The list elements can be
- * removed from the list while doing this.
- */
-#define cds_list_for_each_prev_safe(pos, p, head) \
- for (pos = (head)->prev, p = pos->prev; \
- pos != (head); \
- pos = p, p = pos->prev)
-
-#define cds_list_for_each_entry(pos, head, member) \
- for (pos = cds_list_entry((head)->next, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = cds_list_entry(pos->member.next, __typeof__(*pos), member))
-
-#define cds_list_for_each_entry_reverse(pos, head, member) \
- for (pos = cds_list_entry((head)->prev, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = cds_list_entry(pos->member.prev, __typeof__(*pos), member))
-
-#define cds_list_for_each_entry_safe(pos, p, head, member) \
- for (pos = cds_list_entry((head)->next, __typeof__(*pos), member), \
- p = cds_list_entry(pos->member.next, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = p, p = cds_list_entry(pos->member.next, __typeof__(*pos), member))
-
-/*
- * Same as cds_list_for_each_entry_safe, but starts from "pos" which should
- * point to an entry within the list.
- */
-#define cds_list_for_each_entry_safe_from(pos, p, head, member) \
- for (p = cds_list_entry(pos->member.next, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = p, p = cds_list_entry(pos->member.next, __typeof__(*pos), member))
-
-static inline
-int cds_list_empty(struct cds_list_head *head)
-{
- return head == head->next;
-}
-
-static inline
-void cds_list_replace_init(struct cds_list_head *old,
- struct cds_list_head *_new)
-{
- struct cds_list_head *head = old->next;
-
- cds_list_del(old);
- cds_list_add_tail(_new, head);
- CDS_INIT_LIST_HEAD(old);
-}
-
-#endif /* _CDS_LIST_H */
+++ /dev/null
-#ifndef _URCU_BP_MAP_H
-#define _URCU_BP_MAP_H
-
-/*
- * urcu-map.h
- *
- * Userspace RCU header -- name mapping to allow multiple flavors to be
- * used in the same executable.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-/* Mapping macros to allow multiple flavors in a single binary. */
-
-#define rcu_read_lock rcu_read_lock_bp
-#define _rcu_read_lock _rcu_read_lock_bp
-#define rcu_read_unlock rcu_read_unlock_bp
-#define _rcu_read_unlock _rcu_read_unlock_bp
-#define rcu_read_ongoing rcu_read_ongoing_bp
-#define _rcu_read_ongoing _rcu_read_ongoing_bp
-#define rcu_register_thread rcu_register_thread_bp
-#define rcu_unregister_thread rcu_unregister_thread_bp
-#define rcu_init rcu_init_bp
-#define rcu_exit rcu_exit_bp
-#define synchronize_rcu synchronize_rcu_bp
-#define rcu_reader rcu_reader_bp
-#define rcu_gp rcu_gp_bp
-
-#define get_cpu_call_rcu_data get_cpu_call_rcu_data_bp
-#define get_call_rcu_thread get_call_rcu_thread_bp
-#define create_call_rcu_data create_call_rcu_data_bp
-#define set_cpu_call_rcu_data set_cpu_call_rcu_data_bp
-#define get_default_call_rcu_data get_default_call_rcu_data_bp
-#define get_call_rcu_data get_call_rcu_data_bp
-#define get_thread_call_rcu_data get_thread_call_rcu_data_bp
-#define set_thread_call_rcu_data set_thread_call_rcu_data_bp
-#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_bp
-#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_bp
-#define call_rcu call_rcu_bp
-#define call_rcu_data_free call_rcu_data_free_bp
-#define call_rcu_before_fork call_rcu_before_fork_bp
-#define call_rcu_after_fork_parent call_rcu_after_fork_parent_bp
-#define call_rcu_after_fork_child call_rcu_after_fork_child_bp
-#define rcu_barrier rcu_barrier_bp
-
-#define defer_rcu defer_rcu_bp
-#define rcu_defer_register_thread rcu_defer_register_thread_bp
-#define rcu_defer_unregister_thread rcu_defer_unregister_thread_bp
-#define rcu_defer_barrier rcu_defer_barrier_bp
-#define rcu_defer_barrier_thread rcu_defer_barrier_thread_bp
-#define rcu_defer_exit rcu_defer_exit_bp
-
-#define rcu_flavor rcu_flavor_bp
-
-#define rcu_yield_active rcu_yield_active_bp
-#define rcu_rand_yield rcu_rand_yield_bp
-
-#endif /* _URCU_BP_MAP_H */
+++ /dev/null
-#ifndef _URCU_QSBR_MAP_H
-#define _URCU_QSBR_MAP_H
-
-/*
- * urcu-map.h
- *
- * Userspace RCU header -- name mapping to allow multiple flavors to be
- * used in the same executable.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-/* Mapping macros to allow multiple flavors in a single binary. */
-
-#define rcu_read_lock rcu_read_lock_qsbr
-#define _rcu_read_lock _rcu_read_lock_qsbr
-#define rcu_read_unlock rcu_read_unlock_qsbr
-#define _rcu_read_unlock _rcu_read_unlock_qsbr
-#define rcu_read_ongoing rcu_read_ongoing_qsbr
-#define _rcu_read_ongoing _rcu_read_ongoing_qsbr
-#define rcu_quiescent_state rcu_quiescent_state_qsbr
-#define _rcu_quiescent_state _rcu_quiescent_state_qsbr
-#define rcu_thread_offline rcu_thread_offline_qsbr
-#define rcu_thread_online rcu_thread_online_qsbr
-#define rcu_register_thread rcu_register_thread_qsbr
-#define rcu_unregister_thread rcu_unregister_thread_qsbr
-#define rcu_exit rcu_exit_qsbr
-#define synchronize_rcu synchronize_rcu_qsbr
-#define rcu_reader rcu_reader_qsbr
-#define rcu_gp rcu_gp_qsbr
-
-#define get_cpu_call_rcu_data get_cpu_call_rcu_data_qsbr
-#define get_call_rcu_thread get_call_rcu_thread_qsbr
-#define create_call_rcu_data create_call_rcu_data_qsbr
-#define set_cpu_call_rcu_data set_cpu_call_rcu_data_qsbr
-#define get_default_call_rcu_data get_default_call_rcu_data_qsbr
-#define get_call_rcu_data get_call_rcu_data_qsbr
-#define get_thread_call_rcu_data get_thread_call_rcu_data_qsbr
-#define set_thread_call_rcu_data set_thread_call_rcu_data_qsbr
-#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_qsbr
-#define call_rcu call_rcu_qsbr
-#define call_rcu_data_free call_rcu_data_free_qsbr
-#define call_rcu_before_fork call_rcu_before_fork_qsbr
-#define call_rcu_after_fork_parent call_rcu_after_fork_parent_qsbr
-#define call_rcu_after_fork_child call_rcu_after_fork_child_qsbr
-#define rcu_barrier rcu_barrier_qsbr
-
-#define defer_rcu defer_rcu_qsbr
-#define rcu_defer_register_thread rcu_defer_register_thread_qsbr
-#define rcu_defer_unregister_thread rcu_defer_unregister_thread_qsbr
-#define rcu_defer_barrier rcu_defer_barrier_qsbr
-#define rcu_defer_barrier_thread rcu_defer_barrier_thread_qsbr
-#define rcu_defer_exit rcu_defer_exit_qsbr
-
-#define rcu_flavor rcu_flavor_qsbr
-
-#endif /* _URCU_QSBR_MAP_H */
+++ /dev/null
-#ifndef _URCU_MAP_H
-#define _URCU_MAP_H
-
-/*
- * urcu-map.h
- *
- * Userspace RCU header -- name mapping to allow multiple flavors to be
- * used in the same executable.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * LGPL-compatible code should include this header with :
- *
- * #define _LGPL_SOURCE
- * #include <urcu.h>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-/* Mapping macros to allow multiple flavors in a single binary. */
-
-#if !defined(RCU_MEMBARRIER) && !defined(RCU_SIGNAL) && !defined(RCU_MB)
-#define RCU_MEMBARRIER
-#endif
-
-#ifdef RCU_MEMBARRIER
-
-#define rcu_read_lock rcu_read_lock_memb
-#define _rcu_read_lock _rcu_read_lock_memb
-#define rcu_read_unlock rcu_read_unlock_memb
-#define _rcu_read_unlock _rcu_read_unlock_memb
-#define rcu_read_ongoing rcu_read_ongoing_memb
-#define _rcu_read_ongoing _rcu_read_ongoing_memb
-#define rcu_register_thread rcu_register_thread_memb
-#define rcu_unregister_thread rcu_unregister_thread_memb
-#define rcu_init rcu_init_memb
-#define rcu_exit rcu_exit_memb
-#define synchronize_rcu synchronize_rcu_memb
-#define rcu_reader rcu_reader_memb
-#define rcu_gp rcu_gp_memb
-
-#define get_cpu_call_rcu_data get_cpu_call_rcu_data_memb
-#define get_call_rcu_thread get_call_rcu_thread_memb
-#define create_call_rcu_data create_call_rcu_data_memb
-#define set_cpu_call_rcu_data set_cpu_call_rcu_data_memb
-#define get_default_call_rcu_data get_default_call_rcu_data_memb
-#define get_call_rcu_data get_call_rcu_data_memb
-#define get_thread_call_rcu_data get_thread_call_rcu_data_memb
-#define set_thread_call_rcu_data set_thread_call_rcu_data_memb
-#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_memb
-#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_memb
-#define call_rcu call_rcu_memb
-#define call_rcu_data_free call_rcu_data_free_memb
-#define call_rcu_before_fork call_rcu_before_fork_memb
-#define call_rcu_after_fork_parent call_rcu_after_fork_parent_memb
-#define call_rcu_after_fork_child call_rcu_after_fork_child_memb
-#define rcu_barrier rcu_barrier_memb
-
-#define defer_rcu defer_rcu_memb
-#define rcu_defer_register_thread rcu_defer_register_thread_memb
-#define rcu_defer_unregister_thread rcu_defer_unregister_thread_memb
-#define rcu_defer_barrier rcu_defer_barrier_memb
-#define rcu_defer_barrier_thread rcu_defer_barrier_thread_memb
-#define rcu_defer_exit rcu_defer_exit_memb
-
-#define rcu_flavor rcu_flavor_memb
-
-/* Specific to MEMBARRIER flavor */
-#define rcu_has_sys_membarrier rcu_has_sys_membarrier_memb
-
-#elif defined(RCU_SIGNAL)
-
-#define rcu_read_lock rcu_read_lock_sig
-#define _rcu_read_lock _rcu_read_lock_sig
-#define rcu_read_unlock rcu_read_unlock_sig
-#define _rcu_read_unlock _rcu_read_unlock_sig
-#define rcu_read_ongoing rcu_read_ongoing_sig
-#define _rcu_read_ongoing _rcu_read_ongoing_sig
-#define rcu_register_thread rcu_register_thread_sig
-#define rcu_unregister_thread rcu_unregister_thread_sig
-#define rcu_init rcu_init_sig
-#define rcu_exit rcu_exit_sig
-#define synchronize_rcu synchronize_rcu_sig
-#define rcu_reader rcu_reader_sig
-#define rcu_gp rcu_gp_sig
-
-#define get_cpu_call_rcu_data get_cpu_call_rcu_data_sig
-#define get_call_rcu_thread get_call_rcu_thread_sig
-#define create_call_rcu_data create_call_rcu_data_sig
-#define set_cpu_call_rcu_data set_cpu_call_rcu_data_sig
-#define get_default_call_rcu_data get_default_call_rcu_data_sig
-#define get_call_rcu_data get_call_rcu_data_sig
-#define get_thread_call_rcu_data get_thread_call_rcu_data_sig
-#define set_thread_call_rcu_data set_thread_call_rcu_data_sig
-#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_sig
-#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_sig
-#define call_rcu call_rcu_sig
-#define call_rcu_data_free call_rcu_data_free_sig
-#define call_rcu_before_fork call_rcu_before_fork_sig
-#define call_rcu_after_fork_parent call_rcu_after_fork_parent_sig
-#define call_rcu_after_fork_child call_rcu_after_fork_child_sig
-#define rcu_barrier rcu_barrier_sig
-
-#define defer_rcu defer_rcu_sig
-#define rcu_defer_register_thread rcu_defer_register_thread_sig
-#define rcu_defer_unregister_thread rcu_defer_unregister_thread_sig
-#define rcu_defer_barrier rcu_defer_barrier_sig
-#define rcu_defer_barrier_thread rcu_defer_barrier_thread_sig
-#define rcu_defer_exit rcu_defer_exit_sig
-
-#define rcu_flavor rcu_flavor_sig
-
-#elif defined(RCU_MB)
-
-#define rcu_read_lock rcu_read_lock_mb
-#define _rcu_read_lock _rcu_read_lock_mb
-#define rcu_read_unlock rcu_read_unlock_mb
-#define _rcu_read_unlock _rcu_read_unlock_mb
-#define rcu_read_ongoing rcu_read_ongoing_mb
-#define _rcu_read_ongoing _rcu_read_ongoing_mb
-#define rcu_register_thread rcu_register_thread_mb
-#define rcu_unregister_thread rcu_unregister_thread_mb
-#define rcu_init rcu_init_mb
-#define rcu_exit rcu_exit_mb
-#define synchronize_rcu synchronize_rcu_mb
-#define rcu_reader rcu_reader_mb
-#define rcu_gp rcu_gp_mb
-
-#define get_cpu_call_rcu_data get_cpu_call_rcu_data_mb
-#define get_call_rcu_thread get_call_rcu_thread_mb
-#define create_call_rcu_data create_call_rcu_data_mb
-#define set_cpu_call_rcu_data set_cpu_call_rcu_data_mb
-#define get_default_call_rcu_data get_default_call_rcu_data_mb
-#define get_call_rcu_data get_call_rcu_data_mb
-#define get_thread_call_rcu_data get_thread_call_rcu_data_mb
-#define set_thread_call_rcu_data set_thread_call_rcu_data_mb
-#define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_mb
-#define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_mb
-#define call_rcu call_rcu_mb
-#define call_rcu_data_free call_rcu_data_free_mb
-#define call_rcu_before_fork call_rcu_before_fork_mb
-#define call_rcu_after_fork_parent call_rcu_after_fork_parent_mb
-#define call_rcu_after_fork_child call_rcu_after_fork_child_mb
-#define rcu_barrier rcu_barrier_mb
-
-#define defer_rcu defer_rcu_mb
-#define rcu_defer_register_thread rcu_defer_register_thread_mb
-#define rcu_defer_unregister_thread rcu_defer_unregister_thread_mb
-#define rcu_defer_barrier rcu_defer_barrier_mb
-#define rcu_defer_barrier_thread rcu_defer_barrier_thread_mb
-#define rcu_defer_exit rcu_defer_exit_mb
-
-#define rcu_flavor rcu_flavor_mb
-
-#else
-
-#error "Undefined selection"
-
-#endif
-
-#endif /* _URCU_MAP_H */
+++ /dev/null
-#ifndef _URCU_RAND_COMPAT_H
-#define _URCU_RAND_COMPAT_H
-
-/*
- * urcu/rand-compat.h
- *
- * Userspace RCU library - rand/rand_r Compatibility Header
- *
- * Copyright 1996 - Ulrich Drepper <drepper@cygnus.com >
- * Copyright 2013 - Pierre-Luc St-Charles <pierre-luc.st-charles@polymtl.ca>
- *
- * Note: this file is only used to simplify the code required to
- * use the 'rand_r(...)' system function across multiple platforms,
- * which might not always be referenced the same way.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifdef __ANDROID__
-/*
- * Reentrant random function from POSIX.1c.
- * Copyright (C) 1996, 1999 Free Software Foundation, Inc.
- * This file is part of the GNU C Library.
- * Contributed by Ulrich Drepper <drepper@cygnus.com <mailto:drepper@cygnus.com>>, 1996.
- */
-static inline int rand_r(unsigned int *seed)
-{
- unsigned int next = *seed;
- int result;
-
- next *= 1103515245;
- next += 12345;
- result = (unsigned int) (next / 65536) % 2048;
-
- next *= 1103515245;
- next += 12345;
- result <<= 10;
- result ^= (unsigned int) (next / 65536) % 1024;
-
- next *= 1103515245;
- next += 12345;
- result <<= 10;
- result ^= (unsigned int) (next / 65536) % 1024;
-
- *seed = next;
-
- return result;
-}
-#endif /* __ANDROID__ */
-
-#endif /* _URCU_RAND_COMPAT_H */
+++ /dev/null
-/*
- * Copyright (C) 2002 Free Software Foundation, Inc.
- * (originally part of the GNU C Library)
- * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
- *
- * Copyright (C) 2009 Pierre-Marc Fournier
- * Conversion to RCU list.
- * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifndef _URCU_RCUHLIST_H
-#define _URCU_RCUHLIST_H
-
-#include <urcu/hlist.h>
-#include <urcu/arch.h>
-#include <urcu-pointer.h>
-
-/* Add new element at the head of the list. */
-static inline
-void cds_hlist_add_head_rcu(struct cds_hlist_node *newp,
- struct cds_hlist_head *head)
-{
- newp->next = head->next;
- newp->prev = (struct cds_hlist_node *)head;
- if (head->next)
- head->next->prev = newp;
- rcu_assign_pointer(head->next, newp);
-}
-
-/* Remove element from list. */
-static inline
-void cds_hlist_del_rcu(struct cds_hlist_node *elem)
-{
- if (elem->next)
- elem->next->prev = elem->prev;
- CMM_STORE_SHARED(elem->prev->next, elem->next);
-}
-
-/*
- * Iterate through elements of the list.
- * This must be done while rcu_read_lock() is held.
- */
-#define cds_hlist_for_each_rcu(pos, head) \
- for (pos = rcu_dereference((head)->next); pos != NULL; \
- pos = rcu_dereference(pos->next))
-
-/*
- * cds_hlist_for_each_entry_rcu takes 4 arguments, while the Linux
- * kernel API only takes 3.
- * We implement cds_hlist_for_each_entry_rcu_2() to follow the Linux
- * kernel APIs.
- */
-#define cds_hlist_for_each_entry_rcu(entry, pos, head, member) \
- for (pos = rcu_dereference((head)->next), \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member); \
- pos != NULL; \
- pos = rcu_dereference(pos->next), \
- entry = cds_hlist_entry(pos, __typeof__(*entry), member))
-
-#define cds_hlist_for_each_entry_rcu_2(entry, head, member) \
- for (entry = cds_hlist_entry(rcu_dereference((head)->next), \
- __typeof__(*entry), member); \
- &entry->member != NULL; \
- entry = cds_hlist_entry(rcu_dereference(entry->member.next), \
- __typeof__(*entry), member))
-
-#endif /* _URCU_RCUHLIST_H */
+++ /dev/null
-#ifndef _URCU_RCULFHASH_H
-#define _URCU_RCULFHASH_H
-
-/*
- * urcu/rculfhash.h
- *
- * Userspace RCU library - Lock-Free RCU Hash Table
- *
- * Copyright 2011 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * Include this file _after_ including your URCU flavor.
- */
-
-#include <stdint.h>
-#include <urcu/compiler.h>
-#include <urcu-call-rcu.h>
-#include <urcu-flavor.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * cds_lfht_node: Contains the next pointers and reverse-hash
- * value required for lookup and traversal of the hash table.
- *
- * struct cds_lfht_node should be aligned on 8-bytes boundaries because
- * the three lower bits are used as flags. It is worth noting that the
- * information contained within these three bits could be represented on
- * two bits by re-using the same bit for REMOVAL_OWNER_FLAG and
- * BUCKET_FLAG. This can be done if we ensure that no iterator nor
- * updater check the BUCKET_FLAG after it detects that the REMOVED_FLAG
- * is set. Given the minimum size of struct cds_lfht_node is 8 bytes on
- * 32-bit architectures, we choose to go for simplicity and reserve
- * three bits.
- *
- * struct cds_lfht_node can be embedded into a structure (as a field).
- * caa_container_of() can be used to get the structure from the struct
- * cds_lfht_node after a lookup.
- *
- * The structure which embeds it typically holds the key (or key-value
- * pair) of the object. The caller code is responsible for calculation
- * of the hash value for cds_lfht APIs.
- */
-struct cds_lfht_node {
- struct cds_lfht_node *next; /* ptr | REMOVAL_OWNER_FLAG | BUCKET_FLAG | REMOVED_FLAG */
- unsigned long reverse_hash;
-} __attribute__((aligned(8)));
-
-/* cds_lfht_iter: Used to track state while traversing a hash chain. */
-struct cds_lfht_iter {
- struct cds_lfht_node *node, *next;
-};
-
-static inline
-struct cds_lfht_node *cds_lfht_iter_get_node(struct cds_lfht_iter *iter)
-{
- return iter->node;
-}
-
-struct cds_lfht;
-
-/*
- * Caution !
- * Ensure reader and writer threads are registered as urcu readers.
- */
-
-typedef int (*cds_lfht_match_fct)(struct cds_lfht_node *node, const void *key);
-
-/*
- * cds_lfht_node_init - initialize a hash table node
- * @node: the node to initialize.
- *
- * This function is kept to be eventually used for debugging purposes
- * (detection of memory corruption).
- */
-static inline
-void cds_lfht_node_init(struct cds_lfht_node *node)
-{
-}
-
-/*
- * Hash table creation flags.
- */
-enum {
- CDS_LFHT_AUTO_RESIZE = (1U << 0),
- CDS_LFHT_ACCOUNTING = (1U << 1),
-};
-
-struct cds_lfht_mm_type {
- struct cds_lfht *(*alloc_cds_lfht)(unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets);
- void (*alloc_bucket_table)(struct cds_lfht *ht, unsigned long order);
- void (*free_bucket_table)(struct cds_lfht *ht, unsigned long order);
- struct cds_lfht_node *(*bucket_at)(struct cds_lfht *ht,
- unsigned long index);
-};
-
-extern const struct cds_lfht_mm_type cds_lfht_mm_order;
-extern const struct cds_lfht_mm_type cds_lfht_mm_chunk;
-extern const struct cds_lfht_mm_type cds_lfht_mm_mmap;
-
-/*
- * _cds_lfht_new - API used by cds_lfht_new wrapper. Do not use directly.
- */
-extern
-struct cds_lfht *_cds_lfht_new(unsigned long init_size,
- unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets,
- int flags,
- const struct cds_lfht_mm_type *mm,
- const struct rcu_flavor_struct *flavor,
- pthread_attr_t *attr);
-
-/*
- * cds_lfht_new - allocate a hash table.
- * @init_size: number of buckets to allocate initially. Must be power of two.
- * @min_nr_alloc_buckets: the minimum number of allocated buckets.
- * (must be power of two)
- * @max_nr_buckets: the maximum number of hash table buckets allowed.
- * (must be power of two, 0 is accepted, means
- * "infinite")
- * @flags: hash table creation flags (can be combined with bitwise or: '|').
- * 0: no flags.
- * CDS_LFHT_AUTO_RESIZE: automatically resize hash table.
- * CDS_LFHT_ACCOUNTING: count the number of node addition
- * and removal in the table
- * @attr: optional resize worker thread attributes. NULL for default.
- *
- * Return NULL on error.
- * Note: the RCU flavor must be already included before the hash table header.
- *
- * The programmer is responsible for ensuring that resize operation has a
- * priority equal to hash table updater threads. It should be performed by
- * specifying the appropriate priority in the pthread "attr" argument, and,
- * for CDS_LFHT_AUTO_RESIZE, by ensuring that call_rcu worker threads also have
- * this priority level. Having lower priority for call_rcu and resize threads
- * does not pose any correctness issue, but the resize operations could be
- * starved by updates, thus leading to long hash table bucket chains.
- * Threads calling cds_lfht_new are NOT required to be registered RCU
- * read-side threads. It can be called very early. (e.g. before RCU is
- * initialized)
- */
-static inline
-struct cds_lfht *cds_lfht_new(unsigned long init_size,
- unsigned long min_nr_alloc_buckets,
- unsigned long max_nr_buckets,
- int flags,
- pthread_attr_t *attr)
-{
- return _cds_lfht_new(init_size, min_nr_alloc_buckets, max_nr_buckets,
- flags, NULL, &rcu_flavor, attr);
-}
-
-/*
- * cds_lfht_destroy - destroy a hash table.
- * @ht: the hash table to destroy.
- * @attr: (output) resize worker thread attributes, as received by cds_lfht_new.
- * The caller will typically want to free this pointer if dynamically
- * allocated. The attr point can be NULL if the caller does not
- * need to be informed of the value passed to cds_lfht_new().
- *
- * Return 0 on success, negative error value on error.
- * Threads calling this API need to be registered RCU read-side threads.
- * cds_lfht_destroy should *not* be called from a RCU read-side critical
- * section. It should *not* be called from a call_rcu thread context
- * neither.
- */
-extern
-int cds_lfht_destroy(struct cds_lfht *ht, pthread_attr_t **attr);
-
-/*
- * cds_lfht_count_nodes - count the number of nodes in the hash table.
- * @ht: the hash table.
- * @split_count_before: sample the node count split-counter before traversal.
- * @count: traverse the hash table, count the number of nodes observed.
- * @split_count_after: sample the node count split-counter after traversal.
- *
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- */
-extern
-void cds_lfht_count_nodes(struct cds_lfht *ht,
- long *split_count_before,
- unsigned long *count,
- long *split_count_after);
-
-/*
- * cds_lfht_lookup - lookup a node by key.
- * @ht: the hash table.
- * @hash: the key hash.
- * @match: the key match function.
- * @key: the current node key.
- * @iter: node, if found (output). *iter->node set to NULL if not found.
- *
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function acts as a rcu_dereference() to read the node pointer.
- */
-extern
-void cds_lfht_lookup(struct cds_lfht *ht, unsigned long hash,
- cds_lfht_match_fct match, const void *key,
- struct cds_lfht_iter *iter);
-
-/*
- * cds_lfht_next_duplicate - get the next item with same key, after iterator.
- * @ht: the hash table.
- * @match: the key match function.
- * @key: the current node key.
- * @iter: input: current iterator.
- * output: node, if found. *iter->node set to NULL if not found.
- *
- * Uses an iterator initialized by a lookup or traversal. Important: the
- * iterator _needs_ to be initialized before calling
- * cds_lfht_next_duplicate.
- * Sets *iter-node to the following node with same key.
- * Sets *iter->node to NULL if no following node exists with same key.
- * RCU read-side lock must be held across cds_lfht_lookup and
- * cds_lfht_next calls, and also between cds_lfht_next calls using the
- * node returned by a previous cds_lfht_next.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function acts as a rcu_dereference() to read the node pointer.
- */
-extern
-void cds_lfht_next_duplicate(struct cds_lfht *ht,
- cds_lfht_match_fct match, const void *key,
- struct cds_lfht_iter *iter);
-
-/*
- * cds_lfht_first - get the first node in the table.
- * @ht: the hash table.
- * @iter: First node, if exists (output). *iter->node set to NULL if not found.
- *
- * Output in "*iter". *iter->node set to NULL if table is empty.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function acts as a rcu_dereference() to read the node pointer.
- */
-extern
-void cds_lfht_first(struct cds_lfht *ht, struct cds_lfht_iter *iter);
-
-/*
- * cds_lfht_next - get the next node in the table.
- * @ht: the hash table.
- * @iter: input: current iterator.
- * output: next node, if exists. *iter->node set to NULL if not found.
- *
- * Input/Output in "*iter". *iter->node set to NULL if *iter was
- * pointing to the last table node.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function acts as a rcu_dereference() to read the node pointer.
- */
-extern
-void cds_lfht_next(struct cds_lfht *ht, struct cds_lfht_iter *iter);
-
-/*
- * cds_lfht_add - add a node to the hash table.
- * @ht: the hash table.
- * @hash: the key hash.
- * @node: the node to add.
- *
- * This function supports adding redundant keys into the table.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function issues a full memory barrier before and after its
- * atomic commit.
- */
-extern
-void cds_lfht_add(struct cds_lfht *ht, unsigned long hash,
- struct cds_lfht_node *node);
-
-/*
- * cds_lfht_add_unique - add a node to hash table, if key is not present.
- * @ht: the hash table.
- * @hash: the node's hash.
- * @match: the key match function.
- * @key: the node's key.
- * @node: the node to try adding.
- *
- * Return the node added upon success.
- * Return the unique node already present upon failure. If
- * cds_lfht_add_unique fails, the node passed as parameter should be
- * freed by the caller. In this case, the caller does NOT need to wait
- * for a grace period before freeing or re-using the node.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- *
- * The semantic of this function is that if only this function is used
- * to add keys into the table, no duplicated keys should ever be
- * observable in the table. The same guarantee apply for combination of
- * add_unique and add_replace (see below).
- *
- * Upon success, this function issues a full memory barrier before and
- * after its atomic commit. Upon failure, this function acts like a
- * simple lookup operation: it acts as a rcu_dereference() to read the
- * node pointer. The failure case does not guarantee any other memory
- * barrier.
- */
-extern
-struct cds_lfht_node *cds_lfht_add_unique(struct cds_lfht *ht,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *node);
-
-/*
- * cds_lfht_add_replace - replace or add a node within hash table.
- * @ht: the hash table.
- * @hash: the node's hash.
- * @match: the key match function.
- * @key: the node's key.
- * @node: the node to add.
- *
- * Return the node replaced upon success. If no node matching the key
- * was present, return NULL, which also means the operation succeeded.
- * This replacement operation should never fail.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * After successful replacement, a grace period must be waited for before
- * freeing or re-using the memory reserved for the returned node.
- *
- * The semantic of replacement vs lookups and traversals is the
- * following: if lookups and traversals are performed between a key
- * unique insertion and its removal, we guarantee that the lookups and
- * traversals will always find exactly one instance of the key if it is
- * replaced concurrently with the lookups.
- *
- * Providing this semantic allows us to ensure that replacement-only
- * schemes will never generate duplicated keys. It also allows us to
- * guarantee that a combination of add_replace and add_unique updates
- * will never generate duplicated keys.
- *
- * This function issues a full memory barrier before and after its
- * atomic commit.
- */
-extern
-struct cds_lfht_node *cds_lfht_add_replace(struct cds_lfht *ht,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *node);
-
-/*
- * cds_lfht_replace - replace a node pointed to by iter within hash table.
- * @ht: the hash table.
- * @old_iter: the iterator position of the node to replace.
- * @hash: the node's hash.
- * @match: the key match function.
- * @key: the node's key.
- * @new_node: the new node to use as replacement.
- *
- * Return 0 if replacement is successful, negative value otherwise.
- * Replacing a NULL old node or an already removed node will fail with
- * -ENOENT.
- * If the hash or value of the node to replace and the new node differ,
- * this function returns -EINVAL without proceeding to the replacement.
- * Old node can be looked up with cds_lfht_lookup and cds_lfht_next.
- * RCU read-side lock must be held between lookup and replacement.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * After successful replacement, a grace period must be waited for before
- * freeing or re-using the memory reserved for the old node (which can
- * be accessed with cds_lfht_iter_get_node).
- *
- * The semantic of replacement vs lookups is the same as
- * cds_lfht_add_replace().
- *
- * Upon success, this function issues a full memory barrier before and
- * after its atomic commit. Upon failure, this function does not issue
- * any memory barrier.
- */
-extern
-int cds_lfht_replace(struct cds_lfht *ht,
- struct cds_lfht_iter *old_iter,
- unsigned long hash,
- cds_lfht_match_fct match,
- const void *key,
- struct cds_lfht_node *new_node);
-
-/*
- * cds_lfht_del - remove node pointed to by iterator from hash table.
- * @ht: the hash table.
- * @node: the node to delete.
- *
- * Return 0 if the node is successfully removed, negative value
- * otherwise.
- * Deleting a NULL node or an already removed node will fail with a
- * negative value.
- * Node can be looked up with cds_lfht_lookup and cds_lfht_next,
- * followed by use of cds_lfht_iter_get_node.
- * RCU read-side lock must be held between lookup and removal.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * After successful removal, a grace period must be waited for before
- * freeing or re-using the memory reserved for old node (which can be
- * accessed with cds_lfht_iter_get_node).
- * Upon success, this function issues a full memory barrier before and
- * after its atomic commit. Upon failure, this function does not issue
- * any memory barrier.
- */
-extern
-int cds_lfht_del(struct cds_lfht *ht, struct cds_lfht_node *node);
-
-/*
- * cds_lfht_is_node_deleted - query whether a node is removed from hash table.
- *
- * Return non-zero if the node is deleted from the hash table, 0
- * otherwise.
- * Node can be looked up with cds_lfht_lookup and cds_lfht_next,
- * followed by use of cds_lfht_iter_get_node.
- * RCU read-side lock must be held between lookup and call to this
- * function.
- * Call with rcu_read_lock held.
- * Threads calling this API need to be registered RCU read-side threads.
- * This function does not issue any memory barrier.
- */
-extern
-int cds_lfht_is_node_deleted(struct cds_lfht_node *node);
-
-/*
- * cds_lfht_resize - Force a hash table resize
- * @ht: the hash table.
- * @new_size: update to this hash table size.
- *
- * Threads calling this API need to be registered RCU read-side threads.
- * This function does not (necessarily) issue memory barriers.
- * cds_lfht_resize should *not* be called from a RCU read-side critical
- * section.
- */
-extern
-void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size);
-
-/*
- * Note: it is safe to perform element removal (del), replacement, or
- * any hash table update operation during any of the following hash
- * table traversals.
- * These functions act as rcu_dereference() to read the node pointers.
- */
-#define cds_lfht_for_each(ht, iter, node) \
- for (cds_lfht_first(ht, iter), \
- node = cds_lfht_iter_get_node(iter); \
- node != NULL; \
- cds_lfht_next(ht, iter), \
- node = cds_lfht_iter_get_node(iter))
-
-#define cds_lfht_for_each_duplicate(ht, hash, match, key, iter, node) \
- for (cds_lfht_lookup(ht, hash, match, key, iter), \
- node = cds_lfht_iter_get_node(iter); \
- node != NULL; \
- cds_lfht_next_duplicate(ht, match, key, iter), \
- node = cds_lfht_iter_get_node(iter))
-
-#define cds_lfht_for_each_entry(ht, iter, pos, member) \
- for (cds_lfht_first(ht, iter), \
- pos = caa_container_of(cds_lfht_iter_get_node(iter), \
- __typeof__(*(pos)), member); \
- cds_lfht_iter_get_node(iter) != NULL; \
- cds_lfht_next(ht, iter), \
- pos = caa_container_of(cds_lfht_iter_get_node(iter), \
- __typeof__(*(pos)), member))
-
-#define cds_lfht_for_each_entry_duplicate(ht, hash, match, key, \
- iter, pos, member) \
- for (cds_lfht_lookup(ht, hash, match, key, iter), \
- pos = caa_container_of(cds_lfht_iter_get_node(iter), \
- __typeof__(*(pos)), member); \
- cds_lfht_iter_get_node(iter) != NULL; \
- cds_lfht_next_duplicate(ht, match, key, iter), \
- pos = caa_container_of(cds_lfht_iter_get_node(iter), \
- __typeof__(*(pos)), member))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFHASH_H */
+++ /dev/null
-#ifndef _URCU_RCULFQUEUE_H
-#define _URCU_RCULFQUEUE_H
-
-/*
- * rculfqueue.h
- *
- * Userspace RCU library - Lock-Free RCU Queue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <assert.h>
-#include <urcu-call-rcu.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct cds_lfq_queue_rcu;
-
-struct cds_lfq_node_rcu {
- struct cds_lfq_node_rcu *next;
- int dummy;
-};
-
-struct cds_lfq_queue_rcu {
- struct cds_lfq_node_rcu *head, *tail;
- void (*queue_call_rcu)(struct rcu_head *head,
- void (*func)(struct rcu_head *head));
-};
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/rculfqueue.h>
-
-#define cds_lfq_node_init_rcu _cds_lfq_node_init_rcu
-#define cds_lfq_init_rcu _cds_lfq_init_rcu
-#define cds_lfq_destroy_rcu _cds_lfq_destroy_rcu
-#define cds_lfq_enqueue_rcu _cds_lfq_enqueue_rcu
-#define cds_lfq_dequeue_rcu _cds_lfq_dequeue_rcu
-
-#else /* !_LGPL_SOURCE */
-
-extern void cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node);
-extern void cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
- void queue_call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head)));
-/*
- * The queue should be emptied before calling destroy.
- *
- * Return 0 on success, -EPERM if queue is not empty.
- */
-extern int cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q);
-
-/*
- * Should be called under rcu read lock critical section.
- */
-extern void cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
- struct cds_lfq_node_rcu *node);
-
-/*
- * Should be called under rcu read lock critical section.
- *
- * The caller must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_lfq_node_rcu structure.
- * Returns NULL if queue is empty.
- */
-extern
-struct cds_lfq_node_rcu *cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q);
-
-#endif /* !_LGPL_SOURCE */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFQUEUE_H */
+++ /dev/null
-#ifndef _URCU_RCULFSTACK_H
-#define _URCU_RCULFSTACK_H
-
-/*
- * rculfstack.h
- *
- * Userspace RCU library - Lock-Free RCU Stack
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/compiler.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef CDS_LFS_RCU_DEPRECATED
-#define CDS_LFS_RCU_DEPRECATED \
- CDS_DEPRECATED("urcu/rculfstack.h is deprecated. Please use urcu/lfstack.h instead.")
-#endif
-
-struct cds_lfs_node_rcu {
- struct cds_lfs_node_rcu *next;
-};
-
-struct cds_lfs_stack_rcu {
- struct cds_lfs_node_rcu *head;
-};
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/rculfstack.h>
-
-static inline CDS_LFS_RCU_DEPRECATED
-void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
-{
- _cds_lfs_node_init_rcu(node);
-}
-
-static inline
-void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
-{
- _cds_lfs_init_rcu(s);
-}
-
-static inline CDS_LFS_RCU_DEPRECATED
-int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
- struct cds_lfs_node_rcu *node)
-{
- return _cds_lfs_push_rcu(s, node);
-}
-
-static inline CDS_LFS_RCU_DEPRECATED
-struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
-{
- return _cds_lfs_pop_rcu(s);
-}
-
-#else /* !_LGPL_SOURCE */
-
-extern CDS_LFS_RCU_DEPRECATED
-void cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node);
-extern CDS_LFS_RCU_DEPRECATED
-void cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s);
-extern CDS_LFS_RCU_DEPRECATED
-int cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
- struct cds_lfs_node_rcu *node);
-
-/*
- * Should be called under rcu read lock critical section.
- *
- * The caller must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_lfs_node_rcu structure.
- * Returns NULL if stack is empty.
- */
-extern CDS_LFS_RCU_DEPRECATED
-struct cds_lfs_node_rcu *cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s);
-
-#endif /* !_LGPL_SOURCE */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFSTACK_H */
+++ /dev/null
-/*
- * Copyright (C) 2002 Free Software Foundation, Inc.
- * (originally part of the GNU C Library)
- * Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
- *
- * Copyright (C) 2009 Pierre-Marc Fournier
- * Conversion to RCU list.
- * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#ifndef _URCU_RCULIST_H
-#define _URCU_RCULIST_H
-
-#include <urcu/list.h>
-#include <urcu/arch.h>
-#include <urcu-pointer.h>
-
-/* Add new element at the head of the list. */
-static inline
-void cds_list_add_rcu(struct cds_list_head *newp, struct cds_list_head *head)
-{
- newp->next = head->next;
- newp->prev = head;
- head->next->prev = newp;
- rcu_assign_pointer(head->next, newp);
-}
-
-/* Add new element at the tail of the list. */
-static inline
-void cds_list_add_tail_rcu(struct cds_list_head *newp,
- struct cds_list_head *head)
-{
- newp->next = head;
- newp->prev = head->prev;
- rcu_assign_pointer(head->prev->next, newp);
- head->prev = newp;
-}
-
-/*
- * Replace an old entry atomically with respect to concurrent RCU
- * traversal. Mutual exclusion against concurrent updates is required
- * though.
- */
-static inline
-void cds_list_replace_rcu(struct cds_list_head *old, struct cds_list_head *_new)
-{
- _new->next = old->next;
- _new->prev = old->prev;
- rcu_assign_pointer(_new->prev->next, _new);
- _new->next->prev = _new;
-}
-
-/* Remove element from list. */
-static inline
-void cds_list_del_rcu(struct cds_list_head *elem)
-{
- elem->next->prev = elem->prev;
- CMM_STORE_SHARED(elem->prev->next, elem->next);
-}
-
-/*
- * Iteration through all elements of the list must be done while rcu_read_lock()
- * is held.
- */
-
-/* Iterate forward over the elements of the list. */
-#define cds_list_for_each_rcu(pos, head) \
- for (pos = rcu_dereference((head)->next); pos != (head); \
- pos = rcu_dereference(pos->next))
-
-
-/* Iterate through elements of the list. */
-#define cds_list_for_each_entry_rcu(pos, head, member) \
- for (pos = cds_list_entry(rcu_dereference((head)->next), __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = cds_list_entry(rcu_dereference(pos->member.next), __typeof__(*pos), member))
-
-#endif /* _URCU_RCULIST_H */
+++ /dev/null
-#ifndef _URCU_REF_H
-#define _URCU_REF_H
-
-/*
- * Userspace RCU - Reference counting
- *
- * Copyright (C) 2009 Novell Inc.
- * Copyright (C) 2010 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * Author: Jan Blunck <jblunck@suse.de>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License version 2.1 as
- * published by the Free Software Foundation.
- */
-
-#include <assert.h>
-#include <stdbool.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <urcu/uatomic.h>
-
-struct urcu_ref {
- long refcount; /* ATOMIC */
-};
-
-static inline void urcu_ref_set(struct urcu_ref *ref, long val)
-{
- uatomic_set(&ref->refcount, val);
-}
-
-static inline void urcu_ref_init(struct urcu_ref *ref)
-{
- urcu_ref_set(ref, 1);
-}
-
-static inline bool __attribute__((warn_unused_result))
- urcu_ref_get_safe(struct urcu_ref *ref)
-{
- long old, _new, res;
-
- old = uatomic_read(&ref->refcount);
- for (;;) {
- if (old == LONG_MAX) {
- return false; /* Failure. */
- }
- _new = old + 1;
- res = uatomic_cmpxchg(&ref->refcount, old, _new);
- if (res == old) {
- return true; /* Success. */
- }
- old = res;
- }
-}
-
-static inline void urcu_ref_get(struct urcu_ref *ref)
-{
- if (!urcu_ref_get_safe(ref))
- abort();
-}
-
-static inline void urcu_ref_put(struct urcu_ref *ref,
- void (*release)(struct urcu_ref *))
-{
- long res = uatomic_sub_return(&ref->refcount, 1);
- assert (res >= 0);
- if (res == 0)
- release(ref);
-}
-
-/*
- * urcu_ref_get_unless_zero
- *
- * Allows getting a reference atomically if the reference count is not
- * zero. Returns true if the reference is taken, false otherwise. This
- * needs to be used in conjunction with another synchronization
- * technique (e.g. RCU or mutex) to ensure existence of the reference
- * count. False is also returned in case incrementing the refcount would
- * result in an overflow.
- */
-static inline bool urcu_ref_get_unless_zero(struct urcu_ref *ref)
-{
- long old, _new, res;
-
- old = uatomic_read(&ref->refcount);
- for (;;) {
- if (old == 0 || old == LONG_MAX)
- return false; /* Failure. */
- _new = old + 1;
- res = uatomic_cmpxchg(&ref->refcount, old, _new);
- if (res == old) {
- return true; /* Success. */
- }
- old = res;
- }
-}
-
-#endif /* _URCU_REF_H */
+++ /dev/null
-#ifndef _URCU_STATIC_LFSTACK_H
-#define _URCU_STATIC_LFSTACK_H
-
-/*
- * urcu/static/lfstack.h
- *
- * Userspace RCU library - Lock-Free Stack
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/lfstack.h for
- * linking dynamically with the userspace rcu library.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdbool.h>
-#include <pthread.h>
-#include <assert.h>
-#include <urcu/uatomic.h>
-#include <urcu-pointer.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Lock-free stack.
- *
- * Stack implementing push, pop, pop_all operations, as well as iterator
- * on the stack head returned by pop_all.
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * cds_lfs_push __cds_lfs_pop __cds_lfs_pop_all
- * cds_lfs_push - - -
- * __cds_lfs_pop - X X
- * __cds_lfs_pop_all - X -
- *
- * cds_lfs_pop_blocking and cds_lfs_pop_all_blocking use an internal
- * mutex to provide synchronization.
- */
-
-/*
- * cds_lfs_node_init: initialize lock-free stack node.
- */
-static inline
-void _cds_lfs_node_init(struct cds_lfs_node *node)
-{
-}
-
-/*
- * cds_lfs_init: initialize lock-free stack (with lock). Pair with
- * cds_lfs_destroy().
- */
-static inline
-void _cds_lfs_init(struct cds_lfs_stack *s)
-{
- int ret;
-
- s->head = NULL;
- ret = pthread_mutex_init(&s->lock, NULL);
- assert(!ret);
-}
-
-/*
- * cds_lfs_destroy: destroy lock-free stack (with lock). Pair with
- * cds_lfs_init().
- */
-static inline
-void _cds_lfs_destroy(struct cds_lfs_stack *s)
-{
- int ret = pthread_mutex_destroy(&s->lock);
- assert(!ret);
-}
-
-/*
- * ___cds_lfs_init: initialize lock-free stack (without lock).
- * Don't pair with any destroy function.
- */
-static inline
-void ___cds_lfs_init(struct __cds_lfs_stack *s)
-{
- s->head = NULL;
-}
-
-static inline
-bool ___cds_lfs_empty_head(struct cds_lfs_head *head)
-{
- return head == NULL;
-}
-
-/*
- * cds_lfs_empty: return whether lock-free stack is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- */
-static inline
-bool _cds_lfs_empty(cds_lfs_stack_ptr_t s)
-{
- return ___cds_lfs_empty_head(CMM_LOAD_SHARED(s._s->head));
-}
-
-/*
- * cds_lfs_push: push a node into the stack.
- *
- * Does not require any synchronization with other push nor pop.
- *
- * Lock-free stack push is not subject to ABA problem, so no need to
- * take the RCU read-side lock. Even if "head" changes between two
- * uatomic_cmpxchg() invocations here (being popped, and then pushed
- * again by one or more concurrent threads), the second
- * uatomic_cmpxchg() invocation only cares about pushing a new entry at
- * the head of the stack, ensuring consistency by making sure the new
- * node->next is the same pointer value as the value replaced as head.
- * It does not care about the content of the actual next node, so it can
- * very well be reallocated between the two uatomic_cmpxchg().
- *
- * We take the approach of expecting the stack to be usually empty, so
- * we first try an initial uatomic_cmpxchg() on a NULL old_head, and
- * retry if the old head was non-NULL (the value read by the first
- * uatomic_cmpxchg() is used as old head for the following loop). The
- * upside of this scheme is to minimize the amount of cacheline traffic,
- * always performing an exclusive cacheline access, rather than doing
- * non-exclusive followed by exclusive cacheline access (which would be
- * required if we first read the old head value). This design decision
- * might be revisited after more thorough benchmarking on various
- * platforms.
- *
- * Returns 0 if the stack was empty prior to adding the node.
- * Returns non-zero otherwise.
- */
-static inline
-bool _cds_lfs_push(cds_lfs_stack_ptr_t u_s,
- struct cds_lfs_node *node)
-{
- struct __cds_lfs_stack *s = u_s._s;
- struct cds_lfs_head *head = NULL;
- struct cds_lfs_head *new_head =
- caa_container_of(node, struct cds_lfs_head, node);
-
- for (;;) {
- struct cds_lfs_head *old_head = head;
-
- /*
- * node->next is still private at this point, no need to
- * perform a _CMM_STORE_SHARED().
- */
- node->next = &head->node;
- /*
- * uatomic_cmpxchg() implicit memory barrier orders earlier
- * stores to node before publication.
- */
- head = uatomic_cmpxchg(&s->head, old_head, new_head);
- if (old_head == head)
- break;
- }
- return !___cds_lfs_empty_head(head);
-}
-
-/*
- * __cds_lfs_pop: pop a node from the stack.
- *
- * Returns NULL if stack is empty.
- *
- * __cds_lfs_pop needs to be synchronized using one of the following
- * techniques:
- *
- * 1) Calling __cds_lfs_pop under rcu read lock critical section.
- * Both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
- * grace period to pass before freeing the returned node or pushing
- * the node back into the stack. It is valid to overwrite the content
- * of cds_lfs_node immediately after __cds_lfs_pop and
- * __cds_lfs_pop_all. No RCU read-side critical section is needed
- * around __cds_lfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop
- * and __cds_lfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
- * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-static inline
-struct cds_lfs_node *___cds_lfs_pop(cds_lfs_stack_ptr_t u_s)
-{
- struct __cds_lfs_stack *s = u_s._s;
-
- for (;;) {
- struct cds_lfs_head *head, *next_head;
- struct cds_lfs_node *next;
-
- head = _CMM_LOAD_SHARED(s->head);
- if (___cds_lfs_empty_head(head))
- return NULL; /* Empty stack */
-
- /*
- * Read head before head->next. Matches the implicit
- * memory barrier before uatomic_cmpxchg() in
- * cds_lfs_push.
- */
- cmm_smp_read_barrier_depends();
- next = _CMM_LOAD_SHARED(head->node.next);
- next_head = caa_container_of(next,
- struct cds_lfs_head, node);
- if (uatomic_cmpxchg(&s->head, head, next_head) == head)
- return &head->node;
- /* busy-loop if head changed under us */
- }
-}
-
-/*
- * __cds_lfs_pop_all: pop all nodes from a stack.
- *
- * __cds_lfs_pop_all does not require any synchronization with other
- * push, nor with other __cds_lfs_pop_all, but requires synchronization
- * matching the technique used to synchronize __cds_lfs_pop:
- *
- * 1) If __cds_lfs_pop is called under rcu read lock critical section,
- * both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
- * grace period to pass before freeing the returned node or pushing
- * the node back into the stack. It is valid to overwrite the content
- * of cds_lfs_node immediately after __cds_lfs_pop and
- * __cds_lfs_pop_all. No RCU read-side critical section is needed
- * around __cds_lfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
- * __cds_lfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
- * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-static inline
-struct cds_lfs_head *___cds_lfs_pop_all(cds_lfs_stack_ptr_t u_s)
-{
- struct __cds_lfs_stack *s = u_s._s;
-
- /*
- * Implicit memory barrier after uatomic_xchg() matches implicit
- * memory barrier before uatomic_cmpxchg() in cds_lfs_push. It
- * ensures that all nodes of the returned list are consistent.
- * There is no need to issue memory barriers when iterating on
- * the returned list, because the full memory barrier issued
- * prior to each uatomic_cmpxchg, which each write to head, are
- * taking care to order writes to each node prior to the full
- * memory barrier after this uatomic_xchg().
- */
- return uatomic_xchg(&s->head, NULL);
-}
-
-/*
- * cds_lfs_pop_lock: lock stack pop-protection mutex.
- */
-static inline void _cds_lfs_pop_lock(struct cds_lfs_stack *s)
-{
- int ret;
-
- ret = pthread_mutex_lock(&s->lock);
- assert(!ret);
-}
-
-/*
- * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
- */
-static inline void _cds_lfs_pop_unlock(struct cds_lfs_stack *s)
-{
- int ret;
-
- ret = pthread_mutex_unlock(&s->lock);
- assert(!ret);
-}
-
-/*
- * Call __cds_lfs_pop with an internal pop mutex held.
- */
-static inline
-struct cds_lfs_node *
-_cds_lfs_pop_blocking(struct cds_lfs_stack *s)
-{
- struct cds_lfs_node *retnode;
-
- _cds_lfs_pop_lock(s);
- retnode = ___cds_lfs_pop(s);
- _cds_lfs_pop_unlock(s);
- return retnode;
-}
-
-/*
- * Call __cds_lfs_pop_all with an internal pop mutex held.
- */
-static inline
-struct cds_lfs_head *
-_cds_lfs_pop_all_blocking(struct cds_lfs_stack *s)
-{
- struct cds_lfs_head *rethead;
-
- _cds_lfs_pop_lock(s);
- rethead = ___cds_lfs_pop_all(s);
- _cds_lfs_pop_unlock(s);
- return rethead;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_STATIC_LFSTACK_H */
+++ /dev/null
-#ifndef _URCU_RCULFQUEUE_STATIC_H
-#define _URCU_RCULFQUEUE_STATIC_H
-
-/*
- * rculfqueue-static.h
- *
- * Userspace RCU library - Lock-Free RCU Queue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfqueue.h for linking
- * dynamically with the userspace rcu library.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu-call-rcu.h>
-#include <urcu/uatomic.h>
-#include <urcu-pointer.h>
-#include <assert.h>
-#include <errno.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-struct cds_lfq_node_rcu_dummy {
- struct cds_lfq_node_rcu parent;
- struct rcu_head head;
- struct cds_lfq_queue_rcu *q;
-};
-
-/*
- * Lock-free RCU queue. Enqueue and dequeue operations hold a RCU read
- * lock to deal with cmpxchg ABA problem. This queue is *not* circular:
- * head points to the oldest node, tail points to the newest node.
- * A dummy node is kept to ensure enqueue and dequeue can always proceed
- * concurrently. Keeping a separate head and tail helps with large
- * queues: enqueue and dequeue can proceed concurrently without
- * wrestling for exclusive access to the same variables.
- *
- * Dequeue retry if it detects that it would be dequeueing the last node
- * (it means a dummy node dequeue-requeue is in progress). This ensures
- * that there is always at least one node in the queue.
- *
- * In the dequeue operation, we internally reallocate the dummy node
- * upon dequeue/requeue and use call_rcu to free the old one after a
- * grace period.
- */
-
-static inline
-struct cds_lfq_node_rcu *make_dummy(struct cds_lfq_queue_rcu *q,
- struct cds_lfq_node_rcu *next)
-{
- struct cds_lfq_node_rcu_dummy *dummy;
-
- dummy = malloc(sizeof(struct cds_lfq_node_rcu_dummy));
- assert(dummy);
- dummy->parent.next = next;
- dummy->parent.dummy = 1;
- dummy->q = q;
- return &dummy->parent;
-}
-
-static inline
-void free_dummy_cb(struct rcu_head *head)
-{
- struct cds_lfq_node_rcu_dummy *dummy =
- caa_container_of(head, struct cds_lfq_node_rcu_dummy, head);
- free(dummy);
-}
-
-static inline
-void rcu_free_dummy(struct cds_lfq_node_rcu *node)
-{
- struct cds_lfq_node_rcu_dummy *dummy;
-
- assert(node->dummy);
- dummy = caa_container_of(node, struct cds_lfq_node_rcu_dummy, parent);
- dummy->q->queue_call_rcu(&dummy->head, free_dummy_cb);
-}
-
-static inline
-void free_dummy(struct cds_lfq_node_rcu *node)
-{
- struct cds_lfq_node_rcu_dummy *dummy;
-
- assert(node->dummy);
- dummy = caa_container_of(node, struct cds_lfq_node_rcu_dummy, parent);
- free(dummy);
-}
-
-static inline
-void _cds_lfq_node_init_rcu(struct cds_lfq_node_rcu *node)
-{
- node->next = NULL;
- node->dummy = 0;
-}
-
-static inline
-void _cds_lfq_init_rcu(struct cds_lfq_queue_rcu *q,
- void queue_call_rcu(struct rcu_head *head,
- void (*func)(struct rcu_head *head)))
-{
- q->tail = make_dummy(q, NULL);
- q->head = q->tail;
- q->queue_call_rcu = queue_call_rcu;
-}
-
-/*
- * The queue should be emptied before calling destroy.
- *
- * Return 0 on success, -EPERM if queue is not empty.
- */
-static inline
-int _cds_lfq_destroy_rcu(struct cds_lfq_queue_rcu *q)
-{
- struct cds_lfq_node_rcu *head;
-
- head = rcu_dereference(q->head);
- if (!(head->dummy && head->next == NULL))
- return -EPERM; /* not empty */
- free_dummy(head);
- return 0;
-}
-
-/*
- * Should be called under rcu read lock critical section.
- */
-static inline
-void _cds_lfq_enqueue_rcu(struct cds_lfq_queue_rcu *q,
- struct cds_lfq_node_rcu *node)
-{
- /*
- * uatomic_cmpxchg() implicit memory barrier orders earlier stores to
- * node before publication.
- */
-
- for (;;) {
- struct cds_lfq_node_rcu *tail, *next;
-
- tail = rcu_dereference(q->tail);
- next = uatomic_cmpxchg(&tail->next, NULL, node);
- if (next == NULL) {
- /*
- * Tail was at the end of queue, we successfully
- * appended to it. Now move tail (another
- * enqueue might beat us to it, that's fine).
- */
- (void) uatomic_cmpxchg(&q->tail, tail, node);
- return;
- } else {
- /*
- * Failure to append to current tail.
- * Help moving tail further and retry.
- */
- (void) uatomic_cmpxchg(&q->tail, tail, next);
- continue;
- }
- }
-}
-
-static inline
-void enqueue_dummy(struct cds_lfq_queue_rcu *q)
-{
- struct cds_lfq_node_rcu *node;
-
- /* We need to reallocate to protect from ABA. */
- node = make_dummy(q, NULL);
- _cds_lfq_enqueue_rcu(q, node);
-}
-
-/*
- * Should be called under rcu read lock critical section.
- *
- * The caller must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_lfq_node_rcu structure.
- * Returns NULL if queue is empty.
- */
-static inline
-struct cds_lfq_node_rcu *_cds_lfq_dequeue_rcu(struct cds_lfq_queue_rcu *q)
-{
- for (;;) {
- struct cds_lfq_node_rcu *head, *next;
-
- head = rcu_dereference(q->head);
- next = rcu_dereference(head->next);
- if (head->dummy && next == NULL)
- return NULL; /* empty */
- /*
- * We never, ever allow dequeue to get to a state where
- * the queue is empty (we need at least one node in the
- * queue). This is ensured by checking if the head next
- * is NULL, which means we need to enqueue a dummy node
- * before we can hope dequeuing anything.
- */
- if (!next) {
- enqueue_dummy(q);
- next = rcu_dereference(head->next);
- }
- if (uatomic_cmpxchg(&q->head, head, next) != head)
- continue; /* Concurrently pushed. */
- if (head->dummy) {
- /* Free dummy after grace period. */
- rcu_free_dummy(head);
- continue; /* try again */
- }
- return head;
- }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFQUEUE_STATIC_H */
+++ /dev/null
-#ifndef _URCU_RCULFSTACK_STATIC_H
-#define _URCU_RCULFSTACK_STATIC_H
-
-/*
- * rculfstack-static.h
- *
- * Userspace RCU library - Lock-Free RCU Stack
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See rculfstack.h for linking
- * dynamically with the userspace rcu library.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <urcu/uatomic.h>
-#include <urcu-pointer.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-static inline
-void _cds_lfs_node_init_rcu(struct cds_lfs_node_rcu *node)
-{
-}
-
-static inline
-void _cds_lfs_init_rcu(struct cds_lfs_stack_rcu *s)
-{
- s->head = NULL;
-}
-
-/*
- * Lock-free stack push is not subject to ABA problem, so no need to
- * take the RCU read-side lock. Even if "head" changes between two
- * uatomic_cmpxchg() invocations here (being popped, and then pushed
- * again by one or more concurrent threads), the second
- * uatomic_cmpxchg() invocation only cares about pushing a new entry at
- * the head of the stack, ensuring consistency by making sure the new
- * node->next is the same pointer value as the value replaced as head.
- * It does not care about the content of the actual next node, so it can
- * very well be reallocated between the two uatomic_cmpxchg().
- *
- * We take the approach of expecting the stack to be usually empty, so
- * we first try an initial uatomic_cmpxchg() on a NULL old_head, and
- * retry if the old head was non-NULL (the value read by the first
- * uatomic_cmpxchg() is used as old head for the following loop). The
- * upside of this scheme is to minimize the amount of cacheline traffic,
- * always performing an exclusive cacheline access, rather than doing
- * non-exclusive followed by exclusive cacheline access (which would be
- * required if we first read the old head value). This design decision
- * might be revisited after more throrough benchmarking on various
- * platforms.
- *
- * Returns 0 if the stack was empty prior to adding the node.
- * Returns non-zero otherwise.
- */
-static inline
-int _cds_lfs_push_rcu(struct cds_lfs_stack_rcu *s,
- struct cds_lfs_node_rcu *node)
-{
- struct cds_lfs_node_rcu *head = NULL;
-
- for (;;) {
- struct cds_lfs_node_rcu *old_head = head;
-
- node->next = head;
- /*
- * uatomic_cmpxchg() implicit memory barrier orders earlier
- * stores to node before publication.
- */
- head = uatomic_cmpxchg(&s->head, old_head, node);
- if (old_head == head)
- break;
- }
- return (int) !!((unsigned long) head);
-}
-
-/*
- * Should be called under rcu read-side lock.
- *
- * The caller must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_lfs_node_rcu structure.
- * Returns NULL if stack is empty.
- */
-static inline
-struct cds_lfs_node_rcu *
-_cds_lfs_pop_rcu(struct cds_lfs_stack_rcu *s)
-{
- for (;;) {
- struct cds_lfs_node_rcu *head;
-
- head = rcu_dereference(s->head);
- if (head) {
- struct cds_lfs_node_rcu *next = rcu_dereference(head->next);
-
- if (uatomic_cmpxchg(&s->head, head, next) == head) {
- return head;
- } else {
- /* Concurrent modification. Retry. */
- continue;
- }
- } else {
- /* Empty stack */
- return NULL;
- }
- }
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_RCULFSTACK_STATIC_H */
+++ /dev/null
-#ifndef _URCU_BP_STATIC_H
-#define _URCU_BP_STATIC_H
-
-/*
- * urcu-bp-static.h
- *
- * Userspace RCU header.
- *
- * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
- * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <unistd.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic.h>
-#include <urcu/list.h>
-#include <urcu/tls-compat.h>
-#include <urcu/debug.h>
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-enum rcu_state {
- RCU_READER_ACTIVE_CURRENT,
- RCU_READER_ACTIVE_OLD,
- RCU_READER_INACTIVE,
-};
-
-/*
- * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a
- * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
- */
-#define RCU_GP_COUNT (1UL << 0)
-/* Use the amount of bits equal to half of the architecture long size */
-#define RCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2))
-#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
-
-/*
- * Used internally by _rcu_read_lock.
- */
-extern void rcu_bp_register(void);
-
-struct rcu_gp {
- /*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken.
- * Read by both writer and readers.
- */
- unsigned long ctr;
-} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
-
-extern struct rcu_gp rcu_gp;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- unsigned long ctr;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- pthread_t tid;
- int alloc; /* registry entry allocated */
-};
-
-/*
- * Bulletproof version keeps a pointer to a registry not part of the TLS.
- * Adds a pointer dereference on the read-side, but won't require to unregister
- * the reader thread.
- */
-extern DECLARE_URCU_TLS(struct rcu_reader *, rcu_reader);
-
-extern int urcu_bp_has_sys_membarrier;
-
-static inline void urcu_bp_smp_mb_slave(void)
-{
- if (caa_likely(urcu_bp_has_sys_membarrier))
- cmm_barrier();
- else
- cmm_smp_mb();
-}
-
-static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
-{
- unsigned long v;
-
- if (ctr == NULL)
- return RCU_READER_INACTIVE;
- /*
- * Make sure both tests below are done on the same version of *value
- * to insure consistency.
- */
- v = CMM_LOAD_SHARED(*ctr);
- if (!(v & RCU_GP_CTR_NEST_MASK))
- return RCU_READER_INACTIVE;
- if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
- return RCU_READER_ACTIVE_CURRENT;
- return RCU_READER_ACTIVE_OLD;
-}
-
-/*
- * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as
- * the per-thread rcu_reader.ctr) has the upper bits containing a count of
- * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
- * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
- * _rcu_read_lock() happen before the subsequent read-side critical section.
- */
-static inline void _rcu_read_lock_update(unsigned long tmp)
-{
- if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
- urcu_bp_smp_mb_slave();
- } else
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp + RCU_GP_COUNT);
-}
-
-/*
- * Enter an RCU read-side critical section.
- *
- * The first cmm_barrier() call ensures that the compiler does not reorder
- * the body of _rcu_read_lock() with a mutex.
- *
- * This function and its helper are both less than 10 lines long. The
- * intent is that this function meets the 10-line criterion in LGPL,
- * allowing this function to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_read_lock(void)
-{
- unsigned long tmp;
-
- if (caa_unlikely(!URCU_TLS(rcu_reader)))
- rcu_bp_register(); /* If not yet registered. */
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
- tmp = URCU_TLS(rcu_reader)->ctr;
- urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK);
- _rcu_read_lock_update(tmp);
-}
-
-/*
- * Exit an RCU read-side critical section. This function is less than
- * 10 lines of code, and is intended to be usable by non-LGPL code, as
- * called out in LGPL.
- */
-static inline void _rcu_read_unlock(void)
-{
- unsigned long tmp;
-
- tmp = URCU_TLS(rcu_reader)->ctr;
- urcu_assert(tmp & RCU_GP_CTR_NEST_MASK);
- /* Finish using rcu before decrementing the pointer. */
- urcu_bp_smp_mb_slave();
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader)->ctr, tmp - RCU_GP_COUNT);
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-/*
- * Returns whether within a RCU read-side critical section.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline int _rcu_read_ongoing(void)
-{
- if (caa_unlikely(!URCU_TLS(rcu_reader)))
- rcu_bp_register(); /* If not yet registered. */
- return URCU_TLS(rcu_reader)->ctr & RCU_GP_CTR_NEST_MASK;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_BP_STATIC_H */
+++ /dev/null
-#ifndef _URCU_POINTER_STATIC_H
-#define _URCU_POINTER_STATIC_H
-
-/*
- * urcu-pointer-static.h
- *
- * Userspace RCU header. Operations on pointers.
- *
- * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
- * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * _rcu_dereference - reads (copy) a RCU-protected pointer to a local variable
- * into a RCU read-side critical section. The pointer can later be safely
- * dereferenced within the critical section.
- *
- * This ensures that the pointer copy is invariant thorough the whole critical
- * section.
- *
- * Inserts memory barriers on architectures that require them (currently only
- * Alpha) and documents which pointers are protected by RCU.
- *
- * The compiler memory barrier in CMM_LOAD_SHARED() ensures that value-speculative
- * optimizations (e.g. VSS: Value Speculation Scheduling) does not perform the
- * data read before the pointer read by speculating the value of the pointer.
- * Correct ordering is ensured because the pointer is read as a volatile access.
- * This acts as a global side-effect operation, which forbids reordering of
- * dependent memory operations. Note that such concern about dependency-breaking
- * optimizations will eventually be taken care of by the "memory_order_consume"
- * addition to forthcoming C++ standard.
- *
- * Should match rcu_assign_pointer() or rcu_xchg_pointer().
- *
- * This macro is less than 10 lines long. The intent is that this macro
- * meets the 10-line criterion in LGPL, allowing this function to be
- * expanded directly in non-LGPL code.
- */
-#define _rcu_dereference(p) \
- __extension__ \
- ({ \
- __typeof__(p) _________p1 = CMM_LOAD_SHARED(p); \
- cmm_smp_read_barrier_depends(); \
- (_________p1); \
- })
-
-/**
- * _rcu_cmpxchg_pointer - same as rcu_assign_pointer, but tests if the pointer
- * is as expected by "old". If succeeds, returns the previous pointer to the
- * data structure, which can be safely freed after waiting for a quiescent state
- * using synchronize_rcu(). If fails (unexpected value), returns old (which
- * should not be freed !).
- *
- * This macro is less than 10 lines long. The intent is that this macro
- * meets the 10-line criterion in LGPL, allowing this function to be
- * expanded directly in non-LGPL code.
- */
-#define _rcu_cmpxchg_pointer(p, old, _new) \
- __extension__ \
- ({ \
- __typeof__(*p) _________pold = (old); \
- __typeof__(*p) _________pnew = (_new); \
- if (!__builtin_constant_p(_new) || \
- ((_new) != NULL)) \
- cmm_wmb(); \
- uatomic_cmpxchg(p, _________pold, _________pnew); \
- })
-
-/**
- * _rcu_xchg_pointer - same as rcu_assign_pointer, but returns the previous
- * pointer to the data structure, which can be safely freed after waiting for a
- * quiescent state using synchronize_rcu().
- *
- * This macro is less than 10 lines long. The intent is that this macro
- * meets the 10-line criterion in LGPL, allowing this function to be
- * expanded directly in non-LGPL code.
- */
-#define _rcu_xchg_pointer(p, v) \
- __extension__ \
- ({ \
- __typeof__(*p) _________pv = (v); \
- if (!__builtin_constant_p(v) || \
- ((v) != NULL)) \
- cmm_wmb(); \
- uatomic_xchg(p, _________pv); \
- })
-
-
-#define _rcu_set_pointer(p, v) \
- do { \
- __typeof__(*p) _________pv = (v); \
- if (!__builtin_constant_p(v) || \
- ((v) != NULL)) \
- cmm_wmb(); \
- uatomic_set(p, _________pv); \
- } while (0)
-
-/**
- * _rcu_assign_pointer - assign (publicize) a pointer to a new data structure
- * meant to be read by RCU read-side critical sections. Returns the assigned
- * value.
- *
- * Documents which pointers will be dereferenced by RCU read-side critical
- * sections and adds the required memory barriers on architectures requiring
- * them. It also makes sure the compiler does not reorder code initializing the
- * data structure before its publication.
- *
- * Should match rcu_dereference().
- *
- * This macro is less than 10 lines long. The intent is that this macro
- * meets the 10-line criterion in LGPL, allowing this function to be
- * expanded directly in non-LGPL code.
- */
-#define _rcu_assign_pointer(p, v) _rcu_set_pointer(&(p), v)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_POINTER_STATIC_H */
+++ /dev/null
-#ifndef _URCU_QSBR_STATIC_H
-#define _URCU_QSBR_STATIC_H
-
-/*
- * urcu-qsbr-static.h
- *
- * Userspace RCU QSBR header.
- *
- * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
- * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <limits.h>
-#include <unistd.h>
-#include <stdint.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic.h>
-#include <urcu/list.h>
-#include <urcu/futex.h>
-#include <urcu/tls-compat.h>
-#include <urcu/debug.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-enum rcu_state {
- RCU_READER_ACTIVE_CURRENT,
- RCU_READER_ACTIVE_OLD,
- RCU_READER_INACTIVE,
-};
-
-#define RCU_GP_ONLINE (1UL << 0)
-#define RCU_GP_CTR (1UL << 1)
-
-struct rcu_gp {
- /*
- * Global quiescent period counter with low-order bits unused.
- * Using a int rather than a char to eliminate false register
- * dependencies causing stalls on some architectures.
- */
- unsigned long ctr;
-
- int32_t futex;
-} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
-
-extern struct rcu_gp rcu_gp;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- unsigned long ctr;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- int waiting;
- pthread_t tid;
- /* Reader registered flag, for internal checks. */
- unsigned int registered:1;
-};
-
-extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
-
-/*
- * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
- */
-static inline void wake_up_gp(void)
-{
- if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(rcu_reader).waiting))) {
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).waiting, 0);
- cmm_smp_mb();
- if (uatomic_read(&rcu_gp.futex) != -1)
- return;
- uatomic_set(&rcu_gp.futex, 0);
- /*
- * Ignoring return value until we can make this function
- * return something (because urcu_die() is not publicly
- * exposed).
- */
- (void) futex_noasync(&rcu_gp.futex, FUTEX_WAKE, 1,
- NULL, NULL, 0);
- }
-}
-
-static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
-{
- unsigned long v;
-
- v = CMM_LOAD_SHARED(*ctr);
- if (!v)
- return RCU_READER_INACTIVE;
- if (v == rcu_gp.ctr)
- return RCU_READER_ACTIVE_CURRENT;
- return RCU_READER_ACTIVE_OLD;
-}
-
-/*
- * Enter an RCU read-side critical section.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_read_lock(void)
-{
- urcu_assert(URCU_TLS(rcu_reader).ctr);
-}
-
-/*
- * Exit an RCU read-side critical section.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_read_unlock(void)
-{
- urcu_assert(URCU_TLS(rcu_reader).ctr);
-}
-
-/*
- * Returns whether within a RCU read-side critical section.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline int _rcu_read_ongoing(void)
-{
- return URCU_TLS(rcu_reader).ctr;
-}
-
-/*
- * This is a helper function for _rcu_quiescent_state().
- * The first cmm_smp_mb() ensures memory accesses in the prior read-side
- * critical sections are not reordered with store to
- * URCU_TLS(rcu_reader).ctr, and ensures that mutexes held within an
- * offline section that would happen to end with this
- * rcu_quiescent_state() call are not reordered with
- * store to URCU_TLS(rcu_reader).ctr.
- */
-static inline void _rcu_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
-{
- cmm_smp_mb();
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, gp_ctr);
- cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
- wake_up_gp();
- cmm_smp_mb();
-}
-
-/*
- * Inform RCU of a quiescent state.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- *
- * We skip the memory barriers and gp store if our local ctr already
- * matches the global rcu_gp.ctr value: this is OK because a prior
- * _rcu_quiescent_state() or _rcu_thread_online() already updated it
- * within our thread, so we have no quiescent state to report.
- */
-static inline void _rcu_quiescent_state(void)
-{
- unsigned long gp_ctr;
-
- urcu_assert(URCU_TLS(rcu_reader).registered);
- if ((gp_ctr = CMM_LOAD_SHARED(rcu_gp.ctr)) == URCU_TLS(rcu_reader).ctr)
- return;
- _rcu_quiescent_state_update_and_wakeup(gp_ctr);
-}
-
-/*
- * Take a thread offline, prohibiting it from entering further RCU
- * read-side critical sections.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_thread_offline(void)
-{
- urcu_assert(URCU_TLS(rcu_reader).registered);
- cmm_smp_mb();
- CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, 0);
- cmm_smp_mb(); /* write URCU_TLS(rcu_reader).ctr before read futex */
- wake_up_gp();
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-/*
- * Bring a thread online, allowing it to once again enter RCU
- * read-side critical sections.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_thread_online(void)
-{
- urcu_assert(URCU_TLS(rcu_reader).registered);
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, CMM_LOAD_SHARED(rcu_gp.ctr));
- cmm_smp_mb();
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_QSBR_STATIC_H */
+++ /dev/null
-#ifndef _URCU_STATIC_H
-#define _URCU_STATIC_H
-
-/*
- * urcu-static.h
- *
- * Userspace RCU header.
- *
- * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
- * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- *
- * IBM's contributions to this file may be relicensed under LGPLv2 or later.
- */
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <stdint.h>
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-#include <urcu/system.h>
-#include <urcu/uatomic.h>
-#include <urcu/list.h>
-#include <urcu/futex.h>
-#include <urcu/tls-compat.h>
-#include <urcu/rand-compat.h>
-#include <urcu/debug.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* Default is RCU_MEMBARRIER */
-#if !defined(RCU_MEMBARRIER) && !defined(RCU_MB) && !defined(RCU_SIGNAL)
-#define RCU_MEMBARRIER
-#endif
-
-/*
- * This code section can only be included in LGPL 2.1 compatible source code.
- * See below for the function call wrappers which can be used in code meant to
- * be only linked with the Userspace RCU library. This comes with a small
- * performance degradation on the read-side due to the added function calls.
- * This is required to permit relinking with newer versions of the library.
- */
-
-/*
- * The signal number used by the RCU library can be overridden with
- * -DSIGRCU= when compiling the library.
- * Provide backward compatibility for liburcu 0.3.x SIGURCU.
- */
-#ifdef SIGURCU
-#define SIGRCU SIGURCU
-#endif
-
-#ifndef SIGRCU
-#define SIGRCU SIGUSR1
-#endif
-
-enum rcu_state {
- RCU_READER_ACTIVE_CURRENT,
- RCU_READER_ACTIVE_OLD,
- RCU_READER_INACTIVE,
-};
-
-/*
- * Slave barriers are only guaranteed to be ordered wrt master barriers.
- *
- * The pair ordering is detailed as (O: ordered, X: not ordered) :
- * slave master
- * slave X O
- * master O O
- */
-
-#ifdef RCU_MEMBARRIER
-extern int rcu_has_sys_membarrier;
-
-static inline void smp_mb_slave(void)
-{
- if (caa_likely(rcu_has_sys_membarrier))
- cmm_barrier();
- else
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_MB
-static inline void smp_mb_slave(void)
-{
- cmm_smp_mb();
-}
-#endif
-
-#ifdef RCU_SIGNAL
-static inline void smp_mb_slave(void)
-{
- cmm_barrier();
-}
-#endif
-
-/*
- * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use
- * a full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
- */
-#define RCU_GP_COUNT (1UL << 0)
-/* Use the amount of bits equal to half of the architecture long size */
-#define RCU_GP_CTR_PHASE (1UL << (sizeof(unsigned long) << 2))
-#define RCU_GP_CTR_NEST_MASK (RCU_GP_CTR_PHASE - 1)
-
-struct rcu_gp {
- /*
- * Global grace period counter.
- * Contains the current RCU_GP_CTR_PHASE.
- * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
- * Written to only by writer with mutex taken.
- * Read by both writer and readers.
- */
- unsigned long ctr;
-
- int32_t futex;
-} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
-
-extern struct rcu_gp rcu_gp;
-
-struct rcu_reader {
- /* Data used by both reader and synchronize_rcu() */
- unsigned long ctr;
- char need_mb;
- /* Data used for registry */
- struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
- pthread_t tid;
- /* Reader registered flag, for internal checks. */
- unsigned int registered:1;
-};
-
-extern DECLARE_URCU_TLS(struct rcu_reader, rcu_reader);
-
-/*
- * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
- */
-static inline void wake_up_gp(void)
-{
- if (caa_unlikely(uatomic_read(&rcu_gp.futex) == -1)) {
- uatomic_set(&rcu_gp.futex, 0);
- /*
- * Ignoring return value until we can make this function
- * return something (because urcu_die() is not publicly
- * exposed).
- */
- (void) futex_async(&rcu_gp.futex, FUTEX_WAKE, 1,
- NULL, NULL, 0);
- }
-}
-
-static inline enum rcu_state rcu_reader_state(unsigned long *ctr)
-{
- unsigned long v;
-
- /*
- * Make sure both tests below are done on the same version of *value
- * to insure consistency.
- */
- v = CMM_LOAD_SHARED(*ctr);
- if (!(v & RCU_GP_CTR_NEST_MASK))
- return RCU_READER_INACTIVE;
- if (!((v ^ rcu_gp.ctr) & RCU_GP_CTR_PHASE))
- return RCU_READER_ACTIVE_CURRENT;
- return RCU_READER_ACTIVE_OLD;
-}
-
-/*
- * Helper for _rcu_read_lock(). The format of rcu_gp.ctr (as well as
- * the per-thread rcu_reader.ctr) has the upper bits containing a count of
- * _rcu_read_lock() nesting, and a lower-order bit that contains either zero
- * or RCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
- * _rcu_read_lock() happen before the subsequent read-side critical section.
- */
-static inline void _rcu_read_lock_update(unsigned long tmp)
-{
- if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) {
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, _CMM_LOAD_SHARED(rcu_gp.ctr));
- smp_mb_slave();
- } else
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp + RCU_GP_COUNT);
-}
-
-/*
- * Enter an RCU read-side critical section.
- *
- * The first cmm_barrier() call ensures that the compiler does not reorder
- * the body of _rcu_read_lock() with a mutex.
- *
- * This function and its helper are both less than 10 lines long. The
- * intent is that this function meets the 10-line criterion in LGPL,
- * allowing this function to be invoked directly from non-LGPL code.
- */
-static inline void _rcu_read_lock(void)
-{
- unsigned long tmp;
-
- urcu_assert(URCU_TLS(rcu_reader).registered);
- cmm_barrier();
- tmp = URCU_TLS(rcu_reader).ctr;
- urcu_assert((tmp & RCU_GP_CTR_NEST_MASK) != RCU_GP_CTR_NEST_MASK);
- _rcu_read_lock_update(tmp);
-}
-
-/*
- * This is a helper function for _rcu_read_unlock().
- *
- * The first smp_mb_slave() call ensures that the critical section is
- * seen to precede the store to rcu_reader.ctr.
- * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
- * before reading the update-side futex.
- */
-static inline void _rcu_read_unlock_update_and_wakeup(unsigned long tmp)
-{
- if (caa_likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) {
- smp_mb_slave();
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT);
- smp_mb_slave();
- wake_up_gp();
- } else
- _CMM_STORE_SHARED(URCU_TLS(rcu_reader).ctr, tmp - RCU_GP_COUNT);
-}
-
-/*
- * Exit an RCU read-side crtical section. Both this function and its
- * helper are smaller than 10 lines of code, and are intended to be
- * usable by non-LGPL code, as called out in LGPL.
- */
-static inline void _rcu_read_unlock(void)
-{
- unsigned long tmp;
-
- urcu_assert(URCU_TLS(rcu_reader).registered);
- tmp = URCU_TLS(rcu_reader).ctr;
- urcu_assert(tmp & RCU_GP_CTR_NEST_MASK);
- _rcu_read_unlock_update_and_wakeup(tmp);
- cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
-}
-
-/*
- * Returns whether within a RCU read-side critical section.
- *
- * This function is less than 10 lines long. The intent is that this
- * function meets the 10-line criterion for LGPL, allowing this function
- * to be invoked directly from non-LGPL code.
- */
-static inline int _rcu_read_ongoing(void)
-{
- return URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_STATIC_H */
+++ /dev/null
-#ifndef _URCU_WFCQUEUE_STATIC_H
-#define _URCU_WFCQUEUE_STATIC_H
-
-/*
- * urcu/static/wfcqueue.h
- *
- * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfcqueue.h for
- * linking dynamically with the userspace rcu library.
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <poll.h>
-#include <stdbool.h>
-#include <urcu/compiler.h>
-#include <urcu/uatomic.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Concurrent queue with wait-free enqueue/blocking dequeue.
- *
- * This queue has been designed and implemented collaboratively by
- * Mathieu Desnoyers and Lai Jiangshan. Inspired from
- * half-wait-free/half-blocking queue implementation done by Paul E.
- * McKenney.
- *
- * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * Legend:
- * [1] cds_wfcq_enqueue
- * [2] __cds_wfcq_splice (destination queue)
- * [3] __cds_wfcq_dequeue
- * [4] __cds_wfcq_splice (source queue)
- * [5] __cds_wfcq_first
- * [6] __cds_wfcq_next
- *
- * [1] [2] [3] [4] [5] [6]
- * [1] - - - - - -
- * [2] - - - - - -
- * [3] - - X X X X
- * [4] - - X - X X
- * [5] - - X X - -
- * [6] - - X X - -
- *
- * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
- *
- * For convenience, cds_wfcq_dequeue_blocking() and
- * cds_wfcq_splice_blocking() hold the dequeue lock.
- *
- * Besides locking, mutual exclusion of dequeue, splice and iteration
- * can be ensured by performing all of those operations from a single
- * thread, without requiring any lock.
- */
-
-#define WFCQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
-#define WFCQ_WAIT 10 /* Wait 10 ms if being set */
-
-/*
- * cds_wfcq_node_init: initialize wait-free queue node.
- */
-static inline void _cds_wfcq_node_init(struct cds_wfcq_node *node)
-{
- node->next = NULL;
-}
-
-/*
- * cds_wfcq_init: initialize wait-free queue (with lock). Pair with
- * cds_wfcq_destroy().
- */
-static inline void _cds_wfcq_init(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- int ret;
-
- /* Set queue head and tail */
- _cds_wfcq_node_init(&head->node);
- tail->p = &head->node;
- ret = pthread_mutex_init(&head->lock, NULL);
- assert(!ret);
-}
-
-/*
- * cds_wfcq_destroy: destroy wait-free queue (with lock). Pair with
- * cds_wfcq_init().
- */
-static inline void _cds_wfcq_destroy(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- int ret = pthread_mutex_destroy(&head->lock);
- assert(!ret);
-}
-
-/*
- * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
- * pair with any destroy function.
- */
-static inline void ___cds_wfcq_init(struct __cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- /* Set queue head and tail */
- _cds_wfcq_node_init(&head->node);
- tail->p = &head->node;
-}
-
-/*
- * cds_wfcq_empty: return whether wait-free queue is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- *
- * We perform the test on head->node.next to check if the queue is
- * possibly empty, but we confirm this by checking if the tail pointer
- * points to the head node because the tail pointer is the linearisation
- * point of the enqueuers. Just checking the head next pointer could
- * make a queue appear empty if an enqueuer is preempted for a long time
- * between xchg() and setting the previous node's next pointer.
- */
-static inline bool _cds_wfcq_empty(cds_wfcq_head_ptr_t u_head,
- struct cds_wfcq_tail *tail)
-{
- struct __cds_wfcq_head *head = u_head._h;
- /*
- * Queue is empty if no node is pointed by head->node.next nor
- * tail->p. Even though the tail->p check is sufficient to find
- * out of the queue is empty, we first check head->node.next as a
- * common case to ensure that dequeuers do not frequently access
- * enqueuer's tail->p cache line.
- */
- return CMM_LOAD_SHARED(head->node.next) == NULL
- && CMM_LOAD_SHARED(tail->p) == &head->node;
-}
-
-static inline void _cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- int ret;
-
- ret = pthread_mutex_lock(&head->lock);
- assert(!ret);
-}
-
-static inline void _cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- int ret;
-
- ret = pthread_mutex_unlock(&head->lock);
- assert(!ret);
-}
-
-static inline bool ___cds_wfcq_append(cds_wfcq_head_ptr_t u_head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *new_head,
- struct cds_wfcq_node *new_tail)
-{
- struct __cds_wfcq_head *head = u_head._h;
- struct cds_wfcq_node *old_tail;
-
- /*
- * Implicit memory barrier before uatomic_xchg() orders earlier
- * stores to data structure containing node and setting
- * node->next to NULL before publication.
- */
- old_tail = uatomic_xchg(&tail->p, new_tail);
-
- /*
- * Implicit memory barrier after uatomic_xchg() orders store to
- * q->tail before store to old_tail->next.
- *
- * At this point, dequeuers see a NULL tail->p->next, which
- * indicates that the queue is being appended to. The following
- * store will append "node" to the queue from a dequeuer
- * perspective.
- */
- CMM_STORE_SHARED(old_tail->next, new_head);
- /*
- * Return false if queue was empty prior to adding the node,
- * else return true.
- */
- return old_tail != &head->node;
-}
-
-/*
- * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
- *
- * Issues a full memory barrier before enqueue. No mutual exclusion is
- * required.
- *
- * Returns false if the queue was empty prior to adding the node.
- * Returns true otherwise.
- */
-static inline bool _cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *new_tail)
-{
- return ___cds_wfcq_append(head, tail, new_tail, new_tail);
-}
-
-/*
- * ___cds_wfcq_busy_wait: adaptative busy-wait.
- *
- * Returns 1 if nonblocking and needs to block, 0 otherwise.
- */
-static inline bool
-___cds_wfcq_busy_wait(int *attempt, int blocking)
-{
- if (!blocking)
- return 1;
- if (++(*attempt) >= WFCQ_ADAPT_ATTEMPTS) {
- (void) poll(NULL, 0, WFCQ_WAIT); /* Wait for 10ms */
- *attempt = 0;
- } else {
- caa_cpu_relax();
- }
- return 0;
-}
-
-/*
- * Waiting for enqueuer to complete enqueue and return the next node.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_node_sync_next(struct cds_wfcq_node *node, int blocking)
-{
- struct cds_wfcq_node *next;
- int attempt = 0;
-
- /*
- * Adaptative busy-looping waiting for enqueuer to complete enqueue.
- */
- while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- if (___cds_wfcq_busy_wait(&attempt, blocking))
- return CDS_WFCQ_WOULDBLOCK;
- }
-
- return next;
-}
-
-static inline struct cds_wfcq_node *
-___cds_wfcq_first(cds_wfcq_head_ptr_t u_head,
- struct cds_wfcq_tail *tail,
- int blocking)
-{
- struct __cds_wfcq_head *head = u_head._h;
- struct cds_wfcq_node *node;
-
- if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail))
- return NULL;
- node = ___cds_wfcq_node_sync_next(&head->node, blocking);
- /* Load head->node.next before loading node's content */
- cmm_smp_read_barrier_depends();
- return node;
-}
-
-/*
- * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- *
- * Used by for-like iteration macros in urcu/wfqueue.h:
- * __cds_wfcq_for_each_blocking()
- * __cds_wfcq_for_each_blocking_safe()
- *
- * Returns NULL if queue is empty, first node otherwise.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_first_blocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_first(head, tail, 1);
-}
-
-
-/*
- * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
- *
- * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
- * it needs to block.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_first_nonblocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_first(head, tail, 0);
-}
-
-static inline struct cds_wfcq_node *
-___cds_wfcq_next(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node,
- int blocking)
-{
- struct cds_wfcq_node *next;
-
- /*
- * Even though the following tail->p check is sufficient to find
- * out if we reached the end of the queue, we first check
- * node->next as a common case to ensure that iteration on nodes
- * do not frequently access enqueuer's tail->p cache line.
- */
- if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- /* Load node->next before tail->p */
- cmm_smp_rmb();
- if (CMM_LOAD_SHARED(tail->p) == node)
- return NULL;
- next = ___cds_wfcq_node_sync_next(node, blocking);
- }
- /* Load node->next before loading next's content */
- cmm_smp_read_barrier_depends();
- return next;
-}
-
-/*
- * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- *
- * Used by for-like iteration macros in urcu/wfqueue.h:
- * __cds_wfcq_for_each_blocking()
- * __cds_wfcq_for_each_blocking_safe()
- *
- * Returns NULL if reached end of queue, non-NULL next queue node
- * otherwise.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_next_blocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node)
-{
- return ___cds_wfcq_next(head, tail, node, 1);
-}
-
-/*
- * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
- *
- * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
- * it needs to block.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_next_nonblocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node)
-{
- return ___cds_wfcq_next(head, tail, node, 0);
-}
-
-static inline struct cds_wfcq_node *
-___cds_wfcq_dequeue_with_state(cds_wfcq_head_ptr_t u_head,
- struct cds_wfcq_tail *tail,
- int *state,
- int blocking)
-{
- struct __cds_wfcq_head *head = u_head._h;
- struct cds_wfcq_node *node, *next;
-
- if (state)
- *state = 0;
-
- if (_cds_wfcq_empty(__cds_wfcq_head_cast(head), tail)) {
- return NULL;
- }
-
- node = ___cds_wfcq_node_sync_next(&head->node, blocking);
- if (!blocking && node == CDS_WFCQ_WOULDBLOCK) {
- return CDS_WFCQ_WOULDBLOCK;
- }
-
- if ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- /*
- * @node is probably the only node in the queue.
- * Try to move the tail to &q->head.
- * q->head.next is set to NULL here, and stays
- * NULL if the cmpxchg succeeds. Should the
- * cmpxchg fail due to a concurrent enqueue, the
- * q->head.next will be set to the next node.
- * The implicit memory barrier before
- * uatomic_cmpxchg() orders load node->next
- * before loading q->tail.
- * The implicit memory barrier before uatomic_cmpxchg
- * orders load q->head.next before loading node's
- * content.
- */
- _cds_wfcq_node_init(&head->node);
- if (uatomic_cmpxchg(&tail->p, node, &head->node) == node) {
- if (state)
- *state |= CDS_WFCQ_STATE_LAST;
- return node;
- }
- next = ___cds_wfcq_node_sync_next(node, blocking);
- /*
- * In nonblocking mode, if we would need to block to
- * get node's next, set the head next node pointer
- * (currently NULL) back to its original value.
- */
- if (!blocking && next == CDS_WFCQ_WOULDBLOCK) {
- head->node.next = node;
- return CDS_WFCQ_WOULDBLOCK;
- }
- }
-
- /*
- * Move queue head forward.
- */
- head->node.next = next;
-
- /* Load q->head.next before loading node's content */
- cmm_smp_read_barrier_depends();
- return node;
-}
-
-/*
- * __cds_wfcq_dequeue_with_state_blocking: dequeue node from queue, with state.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * It is valid to reuse and free a dequeued node immediately.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail, int *state)
-{
- return ___cds_wfcq_dequeue_with_state(head, tail, state, 1);
-}
-
-/*
- * ___cds_wfcq_dequeue_blocking: dequeue node from queue.
- *
- * Same as __cds_wfcq_dequeue_with_state_blocking, but without saving
- * state.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_dequeue_blocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
-}
-
-/*
- * __cds_wfcq_dequeue_with_state_nonblocking: dequeue node, with state.
- *
- * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
- * if it needs to block.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_dequeue_with_state_nonblocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail, int *state)
-{
- return ___cds_wfcq_dequeue_with_state(head, tail, state, 0);
-}
-
-/*
- * ___cds_wfcq_dequeue_nonblocking: dequeue node from queue.
- *
- * Same as __cds_wfcq_dequeue_with_state_nonblocking, but without saving
- * state.
- */
-static inline struct cds_wfcq_node *
-___cds_wfcq_dequeue_nonblocking(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, NULL);
-}
-
-/*
- * __cds_wfcq_splice: enqueue all src_q nodes at the end of dest_q.
- *
- * Dequeue all nodes from src_q.
- * dest_q must be already initialized.
- * Mutual exclusion for src_q should be ensured by the caller as
- * specified in the "Synchronisation table".
- * Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue.
- */
-static inline enum cds_wfcq_ret
-___cds_wfcq_splice(
- cds_wfcq_head_ptr_t u_dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t u_src_q_head,
- struct cds_wfcq_tail *src_q_tail,
- int blocking)
-{
- struct __cds_wfcq_head *dest_q_head = u_dest_q_head._h;
- struct __cds_wfcq_head *src_q_head = u_src_q_head._h;
- struct cds_wfcq_node *head, *tail;
- int attempt = 0;
-
- /*
- * Initial emptiness check to speed up cases where queue is
- * empty: only require loads to check if queue is empty.
- */
- if (_cds_wfcq_empty(__cds_wfcq_head_cast(src_q_head), src_q_tail))
- return CDS_WFCQ_RET_SRC_EMPTY;
-
- for (;;) {
- /*
- * Open-coded _cds_wfcq_empty() by testing result of
- * uatomic_xchg, as well as tail pointer vs head node
- * address.
- */
- head = uatomic_xchg(&src_q_head->node.next, NULL);
- if (head)
- break; /* non-empty */
- if (CMM_LOAD_SHARED(src_q_tail->p) == &src_q_head->node)
- return CDS_WFCQ_RET_SRC_EMPTY;
- if (___cds_wfcq_busy_wait(&attempt, blocking))
- return CDS_WFCQ_RET_WOULDBLOCK;
- }
-
- /*
- * Memory barrier implied before uatomic_xchg() orders store to
- * src_q->head before store to src_q->tail. This is required by
- * concurrent enqueue on src_q, which exchanges the tail before
- * updating the previous tail's next pointer.
- */
- tail = uatomic_xchg(&src_q_tail->p, &src_q_head->node);
-
- /*
- * Append the spliced content of src_q into dest_q. Does not
- * require mutual exclusion on dest_q (wait-free).
- */
- if (___cds_wfcq_append(__cds_wfcq_head_cast(dest_q_head), dest_q_tail,
- head, tail))
- return CDS_WFCQ_RET_DEST_NON_EMPTY;
- else
- return CDS_WFCQ_RET_DEST_EMPTY;
-}
-
-/*
- * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Dequeue all nodes from src_q.
- * dest_q must be already initialized.
- * Mutual exclusion for src_q should be ensured by the caller as
- * specified in the "Synchronisation table".
- * Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
- */
-static inline enum cds_wfcq_ret
-___cds_wfcq_splice_blocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
- src_q_head, src_q_tail, 1);
-}
-
-/*
- * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Same as __cds_wfcq_splice_blocking, but returns
- * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
- */
-static inline enum cds_wfcq_ret
-___cds_wfcq_splice_nonblocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- return ___cds_wfcq_splice(dest_q_head, dest_q_tail,
- src_q_head, src_q_tail, 0);
-}
-
-/*
- * cds_wfcq_dequeue_with_state_blocking: dequeue a node from a wait-free queue.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Mutual exclusion with cds_wfcq_splice_blocking and dequeue lock is
- * ensured.
- * It is valid to reuse and free a dequeued node immediately.
- */
-static inline struct cds_wfcq_node *
-_cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail, int *state)
-{
- struct cds_wfcq_node *retval;
-
- _cds_wfcq_dequeue_lock(head, tail);
- retval = ___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_cast(head),
- tail, state);
- _cds_wfcq_dequeue_unlock(head, tail);
- return retval;
-}
-
-/*
- * cds_wfcq_dequeue_blocking: dequeue node from queue.
- *
- * Same as cds_wfcq_dequeue_blocking, but without saving state.
- */
-static inline struct cds_wfcq_node *
-_cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- return _cds_wfcq_dequeue_with_state_blocking(head, tail, NULL);
-}
-
-/*
- * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Dequeue all nodes from src_q.
- * dest_q must be already initialized.
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
- * ensured.
- * Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
- */
-static inline enum cds_wfcq_ret
-_cds_wfcq_splice_blocking(
- struct cds_wfcq_head *dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- struct cds_wfcq_head *src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- enum cds_wfcq_ret ret;
-
- _cds_wfcq_dequeue_lock(src_q_head, src_q_tail);
- ret = ___cds_wfcq_splice_blocking(cds_wfcq_head_cast(dest_q_head), dest_q_tail,
- cds_wfcq_head_cast(src_q_head), src_q_tail);
- _cds_wfcq_dequeue_unlock(src_q_head, src_q_tail);
- return ret;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFCQUEUE_STATIC_H */
+++ /dev/null
-#ifndef _URCU_WFQUEUE_STATIC_H
-#define _URCU_WFQUEUE_STATIC_H
-
-/*
- * wfqueue-static.h
- *
- * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See wfqueue.h for linking
- * dynamically with the userspace rcu library.
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <poll.h>
-#include <urcu/compiler.h>
-#include <urcu/uatomic.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Queue with wait-free enqueue/blocking dequeue.
- * This implementation adds a dummy head node when the queue is empty to ensure
- * we can always update the queue locklessly.
- *
- * Inspired from half-wait-free/half-blocking queue implementation done by
- * Paul E. McKenney.
- */
-
-#define WFQ_ADAPT_ATTEMPTS 10 /* Retry if being set */
-#define WFQ_WAIT 10 /* Wait 10 ms if being set */
-
-static inline void _cds_wfq_node_init(struct cds_wfq_node *node)
-{
- node->next = NULL;
-}
-
-static inline void _cds_wfq_init(struct cds_wfq_queue *q)
-{
- int ret;
-
- _cds_wfq_node_init(&q->dummy);
- /* Set queue head and tail */
- q->head = &q->dummy;
- q->tail = &q->dummy.next;
- ret = pthread_mutex_init(&q->lock, NULL);
- assert(!ret);
-}
-
-static inline void _cds_wfq_destroy(struct cds_wfq_queue *q)
-{
- int ret = pthread_mutex_destroy(&q->lock);
- assert(!ret);
-}
-
-static inline void _cds_wfq_enqueue(struct cds_wfq_queue *q,
- struct cds_wfq_node *node)
-{
- struct cds_wfq_node **old_tail;
-
- /*
- * uatomic_xchg() implicit memory barrier orders earlier stores to data
- * structure containing node and setting node->next to NULL before
- * publication.
- */
- old_tail = uatomic_xchg(&q->tail, &node->next);
- /*
- * At this point, dequeuers see a NULL old_tail->next, which indicates
- * that the queue is being appended to. The following store will append
- * "node" to the queue from a dequeuer perspective.
- */
- CMM_STORE_SHARED(*old_tail, node);
-}
-
-/*
- * Waiting for enqueuer to complete enqueue and return the next node
- */
-static inline struct cds_wfq_node *
-___cds_wfq_node_sync_next(struct cds_wfq_node *node)
-{
- struct cds_wfq_node *next;
- int attempt = 0;
-
- /*
- * Adaptative busy-looping waiting for enqueuer to complete enqueue.
- */
- while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- if (++attempt >= WFQ_ADAPT_ATTEMPTS) {
- (void) poll(NULL, 0, WFQ_WAIT); /* Wait for 10ms */
- attempt = 0;
- } else
- caa_cpu_relax();
- }
-
- return next;
-}
-
-/*
- * It is valid to reuse and free a dequeued node immediately.
- *
- * No need to go on a waitqueue here, as there is no possible state in which the
- * list could cause dequeue to busy-loop needlessly while waiting for another
- * thread to be scheduled. The queue appears empty until tail->next is set by
- * enqueue.
- */
-static inline struct cds_wfq_node *
-___cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- struct cds_wfq_node *node, *next;
-
- /*
- * Queue is empty if it only contains the dummy node.
- */
- if (q->head == &q->dummy && CMM_LOAD_SHARED(q->tail) == &q->dummy.next)
- return NULL;
- node = q->head;
-
- next = ___cds_wfq_node_sync_next(node);
-
- /*
- * Move queue head forward.
- */
- q->head = next;
- /*
- * Requeue dummy node if we just dequeued it.
- */
- if (node == &q->dummy) {
- _cds_wfq_node_init(node);
- _cds_wfq_enqueue(q, node);
- return ___cds_wfq_dequeue_blocking(q);
- }
- return node;
-}
-
-static inline struct cds_wfq_node *
-_cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- struct cds_wfq_node *retnode;
- int ret;
-
- ret = pthread_mutex_lock(&q->lock);
- assert(!ret);
- retnode = ___cds_wfq_dequeue_blocking(q);
- ret = pthread_mutex_unlock(&q->lock);
- assert(!ret);
- return retnode;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFQUEUE_STATIC_H */
+++ /dev/null
-#ifndef _URCU_STATIC_WFSTACK_H
-#define _URCU_STATIC_WFSTACK_H
-
-/*
- * urcu/static/wfstack.h
- *
- * Userspace RCU library - Stack with with wait-free push, blocking traversal.
- *
- * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/wfstack.h for
- * linking dynamically with the userspace rcu library.
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <poll.h>
-#include <stdbool.h>
-#include <urcu/compiler.h>
-#include <urcu/uatomic.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define CDS_WFS_END ((void *) 0x1UL)
-#define CDS_WFS_ADAPT_ATTEMPTS 10 /* Retry if being set */
-#define CDS_WFS_WAIT 10 /* Wait 10 ms if being set */
-
-/*
- * Stack with wait-free push, blocking traversal.
- *
- * Stack implementing push, pop, pop_all operations, as well as iterator
- * on the stack head returned by pop_all.
- *
- * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
- * cds_wfs_first.
- * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
- * iteration on stack head returned by pop_all.
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * cds_wfs_push __cds_wfs_pop __cds_wfs_pop_all
- * cds_wfs_push - - -
- * __cds_wfs_pop - X X
- * __cds_wfs_pop_all - X -
- *
- * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
- * synchronization.
- */
-
-/*
- * cds_wfs_node_init: initialize wait-free stack node.
- */
-static inline
-void _cds_wfs_node_init(struct cds_wfs_node *node)
-{
- node->next = NULL;
-}
-
-/*
- * __cds_wfs_init: initialize wait-free stack. Don't pair with
- * any destroy function.
- */
-static inline void ___cds_wfs_init(struct __cds_wfs_stack *s)
-{
- s->head = CDS_WFS_END;
-}
-
-/*
- * cds_wfs_init: initialize wait-free stack. Pair with
- * cds_wfs_destroy().
- */
-static inline
-void _cds_wfs_init(struct cds_wfs_stack *s)
-{
- int ret;
-
- s->head = CDS_WFS_END;
- ret = pthread_mutex_init(&s->lock, NULL);
- assert(!ret);
-}
-
-/*
- * cds_wfs_destroy: destroy wait-free stack. Pair with
- * cds_wfs_init().
- */
-static inline
-void _cds_wfs_destroy(struct cds_wfs_stack *s)
-{
- int ret = pthread_mutex_destroy(&s->lock);
- assert(!ret);
-}
-
-static inline bool ___cds_wfs_end(void *node)
-{
- return node == CDS_WFS_END;
-}
-
-/*
- * cds_wfs_empty: return whether wait-free stack is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- */
-static inline bool _cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
-{
- struct __cds_wfs_stack *s = u_stack._s;
-
- return ___cds_wfs_end(CMM_LOAD_SHARED(s->head));
-}
-
-/*
- * cds_wfs_push: push a node into the stack.
- *
- * Issues a full memory barrier before push. No mutual exclusion is
- * required.
- *
- * Returns 0 if the stack was empty prior to adding the node.
- * Returns non-zero otherwise.
- */
-static inline
-int _cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
-{
- struct __cds_wfs_stack *s = u_stack._s;
- struct cds_wfs_head *old_head, *new_head;
-
- assert(node->next == NULL);
- new_head = caa_container_of(node, struct cds_wfs_head, node);
- /*
- * uatomic_xchg() implicit memory barrier orders earlier stores
- * to node (setting it to NULL) before publication.
- */
- old_head = uatomic_xchg(&s->head, new_head);
- /*
- * At this point, dequeuers see a NULL node->next, they should
- * busy-wait until node->next is set to old_head.
- */
- CMM_STORE_SHARED(node->next, &old_head->node);
- return !___cds_wfs_end(old_head);
-}
-
-/*
- * Waiting for push to complete enqueue and return the next node.
- */
-static inline struct cds_wfs_node *
-___cds_wfs_node_sync_next(struct cds_wfs_node *node, int blocking)
-{
- struct cds_wfs_node *next;
- int attempt = 0;
-
- /*
- * Adaptative busy-looping waiting for push to complete.
- */
- while ((next = CMM_LOAD_SHARED(node->next)) == NULL) {
- if (!blocking)
- return CDS_WFS_WOULDBLOCK;
- if (++attempt >= CDS_WFS_ADAPT_ATTEMPTS) {
- (void) poll(NULL, 0, CDS_WFS_WAIT); /* Wait for 10ms */
- attempt = 0;
- } else {
- caa_cpu_relax();
- }
- }
-
- return next;
-}
-
-static inline
-struct cds_wfs_node *
-___cds_wfs_pop(cds_wfs_stack_ptr_t u_stack, int *state, int blocking)
-{
- struct cds_wfs_head *head, *new_head;
- struct cds_wfs_node *next;
- struct __cds_wfs_stack *s = u_stack._s;
-
- if (state)
- *state = 0;
- for (;;) {
- head = CMM_LOAD_SHARED(s->head);
- if (___cds_wfs_end(head)) {
- return NULL;
- }
- next = ___cds_wfs_node_sync_next(&head->node, blocking);
- if (!blocking && next == CDS_WFS_WOULDBLOCK) {
- return CDS_WFS_WOULDBLOCK;
- }
- new_head = caa_container_of(next, struct cds_wfs_head, node);
- if (uatomic_cmpxchg(&s->head, head, new_head) == head) {
- if (state && ___cds_wfs_end(new_head))
- *state |= CDS_WFS_STATE_LAST;
- return &head->node;
- }
- if (!blocking) {
- return CDS_WFS_WOULDBLOCK;
- }
- /* busy-loop if head changed under us */
- }
-}
-
-/*
- * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
- *
- * Returns NULL if stack is empty.
- *
- * __cds_wfs_pop_blocking needs to be synchronized using one of the
- * following techniques:
- *
- * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
- * section. The caller must wait for a grace period to pass before
- * freeing the returned node or modifying the cds_wfs_node structure.
- * 2) Using mutual exclusion (e.g. mutexes) to protect
- * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
- * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
- *
- * "state" saves state flags atomically sampled with pop operation.
- */
-static inline
-struct cds_wfs_node *
-___cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, int *state)
-{
- return ___cds_wfs_pop(u_stack, state, 1);
-}
-
-static inline
-struct cds_wfs_node *
-___cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
-{
- return ___cds_wfs_pop_with_state_blocking(u_stack, NULL);
-}
-
-/*
- * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack.
- *
- * Same as __cds_wfs_pop_with_state_blocking, but returns
- * CDS_WFS_WOULDBLOCK if it needs to block.
- *
- * "state" saves state flags atomically sampled with pop operation.
- */
-static inline
-struct cds_wfs_node *
-___cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state)
-{
- return ___cds_wfs_pop(u_stack, state, 0);
-}
-
-/*
- * __cds_wfs_pop_nonblocking: pop a node from the stack.
- *
- * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
- * it needs to block.
- */
-static inline
-struct cds_wfs_node *
-___cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
-{
- return ___cds_wfs_pop_with_state_nonblocking(u_stack, NULL);
-}
-
-/*
- * __cds_wfs_pop_all: pop all nodes from a stack.
- *
- * __cds_wfs_pop_all does not require any synchronization with other
- * push, nor with other __cds_wfs_pop_all, but requires synchronization
- * matching the technique used to synchronize __cds_wfs_pop_blocking:
- *
- * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
- * section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
- * must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_wfs_node structure. However, no RCU
- * read-side critical section is needed around __cds_wfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect
- * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
- * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-static inline
-struct cds_wfs_head *
-___cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
-{
- struct __cds_wfs_stack *s = u_stack._s;
- struct cds_wfs_head *head;
-
- /*
- * Implicit memory barrier after uatomic_xchg() matches implicit
- * memory barrier before uatomic_xchg() in cds_wfs_push. It
- * ensures that all nodes of the returned list are consistent.
- * There is no need to issue memory barriers when iterating on
- * the returned list, because the full memory barrier issued
- * prior to each uatomic_cmpxchg, which each write to head, are
- * taking care to order writes to each node prior to the full
- * memory barrier after this uatomic_xchg().
- */
- head = uatomic_xchg(&s->head, CDS_WFS_END);
- if (___cds_wfs_end(head))
- return NULL;
- return head;
-}
-
-/*
- * cds_wfs_pop_lock: lock stack pop-protection mutex.
- */
-static inline void _cds_wfs_pop_lock(struct cds_wfs_stack *s)
-{
- int ret;
-
- ret = pthread_mutex_lock(&s->lock);
- assert(!ret);
-}
-
-/*
- * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
- */
-static inline void _cds_wfs_pop_unlock(struct cds_wfs_stack *s)
-{
- int ret;
-
- ret = pthread_mutex_unlock(&s->lock);
- assert(!ret);
-}
-
-/*
- * Call __cds_wfs_pop_with_state_blocking with an internal pop mutex held.
- */
-static inline
-struct cds_wfs_node *
-_cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
-{
- struct cds_wfs_node *retnode;
-
- _cds_wfs_pop_lock(s);
- retnode = ___cds_wfs_pop_with_state_blocking(s, state);
- _cds_wfs_pop_unlock(s);
- return retnode;
-}
-
-/*
- * Call _cds_wfs_pop_with_state_blocking without saving any state.
- */
-static inline
-struct cds_wfs_node *
-_cds_wfs_pop_blocking(struct cds_wfs_stack *s)
-{
- return _cds_wfs_pop_with_state_blocking(s, NULL);
-}
-
-/*
- * Call __cds_wfs_pop_all with an internal pop mutex held.
- */
-static inline
-struct cds_wfs_head *
-_cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
-{
- struct cds_wfs_head *rethead;
-
- _cds_wfs_pop_lock(s);
- rethead = ___cds_wfs_pop_all(s);
- _cds_wfs_pop_unlock(s);
- return rethead;
-}
-
-/*
- * cds_wfs_first: get first node of a popped stack.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- *
- * Used by for-like iteration macros in urcu/wfstack.h:
- * cds_wfs_for_each_blocking()
- * cds_wfs_for_each_blocking_safe()
- *
- * Returns NULL if popped stack is empty, top stack node otherwise.
- */
-static inline struct cds_wfs_node *
-_cds_wfs_first(struct cds_wfs_head *head)
-{
- if (___cds_wfs_end(head))
- return NULL;
- return &head->node;
-}
-
-static inline struct cds_wfs_node *
-___cds_wfs_next(struct cds_wfs_node *node, int blocking)
-{
- struct cds_wfs_node *next;
-
- next = ___cds_wfs_node_sync_next(node, blocking);
- /*
- * CDS_WFS_WOULDBLOCK != CSD_WFS_END, so we can check for end
- * even if ___cds_wfs_node_sync_next returns CDS_WFS_WOULDBLOCK,
- * and still return CDS_WFS_WOULDBLOCK.
- */
- if (___cds_wfs_end(next))
- return NULL;
- return next;
-}
-
-/*
- * cds_wfs_next_blocking: get next node of a popped stack.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- *
- * Used by for-like iteration macros in urcu/wfstack.h:
- * cds_wfs_for_each_blocking()
- * cds_wfs_for_each_blocking_safe()
- *
- * Returns NULL if reached end of popped stack, non-NULL next stack
- * node otherwise.
- */
-static inline struct cds_wfs_node *
-_cds_wfs_next_blocking(struct cds_wfs_node *node)
-{
- return ___cds_wfs_next(node, 1);
-}
-
-
-/*
- * cds_wfs_next_nonblocking: get next node of a popped stack.
- *
- * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
- * needs to block.
- */
-static inline struct cds_wfs_node *
-_cds_wfs_next_nonblocking(struct cds_wfs_node *node)
-{
- return ___cds_wfs_next(node, 0);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_STATIC_WFSTACK_H */
+++ /dev/null
-#ifndef _URCU_SYSCALL_COMPAT_H
-#define _URCU_SYSCALL_COMPAT_H
-
-/*
- * urcu/syscall-compat.h
- *
- * Userspace RCU library - Syscall Compatibility Header
- *
- * Copyright 2013 - Pierre-Luc St-Charles <pierre-luc.st-charles@polymtl.ca>
- *
- * Note: this file is only used to simplify the code required to
- * include the 'syscall.h' system header across multiple platforms,
- * which might not always be located at the same place (or needed at all).
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#if defined(__ANDROID__) || defined(__sun__) || defined(__GNU__)
-#include <sys/syscall.h>
-#elif defined(__linux__) || defined(__GLIBC__)
-#include <syscall.h>
-
-#elif defined(__CYGWIN__) || defined(__APPLE__)
-/* Don't include anything on Cygwin or MacOSX. */
-
-#else
-#error "Add platform support to urcu/syscall-compat.h"
-#endif
-
-#endif /* _URCU_SYSCALL_COMPAT_H */
+++ /dev/null
-#ifndef _URCU_SYSTEM_H
-#define _URCU_SYSTEM_H
-
-/*
- * system.h
- *
- * System definitions.
- *
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-
-/*
- * Identify a shared load. A cmm_smp_rmc() or cmm_smp_mc() should come
- * before the load.
- */
-#define _CMM_LOAD_SHARED(p) CMM_ACCESS_ONCE(p)
-
-/*
- * Load a data from shared memory, doing a cache flush if required.
- */
-#define CMM_LOAD_SHARED(p) \
- __extension__ \
- ({ \
- cmm_smp_rmc(); \
- _CMM_LOAD_SHARED(p); \
- })
-
-/*
- * Identify a shared store. A cmm_smp_wmc() or cmm_smp_mc() should
- * follow the store.
- */
-#define _CMM_STORE_SHARED(x, v) __extension__ ({ CMM_ACCESS_ONCE(x) = (v); })
-
-/*
- * Store v into x, where x is located in shared memory. Performs the
- * required cache flush after writing. Returns v.
- */
-#define CMM_STORE_SHARED(x, v) \
- __extension__ \
- ({ \
- __typeof__(x) _v = _CMM_STORE_SHARED(x, v); \
- cmm_smp_wmc(); \
- _v = _v; /* Work around clang "unused result" */ \
- })
-
-#endif /* _URCU_SYSTEM_H */
+++ /dev/null
-#ifndef _URCU_TLS_COMPAT_H
-#define _URCU_TLS_COMPAT_H
-
-/*
- * urcu/tls-compat.h
- *
- * Userspace RCU library - Thread-Local Storage Compatibility Header
- *
- * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <stdlib.h>
-#include <urcu/config.h>
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef CONFIG_RCU_TLS /* Based on ax_tls.m4 */
-
-/*
- * Hint: How to define/declare TLS variables of compound types
- * such as array or function pointers?
- *
- * Answer: Use typedef to assign a type_name to the compound type.
- * Example: Define a TLS variable which is an int array with len=4:
- *
- * typedef int my_int_array_type[4];
- * DEFINE_URCU_TLS(my_int_array_type, var_name);
- *
- * Another example:
- * typedef void (*call_rcu_flavor)(struct rcu_head *, XXXX);
- * DECLARE_URCU_TLS(call_rcu_flavor, p_call_rcu);
- *
- * NOTE: URCU_TLS() is NOT async-signal-safe, you can't use it
- * inside any function which can be called from signal handler.
- *
- * But if pthread_getspecific() is async-signal-safe in your
- * platform, you can make URCU_TLS() async-signal-safe via:
- * ensuring the first call to URCU_TLS() of a given TLS variable of
- * all threads is called earliest from a non-signal handler function.
- *
- * Example: In any thread, the first call of URCU_TLS(rcu_reader)
- * is called from rcu_register_thread(), so we can ensure all later
- * URCU_TLS(rcu_reader) in any thread is async-signal-safe.
- *
- * Moreover, URCU_TLS variables should not be touched from signal
- * handlers setup with with sigaltstack(2).
- */
-
-# define DECLARE_URCU_TLS(type, name) \
- CONFIG_RCU_TLS type name
-
-# define DEFINE_URCU_TLS(type, name) \
- CONFIG_RCU_TLS type name
-
-# define URCU_TLS(name) (name)
-
-#else /* #ifndef CONFIG_RCU_TLS */
-
-/*
- * The *_1() macros ensure macro parameters are expanded.
- */
-
-# include <pthread.h>
-
-struct urcu_tls {
- pthread_key_t key;
- pthread_mutex_t init_mutex;
- int init_done;
-};
-
-# define DECLARE_URCU_TLS_1(type, name) \
- type *__tls_access_ ## name(void)
-# define DECLARE_URCU_TLS(type, name) \
- DECLARE_URCU_TLS_1(type, name)
-
-/*
- * Note: we don't free memory at process exit, since it will be dealt
- * with by the OS.
- */
-# define DEFINE_URCU_TLS_1(type, name) \
- type *__tls_access_ ## name(void) \
- { \
- static struct urcu_tls __tls_ ## name = { \
- .init_mutex = PTHREAD_MUTEX_INITIALIZER,\
- .init_done = 0, \
- }; \
- void *__tls_p; \
- if (!__tls_ ## name.init_done) { \
- /* Mutex to protect concurrent init */ \
- pthread_mutex_lock(&__tls_ ## name.init_mutex); \
- if (!__tls_ ## name.init_done) { \
- (void) pthread_key_create(&__tls_ ## name.key, \
- free); \
- cmm_smp_wmb(); /* create key before write init_done */ \
- __tls_ ## name.init_done = 1; \
- } \
- pthread_mutex_unlock(&__tls_ ## name.init_mutex); \
- } \
- cmm_smp_rmb(); /* read init_done before getting key */ \
- __tls_p = pthread_getspecific(__tls_ ## name.key); \
- if (caa_unlikely(__tls_p == NULL)) { \
- __tls_p = calloc(1, sizeof(type)); \
- (void) pthread_setspecific(__tls_ ## name.key, \
- __tls_p); \
- } \
- return __tls_p; \
- }
-
-# define DEFINE_URCU_TLS(type, name) \
- DEFINE_URCU_TLS_1(type, name)
-
-# define URCU_TLS_1(name) (*__tls_access_ ## name())
-
-# define URCU_TLS(name) URCU_TLS_1(name)
-
-#endif /* #else #ifndef CONFIG_RCU_TLS */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_TLS_COMPAT_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_AARCH64_H
-#define _URCU_ARCH_UATOMIC_AARCH64_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009-2015 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define UATOMIC_HAS_ATOMIC_BYTE
-#define UATOMIC_HAS_ATOMIC_SHORT
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_AARCH64_H */
+++ /dev/null
-#ifndef _URCU_UATOMIC_ARCH_ALPHA_H
-#define _URCU_UATOMIC_ARCH_ALPHA_H
-
-/*
- * Atomic exchange operations for the Alpha architecture. Let GCC do it.
- *
- * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_UATOMIC_ARCH_ALPHA_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_ARM_H
-#define _URCU_ARCH_UATOMIC_ARM_H
-
-/*
- * Atomics for ARM. This approach is usable on kernels back to 2.6.15.
- *
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* xchg */
-#define uatomic_xchg(addr, v) __sync_lock_test_and_set(addr, v)
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_ARM_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_GCC_H
-#define _URCU_ARCH_UATOMIC_GCC_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * If your platform doesn't have a full set of atomics, you will need
- * a separate uatomic_arch_*.h file for your architecture. Otherwise,
- * just rely on the definitions in uatomic/generic.h.
- */
-#define UATOMIC_HAS_ATOMIC_BYTE
-#define UATOMIC_HAS_ATOMIC_SHORT
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_GCC_H */
+++ /dev/null
-#ifndef _URCU_UATOMIC_GENERIC_H
-#define _URCU_UATOMIC_GENERIC_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- * Copyright (c) 2010 Paolo Bonzini
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <stdint.h>
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef uatomic_set
-#define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
-#endif
-
-#ifndef uatomic_read
-#define uatomic_read(addr) CMM_LOAD_SHARED(*(addr))
-#endif
-
-#if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR
-static inline __attribute__((always_inline))
-void _uatomic_link_error(void)
-{
-#ifdef ILLEGAL_INSTR
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__(ILLEGAL_INSTR);
-#else
- __builtin_trap();
-#endif
-}
-
-#else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
-extern void _uatomic_link_error(void);
-#endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */
-
-/* cmpxchg */
-
-#ifndef uatomic_cmpxchg
-static inline __attribute__((always_inline))
-unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
- _new);
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
- _new);
-#endif
- case 4:
- return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
- _new);
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
- _new);
-#endif
- }
- _uatomic_link_error();
- return 0;
-}
-
-
-#define uatomic_cmpxchg(addr, old, _new) \
- ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new),\
- sizeof(*(addr))))
-
-
-/* uatomic_and */
-
-#ifndef uatomic_and
-static inline __attribute__((always_inline))
-void _uatomic_and(void *addr, unsigned long val,
- int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- __sync_and_and_fetch_1((uint8_t *) addr, val);
- return;
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- __sync_and_and_fetch_2((uint16_t *) addr, val);
- return;
-#endif
- case 4:
- __sync_and_and_fetch_4((uint32_t *) addr, val);
- return;
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- __sync_and_and_fetch_8((uint64_t *) addr, val);
- return;
-#endif
- }
- _uatomic_link_error();
-}
-
-#define uatomic_and(addr, v) \
- (_uatomic_and((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
-
-#endif
-
-/* uatomic_or */
-
-#ifndef uatomic_or
-static inline __attribute__((always_inline))
-void _uatomic_or(void *addr, unsigned long val,
- int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- __sync_or_and_fetch_1((uint8_t *) addr, val);
- return;
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- __sync_or_and_fetch_2((uint16_t *) addr, val);
- return;
-#endif
- case 4:
- __sync_or_and_fetch_4((uint32_t *) addr, val);
- return;
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- __sync_or_and_fetch_8((uint64_t *) addr, val);
- return;
-#endif
- }
- _uatomic_link_error();
- return;
-}
-
-#define uatomic_or(addr, v) \
- (_uatomic_or((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
-
-#endif
-
-
-/* uatomic_add_return */
-
-#ifndef uatomic_add_return
-static inline __attribute__((always_inline))
-unsigned long _uatomic_add_return(void *addr, unsigned long val,
- int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- return __sync_add_and_fetch_1((uint8_t *) addr, val);
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- return __sync_add_and_fetch_2((uint16_t *) addr, val);
-#endif
- case 4:
- return __sync_add_and_fetch_4((uint32_t *) addr, val);
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- return __sync_add_and_fetch_8((uint64_t *) addr, val);
-#endif
- }
- _uatomic_link_error();
- return 0;
-}
-
-
-#define uatomic_add_return(addr, v) \
- ((__typeof__(*(addr))) _uatomic_add_return((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#endif /* #ifndef uatomic_add_return */
-
-#ifndef uatomic_xchg
-/* xchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- {
- uint8_t old;
-
- do {
- old = uatomic_read((uint8_t *) addr);
- } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
- old, val));
-
- return old;
- }
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- {
- uint16_t old;
-
- do {
- old = uatomic_read((uint16_t *) addr);
- } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
- old, val));
-
- return old;
- }
-#endif
- case 4:
- {
- uint32_t old;
-
- do {
- old = uatomic_read((uint32_t *) addr);
- } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
- old, val));
-
- return old;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- uint64_t old;
-
- do {
- old = uatomic_read((uint64_t *) addr);
- } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
- old, val));
-
- return old;
- }
-#endif
- }
- _uatomic_link_error();
- return 0;
-}
-
-#define uatomic_xchg(addr, v) \
- ((__typeof__(*(addr))) _uatomic_exchange((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#endif /* #ifndef uatomic_xchg */
-
-#else /* #ifndef uatomic_cmpxchg */
-
-#ifndef uatomic_and
-/* uatomic_and */
-
-static inline __attribute__((always_inline))
-void _uatomic_and(void *addr, unsigned long val, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- {
- uint8_t old, oldt;
-
- oldt = uatomic_read((uint8_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
- } while (oldt != old);
-
- return;
- }
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- {
- uint16_t old, oldt;
-
- oldt = uatomic_read((uint16_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
- } while (oldt != old);
- }
-#endif
- case 4:
- {
- uint32_t old, oldt;
-
- oldt = uatomic_read((uint32_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
- } while (oldt != old);
-
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- uint64_t old, oldt;
-
- oldt = uatomic_read((uint64_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
- } while (oldt != old);
-
- return;
- }
-#endif
- }
- _uatomic_link_error();
-}
-
-#define uatomic_and(addr, v) \
- (_uatomic_and((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
-
-#endif /* #ifndef uatomic_and */
-
-#ifndef uatomic_or
-/* uatomic_or */
-
-static inline __attribute__((always_inline))
-void _uatomic_or(void *addr, unsigned long val, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- {
- uint8_t old, oldt;
-
- oldt = uatomic_read((uint8_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
- } while (oldt != old);
-
- return;
- }
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- {
- uint16_t old, oldt;
-
- oldt = uatomic_read((uint16_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
- } while (oldt != old);
-
- return;
- }
-#endif
- case 4:
- {
- uint32_t old, oldt;
-
- oldt = uatomic_read((uint32_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
- } while (oldt != old);
-
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- uint64_t old, oldt;
-
- oldt = uatomic_read((uint64_t *) addr);
- do {
- old = oldt;
- oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
- } while (oldt != old);
-
- return;
- }
-#endif
- }
- _uatomic_link_error();
-}
-
-#define uatomic_or(addr, v) \
- (_uatomic_or((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
-
-#endif /* #ifndef uatomic_or */
-
-#ifndef uatomic_add_return
-/* uatomic_add_return */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- {
- uint8_t old, oldt;
-
- oldt = uatomic_read((uint8_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint8_t *) addr,
- old, old + val);
- } while (oldt != old);
-
- return old + val;
- }
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- {
- uint16_t old, oldt;
-
- oldt = uatomic_read((uint16_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint16_t *) addr,
- old, old + val);
- } while (oldt != old);
-
- return old + val;
- }
-#endif
- case 4:
- {
- uint32_t old, oldt;
-
- oldt = uatomic_read((uint32_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint32_t *) addr,
- old, old + val);
- } while (oldt != old);
-
- return old + val;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- uint64_t old, oldt;
-
- oldt = uatomic_read((uint64_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint64_t *) addr,
- old, old + val);
- } while (oldt != old);
-
- return old + val;
- }
-#endif
- }
- _uatomic_link_error();
- return 0;
-}
-
-#define uatomic_add_return(addr, v) \
- ((__typeof__(*(addr))) _uatomic_add_return((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#endif /* #ifndef uatomic_add_return */
-
-#ifndef uatomic_xchg
-/* xchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
-{
- switch (len) {
-#ifdef UATOMIC_HAS_ATOMIC_BYTE
- case 1:
- {
- uint8_t old, oldt;
-
- oldt = uatomic_read((uint8_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint8_t *) addr,
- old, val);
- } while (oldt != old);
-
- return old;
- }
-#endif
-#ifdef UATOMIC_HAS_ATOMIC_SHORT
- case 2:
- {
- uint16_t old, oldt;
-
- oldt = uatomic_read((uint16_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint16_t *) addr,
- old, val);
- } while (oldt != old);
-
- return old;
- }
-#endif
- case 4:
- {
- uint32_t old, oldt;
-
- oldt = uatomic_read((uint32_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint32_t *) addr,
- old, val);
- } while (oldt != old);
-
- return old;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- uint64_t old, oldt;
-
- oldt = uatomic_read((uint64_t *) addr);
- do {
- old = oldt;
- oldt = uatomic_cmpxchg((uint64_t *) addr,
- old, val);
- } while (oldt != old);
-
- return old;
- }
-#endif
- }
- _uatomic_link_error();
- return 0;
-}
-
-#define uatomic_xchg(addr, v) \
- ((__typeof__(*(addr))) _uatomic_exchange((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-#endif /* #ifndef uatomic_xchg */
-
-#endif /* #else #ifndef uatomic_cmpxchg */
-
-/* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
-
-#ifndef uatomic_add
-#define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v))
-#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
-#endif
-
-#define uatomic_sub_return(addr, v) \
- uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
-#define uatomic_sub(addr, v) \
- uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
-#define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add()
-#define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add()
-
-#ifndef uatomic_inc
-#define uatomic_inc(addr) uatomic_add((addr), 1)
-#define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add()
-#define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add()
-#endif
-
-#ifndef uatomic_dec
-#define uatomic_dec(addr) uatomic_add((addr), -1)
-#define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add()
-#define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add()
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_UATOMIC_GENERIC_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_HPPA_H
-#define _URCU_ARCH_UATOMIC_HPPA_H
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#define UATOMIC_HAS_ATOMIC_SHORT
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_HPPA_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_IA64_H
-#define _URCU_ARCH_UATOMIC_IA64_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009-2015 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define UATOMIC_HAS_ATOMIC_BYTE
-#define UATOMIC_HAS_ATOMIC_SHORT
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_IA64_H */
+++ /dev/null
-#ifndef _URCU_UATOMIC_ARCH_MIPS_H
-#define _URCU_UATOMIC_ARCH_MIPS_H
-
-/*
- * Atomic exchange operations for the MIPS architecture. Let GCC do it.
- *
- * Copyright (c) 2010 Paolo Bonzini <pbonzini@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_UATOMIC_ARCH_MIPS_H */
+++ /dev/null
-#ifndef _URCU_UATOMIC_ARCH_NIOS2_H
-#define _URCU_UATOMIC_ARCH_NIOS2_H
-
-/*
- * Atomic exchange operations for the NIOS2 architecture. Let GCC do it.
- *
- * Copyright (c) 2016 Marek Vasut <marex@denx.de>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_UATOMIC_ARCH_NIOS2_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_PPC_H
-#define _URCU_ARCH_UATOMIC_PPC_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ILLEGAL_INSTR ".long 0xd00d00"
-
-/*
- * Providing sequential consistency semantic with respect to other
- * instructions for cmpxchg and add_return family of atomic primitives.
- *
- * This is achieved with:
- * lwsync (prior loads can be reordered after following load)
- * lwarx
- * stwcx.
- * test if success (retry)
- * sync
- *
- * Explanation of the sequential consistency provided by this scheme
- * from Paul E. McKenney:
- *
- * The reason we can get away with the lwsync before is that if a prior
- * store reorders with the lwarx, then you have to store to the atomic
- * variable from some other CPU to detect it.
- *
- * And if you do that, the lwarx will lose its reservation, so the stwcx
- * will fail. The atomic operation will retry, so that the caller won't be
- * able to see the misordering.
- */
-
-/* xchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
-{
- switch (len) {
- case 4:
- {
- unsigned int result;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
- "stwcx. %2,0,%1\n" /* else store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- : "=&r"(result)
- : "r"(addr), "r"(val)
- : "memory", "cc");
-
- return result;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long result;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
- "stdcx. %2,0,%1\n" /* else store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- : "=&r"(result)
- : "r"(addr), "r"(val)
- : "memory", "cc");
-
- return result;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__(ILLEGAL_INSTR);
- return 0;
-}
-
-#define uatomic_xchg(addr, v) \
- ((__typeof__(*(addr))) _uatomic_exchange((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-/* cmpxchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- switch (len) {
- case 4:
- {
- unsigned int old_val;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
- "cmpw %0,%3\n" /* if load is not equal to */
- "bne 2f\n" /* old, fail */
- "stwcx. %2,0,%1\n" /* else store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- "2:\n"
- : "=&r"(old_val)
- : "r"(addr), "r"((unsigned int)_new),
- "r"((unsigned int)old)
- : "memory", "cc");
-
- return old_val;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long old_val;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
- "cmpd %0,%3\n" /* if load is not equal to */
- "bne 2f\n" /* old, fail */
- "stdcx. %2,0,%1\n" /* else store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- "2:\n"
- : "=&r"(old_val)
- : "r"(addr), "r"((unsigned long)_new),
- "r"((unsigned long)old)
- : "memory", "cc");
-
- return old_val;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__(ILLEGAL_INSTR);
- return 0;
-}
-
-
-#define uatomic_cmpxchg(addr, old, _new) \
- ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new),\
- sizeof(*(addr))))
-
-/* uatomic_add_return */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_add_return(void *addr, unsigned long val,
- int len)
-{
- switch (len) {
- case 4:
- {
- unsigned int result;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "lwarx %0,0,%1\n" /* load and reserve */
- "add %0,%2,%0\n" /* add val to value loaded */
- "stwcx. %0,0,%1\n" /* store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- : "=&r"(result)
- : "r"(addr), "r"(val)
- : "memory", "cc");
-
- return result;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long result;
-
- __asm__ __volatile__(
- LWSYNC_OPCODE
- "1:\t" "ldarx %0,0,%1\n" /* load and reserve */
- "add %0,%2,%0\n" /* add val to value loaded */
- "stdcx. %0,0,%1\n" /* store conditional */
- "bne- 1b\n" /* retry if lost reservation */
- "sync\n"
- : "=&r"(result)
- : "r"(addr), "r"(val)
- : "memory", "cc");
-
- return result;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__(ILLEGAL_INSTR);
- return 0;
-}
-
-
-#define uatomic_add_return(addr, v) \
- ((__typeof__(*(addr))) _uatomic_add_return((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_PPC_H */
+++ /dev/null
-#ifndef _URCU_UATOMIC_ARCH_S390_H
-#define _URCU_UATOMIC_ARCH_S390_H
-
-/*
- * Atomic exchange operations for the S390 architecture. Based on information
- * taken from the Principles of Operation Appendix A "Conditional Swapping
- * Instructions (CS, CDS)".
- *
- * Copyright (c) 2009 Novell, Inc.
- * Author: Jan Blunck <jblunck@suse.de>
- * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#if __GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ > 2)
-#define COMPILER_HAVE_SHORT_MEM_OPERAND
-#endif
-
-/*
- * MEMOP assembler operand rules:
- * - op refer to MEMOP_IN operand
- * - MEMOP_IN can expand to more than a single operand. Use it at the end of
- * operand list only.
- */
-
-#ifdef COMPILER_HAVE_SHORT_MEM_OPERAND
-
-#define MEMOP_OUT(addr) "=Q" (*(addr))
-#define MEMOP_IN(addr) "Q" (*(addr))
-#define MEMOP_REF(op) #op /* op refer to MEMOP_IN operand */
-
-#else /* !COMPILER_HAVE_SHORT_MEM_OPERAND */
-
-#define MEMOP_OUT(addr) "=m" (*(addr))
-#define MEMOP_IN(addr) "a" (addr), "m" (*(addr))
-#define MEMOP_REF(op) "0(" #op ")" /* op refer to MEMOP_IN operand */
-
-#endif /* !COMPILER_HAVE_SHORT_MEM_OPERAND */
-
-struct __uatomic_dummy {
- unsigned long v[10];
-};
-#define __hp(x) ((struct __uatomic_dummy *)(x))
-
-/* xchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_exchange(volatile void *addr, unsigned long val, int len)
-{
- switch (len) {
- case 4:
- {
- unsigned int old_val;
-
- __asm__ __volatile__(
- "0: cs %0,%2," MEMOP_REF(%3) "\n"
- " brc 4,0b\n"
- : "=&r" (old_val), MEMOP_OUT (__hp(addr))
- : "r" (val), MEMOP_IN (__hp(addr))
- : "memory", "cc");
- return old_val;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long old_val;
-
- __asm__ __volatile__(
- "0: csg %0,%2," MEMOP_REF(%3) "\n"
- " brc 4,0b\n"
- : "=&r" (old_val), MEMOP_OUT (__hp(addr))
- : "r" (val), MEMOP_IN (__hp(addr))
- : "memory", "cc");
- return old_val;
- }
-#endif
- default:
- __asm__ __volatile__(".long 0xd00d00");
- }
-
- return 0;
-}
-
-#define uatomic_xchg(addr, v) \
- (__typeof__(*(addr))) _uatomic_exchange((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr)))
-
-/* cmpxchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- switch (len) {
- case 4:
- {
- unsigned int old_val = (unsigned int)old;
-
- __asm__ __volatile__(
- " cs %0,%2," MEMOP_REF(%3) "\n"
- : "+r" (old_val), MEMOP_OUT (__hp(addr))
- : "r" (_new), MEMOP_IN (__hp(addr))
- : "memory", "cc");
- return old_val;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- " csg %0,%2," MEMOP_REF(%3) "\n"
- : "+r" (old), MEMOP_OUT (__hp(addr))
- : "r" (_new), MEMOP_IN (__hp(addr))
- : "memory", "cc");
- return old;
- }
-#endif
- default:
- __asm__ __volatile__(".long 0xd00d00");
- }
-
- return 0;
-}
-
-#define uatomic_cmpxchg(addr, old, _new) \
- (__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new),\
- sizeof(*(addr)))
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_UATOMIC_ARCH_S390_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_SPARC64_H
-#define _URCU_ARCH_UATOMIC_SPARC64_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
- * Copyright (c) 2009 Mathieu Desnoyers
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* cmpxchg */
-
-static inline __attribute__((always_inline))
-unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- switch (len) {
- case 4:
- {
- __asm__ __volatile__ (
- "membar #StoreLoad | #LoadLoad\n\t"
- "cas [%1],%2,%0\n\t"
- "membar #StoreLoad | #StoreStore\n\t"
- : "+&r" (_new)
- : "r" (addr), "r" (old)
- : "memory");
-
- return _new;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__ (
- "membar #StoreLoad | #LoadLoad\n\t"
- "casx [%1],%2,%0\n\t"
- "membar #StoreLoad | #StoreStore\n\t"
- : "+&r" (_new)
- : "r" (addr), "r" (old)
- : "memory");
-
- return _new;
- }
-#endif
- }
- __builtin_trap();
- return 0;
-}
-
-
-#define uatomic_cmpxchg(addr, old, _new) \
- ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new), \
- sizeof(*(addr))))
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_PPC_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_TILE_H
-#define _URCU_ARCH_UATOMIC_TILE_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009-2015 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define UATOMIC_HAS_ATOMIC_BYTE
-#define UATOMIC_HAS_ATOMIC_SHORT
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_TILE_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_UNKNOWN_H
-#define _URCU_ARCH_UATOMIC_UNKNOWN_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- * Copyright (c) 2010 Paul E. McKenney, IBM Corporation
- * (Adapted from uatomic_arch_ppc.h)
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- */
-
-/* See configure.ac for the list of recognized architectures. */
-#error "Cannot build: unrecognized architecture detected."
-
-#endif /* _URCU_ARCH_UATOMIC_UNKNOWN_H */
+++ /dev/null
-#ifndef _URCU_ARCH_UATOMIC_X86_H
-#define _URCU_ARCH_UATOMIC_X86_H
-
-/*
- * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
- * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
- * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
- * Copyright (c) 2009 Mathieu Desnoyers
- *
- * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
- * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
- *
- * Permission is hereby granted to use or copy this program
- * for any purpose, provided the above notices are retained on all copies.
- * Permission to modify the code and to distribute modified code is granted,
- * provided the above notices are retained, and a notice that the code was
- * modified is included with the above copyright notice.
- *
- * Code inspired from libuatomic_ops-1.2, inherited in part from the
- * Boehm-Demers-Weiser conservative garbage collector.
- */
-
-#include <urcu/compiler.h>
-#include <urcu/system.h>
-
-#define UATOMIC_HAS_ATOMIC_BYTE
-#define UATOMIC_HAS_ATOMIC_SHORT
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Derived from AO_compare_and_swap() and AO_test_and_set_full().
- */
-
-struct __uatomic_dummy {
- unsigned long v[10];
-};
-#define __hp(x) ((struct __uatomic_dummy *)(x))
-
-#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
-
-/* cmpxchg */
-
-static inline __attribute__((always_inline))
-unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len)
-{
- switch (len) {
- case 1:
- {
- unsigned char result = old;
-
- __asm__ __volatile__(
- "lock; cmpxchgb %2, %1"
- : "+a"(result), "+m"(*__hp(addr))
- : "q"((unsigned char)_new)
- : "memory");
- return result;
- }
- case 2:
- {
- unsigned short result = old;
-
- __asm__ __volatile__(
- "lock; cmpxchgw %2, %1"
- : "+a"(result), "+m"(*__hp(addr))
- : "r"((unsigned short)_new)
- : "memory");
- return result;
- }
- case 4:
- {
- unsigned int result = old;
-
- __asm__ __volatile__(
- "lock; cmpxchgl %2, %1"
- : "+a"(result), "+m"(*__hp(addr))
- : "r"((unsigned int)_new)
- : "memory");
- return result;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long result = old;
-
- __asm__ __volatile__(
- "lock; cmpxchgq %2, %1"
- : "+a"(result), "+m"(*__hp(addr))
- : "r"((unsigned long)_new)
- : "memory");
- return result;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return 0;
-}
-
-#define _uatomic_cmpxchg(addr, old, _new) \
- ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new),\
- sizeof(*(addr))))
-
-/* xchg */
-
-static inline __attribute__((always_inline))
-unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
-{
- /* Note: the "xchg" instruction does not need a "lock" prefix. */
- switch (len) {
- case 1:
- {
- unsigned char result;
- __asm__ __volatile__(
- "xchgb %0, %1"
- : "=q"(result), "+m"(*__hp(addr))
- : "0" ((unsigned char)val)
- : "memory");
- return result;
- }
- case 2:
- {
- unsigned short result;
- __asm__ __volatile__(
- "xchgw %0, %1"
- : "=r"(result), "+m"(*__hp(addr))
- : "0" ((unsigned short)val)
- : "memory");
- return result;
- }
- case 4:
- {
- unsigned int result;
- __asm__ __volatile__(
- "xchgl %0, %1"
- : "=r"(result), "+m"(*__hp(addr))
- : "0" ((unsigned int)val)
- : "memory");
- return result;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long result;
- __asm__ __volatile__(
- "xchgq %0, %1"
- : "=r"(result), "+m"(*__hp(addr))
- : "0" ((unsigned long)val)
- : "memory");
- return result;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return 0;
-}
-
-#define _uatomic_xchg(addr, v) \
- ((__typeof__(*(addr))) __uatomic_exchange((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-/* uatomic_add_return */
-
-static inline __attribute__((always_inline))
-unsigned long __uatomic_add_return(void *addr, unsigned long val,
- int len)
-{
- switch (len) {
- case 1:
- {
- unsigned char result = val;
-
- __asm__ __volatile__(
- "lock; xaddb %1, %0"
- : "+m"(*__hp(addr)), "+q" (result)
- :
- : "memory");
- return result + (unsigned char)val;
- }
- case 2:
- {
- unsigned short result = val;
-
- __asm__ __volatile__(
- "lock; xaddw %1, %0"
- : "+m"(*__hp(addr)), "+r" (result)
- :
- : "memory");
- return result + (unsigned short)val;
- }
- case 4:
- {
- unsigned int result = val;
-
- __asm__ __volatile__(
- "lock; xaddl %1, %0"
- : "+m"(*__hp(addr)), "+r" (result)
- :
- : "memory");
- return result + (unsigned int)val;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- unsigned long result = val;
-
- __asm__ __volatile__(
- "lock; xaddq %1, %0"
- : "+m"(*__hp(addr)), "+r" (result)
- :
- : "memory");
- return result + (unsigned long)val;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return 0;
-}
-
-#define _uatomic_add_return(addr, v) \
- ((__typeof__(*(addr))) __uatomic_add_return((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-/* uatomic_and */
-
-static inline __attribute__((always_inline))
-void __uatomic_and(void *addr, unsigned long val, int len)
-{
- switch (len) {
- case 1:
- {
- __asm__ __volatile__(
- "lock; andb %1, %0"
- : "=m"(*__hp(addr))
- : "iq" ((unsigned char)val)
- : "memory");
- return;
- }
- case 2:
- {
- __asm__ __volatile__(
- "lock; andw %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned short)val)
- : "memory");
- return;
- }
- case 4:
- {
- __asm__ __volatile__(
- "lock; andl %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned int)val)
- : "memory");
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- "lock; andq %1, %0"
- : "=m"(*__hp(addr))
- : "er" ((unsigned long)val)
- : "memory");
- return;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return;
-}
-
-#define _uatomic_and(addr, v) \
- (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
-
-/* uatomic_or */
-
-static inline __attribute__((always_inline))
-void __uatomic_or(void *addr, unsigned long val, int len)
-{
- switch (len) {
- case 1:
- {
- __asm__ __volatile__(
- "lock; orb %1, %0"
- : "=m"(*__hp(addr))
- : "iq" ((unsigned char)val)
- : "memory");
- return;
- }
- case 2:
- {
- __asm__ __volatile__(
- "lock; orw %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned short)val)
- : "memory");
- return;
- }
- case 4:
- {
- __asm__ __volatile__(
- "lock; orl %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned int)val)
- : "memory");
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- "lock; orq %1, %0"
- : "=m"(*__hp(addr))
- : "er" ((unsigned long)val)
- : "memory");
- return;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return;
-}
-
-#define _uatomic_or(addr, v) \
- (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
-
-/* uatomic_add */
-
-static inline __attribute__((always_inline))
-void __uatomic_add(void *addr, unsigned long val, int len)
-{
- switch (len) {
- case 1:
- {
- __asm__ __volatile__(
- "lock; addb %1, %0"
- : "=m"(*__hp(addr))
- : "iq" ((unsigned char)val)
- : "memory");
- return;
- }
- case 2:
- {
- __asm__ __volatile__(
- "lock; addw %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned short)val)
- : "memory");
- return;
- }
- case 4:
- {
- __asm__ __volatile__(
- "lock; addl %1, %0"
- : "=m"(*__hp(addr))
- : "ir" ((unsigned int)val)
- : "memory");
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- "lock; addq %1, %0"
- : "=m"(*__hp(addr))
- : "er" ((unsigned long)val)
- : "memory");
- return;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return;
-}
-
-#define _uatomic_add(addr, v) \
- (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
-
-
-/* uatomic_inc */
-
-static inline __attribute__((always_inline))
-void __uatomic_inc(void *addr, int len)
-{
- switch (len) {
- case 1:
- {
- __asm__ __volatile__(
- "lock; incb %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
- case 2:
- {
- __asm__ __volatile__(
- "lock; incw %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
- case 4:
- {
- __asm__ __volatile__(
- "lock; incl %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- "lock; incq %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
-#endif
- }
- /* generate an illegal instruction. Cannot catch this with linker tricks
- * when optimizations are disabled. */
- __asm__ __volatile__("ud2");
- return;
-}
-
-#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
-
-/* uatomic_dec */
-
-static inline __attribute__((always_inline))
-void __uatomic_dec(void *addr, int len)
-{
- switch (len) {
- case 1:
- {
- __asm__ __volatile__(
- "lock; decb %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
- case 2:
- {
- __asm__ __volatile__(
- "lock; decw %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
- case 4:
- {
- __asm__ __volatile__(
- "lock; decl %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
-#if (CAA_BITS_PER_LONG == 64)
- case 8:
- {
- __asm__ __volatile__(
- "lock; decq %0"
- : "=m"(*__hp(addr))
- :
- : "memory");
- return;
- }
-#endif
- }
- /*
- * generate an illegal instruction. Cannot catch this with
- * linker tricks when optimizations are disabled.
- */
- __asm__ __volatile__("ud2");
- return;
-}
-
-#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
-
-#if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
-extern int __rcu_cas_avail;
-extern int __rcu_cas_init(void);
-
-#define UATOMIC_COMPAT(insn) \
- ((caa_likely(__rcu_cas_avail > 0)) \
- ? (_uatomic_##insn) \
- : ((caa_unlikely(__rcu_cas_avail < 0) \
- ? ((__rcu_cas_init() > 0) \
- ? (_uatomic_##insn) \
- : (compat_uatomic_##insn)) \
- : (compat_uatomic_##insn))))
-
-/*
- * We leave the return value so we don't break the ABI, but remove the
- * return value from the API.
- */
-extern unsigned long _compat_uatomic_set(void *addr,
- unsigned long _new, int len);
-#define compat_uatomic_set(addr, _new) \
- ((void) _compat_uatomic_set((addr), \
- caa_cast_long_keep_sign(_new), \
- sizeof(*(addr))))
-
-
-extern unsigned long _compat_uatomic_xchg(void *addr,
- unsigned long _new, int len);
-#define compat_uatomic_xchg(addr, _new) \
- ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
- caa_cast_long_keep_sign(_new), \
- sizeof(*(addr))))
-
-extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
- unsigned long _new, int len);
-#define compat_uatomic_cmpxchg(addr, old, _new) \
- ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
- caa_cast_long_keep_sign(old), \
- caa_cast_long_keep_sign(_new), \
- sizeof(*(addr))))
-
-extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
-#define compat_uatomic_and(addr, v) \
- (_compat_uatomic_and((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
-#define compat_uatomic_or(addr, v) \
- (_compat_uatomic_or((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-extern unsigned long _compat_uatomic_add_return(void *addr,
- unsigned long _new, int len);
-#define compat_uatomic_add_return(addr, v) \
- ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
- caa_cast_long_keep_sign(v), \
- sizeof(*(addr))))
-
-#define compat_uatomic_add(addr, v) \
- ((void)compat_uatomic_add_return((addr), (v)))
-#define compat_uatomic_inc(addr) \
- (compat_uatomic_add((addr), 1))
-#define compat_uatomic_dec(addr) \
- (compat_uatomic_add((addr), -1))
-
-#else
-#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
-#endif
-
-/* Read is atomic even in compat mode */
-#define uatomic_set(addr, v) \
- UATOMIC_COMPAT(set(addr, v))
-
-#define uatomic_cmpxchg(addr, old, _new) \
- UATOMIC_COMPAT(cmpxchg(addr, old, _new))
-#define uatomic_xchg(addr, v) \
- UATOMIC_COMPAT(xchg(addr, v))
-
-#define uatomic_and(addr, v) \
- UATOMIC_COMPAT(and(addr, v))
-#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
-
-#define uatomic_or(addr, v) \
- UATOMIC_COMPAT(or(addr, v))
-#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
-
-#define uatomic_add_return(addr, v) \
- UATOMIC_COMPAT(add_return(addr, v))
-
-#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
-#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
-
-#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
-#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
-
-#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
-#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
-#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
-
-#ifdef __cplusplus
-}
-#endif
-
-#include <urcu/uatomic/generic.h>
-
-#endif /* _URCU_ARCH_UATOMIC_X86_H */
+++ /dev/null
-#warning "urcu/uatomic_arch.h is deprecated. Please include urcu/uatomic.h instead."
-#include <urcu/uatomic.h>
+++ /dev/null
-#warning "urcu/urcu-futex.h is deprecated. Please include urcu/futex.h instead."
-#include <urcu/futex.h>
+++ /dev/null
-#warning "urcu/urcu_ref.h is deprecated. Please include urcu/ref.h instead."
-#include <urcu/ref.h>
+++ /dev/null
-#ifndef _URCU_WFCQUEUE_H
-#define _URCU_WFCQUEUE_H
-
-/*
- * urcu/wfcqueue.h
- *
- * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <stdbool.h>
-#include <urcu/compiler.h>
-#include <urcu/arch.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Concurrent queue with wait-free enqueue/blocking dequeue.
- *
- * This queue has been designed and implemented collaboratively by
- * Mathieu Desnoyers and Lai Jiangshan. Inspired from
- * half-wait-free/half-blocking queue implementation done by Paul E.
- * McKenney.
- */
-
-#define CDS_WFCQ_WOULDBLOCK ((struct cds_wfcq_node *) -1UL)
-
-enum cds_wfcq_ret {
- CDS_WFCQ_RET_WOULDBLOCK = -1,
- CDS_WFCQ_RET_DEST_EMPTY = 0,
- CDS_WFCQ_RET_DEST_NON_EMPTY = 1,
- CDS_WFCQ_RET_SRC_EMPTY = 2,
-};
-
-enum cds_wfcq_state {
- CDS_WFCQ_STATE_LAST = (1U << 0),
-};
-
-struct cds_wfcq_node {
- struct cds_wfcq_node *next;
-};
-
-/*
- * Do not put head and tail on the same cache-line if concurrent
- * enqueue/dequeue are expected from many CPUs. This eliminates
- * false-sharing between enqueue and dequeue.
- */
-struct __cds_wfcq_head {
- struct cds_wfcq_node node;
-};
-
-struct cds_wfcq_head {
- struct cds_wfcq_node node;
- pthread_mutex_t lock;
-};
-
-#ifndef __cplusplus
-/*
- * The transparent union allows calling functions that work on both
- * struct cds_wfcq_head and struct __cds_wfcq_head on any of those two
- * types.
- */
-typedef union {
- struct __cds_wfcq_head *_h;
- struct cds_wfcq_head *h;
-} __attribute__((__transparent_union__)) cds_wfcq_head_ptr_t;
-
-/*
- * This static inline is only present for compatibility with C++. It is
- * effect-less in C.
- */
-static inline struct __cds_wfcq_head *__cds_wfcq_head_cast(struct __cds_wfcq_head *head)
-{
- return head;
-}
-
-/*
- * This static inline is only present for compatibility with C++. It is
- * effect-less in C.
- */
-static inline struct cds_wfcq_head *cds_wfcq_head_cast(struct cds_wfcq_head *head)
-{
- return head;
-}
-#else /* #ifndef __cplusplus */
-
-/* C++ ignores transparent union. */
-typedef union {
- struct __cds_wfcq_head *_h;
- struct cds_wfcq_head *h;
-} cds_wfcq_head_ptr_t;
-
-/* C++ ignores transparent union. Requires an explicit conversion. */
-static inline cds_wfcq_head_ptr_t __cds_wfcq_head_cast(struct __cds_wfcq_head *head)
-{
- cds_wfcq_head_ptr_t ret = { ._h = head };
- return ret;
-}
-/* C++ ignores transparent union. Requires an explicit conversion. */
-static inline cds_wfcq_head_ptr_t cds_wfcq_head_cast(struct cds_wfcq_head *head)
-{
- cds_wfcq_head_ptr_t ret = { .h = head };
- return ret;
-}
-#endif /* #else #ifndef __cplusplus */
-
-struct cds_wfcq_tail {
- struct cds_wfcq_node *p;
-};
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/wfcqueue.h>
-
-#define cds_wfcq_node_init _cds_wfcq_node_init
-#define cds_wfcq_init _cds_wfcq_init
-#define __cds_wfcq_init ___cds_wfcq_init
-#define cds_wfcq_destroy _cds_wfcq_destroy
-#define cds_wfcq_empty _cds_wfcq_empty
-#define cds_wfcq_enqueue _cds_wfcq_enqueue
-
-/* Dequeue locking */
-#define cds_wfcq_dequeue_lock _cds_wfcq_dequeue_lock
-#define cds_wfcq_dequeue_unlock _cds_wfcq_dequeue_unlock
-
-/* Locking performed within cds_wfcq calls. */
-#define cds_wfcq_dequeue_blocking _cds_wfcq_dequeue_blocking
-#define cds_wfcq_dequeue_with_state_blocking \
- _cds_wfcq_dequeue_with_state_blocking
-#define cds_wfcq_splice_blocking _cds_wfcq_splice_blocking
-#define cds_wfcq_first_blocking _cds_wfcq_first_blocking
-#define cds_wfcq_next_blocking _cds_wfcq_next_blocking
-
-/* Locking ensured by caller by holding cds_wfcq_dequeue_lock() */
-#define __cds_wfcq_dequeue_blocking ___cds_wfcq_dequeue_blocking
-#define __cds_wfcq_dequeue_with_state_blocking \
- ___cds_wfcq_dequeue_with_state_blocking
-#define __cds_wfcq_splice_blocking ___cds_wfcq_splice_blocking
-#define __cds_wfcq_first_blocking ___cds_wfcq_first_blocking
-#define __cds_wfcq_next_blocking ___cds_wfcq_next_blocking
-
-/*
- * Locking ensured by caller by holding cds_wfcq_dequeue_lock().
- * Non-blocking: deque, first, next return CDS_WFCQ_WOULDBLOCK if they
- * need to block. splice returns nonzero if it needs to block.
- */
-#define __cds_wfcq_dequeue_nonblocking ___cds_wfcq_dequeue_nonblocking
-#define __cds_wfcq_dequeue_with_state_nonblocking \
- ___cds_wfcq_dequeue_with_state_nonblocking
-#define __cds_wfcq_splice_nonblocking ___cds_wfcq_splice_nonblocking
-#define __cds_wfcq_first_nonblocking ___cds_wfcq_first_nonblocking
-#define __cds_wfcq_next_nonblocking ___cds_wfcq_next_nonblocking
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * Mutual exclusion of cds_wfcq_* / __cds_wfcq_* API
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * Legend:
- * [1] cds_wfcq_enqueue
- * [2] __cds_wfcq_splice (destination queue)
- * [3] __cds_wfcq_dequeue
- * [4] __cds_wfcq_splice (source queue)
- * [5] __cds_wfcq_first
- * [6] __cds_wfcq_next
- *
- * [1] [2] [3] [4] [5] [6]
- * [1] - - - - - -
- * [2] - - - - - -
- * [3] - - X X X X
- * [4] - - X - X X
- * [5] - - X X - -
- * [6] - - X X - -
- *
- * Mutual exclusion can be ensured by holding cds_wfcq_dequeue_lock().
- *
- * For convenience, cds_wfcq_dequeue_blocking() and
- * cds_wfcq_splice_blocking() hold the dequeue lock.
- *
- * Besides locking, mutual exclusion of dequeue, splice and iteration
- * can be ensured by performing all of those operations from a single
- * thread, without requiring any lock.
- */
-
-/*
- * cds_wfcq_node_init: initialize wait-free queue node.
- */
-extern void cds_wfcq_node_init(struct cds_wfcq_node *node);
-
-/*
- * cds_wfcq_init: initialize wait-free queue. Pair with
- * cds_wfcq_destroy().
- */
-extern void cds_wfcq_init(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_destroy: destroy wait-free queue. Pair with
- * cds_wfcq_init().
- */
-extern void cds_wfcq_destroy(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * __cds_wfcq_init: initialize wait-free queue (without lock). Don't
- * pair with any destroy function.
- */
-extern void __cds_wfcq_init(struct __cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_empty: return whether wait-free queue is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- */
-extern bool cds_wfcq_empty(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_dequeue_lock: take the dequeue mutual exclusion lock.
- */
-extern void cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_dequeue_unlock: release the dequeue mutual exclusion lock.
- */
-extern void cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_enqueue: enqueue a node into a wait-free queue.
- *
- * Issues a full memory barrier before enqueue. No mutual exclusion is
- * required.
- *
- * Returns false if the queue was empty prior to adding the node.
- * Returns true otherwise.
- */
-extern bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node);
-
-/*
- * cds_wfcq_dequeue_blocking: dequeue a node from a wait-free queue.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * It is valid to reuse and free a dequeued node immediately.
- * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
- * ensured.
- */
-extern struct cds_wfcq_node *cds_wfcq_dequeue_blocking(
- struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail);
-
-/*
- * cds_wfcq_dequeue_with_state_blocking: dequeue with state.
- *
- * Same as cds_wfcq_dequeue_blocking, but saves whether dequeueing the
- * last node of the queue into state (CDS_WFCQ_STATE_LAST).
- */
-extern struct cds_wfcq_node *cds_wfcq_dequeue_with_state_blocking(
- struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail,
- int *state);
-
-/*
- * cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Dequeue all nodes from src_q.
- * dest_q must be already initialized.
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Mutual exclusion with cds_wfcq_dequeue_blocking and dequeue lock is
- * ensured.
- *
- * Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue.
- */
-extern enum cds_wfcq_ret cds_wfcq_splice_blocking(
- struct cds_wfcq_head *dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- struct cds_wfcq_head *src_q_head,
- struct cds_wfcq_tail *src_q_tail);
-
-/*
- * __cds_wfcq_dequeue_blocking: dequeue a node from a wait-free queue.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * It is valid to reuse and free a dequeued node immediately.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- */
-extern struct cds_wfcq_node *__cds_wfcq_dequeue_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail);
-
-/*
- * __cds_wfcq_dequeue_with_state_blocking: dequeue with state.
- *
- * Same as __cds_wfcq_dequeue_blocking, but saves whether dequeueing the
- * last node of the queue into state (CDS_WFCQ_STATE_LAST).
- */
-extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- int *state);
-
-/*
- * __cds_wfcq_dequeue_nonblocking: dequeue a node from a wait-free queue.
- *
- * Same as __cds_wfcq_dequeue_blocking, but returns CDS_WFCQ_WOULDBLOCK
- * if it needs to block.
- */
-extern struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail);
-
-/*
- * __cds_wfcq_dequeue_with_state_blocking: dequeue with state.
- *
- * Same as __cds_wfcq_dequeue_nonblocking, but saves whether dequeueing
- * the last node of the queue into state (CDS_WFCQ_STATE_LAST).
- */
-extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- int *state);
-
-/*
- * __cds_wfcq_splice_blocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Dequeue all nodes from src_q.
- * dest_q must be already initialized.
- * Mutual exclusion for src_q should be ensured by the caller as
- * specified in the "Synchronisation table".
- * Returns enum cds_wfcq_ret which indicates the state of the src or
- * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK.
- */
-extern enum cds_wfcq_ret __cds_wfcq_splice_blocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail);
-
-/*
- * __cds_wfcq_splice_nonblocking: enqueue all src_q nodes at the end of dest_q.
- *
- * Same as __cds_wfcq_splice_blocking, but returns
- * CDS_WFCQ_RET_WOULDBLOCK if it needs to block.
- */
-extern enum cds_wfcq_ret __cds_wfcq_splice_nonblocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail);
-
-/*
- * __cds_wfcq_first_blocking: get first node of a queue, without dequeuing.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- *
- * Used by for-like iteration macros:
- * __cds_wfcq_for_each_blocking()
- * __cds_wfcq_for_each_blocking_safe()
- *
- * Returns NULL if queue is empty, first node otherwise.
- */
-extern struct cds_wfcq_node *__cds_wfcq_first_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail);
-
-/*
- * __cds_wfcq_first_nonblocking: get first node of a queue, without dequeuing.
- *
- * Same as __cds_wfcq_first_blocking, but returns CDS_WFCQ_WOULDBLOCK if
- * it needs to block.
- */
-extern struct cds_wfcq_node *__cds_wfcq_first_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail);
-
-/*
- * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- *
- * Used by for-like iteration macros:
- * __cds_wfcq_for_each_blocking()
- * __cds_wfcq_for_each_blocking_safe()
- *
- * Returns NULL if reached end of queue, non-NULL next queue node
- * otherwise.
- */
-extern struct cds_wfcq_node *__cds_wfcq_next_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node);
-
-/*
- * __cds_wfcq_next_blocking: get next node of a queue, without dequeuing.
- *
- * Same as __cds_wfcq_next_blocking, but returns CDS_WFCQ_WOULDBLOCK if
- * it needs to block.
- */
-extern struct cds_wfcq_node *__cds_wfcq_next_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node);
-
-#endif /* !_LGPL_SOURCE */
-
-/*
- * __cds_wfcq_for_each_blocking: Iterate over all nodes in a queue,
- * without dequeuing them.
- * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
- * @tail: tail of the queue (struct cds_wfcq_tail pointer).
- * @node: iterator on the queue (struct cds_wfcq_node pointer).
- *
- * Content written into each node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- */
-#define __cds_wfcq_for_each_blocking(head, tail, node) \
- for (node = __cds_wfcq_first_blocking(head, tail); \
- node != NULL; \
- node = __cds_wfcq_next_blocking(head, tail, node))
-
-/*
- * __cds_wfcq_for_each_blocking_safe: Iterate over all nodes in a queue,
- * without dequeuing them. Safe against deletion.
- * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer).
- * @tail: tail of the queue (struct cds_wfcq_tail pointer).
- * @node: iterator on the queue (struct cds_wfcq_node pointer).
- * @n: struct cds_wfcq_node pointer holding the next pointer (used
- * internally).
- *
- * Content written into each node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- * Dequeue/splice/iteration mutual exclusion should be ensured by the
- * caller.
- */
-#define __cds_wfcq_for_each_blocking_safe(head, tail, node, n) \
- for (node = __cds_wfcq_first_blocking(head, tail), \
- n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL); \
- node != NULL; \
- node = n, n = (node ? __cds_wfcq_next_blocking(head, tail, node) : NULL))
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFCQUEUE_H */
+++ /dev/null
-#ifndef _URCU_WFQUEUE_H
-#define _URCU_WFQUEUE_H
-
-/*
- * wfqueue.h
- *
- * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <urcu/compiler.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifndef CDS_WFQ_DEPRECATED
-#define CDS_WFQ_DEPRECATED \
- CDS_DEPRECATED("urcu/wfqueue.h is deprecated. Please use urcu/wfcqueue.h instead.")
-#endif
-
-/*
- * Queue with wait-free enqueue/blocking dequeue.
- * This implementation adds a dummy head node when the queue is empty to ensure
- * we can always update the queue locklessly.
- *
- * Inspired from half-wait-free/half-blocking queue implementation done by
- * Paul E. McKenney.
- */
-
-struct cds_wfq_node {
- struct cds_wfq_node *next;
-};
-
-struct cds_wfq_queue {
- struct cds_wfq_node *head, **tail;
- struct cds_wfq_node dummy; /* Dummy node */
- pthread_mutex_t lock;
-};
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/wfqueue.h>
-
-static inline CDS_WFQ_DEPRECATED
-void cds_wfq_node_init(struct cds_wfq_node *node)
-{
- _cds_wfq_node_init(node);
-}
-
-static inline CDS_WFQ_DEPRECATED
-void cds_wfq_init(struct cds_wfq_queue *q)
-{
- _cds_wfq_init(q);
-}
-
-static inline CDS_WFQ_DEPRECATED
-void cds_wfq_destroy(struct cds_wfq_queue *q)
-{
- _cds_wfq_destroy(q);
-}
-
-static inline CDS_WFQ_DEPRECATED
-void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node)
-{
- _cds_wfq_enqueue(q, node);
-}
-
-static inline CDS_WFQ_DEPRECATED
-struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- return ___cds_wfq_dequeue_blocking(q);
-}
-
-static inline CDS_WFQ_DEPRECATED
-struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- return _cds_wfq_dequeue_blocking(q);
-}
-
-#else /* !_LGPL_SOURCE */
-
-extern CDS_WFQ_DEPRECATED
-void cds_wfq_node_init(struct cds_wfq_node *node);
-
-extern CDS_WFQ_DEPRECATED
-void cds_wfq_init(struct cds_wfq_queue *q);
-
-extern CDS_WFQ_DEPRECATED
-void cds_wfq_destroy(struct cds_wfq_queue *q);
-
-extern CDS_WFQ_DEPRECATED
-void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node);
-
-/* __cds_wfq_dequeue_blocking: caller ensures mutual exclusion between dequeues */
-extern CDS_WFQ_DEPRECATED
-struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q);
-
-extern CDS_WFQ_DEPRECATED
-struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q);
-
-#endif /* !_LGPL_SOURCE */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _URCU_WFQUEUE_H */
+++ /dev/null
-#ifndef _URCU_WFSTACK_H
-#define _URCU_WFSTACK_H
-
-/*
- * urcu/wfstack.h
- *
- * Userspace RCU library - Stack with wait-free push, blocking traversal.
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-#include <pthread.h>
-#include <assert.h>
-#include <stdbool.h>
-#include <urcu/compiler.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- * Stack with wait-free push, blocking traversal.
- *
- * Stack implementing push, pop, pop_all operations, as well as iterator
- * on the stack head returned by pop_all.
- *
- * Wait-free operations: cds_wfs_push, __cds_wfs_pop_all, cds_wfs_empty,
- * cds_wfs_first.
- * Blocking operations: cds_wfs_pop, cds_wfs_pop_all, cds_wfs_next,
- * iteration on stack head returned by pop_all.
- *
- * Synchronization table:
- *
- * External synchronization techniques described in the API below is
- * required between pairs marked with "X". No external synchronization
- * required between pairs marked with "-".
- *
- * cds_wfs_push __cds_wfs_pop __cds_wfs_pop_all
- * cds_wfs_push - - -
- * __cds_wfs_pop - X X
- * __cds_wfs_pop_all - X -
- *
- * cds_wfs_pop and cds_wfs_pop_all use an internal mutex to provide
- * synchronization.
- */
-
-#define CDS_WFS_WOULDBLOCK ((void *) -1UL)
-
-enum cds_wfs_state {
- CDS_WFS_STATE_LAST = (1U << 0),
-};
-
-/*
- * struct cds_wfs_node is returned by __cds_wfs_pop, and also used as
- * iterator on stack. It is not safe to dereference the node next
- * pointer when returned by __cds_wfs_pop_blocking.
- */
-struct cds_wfs_node {
- struct cds_wfs_node *next;
-};
-
-/*
- * struct cds_wfs_head is returned by __cds_wfs_pop_all, and can be used
- * to begin iteration on the stack. "node" needs to be the first field of
- * cds_wfs_head, so the end-of-stack pointer value can be used for both
- * types.
- */
-struct cds_wfs_head {
- struct cds_wfs_node node;
-};
-
-struct __cds_wfs_stack {
- struct cds_wfs_head *head;
-};
-
-struct cds_wfs_stack {
- struct cds_wfs_head *head;
- pthread_mutex_t lock;
-};
-
-/*
- * The transparent union allows calling functions that work on both
- * struct cds_wfs_stack and struct __cds_wfs_stack on any of those two
- * types.
- */
-typedef union {
- struct __cds_wfs_stack *_s;
- struct cds_wfs_stack *s;
-} __attribute__((__transparent_union__)) cds_wfs_stack_ptr_t;
-
-#ifdef _LGPL_SOURCE
-
-#include <urcu/static/wfstack.h>
-
-#define cds_wfs_node_init _cds_wfs_node_init
-#define cds_wfs_init _cds_wfs_init
-#define cds_wfs_destroy _cds_wfs_destroy
-#define __cds_wfs_init ___cds_wfs_init
-#define cds_wfs_empty _cds_wfs_empty
-#define cds_wfs_push _cds_wfs_push
-
-/* Locking performed internally */
-#define cds_wfs_pop_blocking _cds_wfs_pop_blocking
-#define cds_wfs_pop_with_state_blocking _cds_wfs_pop_with_state_blocking
-#define cds_wfs_pop_all_blocking _cds_wfs_pop_all_blocking
-
-/*
- * For iteration on cds_wfs_head returned by __cds_wfs_pop_all or
- * cds_wfs_pop_all_blocking.
- */
-#define cds_wfs_first _cds_wfs_first
-#define cds_wfs_next_blocking _cds_wfs_next_blocking
-#define cds_wfs_next_nonblocking _cds_wfs_next_nonblocking
-
-/* Pop locking with internal mutex */
-#define cds_wfs_pop_lock _cds_wfs_pop_lock
-#define cds_wfs_pop_unlock _cds_wfs_pop_unlock
-
-/* Synchronization ensured by the caller. See synchronization table. */
-#define __cds_wfs_pop_blocking ___cds_wfs_pop_blocking
-#define __cds_wfs_pop_with_state_blocking \
- ___cds_wfs_pop_with_state_blocking
-#define __cds_wfs_pop_nonblocking ___cds_wfs_pop_nonblocking
-#define __cds_wfs_pop_with_state_nonblocking \
- ___cds_wfs_pop_with_state_nonblocking
-#define __cds_wfs_pop_all ___cds_wfs_pop_all
-
-#else /* !_LGPL_SOURCE */
-
-/*
- * cds_wfs_node_init: initialize wait-free stack node.
- */
-extern void cds_wfs_node_init(struct cds_wfs_node *node);
-
-/*
- * cds_wfs_init: initialize wait-free stack (with lock). Pair with
- * cds_wfs_destroy().
- */
-extern void cds_wfs_init(struct cds_wfs_stack *s);
-
-/*
- * cds_wfs_destroy: destroy wait-free stack (with lock). Pair with
- * cds_wfs_init().
- */
-extern void cds_wfs_destroy(struct cds_wfs_stack *s);
-
-/*
- * __cds_wfs_init: initialize wait-free stack (no lock). Don't pair with
- * any destroy function.
- */
-extern void __cds_wfs_init(struct __cds_wfs_stack *s);
-
-/*
- * cds_wfs_empty: return whether wait-free stack is empty.
- *
- * No memory barrier is issued. No mutual exclusion is required.
- */
-extern bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack);
-
-/*
- * cds_wfs_push: push a node into the stack.
- *
- * Issues a full memory barrier before push. No mutual exclusion is
- * required.
- *
- * Returns 0 if the stack was empty prior to adding the node.
- * Returns non-zero otherwise.
- */
-extern int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node);
-
-/*
- * cds_wfs_pop_blocking: pop a node from the stack.
- *
- * Calls __cds_wfs_pop_blocking with an internal pop mutex held.
- */
-extern struct cds_wfs_node *cds_wfs_pop_blocking(struct cds_wfs_stack *s);
-
-/*
- * cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
- *
- * Same as cds_wfs_pop_blocking, but stores whether the stack was
- * empty into state (CDS_WFS_STATE_LAST).
- */
-extern struct cds_wfs_node *
- cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state);
-
-/*
- * cds_wfs_pop_all_blocking: pop all nodes from a stack.
- *
- * Calls __cds_wfs_pop_all with an internal pop mutex held.
- */
-extern struct cds_wfs_head *cds_wfs_pop_all_blocking(struct cds_wfs_stack *s);
-
-/*
- * cds_wfs_first: get first node of a popped stack.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- *
- * Used by for-like iteration macros in urcu/wfstack.h:
- * cds_wfs_for_each_blocking()
- * cds_wfs_for_each_blocking_safe()
- *
- * Returns NULL if popped stack is empty, top stack node otherwise.
- */
-extern struct cds_wfs_node *cds_wfs_first(struct cds_wfs_head *head);
-
-/*
- * cds_wfs_next_blocking: get next node of a popped stack.
- *
- * Content written into the node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- *
- * Used by for-like iteration macros in urcu/wfstack.h:
- * cds_wfs_for_each_blocking()
- * cds_wfs_for_each_blocking_safe()
- *
- * Returns NULL if reached end of popped stack, non-NULL next stack
- * node otherwise.
- */
-extern struct cds_wfs_node *cds_wfs_next_blocking(struct cds_wfs_node *node);
-
-/*
- * cds_wfs_next_nonblocking: get next node of a popped stack.
- *
- * Same as cds_wfs_next_blocking, but returns CDS_WFS_WOULDBLOCK if it
- * needs to block.
- */
-extern struct cds_wfs_node *cds_wfs_next_nonblocking(struct cds_wfs_node *node);
-
-/*
- * cds_wfs_pop_lock: lock stack pop-protection mutex.
- */
-extern void cds_wfs_pop_lock(struct cds_wfs_stack *s);
-
-/*
- * cds_wfs_pop_unlock: unlock stack pop-protection mutex.
- */
-extern void cds_wfs_pop_unlock(struct cds_wfs_stack *s);
-
-/*
- * __cds_wfs_pop_blocking: pop a node from the stack.
- *
- * Returns NULL if stack is empty.
- *
- * __cds_wfs_pop_blocking needs to be synchronized using one of the
- * following techniques:
- *
- * 1) Calling __cds_wfs_pop_blocking under rcu read lock critical
- * section. The caller must wait for a grace period to pass before
- * freeing the returned node or modifying the cds_wfs_node structure.
- * 2) Using mutual exclusion (e.g. mutexes) to protect
- * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
- * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-extern struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack);
-
-/*
- * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state.
- *
- * Same as __cds_wfs_pop_blocking, but stores whether the stack was
- * empty into state (CDS_WFS_STATE_LAST).
- */
-extern struct cds_wfs_node *
- __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack,
- int *state);
-
-/*
- * __cds_wfs_pop_nonblocking: pop a node from the stack.
- *
- * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if
- * it needs to block.
- */
-extern struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack);
-
-/*
- * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack, with state.
- *
- * Same as __cds_wfs_pop_nonblocking, but stores whether the stack was
- * empty into state (CDS_WFS_STATE_LAST).
- */
-extern struct cds_wfs_node *
- __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack,
- int *state);
-
-/*
- * __cds_wfs_pop_all: pop all nodes from a stack.
- *
- * __cds_wfs_pop_all does not require any synchronization with other
- * push, nor with other __cds_wfs_pop_all, but requires synchronization
- * matching the technique used to synchronize __cds_wfs_pop_blocking:
- *
- * 1) If __cds_wfs_pop_blocking is called under rcu read lock critical
- * section, both __cds_wfs_pop_blocking and cds_wfs_pop_all callers
- * must wait for a grace period to pass before freeing the returned
- * node or modifying the cds_wfs_node structure. However, no RCU
- * read-side critical section is needed around __cds_wfs_pop_all.
- * 2) Using mutual exclusion (e.g. mutexes) to protect
- * __cds_wfs_pop_blocking and __cds_wfs_pop_all callers.
- * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking()
- * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme).
- */
-extern struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack);
-
-#endif /* !_LGPL_SOURCE */
-
-#ifdef __cplusplus
-}
-#endif
-
-/*
- * cds_wfs_for_each_blocking: Iterate over all nodes returned by
- * __cds_wfs_pop_all().
- * @head: head of the queue (struct cds_wfs_head pointer).
- * @node: iterator (struct cds_wfs_node pointer).
- *
- * Content written into each node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- */
-#define cds_wfs_for_each_blocking(head, node) \
- for (node = cds_wfs_first(head); \
- node != NULL; \
- node = cds_wfs_next_blocking(node))
-
-/*
- * cds_wfs_for_each_blocking_safe: Iterate over all nodes returned by
- * __cds_wfs_pop_all(). Safe against deletion.
- * @head: head of the queue (struct cds_wfs_head pointer).
- * @node: iterator (struct cds_wfs_node pointer).
- * @n: struct cds_wfs_node pointer holding the next pointer (used
- * internally).
- *
- * Content written into each node before enqueue is guaranteed to be
- * consistent, but no other memory ordering is ensured.
- */
-#define cds_wfs_for_each_blocking_safe(head, node, n) \
- for (node = cds_wfs_first(head), \
- n = (node ? cds_wfs_next_blocking(node) : NULL); \
- node != NULL; \
- node = n, n = (node ? cds_wfs_next_blocking(node) : NULL))
-
-#endif /* _URCU_WFSTACK_H */
+++ /dev/null
-/*
- * wfcqueue.c
- *
- * Userspace RCU library - Concurrent Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * Copyright 2011-2012 - Lai Jiangshan <laijs@cn.fujitsu.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#include "urcu/wfcqueue.h"
-#include "urcu/static/wfcqueue.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void cds_wfcq_node_init(struct cds_wfcq_node *node)
-{
- _cds_wfcq_node_init(node);
-}
-
-void cds_wfcq_init(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- _cds_wfcq_init(head, tail);
-}
-
-void cds_wfcq_destroy(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- _cds_wfcq_destroy(head, tail);
-}
-
-void __cds_wfcq_init(struct __cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- ___cds_wfcq_init(head, tail);
-}
-
-bool cds_wfcq_empty(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-
-{
- return _cds_wfcq_empty(head, tail);
-}
-
-bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node)
-{
- return _cds_wfcq_enqueue(head, tail, node);
-}
-
-void cds_wfcq_dequeue_lock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- _cds_wfcq_dequeue_lock(head, tail);
-}
-
-void cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- _cds_wfcq_dequeue_unlock(head, tail);
-}
-
-struct cds_wfcq_node *cds_wfcq_dequeue_blocking(
- struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail)
-{
- return _cds_wfcq_dequeue_blocking(head, tail);
-}
-
-struct cds_wfcq_node *cds_wfcq_dequeue_with_state_blocking(
- struct cds_wfcq_head *head,
- struct cds_wfcq_tail *tail,
- int *state)
-{
- return _cds_wfcq_dequeue_with_state_blocking(head, tail, state);
-}
-
-enum cds_wfcq_ret cds_wfcq_splice_blocking(
- struct cds_wfcq_head *dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- struct cds_wfcq_head *src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- return _cds_wfcq_splice_blocking(dest_q_head, dest_q_tail,
- src_q_head, src_q_tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_dequeue_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_dequeue_blocking(head, tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- int *state)
-{
- return ___cds_wfcq_dequeue_with_state_blocking(head, tail, state);
-}
-
-struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_dequeue_nonblocking(head, tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- int *state)
-{
- return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, state);
-}
-
-enum cds_wfcq_ret __cds_wfcq_splice_blocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- return ___cds_wfcq_splice_blocking(dest_q_head, dest_q_tail,
- src_q_head, src_q_tail);
-}
-
-enum cds_wfcq_ret __cds_wfcq_splice_nonblocking(
- cds_wfcq_head_ptr_t dest_q_head,
- struct cds_wfcq_tail *dest_q_tail,
- cds_wfcq_head_ptr_t src_q_head,
- struct cds_wfcq_tail *src_q_tail)
-{
- return ___cds_wfcq_splice_nonblocking(dest_q_head, dest_q_tail,
- src_q_head, src_q_tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_first_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_first_blocking(head, tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_first_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail)
-{
- return ___cds_wfcq_first_nonblocking(head, tail);
-}
-
-struct cds_wfcq_node *__cds_wfcq_next_blocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node)
-{
- return ___cds_wfcq_next_blocking(head, tail, node);
-}
-
-struct cds_wfcq_node *__cds_wfcq_next_nonblocking(
- cds_wfcq_head_ptr_t head,
- struct cds_wfcq_tail *tail,
- struct cds_wfcq_node *node)
-{
- return ___cds_wfcq_next_nonblocking(head, tail, node);
-}
+++ /dev/null
-/*
- * wfqueue.c
- *
- * Userspace RCU library - Queue with Wait-Free Enqueue/Blocking Dequeue
- *
- * Copyright 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Remove deprecation warnings from LGPL wrapper build. */
-#define CDS_WFQ_DEPRECATED
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#include "urcu/wfqueue.h"
-#include "urcu/static/wfqueue.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void cds_wfq_node_init(struct cds_wfq_node *node)
-{
- _cds_wfq_node_init(node);
-}
-
-void cds_wfq_init(struct cds_wfq_queue *q)
-{
- _cds_wfq_init(q);
-}
-
-void cds_wfq_destroy(struct cds_wfq_queue *q)
-{
- _cds_wfq_destroy(q);
-}
-
-void cds_wfq_enqueue(struct cds_wfq_queue *q, struct cds_wfq_node *node)
-{
- _cds_wfq_enqueue(q, node);
-}
-
-struct cds_wfq_node *__cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- return ___cds_wfq_dequeue_blocking(q);
-}
-
-struct cds_wfq_node *cds_wfq_dequeue_blocking(struct cds_wfq_queue *q)
-{
- return _cds_wfq_dequeue_blocking(q);
-}
+++ /dev/null
-/*
- * wfstack.c
- *
- * Userspace RCU library - Stack with wait-free push, blocking traversal.
- *
- * Copyright 2010-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
-#include "urcu/wfstack.h"
-#include "urcu/static/wfstack.h"
-
-/*
- * library wrappers to be used by non-LGPL compatible source code.
- */
-
-void cds_wfs_node_init(struct cds_wfs_node *node)
-{
- _cds_wfs_node_init(node);
-}
-
-void cds_wfs_init(struct cds_wfs_stack *s)
-{
- _cds_wfs_init(s);
-}
-
-void cds_wfs_destroy(struct cds_wfs_stack *s)
-{
- _cds_wfs_destroy(s);
-}
-
-void __cds_wfs_init(struct __cds_wfs_stack *s)
-{
- ___cds_wfs_init(s);
-}
-
-bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack)
-{
- return _cds_wfs_empty(u_stack);
-}
-
-int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node)
-{
- return _cds_wfs_push(u_stack, node);
-}
-
-struct cds_wfs_node *cds_wfs_pop_blocking(struct cds_wfs_stack *s)
-{
- return _cds_wfs_pop_blocking(s);
-}
-
-struct cds_wfs_node *
- cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state)
-{
- return _cds_wfs_pop_with_state_blocking(s, state);
-}
-
-struct cds_wfs_head *cds_wfs_pop_all_blocking(struct cds_wfs_stack *s)
-{
- return _cds_wfs_pop_all_blocking(s);
-}
-
-struct cds_wfs_node *cds_wfs_first(struct cds_wfs_head *head)
-{
- return _cds_wfs_first(head);
-}
-
-struct cds_wfs_node *cds_wfs_next_blocking(struct cds_wfs_node *node)
-{
- return _cds_wfs_next_blocking(node);
-}
-
-struct cds_wfs_node *cds_wfs_next_nonblocking(struct cds_wfs_node *node)
-{
- return _cds_wfs_next_nonblocking(node);
-}
-
-void cds_wfs_pop_lock(struct cds_wfs_stack *s)
-{
- _cds_wfs_pop_lock(s);
-}
-
-void cds_wfs_pop_unlock(struct cds_wfs_stack *s)
-{
- _cds_wfs_pop_unlock(s);
-}
-
-struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack)
-{
- return ___cds_wfs_pop_blocking(u_stack);
-}
-
-struct cds_wfs_node *
- __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack,
- int *state)
-{
- return ___cds_wfs_pop_with_state_blocking(u_stack, state);
-}
-
-struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack)
-{
- return ___cds_wfs_pop_nonblocking(u_stack);
-}
-
-struct cds_wfs_node *
- __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack,
- int *state)
-{
- return ___cds_wfs_pop_with_state_nonblocking(u_stack, state);
-}
-
-struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack)
-{
- return ___cds_wfs_pop_all(u_stack);
-}
projects / urcu.git / commitdiff
