From 0376e7b2f8d26778ebc4750b34e718c062c92764 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 3 Jun 2011 12:47:56 -0400 Subject: [PATCH] Make defer_rcu() usable from library using multiple URCU implementations [ Edit: minor tab to whitespace cleanup ] Signed-off-by: Paul E. McKenney Signed-off-by: Mathieu Desnoyers --- Makefile.am | 4 +- README | 11 ++- tests/Makefile.am | 2 +- urcu-bp-map.h | 6 ++ urcu-bp.c | 1 + urcu-bp.h | 1 + urcu-defer.c => urcu-defer-impl.h | 131 ++++++++++++++++++++++-------- urcu-defer-static.h | 108 ------------------------ urcu-map.h | 26 ++++++ urcu-qsbr-map.h | 6 ++ urcu-qsbr.c | 1 + urcu-qsbr.h | 1 + urcu.c | 1 + urcu.h | 1 + 14 files changed, 149 insertions(+), 151 deletions(-) rename urcu-defer.c => urcu-defer-impl.h (72%) delete mode 100644 urcu-defer-static.h diff --git a/Makefile.am b/Makefile.am index 4b70644..1589884 100644 --- a/Makefile.am +++ b/Makefile.am @@ -30,7 +30,7 @@ COMPAT+=compat_futex.c endif lib_LTLIBRARIES = liburcu.la liburcu-qsbr.la liburcu-mb.la liburcu-signal.la \ - liburcu-bp.la liburcu-defer.la \ + liburcu-bp.la \ libwfqueue.la libwfstack.la librculfqueue.la librculfstack.la liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) @@ -45,8 +45,6 @@ liburcu_signal_la_CFLAGS = -DRCU_SIGNAL liburcu_bp_la_SOURCES = urcu-bp.c urcu-pointer.c $(COMPAT) -liburcu_defer_la_SOURCES = urcu-defer.c $(COMPAT) - libwfqueue_la_SOURCES = wfqueue.c $(COMPAT) libwfstack_la_SOURCES = wfstack.c $(COMPAT) librculfqueue_la_SOURCES = rculfqueue.c $(COMPAT) diff --git a/README b/README index 659511f..7d97f19 100644 --- a/README +++ b/README @@ -124,16 +124,15 @@ Writing Usage of liburcu-defer - * #include - * Link with "-lurcu-defer", and also with one of the urcu library - (either urcu, urcu-bp, urcu-mb or urcu-qsbr). + * Follow instructions for either liburcu, liburcu-qsbr, + liburcu-mb, liburcu-signal, or liburcu-bp above. + The liburcu-defer functionality is pulled into each of + those library modules. * Provides defer_rcu() primitive to enqueue delayed callbacks. Queued callbacks are executed in batch periodically after a grace period. Do _not_ use defer_rcu() within a read-side critical section, because it may call synchronize_rcu() if the thread queue is full. - * Provides defer_rcu_ratelimit() primitive, which acts just like - defer_rcu(), but takes an additional rate limiter callback forcing - synchronized callback execution of the limiter returns non-zero. + This can lead to deadlock or worse. * Requires that rcu_defer_barrier() must be called in library destructor if a library queues callbacks and is expected to be unloaded with dlclose(). diff --git a/tests/Makefile.am b/tests/Makefile.am index 8dacb11..5598689 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -35,7 +35,7 @@ URCU_MB=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfqueue. # URCU_SIGNAL uses urcu.c but -DRCU_SIGNAL must be defined URCU_SIGNAL=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfqueue.c $(COMPAT) URCU_BP=$(top_srcdir)/urcu-bp.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfqueue.c $(COMPAT) -URCU_DEFER=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-defer.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfqueue.c $(COMPAT) +URCU_DEFER=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfqueue.c $(COMPAT) URCU_LIB=$(top_builddir)/liburcu.la URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la diff --git a/urcu-bp-map.h b/urcu-bp-map.h index 6321802..4abe8dc 100644 --- a/urcu-bp-map.h +++ b/urcu-bp-map.h @@ -58,4 +58,10 @@ #define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_bp #define call_rcu call_rcu_bp +#define defer_rcu defer_rcu_bp +#define rcu_defer_register_thread rcu_defer_register_thread_bp +#define rcu_defer_unregister_thread rcu_defer_unregister_thread_bp +#define rcu_defer_barrier rcu_defer_barrier_bp +#define rcu_defer_barrier_thread rcu_defer_barrier_thread_bp + #endif /* _URCU_BP_MAP_H */ diff --git a/urcu-bp.c b/urcu-bp.c index 5474f9f..14b2001 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -379,3 +379,4 @@ void rcu_bp_after_fork_child(void) } #include "urcu-call-rcu-impl.h" +#include "urcu-defer-impl.h" diff --git a/urcu-bp.h b/urcu-bp.h index bc2dbc3..21fe66a 100644 --- a/urcu-bp.h +++ b/urcu-bp.h @@ -118,5 +118,6 @@ static inline void rcu_init(void) #endif #include "urcu-call-rcu.h" +#include "urcu-defer.h" #endif /* _URCU_BP_H */ diff --git a/urcu-defer.c b/urcu-defer-impl.h similarity index 72% rename from urcu-defer.c rename to urcu-defer-impl.h index 3f596ae..0aedd53 100644 --- a/urcu-defer.c +++ b/urcu-defer-impl.h @@ -1,9 +1,16 @@ +#ifndef _URCU_DEFER_IMPL_H +#define _URCU_DEFER_IMPL_H + /* - * urcu-defer.c + * urcu-defer-impl.h + * + * Userspace RCU header - memory reclamation. * - * Userspace RCU library - batch memory reclamation + * TO BE INCLUDED ONLY FROM URCU LIBRARY CODE. See urcu-defer.h for linking + * dynamically with the userspace rcu reclamation library. * * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -18,13 +25,15 @@ * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. */ -#include +#include #include +#include #include #include -#include #include #include #include @@ -33,7 +42,72 @@ #include #include "urcu/urcu-futex.h" -#include "urcu-defer-static.h" + +#include +#include +#include +#include +#include + +/* + * Number of entries in the per-thread defer queue. Must be power of 2. + */ +#define DEFER_QUEUE_SIZE (1 << 12) +#define DEFER_QUEUE_MASK (DEFER_QUEUE_SIZE - 1) + +/* + * Typically, data is aligned at least on the architecture size. + * Use lowest bit to indicate that the current callback is changing. + * Assumes that (void *)-2L is not used often. Used to encode non-aligned + * functions and non-aligned data using extra space. + * We encode the (void *)-2L fct as: -2L, fct, data. + * We encode the (void *)-2L data as: -2L, fct, data. + * Here, DQ_FCT_MARK == ~DQ_FCT_BIT. Required for the test order. + */ +#define DQ_FCT_BIT (1 << 0) +#define DQ_IS_FCT_BIT(x) ((unsigned long)(x) & DQ_FCT_BIT) +#define DQ_SET_FCT_BIT(x) \ + (x = (void *)((unsigned long)(x) | DQ_FCT_BIT)) +#define DQ_CLEAR_FCT_BIT(x) \ + (x = (void *)((unsigned long)(x) & ~DQ_FCT_BIT)) +#define DQ_FCT_MARK ((void *)(~DQ_FCT_BIT)) + +/* + * This code section can only be included in LGPL 2.1 compatible source code. + * See below for the function call wrappers which can be used in code meant to + * be only linked with the Userspace RCU library. This comes with a small + * performance degradation on the read-side due to the added function calls. + * This is required to permit relinking with newer versions of the library. + */ + +#ifdef DEBUG_RCU +#define rcu_assert(args...) assert(args) +#else +#define rcu_assert(args...) +#endif + +/* + * defer queue. + * Contains pointers. Encoded to save space when same callback is often used. + * When looking up the next item: + * - if DQ_FCT_BIT is set, set the current callback to DQ_CLEAR_FCT_BIT(ptr) + * - next element contains pointer to data. + * - else if item == DQ_FCT_MARK + * - set the current callback to next element ptr + * - following next element contains pointer to data. + * - else current element contains data + */ +struct defer_queue { + unsigned long head; /* add element at head */ + void *last_fct_in; /* last fct pointer encoded */ + unsigned long tail; /* next element to remove at tail */ + void *last_fct_out; /* last fct pointer encoded */ + void **q; + /* registry information */ + unsigned long last_head; + struct cds_list_head list; /* list of thread queues */ +}; + /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #include "urcu-defer.h" @@ -54,10 +128,10 @@ static int defer_thread_futex; * the reclamation tread. */ static struct defer_queue __thread defer_queue; -static CDS_LIST_HEAD(registry); +static CDS_LIST_HEAD(registry_defer); static pthread_t tid_defer; -static void mutex_lock(pthread_mutex_t *mutex) +static void mutex_lock_defer(pthread_mutex_t *mutex) { int ret; @@ -80,17 +154,6 @@ static void mutex_lock(pthread_mutex_t *mutex) #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */ } -static void mutex_unlock(pthread_mutex_t *mutex) -{ - int ret; - - ret = pthread_mutex_unlock(mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - /* * Wake-up any waiting defer thread. Called from many concurrent threads. */ @@ -108,8 +171,8 @@ static unsigned long rcu_defer_num_callbacks(void) unsigned long num_items = 0, head; struct defer_queue *index; - mutex_lock(&rcu_defer_mutex); - cds_list_for_each_entry(index, ®istry, list) { + mutex_lock_defer(&rcu_defer_mutex); + cds_list_for_each_entry(index, ®istry_defer, list) { head = CMM_LOAD_SHARED(index->head); num_items += head - index->tail; } @@ -184,7 +247,7 @@ static void _rcu_defer_barrier_thread(void) void rcu_defer_barrier_thread(void) { - mutex_lock(&rcu_defer_mutex); + mutex_lock_defer(&rcu_defer_mutex); _rcu_defer_barrier_thread(); mutex_unlock(&rcu_defer_mutex); } @@ -207,11 +270,11 @@ void rcu_defer_barrier(void) struct defer_queue *index; unsigned long num_items = 0; - if (cds_list_empty(®istry)) + if (cds_list_empty(®istry_defer)) return; - mutex_lock(&rcu_defer_mutex); - cds_list_for_each_entry(index, ®istry, list) { + mutex_lock_defer(&rcu_defer_mutex); + cds_list_for_each_entry(index, ®istry_defer, list) { index->last_head = CMM_LOAD_SHARED(index->head); num_items += index->last_head - index->tail; } @@ -223,7 +286,7 @@ void rcu_defer_barrier(void) goto end; } synchronize_rcu(); - cds_list_for_each_entry(index, ®istry, list) + cds_list_for_each_entry(index, ®istry_defer, list) rcu_defer_barrier_queue(index, index->last_head); end: mutex_unlock(&rcu_defer_mutex); @@ -349,10 +412,10 @@ int rcu_defer_register_thread(void) if (!defer_queue.q) return -ENOMEM; - mutex_lock(&defer_thread_mutex); - mutex_lock(&rcu_defer_mutex); - was_empty = cds_list_empty(®istry); - cds_list_add(&defer_queue.list, ®istry); + mutex_lock_defer(&defer_thread_mutex); + mutex_lock_defer(&rcu_defer_mutex); + was_empty = cds_list_empty(®istry_defer); + cds_list_add(&defer_queue.list, ®istry_defer); mutex_unlock(&rcu_defer_mutex); if (was_empty) @@ -365,13 +428,13 @@ void rcu_defer_unregister_thread(void) { int is_empty; - mutex_lock(&defer_thread_mutex); - mutex_lock(&rcu_defer_mutex); + mutex_lock_defer(&defer_thread_mutex); + mutex_lock_defer(&rcu_defer_mutex); cds_list_del(&defer_queue.list); _rcu_defer_barrier_thread(); free(defer_queue.q); defer_queue.q = NULL; - is_empty = cds_list_empty(®istry); + is_empty = cds_list_empty(®istry_defer); mutex_unlock(&rcu_defer_mutex); if (is_empty) @@ -381,5 +444,7 @@ void rcu_defer_unregister_thread(void) void rcu_defer_exit(void) { - assert(cds_list_empty(®istry)); + assert(cds_list_empty(®istry_defer)); } + +#endif /* _URCU_DEFER_IMPL_H */ diff --git a/urcu-defer-static.h b/urcu-defer-static.h deleted file mode 100644 index 3e5df3e..0000000 --- a/urcu-defer-static.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef _URCU_DEFER_STATIC_H -#define _URCU_DEFER_STATIC_H - -/* - * urcu-defer-static.h - * - * Userspace RCU header - memory reclamation. - * - * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-defer.h for linking - * dynamically with the userspace rcu reclamation library. - * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * IBM's contributions to this file may be relicensed under LGPLv2 or later. - */ - -#include -#include - -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Number of entries in the per-thread defer queue. Must be power of 2. - */ -#define DEFER_QUEUE_SIZE (1 << 12) -#define DEFER_QUEUE_MASK (DEFER_QUEUE_SIZE - 1) - -/* - * Typically, data is aligned at least on the architecture size. - * Use lowest bit to indicate that the current callback is changing. - * Assumes that (void *)-2L is not used often. Used to encode non-aligned - * functions and non-aligned data using extra space. - * We encode the (void *)-2L fct as: -2L, fct, data. - * We encode the (void *)-2L data as: -2L, fct, data. - * Here, DQ_FCT_MARK == ~DQ_FCT_BIT. Required for the test order. - */ -#define DQ_FCT_BIT (1 << 0) -#define DQ_IS_FCT_BIT(x) ((unsigned long)(x) & DQ_FCT_BIT) -#define DQ_SET_FCT_BIT(x) \ - (x = (void *)((unsigned long)(x) | DQ_FCT_BIT)) -#define DQ_CLEAR_FCT_BIT(x) \ - (x = (void *)((unsigned long)(x) & ~DQ_FCT_BIT)) -#define DQ_FCT_MARK ((void *)(~DQ_FCT_BIT)) - -/* - * This code section can only be included in LGPL 2.1 compatible source code. - * See below for the function call wrappers which can be used in code meant to - * be only linked with the Userspace RCU library. This comes with a small - * performance degradation on the read-side due to the added function calls. - * This is required to permit relinking with newer versions of the library. - */ - -#ifdef DEBUG_RCU -#define rcu_assert(args...) assert(args) -#else -#define rcu_assert(args...) -#endif - -/* - * defer queue. - * Contains pointers. Encoded to save space when same callback is often used. - * When looking up the next item: - * - if DQ_FCT_BIT is set, set the current callback to DQ_CLEAR_FCT_BIT(ptr) - * - next element contains pointer to data. - * - else if item == DQ_FCT_MARK - * - set the current callback to next element ptr - * - following next element contains pointer to data. - * - else current element contains data - */ -struct defer_queue { - unsigned long head; /* add element at head */ - void *last_fct_in; /* last fct pointer encoded */ - unsigned long tail; /* next element to remove at tail */ - void *last_fct_out; /* last fct pointer encoded */ - void **q; - /* registry information */ - unsigned long last_head; - struct cds_list_head list; /* list of thread queues */ -}; - -#ifdef __cplusplus -} -#endif - -#endif /* _URCU_DEFER_STATIC_H */ diff --git a/urcu-map.h b/urcu-map.h index eccaac2..b55e304 100644 --- a/urcu-map.h +++ b/urcu-map.h @@ -34,6 +34,10 @@ /* Mapping macros to allow multiple flavors in a single binary. */ +#if !defined(RCU_MEMBARRIER) && !defined(RCU_SIGNAL) && !defined(RCU_MB) +#define RCU_MB +#endif + #ifdef RCU_MEMBARRIER #define rcu_read_lock rcu_read_lock_memb @@ -60,6 +64,12 @@ #define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_memb #define call_rcu call_rcu_memb +#define defer_rcu defer_rcu_memb +#define rcu_defer_register_thread rcu_defer_register_thread_memb +#define rcu_defer_unregister_thread rcu_defer_unregister_thread_memb +#define rcu_defer_barrier rcu_defer_barrier_memb +#define rcu_defer_barrier_thread rcu_defer_barrier_thread_memb + #elif defined(RCU_SIGNAL) #define rcu_read_lock rcu_read_lock_sig @@ -86,6 +96,12 @@ #define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_sig #define call_rcu call_rcu_sig +#define defer_rcu defer_rcu_sig +#define rcu_defer_register_thread rcu_defer_register_thread_sig +#define rcu_defer_unregister_thread rcu_defer_unregister_thread_sig +#define rcu_defer_barrier rcu_defer_barrier_sig +#define rcu_defer_barrier_thread rcu_defer_barrier_thread_sig + #elif defined(RCU_MB) #define rcu_read_lock rcu_read_lock_mb @@ -112,6 +128,16 @@ #define free_all_cpu_call_rcu_data free_all_cpu_call_rcu_data_mb #define call_rcu call_rcu_mb +#define defer_rcu defer_rcu_mb +#define rcu_defer_register_thread rcu_defer_register_thread_mb +#define rcu_defer_unregister_thread rcu_defer_unregister_thread_mb +#define rcu_defer_barrier rcu_defer_barrier_mb +#define rcu_defer_barrier_thread rcu_defer_barrier_thread_mb + +#else + +#error "Undefined selection" + #endif #endif /* _URCU_MAP_H */ diff --git a/urcu-qsbr-map.h b/urcu-qsbr-map.h index 2bce1b6..0d88d83 100644 --- a/urcu-qsbr-map.h +++ b/urcu-qsbr-map.h @@ -60,4 +60,10 @@ #define create_all_cpu_call_rcu_data create_all_cpu_call_rcu_data_qsbr #define call_rcu call_rcu_qsbr +#define defer_rcu defer_rcu_qsbr +#define rcu_defer_register_thread rcu_defer_register_thread_qsbr +#define rcu_defer_unregister_thread rcu_defer_unregister_thread_qsbr +#define rcu_defer_barrier rcu_defer_barrier_qsbr +#define rcu_defer_barrier_thread rcu_defer_barrier_thread_qsbr + #endif /* _URCU_QSBR_MAP_H */ diff --git a/urcu-qsbr.c b/urcu-qsbr.c index 8dcad33..cf8b5ce 100644 --- a/urcu-qsbr.c +++ b/urcu-qsbr.c @@ -334,3 +334,4 @@ void rcu_exit(void) } #include "urcu-call-rcu-impl.h" +#include "urcu-defer-impl.h" diff --git a/urcu-qsbr.h b/urcu-qsbr.h index 7ef1bfe..a691c52 100644 --- a/urcu-qsbr.h +++ b/urcu-qsbr.h @@ -125,5 +125,6 @@ extern void rcu_unregister_thread(void); #endif #include "urcu-call-rcu.h" +#include "urcu-defer.h" #endif /* _URCU_QSBR_H */ diff --git a/urcu.c b/urcu.c index 4ee9e3b..d356f54 100644 --- a/urcu.c +++ b/urcu.c @@ -434,3 +434,4 @@ void rcu_exit(void) #endif /* #ifdef RCU_SIGNAL */ #include "urcu-call-rcu-impl.h" +#include "urcu-defer-impl.h" diff --git a/urcu.h b/urcu.h index 417e609..15c8c38 100644 --- a/urcu.h +++ b/urcu.h @@ -111,5 +111,6 @@ extern void rcu_init(void); #endif #include "urcu-call-rcu.h" +#include "urcu-defer.h" #endif /* _URCU_H */ -- 2.34.1