[userspace-rcu.git] / urcu-bp.c

/*
 * urcu-bp.c
 *
 * Userspace RCU library, "bulletproof" version.
 *
 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
 */

#define _GNU_SOURCE
#define _LGPL_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <assert.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#include <unistd.h>
#include <sys/mman.h>

#include "urcu/wfcqueue.h"
#include "urcu/map/urcu-bp.h"
#include "urcu/static/urcu-bp.h"
#include "urcu-pointer.h"
#include "urcu/tls-compat.h"

#include "urcu-die.h"

/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
#undef _LGPL_SOURCE
#include "urcu-bp.h"
#define _LGPL_SOURCE

#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif

#ifdef __linux__
static
void *mremap_wrapper(void *old_address, size_t old_size,
		size_t new_size, int flags)
{
	return mremap(old_address, old_size, new_size, flags);
}
#else

#define MREMAP_MAYMOVE	1
#define MREMAP_FIXED	2

/*
 * mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
 * This is not generic.
*/
static
void *mremap_wrapper(void *old_address, size_t old_size,
		size_t new_size, int flags)
{
	void *new_address;

	assert(flags & MREMAP_MAYMOVE);
	assert(!(flags & MREMAP_FIXED));
	new_address = mmap(old_address, new_size,
			   PROT_READ | PROT_WRITE,
			   MAP_ANONYMOUS | MAP_PRIVATE,
			   -1, 0);
	if (new_address == MAP_FAILED)
		return MAP_FAILED;
	if (old_address) {
		memcpy(new_address, old_address, old_size);
		munmap(old_address, old_size);
	}
	return new_address;
}
#endif

/* Sleep delay in us */
#define RCU_SLEEP_DELAY		1000
#define ARENA_INIT_ALLOC	16

/*
 * Active attempts to check for reader Q.S. before calling sleep().
 */
#define RCU_QS_ACTIVE_ATTEMPTS 100

void __attribute__((destructor)) rcu_bp_exit(void);

static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;

#ifdef DEBUG_YIELD
unsigned int rcu_yield_active;
DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);
#endif

struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };

/*
 * Pointer to registry elements. Written to only by each individual reader. Read
 * by both the reader and the writers.
 */
DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);

static CDS_LIST_HEAD(registry);

struct registry_arena {
	void *p;
	size_t len;
	size_t used;
};

static struct registry_arena registry_arena;

/* Saved fork signal mask, protected by rcu_gp_lock */
static sigset_t saved_fork_signal_mask;

static void rcu_gc_registry(void);

static void mutex_lock(pthread_mutex_t *mutex)
{
	int ret;

#ifndef DISTRUST_SIGNALS_EXTREME
	ret = pthread_mutex_lock(mutex);
	if (ret)
		urcu_die(ret);
#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
		if (ret != EBUSY && ret != EINTR)
			urcu_die(ret);
		poll(NULL,0,10);
	}
#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
}

static void mutex_unlock(pthread_mutex_t *mutex)
{
	int ret;

	ret = pthread_mutex_unlock(mutex);
	if (ret)
		urcu_die(ret);
}

static void wait_for_readers(struct cds_list_head *input_readers,
			struct cds_list_head *cur_snap_readers,
			struct cds_list_head *qsreaders)
{
	int wait_loops = 0;
	struct rcu_reader *index, *tmp;

	/*
	 * Wait for each thread URCU_TLS(rcu_reader).ctr to either
	 * indicate quiescence (not nested), or observe the current
	 * rcu_gp.ctr value.
	 */
	for (;;) {
		wait_loops++;
		cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
			switch (rcu_reader_state(&index->ctr)) {
			case RCU_READER_ACTIVE_CURRENT:
				if (cur_snap_readers) {
					cds_list_move(&index->node,
						cur_snap_readers);
					break;
				}
				/* Fall-through */
			case RCU_READER_INACTIVE:
				cds_list_move(&index->node, qsreaders);
				break;
			case RCU_READER_ACTIVE_OLD:
				/*
				 * Old snapshot. Leaving node in
				 * input_readers will make us busy-loop
				 * until the snapshot becomes current or
				 * the reader becomes inactive.
				 */
				break;
			}
		}

		if (cds_list_empty(input_readers)) {
			break;
		} else {
			if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
				usleep(RCU_SLEEP_DELAY);
			else
				caa_cpu_relax();
		}
	}
}

void synchronize_rcu(void)
{
	CDS_LIST_HEAD(cur_snap_readers);
	CDS_LIST_HEAD(qsreaders);
	sigset_t newmask, oldmask;
	int ret;

	ret = sigfillset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	assert(!ret);

	mutex_lock(&rcu_gp_lock);

	if (cds_list_empty(&registry))
		goto out;

	/* All threads should read qparity before accessing data structure
	 * where new ptr points to. */
	/* Write new ptr before changing the qparity */
	cmm_smp_mb();

	/* Remove old registry elements */
	rcu_gc_registry();

	/*
	 * Wait for readers to observe original parity or be quiescent.
	 */
	wait_for_readers(&registry, &cur_snap_readers, &qsreaders);

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/* Switch parity: 0 -> 1, 1 -> 0 */
	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);

	/*
	 * Must commit qparity update to memory before waiting for other parity
	 * quiescent state. Failure to do so could result in the writer waiting
	 * forever while new readers are always accessing data (no progress).
	 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
	 */

	/*
	 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	 * model easier to understand. It does not have a big performance impact
	 * anyway, given this is the write-side.
	 */
	cmm_smp_mb();

	/*
	 * Wait for readers to observe new parity or be quiescent.
	 */
	wait_for_readers(&cur_snap_readers, NULL, &qsreaders);

	/*
	 * Put quiescent reader list back into registry.
	 */
	cds_list_splice(&qsreaders, &registry);

	/*
	 * Finish waiting for reader threads before letting the old ptr being
	 * freed.
	 */
	cmm_smp_mb();
out:
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

/*
 * library wrappers to be used by non-LGPL compatible source code.
 */

void rcu_read_lock(void)
{
	_rcu_read_lock();
}

void rcu_read_unlock(void)
{
	_rcu_read_unlock();
}

int rcu_read_ongoing(void)
{
	return _rcu_read_ongoing();
}

/*
 * only grow for now.
 */
static void resize_arena(struct registry_arena *arena, size_t len)
{
	void *new_arena;

	if (!arena->p)
		new_arena = mmap(arena->p, len,
				 PROT_READ | PROT_WRITE,
				 MAP_ANONYMOUS | MAP_PRIVATE,
				 -1, 0);
	else
		new_arena = mremap_wrapper(arena->p, arena->len,
				   	len, MREMAP_MAYMOVE);
	assert(new_arena != MAP_FAILED);

	/*
	 * re-used the same region ?
	 */
	if (new_arena == arena->p)
		return;

	bzero(new_arena + arena->len, len - arena->len);
	arena->p = new_arena;
}

/* Called with signals off and mutex locked */
static void add_thread(void)
{
	struct rcu_reader *rcu_reader_reg;

	if (registry_arena.len
	    < registry_arena.used + sizeof(struct rcu_reader))
		resize_arena(&registry_arena,
		caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
	/*
	 * Find a free spot.
	 */
	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			break;
	}
	rcu_reader_reg->alloc = 1;
	registry_arena.used += sizeof(struct rcu_reader);

	/* Add to registry */
	rcu_reader_reg->tid = pthread_self();
	assert(rcu_reader_reg->ctr == 0);
	cds_list_add(&rcu_reader_reg->node, &registry);
	URCU_TLS(rcu_reader) = rcu_reader_reg;
}

/* Called with signals off and mutex locked */
static void rcu_gc_registry(void)
{
	struct rcu_reader *rcu_reader_reg;
	pthread_t tid;
	int ret;

	for (rcu_reader_reg = registry_arena.p;
	     (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	     rcu_reader_reg++) {
		if (!rcu_reader_reg->alloc)
			continue;
		tid = rcu_reader_reg->tid;
		ret = pthread_kill(tid, 0);
		assert(ret != EINVAL);
		if (ret == ESRCH) {
			cds_list_del(&rcu_reader_reg->node);
			rcu_reader_reg->ctr = 0;
			rcu_reader_reg->alloc = 0;
			registry_arena.used -= sizeof(struct rcu_reader);
		}
	}
}

/* Disable signals, take mutex, add to registry */
void rcu_bp_register(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigfillset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	assert(!ret);

	/*
	 * Check if a signal concurrently registered our thread since
	 * the check in rcu_read_lock(). */
	if (URCU_TLS(rcu_reader))
		goto end;

	mutex_lock(&rcu_gp_lock);
	add_thread();
	mutex_unlock(&rcu_gp_lock);
end:
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_exit(void)
{
	if (registry_arena.p)
		munmap(registry_arena.p, registry_arena.len);
}

/*
 * Holding the rcu_gp_lock across fork will make sure we fork() don't race with
 * a concurrent thread executing with this same lock held. This ensures that the
 * registry is in a coherent state in the child.
 */
void rcu_bp_before_fork(void)
{
	sigset_t newmask, oldmask;
	int ret;

	ret = sigfillset(&newmask);
	assert(!ret);
	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
	assert(!ret);
	mutex_lock(&rcu_gp_lock);
	saved_fork_signal_mask = oldmask;
}

void rcu_bp_after_fork_parent(void)
{
	sigset_t oldmask;
	int ret;

	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void rcu_bp_after_fork_child(void)
{
	sigset_t oldmask;
	int ret;

	rcu_gc_registry();
	oldmask = saved_fork_signal_mask;
	mutex_unlock(&rcu_gp_lock);
	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	assert(!ret);
}

void *rcu_dereference_sym_bp(void *p)
{
	return _rcu_dereference(p);
}

void *rcu_set_pointer_sym_bp(void **p, void *v)
{
	cmm_wmb();
	uatomic_set(p, v);
	return v;
}

void *rcu_xchg_pointer_sym_bp(void **p, void *v)
{
	cmm_wmb();
	return uatomic_xchg(p, v);
}

void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new)
{
	cmm_wmb();
	return uatomic_cmpxchg(p, old, _new);
}

DEFINE_RCU_FLAVOR(rcu_flavor);

#include "urcu-call-rcu-impl.h"
#include "urcu-defer-impl.h"
Commit	Line	Data
fdee2e6d MD	1	/*
	2	* urcu-bp.c
	3	*
	4	* Userspace RCU library, "bulletproof" version.
	5	*
6982d6d7	6	* Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d MD	7	* Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
	8	*
	9	* This library is free software; you can redistribute it and/or
	10	* modify it under the terms of the GNU Lesser General Public
	11	* License as published by the Free Software Foundation; either
	12	* version 2.1 of the License, or (at your option) any later version.
	13	*
	14	* This library is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	17	* Lesser General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU Lesser General Public
	20	* License along with this library; if not, write to the Free Software
	21	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	22	*
	23	* IBM's contributions to this file may be relicensed under LGPLv2 or later.
	24	*/
	25
0617bf4c	26	#define _GNU_SOURCE
71c811bf	27	#define _LGPL_SOURCE
fdee2e6d MD	28	#include <stdio.h>
	29	#include <pthread.h>
	30	#include <signal.h>
	31	#include <assert.h>
	32	#include <stdlib.h>
	33	#include <string.h>
	34	#include <errno.h>
	35	#include <poll.h>
	36	#include <unistd.h>
	37	#include <sys/mman.h>
	38
d73fb81f	39	#include "urcu/wfcqueue.h"
57760d44	40	#include "urcu/map/urcu-bp.h"
af7c2dbe	41	#include "urcu/static/urcu-bp.h"
618b2595	42	#include "urcu-pointer.h"
bd252a04	43	#include "urcu/tls-compat.h"
71c811bf	44
4a6d7378 MD	45	#include "urcu-die.h"
4a6d7378 MD	46
fdee2e6d	47	/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
71c811bf	48	#undef _LGPL_SOURCE
fdee2e6d	49	#include "urcu-bp.h"
71c811bf	50	#define _LGPL_SOURCE
fdee2e6d	51
4c1ae2ea MD	52	#ifndef MAP_ANONYMOUS
	53	#define MAP_ANONYMOUS MAP_ANON
	54	#endif
	55
c7eaf61c MD	56	#ifdef __linux__
	57	static
	58	void mremap_wrapper(void old_address, size_t old_size,
	59	size_t new_size, int flags)
	60	{
	61	return mremap(old_address, old_size, new_size, flags);
	62	}
	63	#else
45a4872f MD	64
	65	#define MREMAP_MAYMOVE 1
	66	#define MREMAP_FIXED 2
	67
	68	/*
	69	* mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping.
	70	* This is not generic.
	71	*/
c7eaf61c MD	72	static
	73	void mremap_wrapper(void old_address, size_t old_size,
	74	size_t new_size, int flags)
45a4872f	75	{
dc745ef6	76	void *new_address;
45a4872f MD	77
	78	assert(flags & MREMAP_MAYMOVE);
	79	assert(!(flags & MREMAP_FIXED));
	80	new_address = mmap(old_address, new_size,
	81	PROT_READ \| PROT_WRITE,
	82	MAP_ANONYMOUS \| MAP_PRIVATE,
	83	-1, 0);
	84	if (new_address == MAP_FAILED)
	85	return MAP_FAILED;
	86	if (old_address) {
	87	memcpy(new_address, old_address, old_size);
	88	munmap(old_address, old_size);
	89	}
	90	return new_address;
	91	}
	92	#endif
	93
fdee2e6d MD	94	/* Sleep delay in us */
	95	#define RCU_SLEEP_DELAY 1000
	96	#define ARENA_INIT_ALLOC 16
	97
b7b6a8f5 PB	98	/*
	99	* Active attempts to check for reader Q.S. before calling sleep().
	100	*/
	101	#define RCU_QS_ACTIVE_ATTEMPTS 100
	102
02be5561	103	void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d	104
6abb4bd5	105	static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d MD	106
fdee2e6d MD	107	#ifdef DEBUG_YIELD
1de4df4b MD	108	unsigned int rcu_yield_active;
1de4df4b MD	109	DEFINE_URCU_TLS(unsigned int, rcu_rand_yield);
fdee2e6d MD	110	#endif
fdee2e6d MD	111
c13c2e55	112	struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT };
fdee2e6d MD	113
	114	/*
	115	* Pointer to registry elements. Written to only by each individual reader. Read
	116	* by both the reader and the writers.
	117	*/
bd252a04	118	DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader);
fdee2e6d	119
16aa9ee8	120	static CDS_LIST_HEAD(registry);
fdee2e6d MD	121
	122	struct registry_arena {
	123	void *p;
	124	size_t len;
	125	size_t used;
	126	};
	127
	128	static struct registry_arena registry_arena;
	129
4cf1675f MD	130	/* Saved fork signal mask, protected by rcu_gp_lock */
	131	static sigset_t saved_fork_signal_mask;
	132
fdee2e6d MD	133	static void rcu_gc_registry(void);
fdee2e6d MD	134
6abb4bd5	135	static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d MD	136	{
	137	int ret;
	138
	139	#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5	140	ret = pthread_mutex_lock(mutex);
4a6d7378 MD	141	if (ret)
4a6d7378 MD	142	urcu_die(ret);
fdee2e6d	143	#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5	144	while ((ret = pthread_mutex_trylock(mutex)) != 0) {
4a6d7378 MD	145	if (ret != EBUSY && ret != EINTR)
4a6d7378 MD	146	urcu_die(ret);
fdee2e6d MD	147	poll(NULL,0,10);
	148	}
	149	#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
	150	}
	151
6abb4bd5	152	static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d MD	153	{
	154	int ret;
	155
6abb4bd5	156	ret = pthread_mutex_unlock(mutex);
4a6d7378 MD	157	if (ret)
4a6d7378 MD	158	urcu_die(ret);
fdee2e6d MD	159	}
fdee2e6d MD	160
52c75091 MD	161	static void wait_for_readers(struct cds_list_head *input_readers,
	162	struct cds_list_head *cur_snap_readers,
	163	struct cds_list_head *qsreaders)
fdee2e6d	164	{
fdee2e6d	165	int wait_loops = 0;
02be5561	166	struct rcu_reader index, tmp;
fdee2e6d	167
fdee2e6d	168	/*
dd61d077 MD	169	* Wait for each thread URCU_TLS(rcu_reader).ctr to either
dd61d077 MD	170	* indicate quiescence (not nested), or observe the current
c13c2e55	171	* rcu_gp.ctr value.
fdee2e6d MD	172	*/
	173	for (;;) {
	174	wait_loops++;
52c75091 MD	175	cds_list_for_each_entry_safe(index, tmp, input_readers, node) {
	176	switch (rcu_reader_state(&index->ctr)) {
	177	case RCU_READER_ACTIVE_CURRENT:
	178	if (cur_snap_readers) {
	179	cds_list_move(&index->node,
	180	cur_snap_readers);
	181	break;
	182	}
	183	/* Fall-through */
	184	case RCU_READER_INACTIVE:
	185	cds_list_move(&index->node, qsreaders);
	186	break;
	187	case RCU_READER_ACTIVE_OLD:
	188	/*
	189	* Old snapshot. Leaving node in
	190	* input_readers will make us busy-loop
	191	* until the snapshot becomes current or
	192	* the reader becomes inactive.
	193	*/
	194	break;
	195	}
fdee2e6d MD	196	}
fdee2e6d MD	197
52c75091	198	if (cds_list_empty(input_readers)) {
fdee2e6d MD	199	break;
	200	} else {
	201	if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
	202	usleep(RCU_SLEEP_DELAY);
	203	else
06f22bdb	204	caa_cpu_relax();
fdee2e6d MD	205	}
fdee2e6d MD	206	}
fdee2e6d MD	207	}
	208
	209	void synchronize_rcu(void)
	210	{
52c75091 MD	211	CDS_LIST_HEAD(cur_snap_readers);
52c75091 MD	212	CDS_LIST_HEAD(qsreaders);
fdee2e6d MD	213	sigset_t newmask, oldmask;
	214	int ret;
	215
6ed4b2e6	216	ret = sigfillset(&newmask);
fdee2e6d	217	assert(!ret);
6ed4b2e6	218	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
fdee2e6d MD	219	assert(!ret);
fdee2e6d MD	220
6abb4bd5	221	mutex_lock(&rcu_gp_lock);
fdee2e6d	222
16aa9ee8	223	if (cds_list_empty(&registry))
2dfb8b5e	224	goto out;
fdee2e6d MD	225
fdee2e6d MD	226	/* All threads should read qparity before accessing data structure
2dfb8b5e	227	* where new ptr points to. */
fdee2e6d	228	/* Write new ptr before changing the qparity */
5481ddb3	229	cmm_smp_mb();
fdee2e6d	230
2dfb8b5e MD	231	/* Remove old registry elements */
2dfb8b5e MD	232	rcu_gc_registry();
fdee2e6d MD	233
fdee2e6d MD	234	/*
dd61d077 MD	235	* Wait for readers to observe original parity or be quiescent.
dd61d077 MD	236	*/
52c75091	237	wait_for_readers(&registry, &cur_snap_readers, &qsreaders);
dd61d077 MD	238
	239	/*
	240	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
	241	* model easier to understand. It does not have a big performance impact
	242	* anyway, given this is the write-side.
	243	*/
	244	cmm_smp_mb();
	245
	246	/* Switch parity: 0 -> 1, 1 -> 0 */
c13c2e55	247	CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ RCU_GP_CTR_PHASE);
dd61d077 MD	248
	249	/*
	250	* Must commit qparity update to memory before waiting for other parity
	251	* quiescent state. Failure to do so could result in the writer waiting
	252	* forever while new readers are always accessing data (no progress).
	253	* Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
fdee2e6d	254	*/
fdee2e6d MD	255
fdee2e6d MD	256	/*
5481ddb3	257	* Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d MD	258	* model easier to understand. It does not have a big performance impact
	259	* anyway, given this is the write-side.
	260	*/
5481ddb3	261	cmm_smp_mb();
fdee2e6d	262
fdee2e6d	263	/*
dd61d077	264	* Wait for readers to observe new parity or be quiescent.
fdee2e6d	265	*/
52c75091 MD	266	wait_for_readers(&cur_snap_readers, NULL, &qsreaders);
	267
	268	/*
	269	* Put quiescent reader list back into registry.
	270	*/
	271	cds_list_splice(&qsreaders, &registry);
fdee2e6d MD	272
fdee2e6d MD	273	/*
2dfb8b5e MD	274	* Finish waiting for reader threads before letting the old ptr being
2dfb8b5e MD	275	* freed.
fdee2e6d	276	*/
5481ddb3	277	cmm_smp_mb();
2dfb8b5e	278	out:
6abb4bd5	279	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	280	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	281	assert(!ret);
	282	}
	283
	284	/*
	285	* library wrappers to be used by non-LGPL compatible source code.
	286	*/
	287
	288	void rcu_read_lock(void)
	289	{
	290	_rcu_read_lock();
	291	}
	292
	293	void rcu_read_unlock(void)
	294	{
	295	_rcu_read_unlock();
	296	}
	297
882f3357 MD	298	int rcu_read_ongoing(void)
	299	{
	300	return _rcu_read_ongoing();
	301	}
	302
fdee2e6d MD	303	/*
	304	* only grow for now.
	305	*/
	306	static void resize_arena(struct registry_arena *arena, size_t len)
	307	{
	308	void *new_arena;
	309
0617bf4c MD	310	if (!arena->p)
	311	new_arena = mmap(arena->p, len,
	312	PROT_READ \| PROT_WRITE,
	313	MAP_ANONYMOUS \| MAP_PRIVATE,
	314	-1, 0);
	315	else
c7eaf61c MD	316	new_arena = mremap_wrapper(arena->p, arena->len,
c7eaf61c MD	317	len, MREMAP_MAYMOVE);
0617bf4c MD	318	assert(new_arena != MAP_FAILED);
0617bf4c MD	319
fdee2e6d MD	320	/*
	321	* re-used the same region ?
	322	*/
	323	if (new_arena == arena->p)
	324	return;
	325
fdee2e6d MD	326	bzero(new_arena + arena->len, len - arena->len);
	327	arena->p = new_arena;
	328	}
	329
	330	/* Called with signals off and mutex locked */
	331	static void add_thread(void)
	332	{
02be5561	333	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	334
fdee2e6d MD	335	if (registry_arena.len
02be5561	336	< registry_arena.used + sizeof(struct rcu_reader))
fdee2e6d	337	resize_arena(&registry_arena,
2f8a5ae7	338	caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC));
fdee2e6d MD	339	/*
	340	* Find a free spot.
	341	*/
02be5561 MD	342	for (rcu_reader_reg = registry_arena.p;
	343	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	344	rcu_reader_reg++) {
	345	if (!rcu_reader_reg->alloc)
fdee2e6d MD	346	break;
fdee2e6d MD	347	}
02be5561 MD	348	rcu_reader_reg->alloc = 1;
02be5561 MD	349	registry_arena.used += sizeof(struct rcu_reader);
fdee2e6d MD	350
fdee2e6d MD	351	/* Add to registry */
02be5561 MD	352	rcu_reader_reg->tid = pthread_self();
02be5561 MD	353	assert(rcu_reader_reg->ctr == 0);
16aa9ee8	354	cds_list_add(&rcu_reader_reg->node, &registry);
bd252a04	355	URCU_TLS(rcu_reader) = rcu_reader_reg;
fdee2e6d MD	356	}
	357
	358	/* Called with signals off and mutex locked */
	359	static void rcu_gc_registry(void)
	360	{
02be5561	361	struct rcu_reader *rcu_reader_reg;
fdee2e6d MD	362	pthread_t tid;
	363	int ret;
	364
02be5561 MD	365	for (rcu_reader_reg = registry_arena.p;
	366	(void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
	367	rcu_reader_reg++) {
	368	if (!rcu_reader_reg->alloc)
fdee2e6d	369	continue;
02be5561	370	tid = rcu_reader_reg->tid;
fdee2e6d MD	371	ret = pthread_kill(tid, 0);
	372	assert(ret != EINVAL);
	373	if (ret == ESRCH) {
16aa9ee8	374	cds_list_del(&rcu_reader_reg->node);
79266051	375	rcu_reader_reg->ctr = 0;
02be5561 MD	376	rcu_reader_reg->alloc = 0;
02be5561 MD	377	registry_arena.used -= sizeof(struct rcu_reader);
fdee2e6d MD	378	}
	379	}
	380	}
	381
	382	/* Disable signals, take mutex, add to registry */
	383	void rcu_bp_register(void)
	384	{
	385	sigset_t newmask, oldmask;
	386	int ret;
	387
6ed4b2e6	388	ret = sigfillset(&newmask);
fdee2e6d	389	assert(!ret);
6ed4b2e6	390	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
fdee2e6d MD	391	assert(!ret);
	392
	393	/*
	394	* Check if a signal concurrently registered our thread since
	395	* the check in rcu_read_lock(). */
bd252a04	396	if (URCU_TLS(rcu_reader))
fdee2e6d MD	397	goto end;
fdee2e6d MD	398
6abb4bd5	399	mutex_lock(&rcu_gp_lock);
fdee2e6d	400	add_thread();
6abb4bd5	401	mutex_unlock(&rcu_gp_lock);
fdee2e6d MD	402	end:
	403	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	404	assert(!ret);
	405	}
	406
9380711a	407	void rcu_bp_exit(void)
fdee2e6d	408	{
9380711a MD	409	if (registry_arena.p)
9380711a MD	410	munmap(registry_arena.p, registry_arena.len);
fdee2e6d	411	}
4cf1675f MD	412
	413	/*
	414	* Holding the rcu_gp_lock across fork will make sure we fork() don't race with
	415	* a concurrent thread executing with this same lock held. This ensures that the
	416	* registry is in a coherent state in the child.
	417	*/
	418	void rcu_bp_before_fork(void)
	419	{
	420	sigset_t newmask, oldmask;
	421	int ret;
	422
6ed4b2e6	423	ret = sigfillset(&newmask);
4cf1675f	424	assert(!ret);
6ed4b2e6	425	ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
4cf1675f MD	426	assert(!ret);
	427	mutex_lock(&rcu_gp_lock);
	428	saved_fork_signal_mask = oldmask;
	429	}
	430
	431	void rcu_bp_after_fork_parent(void)
	432	{
	433	sigset_t oldmask;
	434	int ret;
	435
	436	oldmask = saved_fork_signal_mask;
	437	mutex_unlock(&rcu_gp_lock);
	438	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	439	assert(!ret);
	440	}
	441
	442	void rcu_bp_after_fork_child(void)
	443	{
	444	sigset_t oldmask;
	445	int ret;
	446
	447	rcu_gc_registry();
	448	oldmask = saved_fork_signal_mask;
	449	mutex_unlock(&rcu_gp_lock);
	450	ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
	451	assert(!ret);
	452	}
5e77fc1f	453
9b7981bb MD	454	void rcu_dereference_sym_bp(void p)
	455	{
	456	return _rcu_dereference(p);
	457	}
	458
5efd3cd2 MD	459	void rcu_set_pointer_sym_bp(void p, void v)
	460	{
	461	cmm_wmb();
424d4ed5 MD	462	uatomic_set(p, v);
424d4ed5 MD	463	return v;
5efd3cd2 MD	464	}
	465
	466	void rcu_xchg_pointer_sym_bp(void p, void v)
	467	{
	468	cmm_wmb();
	469	return uatomic_xchg(p, v);
	470	}
	471
	472	void rcu_cmpxchg_pointer_sym_bp(void p, void old, void *_new)
	473	{
	474	cmm_wmb();
	475	return uatomic_cmpxchg(p, old, _new);
	476	}
	477
5e6b23a6	478	DEFINE_RCU_FLAVOR(rcu_flavor);
541d828d	479
5e77fc1f	480	#include "urcu-call-rcu-impl.h"
0376e7b2	481	#include "urcu-defer-impl.h"