[userspace-rcu.git] / src / urcu-wait.h

#ifndef _URCU_WAIT_H
#define _URCU_WAIT_H

/*
 * urcu-wait.h
 *
 * Userspace RCU library wait/wakeup management
 *
 * Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <urcu/uatomic.h>
#include <urcu/wfstack.h>
#include "urcu-die.h"

/*
 * Number of busy-loop attempts before waiting on futex for grace period
 * batching.
 */
#define URCU_WAIT_ATTEMPTS 1000

enum urcu_wait_state {
	/* URCU_WAIT_WAITING is compared directly (futex compares it). */
	URCU_WAIT_WAITING =	0,
	/* non-zero are used as masks. */
	URCU_WAIT_WAKEUP =	(1 << 0),
	URCU_WAIT_RUNNING =	(1 << 1),
	URCU_WAIT_TEARDOWN =	(1 << 2),
};

struct urcu_wait_node {
	struct cds_wfs_node node;
	int32_t state;	/* enum urcu_wait_state */
};

#define URCU_WAIT_NODE_INIT(name, _state)		\
	{ .state = _state }

#define DEFINE_URCU_WAIT_NODE(name, state)		\
	struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)

#define DECLARE_URCU_WAIT_NODE(name)			\
	struct urcu_wait_node name

struct urcu_wait_queue {
	struct cds_wfs_stack stack;
};

#define URCU_WAIT_QUEUE_HEAD_INIT(name)				\
	{							\
		 .stack = {					\
			.head = CDS_WFS_END,			\
			.lock = PTHREAD_MUTEX_INITIALIZER,	\
		},						\
	}

#define DECLARE_URCU_WAIT_QUEUE(name)			\
	struct urcu_wait_queue name

#define DEFINE_URCU_WAIT_QUEUE(name)			\
	struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)

struct urcu_waiters {
	struct cds_wfs_head *head;
};

/*
 * Add ourself atomically to a wait queue. Return 0 if queue was
 * previously empty, else return 1.
 * A full memory barrier is issued before being added to the wait queue.
 */
static inline
bool urcu_wait_add(struct urcu_wait_queue *queue,
		struct urcu_wait_node *node)
{
	return cds_wfs_push(&queue->stack, &node->node);
}

/*
 * Atomically move all waiters from wait queue into our local struct
 * urcu_waiters.
 */
static inline
void urcu_move_waiters(struct urcu_waiters *waiters,
		struct urcu_wait_queue *queue)
{
	waiters->head = __cds_wfs_pop_all(&queue->stack);
}

static inline
void urcu_wait_set_state(struct urcu_wait_node *node,
		enum urcu_wait_state state)
{
	node->state = state;
}

static inline
void urcu_wait_node_init(struct urcu_wait_node *node,
		enum urcu_wait_state state)
{
	urcu_wait_set_state(node, state);
	cds_wfs_node_init(&node->node);
}

/*
 * Note: urcu_adaptative_wake_up needs "value" to stay allocated
 * throughout its execution. In this scheme, the waiter owns the node
 * memory, and we only allow it to free this memory when it receives the
 * URCU_WAIT_TEARDOWN flag.
 */
static inline
void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
{
	cmm_smp_mb();
	assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
	uatomic_set(&wait->state, URCU_WAIT_WAKEUP);
	if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
		if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
				NULL, NULL, 0) < 0)
			urcu_die(errno);
	}
	/* Allow teardown of struct urcu_wait memory. */
	uatomic_or(&wait->state, URCU_WAIT_TEARDOWN);
}

/*
 * Caller must initialize "value" to URCU_WAIT_WAITING before passing its
 * memory to waker thread.
 */
static inline
void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
{
	unsigned int i;

	/* Load and test condition before read state */
	cmm_smp_rmb();
	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
		if (uatomic_read(&wait->state) != URCU_WAIT_WAITING)
			goto skip_futex_wait;
		caa_cpu_relax();
	}
	while (uatomic_read(&wait->state) == URCU_WAIT_WAITING) {
		if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) {
			/*
			 * Prior queued wakeups queued by unrelated code
			 * using the same address can cause futex wait to
			 * return 0 even through the futex value is still
			 * URCU_WAIT_WAITING (spurious wakeups). Check
			 * the value again in user-space to validate
			 * whether it really differs from
			 * URCU_WAIT_WAITING.
			 */
			continue;
		}
		switch (errno) {
		case EAGAIN:
			/* Value already changed. */
			goto skip_futex_wait;
		case EINTR:
			/* Retry if interrupted by signal. */
			break;	/* Get out of switch. Check again. */
		default:
			/* Unexpected error. */
			urcu_die(errno);
		}
	}
skip_futex_wait:

	/* Tell waker thread than we are running. */
	uatomic_or(&wait->state, URCU_WAIT_RUNNING);

	/*
	 * Wait until waker thread lets us know it's ok to tear down
	 * memory allocated for struct urcu_wait.
	 */
	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
		if (uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN)
			break;
		caa_cpu_relax();
	}
	while (!(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN))
		poll(NULL, 0, 10);
	assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
}

static inline
void urcu_wake_all_waiters(struct urcu_waiters *waiters)
{
	struct cds_wfs_node *iter, *iter_n;

	/* Wake all waiters in our stack head */
	cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
		struct urcu_wait_node *wait_node =
			caa_container_of(iter, struct urcu_wait_node, node);

		/* Don't wake already running threads */
		if (wait_node->state & URCU_WAIT_RUNNING)
			continue;
		urcu_adaptative_wake_up(wait_node);
	}
}

#endif /* _URCU_WAIT_H */
Commit	Line	Data
cba82d7b MD	1	#ifndef _URCU_WAIT_H
	2	#define _URCU_WAIT_H
	3
	4	/*
	5	* urcu-wait.h
	6	*
	7	* Userspace RCU library wait/wakeup management
	8	*
	9	* Copyright (c) 2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2.1 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
	26	#include <urcu/uatomic.h>
bf6822a6	27	#include <urcu/wfstack.h>
b0a841b4	28	#include "urcu-die.h"
cba82d7b MD	29
	30	/*
	31	* Number of busy-loop attempts before waiting on futex for grace period
	32	* batching.
	33	*/
	34	#define URCU_WAIT_ATTEMPTS 1000
	35
	36	enum urcu_wait_state {
	37	/* URCU_WAIT_WAITING is compared directly (futex compares it). */
	38	URCU_WAIT_WAITING = 0,
	39	/* non-zero are used as masks. */
	40	URCU_WAIT_WAKEUP = (1 << 0),
bf6822a6	41	URCU_WAIT_RUNNING = (1 << 1),
cba82d7b MD	42	URCU_WAIT_TEARDOWN = (1 << 2),
	43	};
	44
bf6822a6 MD	45	struct urcu_wait_node {
	46	struct cds_wfs_node node;
	47	int32_t state; /* enum urcu_wait_state */
cba82d7b MD	48	};
cba82d7b MD	49
bf6822a6 MD	50	#define URCU_WAIT_NODE_INIT(name, _state) \
	51	{ .state = _state }
	52
	53	#define DEFINE_URCU_WAIT_NODE(name, state) \
	54	struct urcu_wait_node name = URCU_WAIT_NODE_INIT(name, state)
	55
	56	#define DECLARE_URCU_WAIT_NODE(name) \
	57	struct urcu_wait_node name
	58
	59	struct urcu_wait_queue {
	60	struct cds_wfs_stack stack;
	61	};
	62
cafe8ce1 MD	63	#define URCU_WAIT_QUEUE_HEAD_INIT(name) \
	64	{ \
	65	.stack = { \
	66	.head = CDS_WFS_END, \
	67	.lock = PTHREAD_MUTEX_INITIALIZER, \
	68	}, \
	69	}
bf6822a6 MD	70
	71	#define DECLARE_URCU_WAIT_QUEUE(name) \
	72	struct urcu_wait_queue name
	73
	74	#define DEFINE_URCU_WAIT_QUEUE(name) \
	75	struct urcu_wait_queue name = URCU_WAIT_QUEUE_HEAD_INIT(name)
	76
	77	struct urcu_waiters {
	78	struct cds_wfs_head *head;
	79	};
	80
	81	/*
	82	* Add ourself atomically to a wait queue. Return 0 if queue was
	83	* previously empty, else return 1.
	84	* A full memory barrier is issued before being added to the wait queue.
	85	*/
	86	static inline
	87	bool urcu_wait_add(struct urcu_wait_queue *queue,
	88	struct urcu_wait_node *node)
	89	{
	90	return cds_wfs_push(&queue->stack, &node->node);
	91	}
	92
	93	/*
	94	* Atomically move all waiters from wait queue into our local struct
	95	* urcu_waiters.
	96	*/
	97	static inline
	98	void urcu_move_waiters(struct urcu_waiters *waiters,
	99	struct urcu_wait_queue *queue)
	100	{
	101	waiters->head = __cds_wfs_pop_all(&queue->stack);
	102	}
	103
	104	static inline
	105	void urcu_wait_set_state(struct urcu_wait_node *node,
	106	enum urcu_wait_state state)
	107	{
	108	node->state = state;
	109	}
	110
cba82d7b	111	static inline
bf6822a6 MD	112	void urcu_wait_node_init(struct urcu_wait_node *node,
bf6822a6 MD	113	enum urcu_wait_state state)
cba82d7b	114	{
bf6822a6 MD	115	urcu_wait_set_state(node, state);
bf6822a6 MD	116	cds_wfs_node_init(&node->node);
cba82d7b MD	117	}
	118
	119	/*
	120	* Note: urcu_adaptative_wake_up needs "value" to stay allocated
bf6822a6	121	* throughout its execution. In this scheme, the waiter owns the node
cba82d7b MD	122	* memory, and we only allow it to free this memory when it receives the
	123	* URCU_WAIT_TEARDOWN flag.
	124	*/
	125	static inline
bf6822a6	126	void urcu_adaptative_wake_up(struct urcu_wait_node *wait)
cba82d7b MD	127	{
cba82d7b MD	128	cmm_smp_mb();
bf6822a6 MD	129	assert(uatomic_read(&wait->state) == URCU_WAIT_WAITING);
bf6822a6 MD	130	uatomic_set(&wait->state, URCU_WAIT_WAKEUP);
b0a841b4 MD	131	if (!(uatomic_read(&wait->state) & URCU_WAIT_RUNNING)) {
	132	if (futex_noasync(&wait->state, FUTEX_WAKE, 1,
	133	NULL, NULL, 0) < 0)
	134	urcu_die(errno);
	135	}
cba82d7b	136	/* Allow teardown of struct urcu_wait memory. */
bf6822a6	137	uatomic_or(&wait->state, URCU_WAIT_TEARDOWN);
cba82d7b MD	138	}
	139
	140	/*
	141	* Caller must initialize "value" to URCU_WAIT_WAITING before passing its
	142	* memory to waker thread.
	143	*/
bf6822a6 MD	144	static inline
bf6822a6 MD	145	void urcu_adaptative_busy_wait(struct urcu_wait_node *wait)
cba82d7b MD	146	{
	147	unsigned int i;
	148
bf6822a6	149	/* Load and test condition before read state */
cba82d7b MD	150	cmm_smp_rmb();
cba82d7b MD	151	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
bf6822a6	152	if (uatomic_read(&wait->state) != URCU_WAIT_WAITING)
cba82d7b MD	153	goto skip_futex_wait;
	154	caa_cpu_relax();
	155	}
b47d409e MD	156	while (uatomic_read(&wait->state) == URCU_WAIT_WAITING) {
	157	if (!futex_noasync(&wait->state, FUTEX_WAIT, URCU_WAIT_WAITING, NULL, NULL, 0)) {
	158	/*
	159	* Prior queued wakeups queued by unrelated code
	160	* using the same address can cause futex wait to
	161	* return 0 even through the futex value is still
	162	* URCU_WAIT_WAITING (spurious wakeups). Check
	163	* the value again in user-space to validate
	164	* whether it really differs from
	165	* URCU_WAIT_WAITING.
	166	*/
	167	continue;
	168	}
b0a841b4	169	switch (errno) {
b47d409e	170	case EAGAIN:
b0a841b4 MD	171	/* Value already changed. */
	172	goto skip_futex_wait;
	173	case EINTR:
	174	/* Retry if interrupted by signal. */
b47d409e	175	break; /* Get out of switch. Check again. */
b0a841b4 MD	176	default:
	177	/* Unexpected error. */
	178	urcu_die(errno);
	179	}
	180	}
cba82d7b MD	181	skip_futex_wait:
cba82d7b MD	182
ffa11a18	183	/* Tell waker thread than we are running. */
bf6822a6	184	uatomic_or(&wait->state, URCU_WAIT_RUNNING);
cba82d7b MD	185
	186	/*
	187	* Wait until waker thread lets us know it's ok to tear down
	188	* memory allocated for struct urcu_wait.
	189	*/
	190	for (i = 0; i < URCU_WAIT_ATTEMPTS; i++) {
bf6822a6	191	if (uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN)
cba82d7b MD	192	break;
	193	caa_cpu_relax();
	194	}
bf6822a6	195	while (!(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN))
cba82d7b	196	poll(NULL, 0, 10);
bf6822a6 MD	197	assert(uatomic_read(&wait->state) & URCU_WAIT_TEARDOWN);
	198	}
	199
	200	static inline
	201	void urcu_wake_all_waiters(struct urcu_waiters *waiters)
	202	{
	203	struct cds_wfs_node iter, iter_n;
	204
	205	/* Wake all waiters in our stack head */
	206	cds_wfs_for_each_blocking_safe(waiters->head, iter, iter_n) {
	207	struct urcu_wait_node *wait_node =
	208	caa_container_of(iter, struct urcu_wait_node, node);
	209
	210	/* Don't wake already running threads */
	211	if (wait_node->state & URCU_WAIT_RUNNING)
	212	continue;
	213	urcu_adaptative_wake_up(wait_node);
	214	}
cba82d7b MD	215	}
	216
	217	#endif /* _URCU_WAIT_H */