1 // SPDX-FileCopyrightText: 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
3 // SPDX-License-Identifier: LGPL-2.1-or-later
5 #ifndef _URCU_STATIC_LFSTACK_H
6 #define _URCU_STATIC_LFSTACK_H
9 * Userspace RCU library - Lock-Free Stack
11 * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu/lfstack.h for
12 * linking dynamically with the userspace rcu library.
17 #include <urcu/assert.h>
18 #include <urcu/uatomic.h>
19 #include <urcu-pointer.h>
28 * Stack implementing push, pop, pop_all operations, as well as iterator
29 * on the stack head returned by pop_all.
31 * Synchronization table:
33 * External synchronization techniques described in the API below is
34 * required between pairs marked with "X". No external synchronization
35 * required between pairs marked with "-".
37 * cds_lfs_push __cds_lfs_pop __cds_lfs_pop_all
40 * __cds_lfs_pop_all - X -
42 * cds_lfs_pop_blocking and cds_lfs_pop_all_blocking use an internal
43 * mutex to provide synchronization.
47 * cds_lfs_node_init: initialize lock-free stack node.
50 void _cds_lfs_node_init(struct cds_lfs_node
*node
__attribute__((unused
)))
55 * cds_lfs_init: initialize lock-free stack (with lock). Pair with
59 void _cds_lfs_init(struct cds_lfs_stack
*s
)
64 ret
= pthread_mutex_init(&s
->lock
, NULL
);
65 urcu_posix_assert(!ret
);
69 * cds_lfs_destroy: destroy lock-free stack (with lock). Pair with
73 void _cds_lfs_destroy(struct cds_lfs_stack
*s
)
75 int ret
= pthread_mutex_destroy(&s
->lock
);
76 urcu_posix_assert(!ret
);
80 * ___cds_lfs_init: initialize lock-free stack (without lock).
81 * Don't pair with any destroy function.
84 void ___cds_lfs_init(struct __cds_lfs_stack
*s
)
90 bool ___cds_lfs_empty_head(struct cds_lfs_head
*head
)
96 * cds_lfs_empty: return whether lock-free stack is empty.
98 * No memory barrier is issued. No mutual exclusion is required.
101 bool _cds_lfs_empty(cds_lfs_stack_ptr_t s
)
103 return ___cds_lfs_empty_head(CMM_LOAD_SHARED(s
._s
->head
));
107 * cds_lfs_push: push a node into the stack.
109 * Does not require any synchronization with other push nor pop.
111 * Lock-free stack push is not subject to ABA problem, so no need to
112 * take the RCU read-side lock. Even if "head" changes between two
113 * uatomic_cmpxchg() invocations here (being popped, and then pushed
114 * again by one or more concurrent threads), the second
115 * uatomic_cmpxchg() invocation only cares about pushing a new entry at
116 * the head of the stack, ensuring consistency by making sure the new
117 * node->next is the same pointer value as the value replaced as head.
118 * It does not care about the content of the actual next node, so it can
119 * very well be reallocated between the two uatomic_cmpxchg().
121 * We take the approach of expecting the stack to be usually empty, so
122 * we first try an initial uatomic_cmpxchg() on a NULL old_head, and
123 * retry if the old head was non-NULL (the value read by the first
124 * uatomic_cmpxchg() is used as old head for the following loop). The
125 * upside of this scheme is to minimize the amount of cacheline traffic,
126 * always performing an exclusive cacheline access, rather than doing
127 * non-exclusive followed by exclusive cacheline access (which would be
128 * required if we first read the old head value). This design decision
129 * might be revisited after more thorough benchmarking on various
132 * Returns 0 if the stack was empty prior to adding the node.
133 * Returns non-zero otherwise.
136 bool _cds_lfs_push(cds_lfs_stack_ptr_t u_s
,
137 struct cds_lfs_node
*node
)
139 struct __cds_lfs_stack
*s
= u_s
._s
;
140 struct cds_lfs_head
*head
= NULL
;
141 struct cds_lfs_head
*new_head
=
142 caa_container_of(node
, struct cds_lfs_head
, node
);
145 struct cds_lfs_head
*old_head
= head
;
148 * node->next is still private at this point, no need to
149 * perform a _CMM_STORE_SHARED().
151 node
->next
= &head
->node
;
153 * uatomic_cmpxchg() implicit memory barrier orders earlier
154 * stores to node before publication.
156 head
= uatomic_cmpxchg(&s
->head
, old_head
, new_head
);
157 if (old_head
== head
)
160 return !___cds_lfs_empty_head(head
);
164 * __cds_lfs_pop: pop a node from the stack.
166 * Returns NULL if stack is empty.
168 * __cds_lfs_pop needs to be synchronized using one of the following
171 * 1) Calling __cds_lfs_pop under rcu read lock critical section.
172 * Both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
173 * grace period to pass before freeing the returned node or pushing
174 * the node back into the stack. It is valid to overwrite the content
175 * of cds_lfs_node immediately after __cds_lfs_pop and
176 * __cds_lfs_pop_all. No RCU read-side critical section is needed
177 * around __cds_lfs_pop_all.
178 * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop
179 * and __cds_lfs_pop_all callers.
180 * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
181 * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
184 struct cds_lfs_node
*___cds_lfs_pop(cds_lfs_stack_ptr_t u_s
)
186 struct __cds_lfs_stack
*s
= u_s
._s
;
189 struct cds_lfs_head
*head
, *next_head
;
190 struct cds_lfs_node
*next
;
192 head
= _CMM_LOAD_SHARED(s
->head
);
193 if (___cds_lfs_empty_head(head
))
194 return NULL
; /* Empty stack */
197 * Read head before head->next. Matches the implicit
198 * memory barrier before uatomic_cmpxchg() in
201 cmm_smp_read_barrier_depends();
202 next
= _CMM_LOAD_SHARED(head
->node
.next
);
203 next_head
= caa_container_of(next
,
204 struct cds_lfs_head
, node
);
205 if (uatomic_cmpxchg(&s
->head
, head
, next_head
) == head
)
207 /* busy-loop if head changed under us */
212 * __cds_lfs_pop_all: pop all nodes from a stack.
214 * __cds_lfs_pop_all does not require any synchronization with other
215 * push, nor with other __cds_lfs_pop_all, but requires synchronization
216 * matching the technique used to synchronize __cds_lfs_pop:
218 * 1) If __cds_lfs_pop is called under rcu read lock critical section,
219 * both __cds_lfs_pop and __cds_lfs_pop_all callers must wait for a
220 * grace period to pass before freeing the returned node or pushing
221 * the node back into the stack. It is valid to overwrite the content
222 * of cds_lfs_node immediately after __cds_lfs_pop and
223 * __cds_lfs_pop_all. No RCU read-side critical section is needed
224 * around __cds_lfs_pop_all.
225 * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
226 * __cds_lfs_pop_all callers.
227 * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
228 * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
231 struct cds_lfs_head
*___cds_lfs_pop_all(cds_lfs_stack_ptr_t u_s
)
233 struct __cds_lfs_stack
*s
= u_s
._s
;
236 * Implicit memory barrier after uatomic_xchg() matches implicit
237 * memory barrier before uatomic_cmpxchg() in cds_lfs_push. It
238 * ensures that all nodes of the returned list are consistent.
239 * There is no need to issue memory barriers when iterating on
240 * the returned list, because the full memory barrier issued
241 * prior to each uatomic_cmpxchg, which each write to head, are
242 * taking care to order writes to each node prior to the full
243 * memory barrier after this uatomic_xchg().
245 return uatomic_xchg(&s
->head
, NULL
);
249 * cds_lfs_pop_lock: lock stack pop-protection mutex.
251 static inline void _cds_lfs_pop_lock(struct cds_lfs_stack
*s
)
255 ret
= pthread_mutex_lock(&s
->lock
);
256 urcu_posix_assert(!ret
);
260 * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
262 static inline void _cds_lfs_pop_unlock(struct cds_lfs_stack
*s
)
266 ret
= pthread_mutex_unlock(&s
->lock
);
267 urcu_posix_assert(!ret
);
271 * Call __cds_lfs_pop with an internal pop mutex held.
274 struct cds_lfs_node
*
275 _cds_lfs_pop_blocking(struct cds_lfs_stack
*s
)
277 struct cds_lfs_node
*retnode
;
278 cds_lfs_stack_ptr_t stack
;
280 _cds_lfs_pop_lock(s
);
282 retnode
= ___cds_lfs_pop(stack
);
283 _cds_lfs_pop_unlock(s
);
288 * Call __cds_lfs_pop_all with an internal pop mutex held.
291 struct cds_lfs_head
*
292 _cds_lfs_pop_all_blocking(struct cds_lfs_stack
*s
)
294 struct cds_lfs_head
*rethead
;
295 cds_lfs_stack_ptr_t stack
;
297 _cds_lfs_pop_lock(s
);
299 rethead
= ___cds_lfs_pop_all(stack
);
300 _cds_lfs_pop_unlock(s
);
308 #endif /* _URCU_STATIC_LFSTACK_H */