+/*
+ * __cds_lfs_pop_all: pop all nodes from a stack.
+ *
+ * __cds_lfs_pop_all does not require any synchronization with other
+ * push, nor with other __cds_lfs_pop_all, but requires synchronization
+ * matching the technique used to synchronize __cds_lfs_pop:
+ *
+ * 1) If __cds_lfs_pop is called under rcu read lock critical section,
+ * both __cds_lfs_pop and cds_lfs_pop_all callers must wait for a
+ * grace period to pass before freeing the returned node or modifying
+ * the cds_lfs_node structure. However, no RCU read-side critical
+ * section is needed around __cds_lfs_pop_all.
+ * 2) Using mutual exclusion (e.g. mutexes) to protect __cds_lfs_pop and
+ * __cds_lfs_pop_all callers.
+ * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and
+ * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme).
+ */
+static inline
+struct cds_lfs_head *___cds_lfs_pop_all(struct cds_lfs_stack *s)
+{
+ /*
+ * Implicit memory barrier after uatomic_xchg() matches implicit
+ * memory barrier before uatomic_cmpxchg() in cds_lfs_push. It
+ * ensures that all nodes of the returned list are consistent.
+ * There is no need to issue memory barriers when iterating on
+ * the returned list, because the full memory barrier issued
+ * prior to each uatomic_cmpxchg, which each write to head, are
+ * taking care to order writes to each node prior to the full
+ * memory barrier after this uatomic_xchg().
+ */
+ return uatomic_xchg(&s->head, NULL);
+}
+
+/*
+ * cds_lfs_pop_lock: lock stack pop-protection mutex.
+ */
+static inline void _cds_lfs_pop_lock(struct cds_lfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_lock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * cds_lfs_pop_unlock: unlock stack pop-protection mutex.
+ */
+static inline void _cds_lfs_pop_unlock(struct cds_lfs_stack *s)
+{
+ int ret;
+
+ ret = pthread_mutex_unlock(&s->lock);
+ assert(!ret);
+}
+
+/*
+ * Call __cds_lfs_pop with an internal pop mutex held.
+ */
+static inline
+struct cds_lfs_node *
+_cds_lfs_pop_blocking(struct cds_lfs_stack *s)
+{
+ struct cds_lfs_node *retnode;
+
+ _cds_lfs_pop_lock(s);
+ retnode = ___cds_lfs_pop(s);
+ _cds_lfs_pop_unlock(s);
+ return retnode;
+}
+
+/*
+ * Call __cds_lfs_pop_all with an internal pop mutex held.
+ */
+static inline
+struct cds_lfs_head *
+_cds_lfs_pop_all_blocking(struct cds_lfs_stack *s)
+{
+ struct cds_lfs_head *rethead;
+
+ _cds_lfs_pop_lock(s);
+ rethead = ___cds_lfs_pop_all(s);
+ _cds_lfs_pop_unlock(s);
+ return rethead;
+}
+