2 * SPDX-License-Identifier: LGPL-2.1-or-later
4 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
7 * Userspace RCU header.
9 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
10 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
13 #ifndef _LTTNG_UST_URCU_STATIC_H
14 #define _LTTNG_UST_URCU_STATIC_H
20 #include <urcu/config.h>
21 #include <urcu/compiler.h>
22 #include <urcu/arch.h>
23 #include <urcu/system.h>
24 #include <urcu/uatomic.h>
25 #include <urcu/list.h>
26 #include <urcu/tls-compat.h>
27 #include <urcu/debug.h>
30 * This code section can only be included in LGPL 2.1 compatible source code.
31 * See below for the function call wrappers which can be used in code meant to
32 * be only linked with the Userspace RCU library. This comes with a small
33 * performance degradation on the read-side due to the added function calls.
34 * This is required to permit relinking with newer versions of the library.
41 enum lttng_ust_urcu_state
{
42 LTTNG_UST_URCU_READER_ACTIVE_CURRENT
,
43 LTTNG_UST_URCU_READER_ACTIVE_OLD
,
44 LTTNG_UST_URCU_READER_INACTIVE
,
48 * The trick here is that LTTNG_UST_URCU_GP_CTR_PHASE must be a multiple of 8 so we can use a
49 * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits.
51 #define LTTNG_UST_URCU_GP_COUNT (1UL << 0)
52 /* Use the amount of bits equal to half of the architecture long size */
53 #define LTTNG_UST_URCU_GP_CTR_PHASE (1UL << (sizeof(long) << 2))
54 #define LTTNG_UST_URCU_GP_CTR_NEST_MASK (LTTNG_UST_URCU_GP_CTR_PHASE - 1)
57 * Used internally by _lttng_ust_urcu_read_lock.
59 extern void lttng_ust_urcu_register(void);
61 struct lttng_ust_urcu_gp
{
63 * Global grace period counter.
64 * Contains the current LTTNG_UST_URCU_GP_CTR_PHASE.
65 * Also has a LTTNG_UST_URCU_GP_COUNT of 1, to accelerate the reader fast path.
66 * Written to only by writer with mutex taken.
67 * Read by both writer and readers.
70 } __attribute__((aligned(CAA_CACHE_LINE_SIZE
)));
72 extern struct lttng_ust_urcu_gp lttng_ust_urcu_gp
;
74 struct lttng_ust_urcu_reader
{
75 /* Data used by both reader and lttng_ust_urcu_synchronize_rcu() */
77 /* Data used for registry */
78 struct cds_list_head node
__attribute__((aligned(CAA_CACHE_LINE_SIZE
)));
80 int alloc
; /* registry entry allocated */
84 * Bulletproof version keeps a pointer to a registry not part of the TLS.
85 * Adds a pointer dereference on the read-side, but won't require to unregister
88 extern DECLARE_URCU_TLS(struct lttng_ust_urcu_reader
*, lttng_ust_urcu_reader
);
90 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
91 #define lttng_ust_urcu_has_sys_membarrier 1
93 extern int lttng_ust_urcu_has_sys_membarrier
;
96 static inline void lttng_ust_urcu_smp_mb_slave(void)
98 if (caa_likely(lttng_ust_urcu_has_sys_membarrier
))
104 static inline enum lttng_ust_urcu_state
lttng_ust_urcu_reader_state(unsigned long *ctr
)
109 return LTTNG_UST_URCU_READER_INACTIVE
;
111 * Make sure both tests below are done on the same version of *value
112 * to insure consistency.
114 v
= CMM_LOAD_SHARED(*ctr
);
115 if (!(v
& LTTNG_UST_URCU_GP_CTR_NEST_MASK
))
116 return LTTNG_UST_URCU_READER_INACTIVE
;
117 if (!((v
^ lttng_ust_urcu_gp
.ctr
) & LTTNG_UST_URCU_GP_CTR_PHASE
))
118 return LTTNG_UST_URCU_READER_ACTIVE_CURRENT
;
119 return LTTNG_UST_URCU_READER_ACTIVE_OLD
;
123 * Helper for _lttng_ust_urcu_read_lock(). The format of lttng_ust_urcu_gp.ctr (as well as
124 * the per-thread rcu_reader.ctr) has the upper bits containing a count of
125 * _lttng_ust_urcu_read_lock() nesting, and a lower-order bit that contains either zero
126 * or LTTNG_UST_URCU_GP_CTR_PHASE. The smp_mb_slave() ensures that the accesses in
127 * _lttng_ust_urcu_read_lock() happen before the subsequent read-side critical section.
129 static inline void _lttng_ust_urcu_read_lock_update(unsigned long tmp
)
131 if (caa_likely(!(tmp
& LTTNG_UST_URCU_GP_CTR_NEST_MASK
))) {
132 _CMM_STORE_SHARED(URCU_TLS(lttng_ust_urcu_reader
)->ctr
, _CMM_LOAD_SHARED(lttng_ust_urcu_gp
.ctr
));
133 lttng_ust_urcu_smp_mb_slave();
135 _CMM_STORE_SHARED(URCU_TLS(lttng_ust_urcu_reader
)->ctr
, tmp
+ LTTNG_UST_URCU_GP_COUNT
);
139 * Enter an RCU read-side critical section.
141 * The first cmm_barrier() call ensures that the compiler does not reorder
142 * the body of _lttng_ust_urcu_read_lock() with a mutex.
144 * This function and its helper are both less than 10 lines long. The
145 * intent is that this function meets the 10-line criterion in LGPL,
146 * allowing this function to be invoked directly from non-LGPL code.
148 static inline void _lttng_ust_urcu_read_lock(void)
152 if (caa_unlikely(!URCU_TLS(lttng_ust_urcu_reader
)))
153 lttng_ust_urcu_register(); /* If not yet registered. */
154 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
155 tmp
= URCU_TLS(lttng_ust_urcu_reader
)->ctr
;
156 urcu_assert((tmp
& LTTNG_UST_URCU_GP_CTR_NEST_MASK
) != LTTNG_UST_URCU_GP_CTR_NEST_MASK
);
157 _lttng_ust_urcu_read_lock_update(tmp
);
161 * Exit an RCU read-side critical section. This function is less than
162 * 10 lines of code, and is intended to be usable by non-LGPL code, as
163 * called out in LGPL.
165 static inline void _lttng_ust_urcu_read_unlock(void)
169 tmp
= URCU_TLS(lttng_ust_urcu_reader
)->ctr
;
170 urcu_assert(tmp
& LTTNG_UST_URCU_GP_CTR_NEST_MASK
);
171 /* Finish using rcu before decrementing the pointer. */
172 lttng_ust_urcu_smp_mb_slave();
173 _CMM_STORE_SHARED(URCU_TLS(lttng_ust_urcu_reader
)->ctr
, tmp
- LTTNG_UST_URCU_GP_COUNT
);
174 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
178 * Returns whether within a RCU read-side critical section.
180 * This function is less than 10 lines long. The intent is that this
181 * function meets the 10-line criterion for LGPL, allowing this function
182 * to be invoked directly from non-LGPL code.
184 static inline int _lttng_ust_urcu_read_ongoing(void)
186 if (caa_unlikely(!URCU_TLS(lttng_ust_urcu_reader
)))
187 lttng_ust_urcu_register(); /* If not yet registered. */
188 return URCU_TLS(lttng_ust_urcu_reader
)->ctr
& LTTNG_UST_URCU_GP_CTR_NEST_MASK
;
195 #endif /* _LTTNG_UST_URCU_STATIC_H */