Commit | Line | Data |
---|---|---|
ce29b371 MJ |
1 | // SPDX-FileCopyrightText: 2006 Paul E. McKenney, IBM. |
2 | // | |
3 | // SPDX-License-Identifier: GPL-2.0-only | |
4 | ||
1a43bbd8 | 5 | #ifndef _INCLUDE_API_H |
0578089f PM |
6 | #define _INCLUDE_API_H |
7 | ||
8 | /* | |
9 | * common.h: Common Linux kernel-isms. | |
10 | * | |
0578089f PM |
11 | * Much code taken from the Linux kernel. For such code, the option |
12 | * to redistribute under later versions of GPL might not be available. | |
13 | */ | |
14 | ||
78bec10c | 15 | #include <urcu/compiler.h> |
3c570c9c | 16 | #include <urcu/arch.h> |
c8c32a52 | 17 | #include <urcu/uatomic.h> |
3c570c9c | 18 | |
0578089f PM |
19 | /* |
20 | * Machine parameters. | |
21 | */ | |
22 | ||
0578089f | 23 | #define ____cacheline_internodealigned_in_smp \ |
901bbb44 | 24 | __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) |
6ee91d83 | 25 | |
0578089f PM |
26 | |
27 | #include <stdio.h> | |
28 | #include <stdlib.h> | |
29 | #include <errno.h> | |
30 | #include <limits.h> | |
31 | #include <sys/types.h> | |
0578089f | 32 | #include <pthread.h> |
0578089f PM |
33 | #include <sys/param.h> |
34 | /* #include "atomic.h" */ | |
35 | ||
0578089f PM |
36 | /* |
37 | * Exclusive locking primitives. | |
38 | */ | |
39 | ||
40 | typedef pthread_mutex_t spinlock_t; | |
41 | ||
42 | #define DEFINE_SPINLOCK(lock) spinlock_t lock = PTHREAD_MUTEX_INITIALIZER; | |
43 | #define __SPIN_LOCK_UNLOCKED(lockp) PTHREAD_MUTEX_INITIALIZER | |
44 | ||
45 | static void spin_lock_init(spinlock_t *sp) | |
46 | { | |
47 | if (pthread_mutex_init(sp, NULL) != 0) { | |
48 | perror("spin_lock_init:pthread_mutex_init"); | |
49 | exit(-1); | |
50 | } | |
51 | } | |
52 | ||
53 | static void spin_lock(spinlock_t *sp) | |
54 | { | |
55 | if (pthread_mutex_lock(sp) != 0) { | |
56 | perror("spin_lock:pthread_mutex_lock"); | |
57 | exit(-1); | |
58 | } | |
59 | } | |
60 | ||
61 | static void spin_unlock(spinlock_t *sp) | |
62 | { | |
63 | if (pthread_mutex_unlock(sp) != 0) { | |
64 | perror("spin_unlock:pthread_mutex_unlock"); | |
65 | exit(-1); | |
66 | } | |
67 | } | |
68 | ||
69 | #define spin_lock_irqsave(l, f) do { f = 1; spin_lock(l); } while (0) | |
70 | #define spin_unlock_irqrestore(l, f) do { f = 0; spin_unlock(l); } while (0) | |
71 | ||
72 | /* | |
73 | * Thread creation/destruction primitives. | |
74 | */ | |
75 | ||
76 | typedef pthread_t thread_id_t; | |
77 | ||
af9987a9 | 78 | #define NR_THREADS 4096 |
0578089f | 79 | |
3c570c9c PB |
80 | #define __THREAD_ID_MAP_EMPTY ((thread_id_t) 0) |
81 | #define __THREAD_ID_MAP_WAITING ((thread_id_t) 1) | |
0578089f PM |
82 | thread_id_t __thread_id_map[NR_THREADS]; |
83 | spinlock_t __thread_id_map_mutex; | |
84 | ||
85 | #define for_each_thread(t) \ | |
86 | for (t = 0; t < NR_THREADS; t++) | |
87 | ||
88 | #define for_each_running_thread(t) \ | |
89 | for (t = 0; t < NR_THREADS; t++) \ | |
90 | if ((__thread_id_map[t] != __THREAD_ID_MAP_EMPTY) && \ | |
91 | (__thread_id_map[t] != __THREAD_ID_MAP_WAITING)) | |
92 | ||
3c570c9c PB |
93 | #define for_each_tid(t, tid) \ |
94 | for (t = 0; t < NR_THREADS; t++) \ | |
95 | if ((((tid) = __thread_id_map[t]) != __THREAD_ID_MAP_EMPTY) && \ | |
96 | ((tid) != __THREAD_ID_MAP_WAITING)) | |
97 | ||
0578089f PM |
98 | pthread_key_t thread_id_key; |
99 | ||
100 | static int __smp_thread_id(void) | |
101 | { | |
102 | int i; | |
103 | thread_id_t tid = pthread_self(); | |
104 | ||
105 | for (i = 0; i < NR_THREADS; i++) { | |
c8c32a52 | 106 | if (uatomic_read(&__thread_id_map[i]) == tid) { |
0578089f PM |
107 | long v = i + 1; /* must be non-NULL. */ |
108 | ||
109 | if (pthread_setspecific(thread_id_key, (void *)v) != 0) { | |
110 | perror("pthread_setspecific"); | |
111 | exit(-1); | |
112 | } | |
113 | return i; | |
114 | } | |
115 | } | |
116 | spin_lock(&__thread_id_map_mutex); | |
117 | for (i = 0; i < NR_THREADS; i++) { | |
8ff97cc3 | 118 | if (__thread_id_map[i] == tid) { |
0578089f PM |
119 | spin_unlock(&__thread_id_map_mutex); |
120 | return i; | |
8ff97cc3 | 121 | } |
0578089f PM |
122 | } |
123 | spin_unlock(&__thread_id_map_mutex); | |
665eb3ef MD |
124 | fprintf(stderr, "smp_thread_id: Rogue thread, id: %lu(%#lx)\n", |
125 | (unsigned long) tid, (unsigned long) tid); | |
0578089f PM |
126 | exit(-1); |
127 | } | |
128 | ||
129 | static int smp_thread_id(void) | |
130 | { | |
131 | void *id; | |
132 | ||
133 | id = pthread_getspecific(thread_id_key); | |
134 | if (id == NULL) | |
135 | return __smp_thread_id(); | |
153b081a | 136 | return ((long) id - 1); |
0578089f PM |
137 | } |
138 | ||
139 | static thread_id_t create_thread(void *(*func)(void *), void *arg) | |
140 | { | |
141 | thread_id_t tid; | |
142 | int i; | |
143 | ||
144 | spin_lock(&__thread_id_map_mutex); | |
145 | for (i = 0; i < NR_THREADS; i++) { | |
146 | if (__thread_id_map[i] == __THREAD_ID_MAP_EMPTY) | |
147 | break; | |
148 | } | |
149 | if (i >= NR_THREADS) { | |
150 | spin_unlock(&__thread_id_map_mutex); | |
151 | fprintf(stderr, "Thread limit of %d exceeded!\n", NR_THREADS); | |
152 | exit(-1); | |
153 | } | |
154 | __thread_id_map[i] = __THREAD_ID_MAP_WAITING; | |
c8c32a52 | 155 | |
0578089f PM |
156 | if (pthread_create(&tid, NULL, func, arg) != 0) { |
157 | perror("create_thread:pthread_create"); | |
158 | exit(-1); | |
159 | } | |
c8c32a52 OD |
160 | uatomic_set(&__thread_id_map[i], tid); |
161 | spin_unlock(&__thread_id_map_mutex); | |
0578089f PM |
162 | return tid; |
163 | } | |
164 | ||
165 | static void *wait_thread(thread_id_t tid) | |
166 | { | |
167 | int i; | |
168 | void *vp; | |
169 | ||
170 | for (i = 0; i < NR_THREADS; i++) { | |
c8c32a52 | 171 | if (uatomic_read(&__thread_id_map[i]) == tid) |
0578089f PM |
172 | break; |
173 | } | |
174 | if (i >= NR_THREADS){ | |
665eb3ef MD |
175 | fprintf(stderr, "wait_thread: bad tid = %lu(%#lx)\n", |
176 | (unsigned long)tid, (unsigned long)tid); | |
0578089f PM |
177 | exit(-1); |
178 | } | |
179 | if (pthread_join(tid, &vp) != 0) { | |
180 | perror("wait_thread:pthread_join"); | |
181 | exit(-1); | |
182 | } | |
c8c32a52 | 183 | uatomic_set(&__thread_id_map[i], __THREAD_ID_MAP_EMPTY); |
0578089f PM |
184 | return vp; |
185 | } | |
186 | ||
187 | static void wait_all_threads(void) | |
188 | { | |
189 | int i; | |
190 | thread_id_t tid; | |
191 | ||
192 | for (i = 1; i < NR_THREADS; i++) { | |
193 | tid = __thread_id_map[i]; | |
194 | if (tid != __THREAD_ID_MAP_EMPTY && | |
195 | tid != __THREAD_ID_MAP_WAITING) | |
196 | (void)wait_thread(tid); | |
197 | } | |
198 | } | |
199 | ||
a142df4e | 200 | #ifdef HAVE_SCHED_SETAFFINITY |
0578089f PM |
201 | static void run_on(int cpu) |
202 | { | |
203 | cpu_set_t mask; | |
204 | ||
205 | CPU_ZERO(&mask); | |
206 | CPU_SET(cpu, &mask); | |
207 | sched_setaffinity(0, sizeof(mask), &mask); | |
208 | } | |
a142df4e MJ |
209 | #else |
210 | ||
211 | static void run_on(int cpu __attribute__((unused))) | |
212 | {} | |
213 | #endif /* HAVE_SCHED_SETAFFINITY */ | |
0578089f PM |
214 | |
215 | /* | |
216 | * timekeeping -- very crude -- should use MONOTONIC... | |
217 | */ | |
218 | ||
61c3fb60 | 219 | static inline |
0578089f PM |
220 | long long get_microseconds(void) |
221 | { | |
222 | struct timeval tv; | |
223 | ||
224 | if (gettimeofday(&tv, NULL) != 0) | |
225 | abort(); | |
226 | return ((long long)tv.tv_sec) * 1000000LL + (long long)tv.tv_usec; | |
227 | } | |
228 | ||
229 | /* | |
230 | * Per-thread variables. | |
231 | */ | |
232 | ||
233 | #define DEFINE_PER_THREAD(type, name) \ | |
234 | struct { \ | |
235 | __typeof__(type) v \ | |
06f22bdb | 236 | __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))); \ |
e1789ce2 | 237 | } __per_thread_##name[NR_THREADS] |
0578089f PM |
238 | #define DECLARE_PER_THREAD(type, name) extern DEFINE_PER_THREAD(type, name) |
239 | ||
240 | #define per_thread(name, thread) __per_thread_##name[thread].v | |
241 | #define __get_thread_var(name) per_thread(name, smp_thread_id()) | |
242 | ||
243 | #define init_per_thread(name, v) \ | |
244 | do { \ | |
245 | int __i_p_t_i; \ | |
246 | for (__i_p_t_i = 0; __i_p_t_i < NR_THREADS; __i_p_t_i++) \ | |
247 | per_thread(name, __i_p_t_i) = v; \ | |
248 | } while (0) | |
249 | ||
0578089f PM |
250 | DEFINE_PER_THREAD(int, smp_processor_id); |
251 | ||
0578089f PM |
252 | /* |
253 | * Bug checks. | |
254 | */ | |
255 | ||
256 | #define BUG_ON(c) do { if (!(c)) abort(); } while (0) | |
257 | ||
258 | /* | |
259 | * Initialization -- Must be called before calling any primitives. | |
260 | */ | |
261 | ||
262 | static void smp_init(void) | |
263 | { | |
264 | int i; | |
265 | ||
266 | spin_lock_init(&__thread_id_map_mutex); | |
267 | __thread_id_map[0] = pthread_self(); | |
268 | for (i = 1; i < NR_THREADS; i++) | |
269 | __thread_id_map[i] = __THREAD_ID_MAP_EMPTY; | |
270 | init_per_thread(smp_processor_id, 0); | |
271 | if (pthread_key_create(&thread_id_key, NULL) != 0) { | |
272 | perror("pthread_key_create"); | |
273 | exit(-1); | |
274 | } | |
275 | } | |
276 | ||
1a43bbd8 | 277 | #endif |