Commit | Line | Data |
---|---|---|
8760d94e PB |
1 | #ifndef _URCU_UATOMIC_GENERIC_H |
2 | #define _URCU_UATOMIC_GENERIC_H | |
3 | ||
4 | /* | |
5 | * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. | |
6 | * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. | |
7 | * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. | |
8 | * Copyright (c) 2009 Mathieu Desnoyers | |
9 | * Copyright (c) 2010 Paolo Bonzini | |
10 | * | |
11 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED | |
12 | * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. | |
13 | * | |
14 | * Permission is hereby granted to use or copy this program | |
15 | * for any purpose, provided the above notices are retained on all copies. | |
16 | * Permission to modify the code and to distribute modified code is granted, | |
17 | * provided the above notices are retained, and a notice that the code was | |
18 | * modified is included with the above copyright notice. | |
19 | * | |
20 | * Code inspired from libuatomic_ops-1.2, inherited in part from the | |
21 | * Boehm-Demers-Weiser conservative garbage collector. | |
22 | */ | |
23 | ||
2917c006 | 24 | #include <stdint.h> |
8760d94e PB |
25 | #include <urcu/compiler.h> |
26 | #include <urcu/system.h> | |
27 | ||
28 | #ifdef __cplusplus | |
29 | extern "C" { | |
30 | #endif | |
31 | ||
8760d94e | 32 | #ifndef uatomic_set |
3daae22a | 33 | #define uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v))) |
8760d94e PB |
34 | #endif |
35 | ||
36 | #ifndef uatomic_read | |
6cf3827c | 37 | #define uatomic_read(addr) CMM_LOAD_SHARED(*(addr)) |
8760d94e PB |
38 | #endif |
39 | ||
40 | #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR | |
41 | static inline __attribute__((always_inline)) | |
5c28497b | 42 | void _uatomic_link_error(void) |
8760d94e PB |
43 | { |
44 | #ifdef ILLEGAL_INSTR | |
d0bbd9c2 MD |
45 | /* |
46 | * generate an illegal instruction. Cannot catch this with | |
47 | * linker tricks when optimizations are disabled. | |
48 | */ | |
8760d94e PB |
49 | __asm__ __volatile__(ILLEGAL_INSTR); |
50 | #else | |
5c28497b | 51 | __builtin_trap(); |
8760d94e PB |
52 | #endif |
53 | } | |
54 | ||
55 | #else /* #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */ | |
5c28497b | 56 | extern void _uatomic_link_error(void); |
8760d94e PB |
57 | #endif /* #else #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR */ |
58 | ||
59 | /* cmpxchg */ | |
60 | ||
61 | #ifndef uatomic_cmpxchg | |
62 | static inline __attribute__((always_inline)) | |
63 | unsigned long _uatomic_cmpxchg(void *addr, unsigned long old, | |
64 | unsigned long _new, int len) | |
65 | { | |
66 | switch (len) { | |
f469d839 PB |
67 | #ifdef UATOMIC_HAS_ATOMIC_BYTE |
68 | case 1: | |
2917c006 KR |
69 | return __sync_val_compare_and_swap_1((uint8_t *) addr, old, |
70 | _new); | |
f469d839 PB |
71 | #endif |
72 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
73 | case 2: | |
2917c006 KR |
74 | return __sync_val_compare_and_swap_2((uint16_t *) addr, old, |
75 | _new); | |
f469d839 | 76 | #endif |
8760d94e | 77 | case 4: |
2917c006 KR |
78 | return __sync_val_compare_and_swap_4((uint32_t *) addr, old, |
79 | _new); | |
b39e1761 | 80 | #if (CAA_BITS_PER_LONG == 64) |
8760d94e | 81 | case 8: |
2917c006 KR |
82 | return __sync_val_compare_and_swap_8((uint64_t *) addr, old, |
83 | _new); | |
8760d94e PB |
84 | #endif |
85 | } | |
86 | _uatomic_link_error(); | |
87 | return 0; | |
88 | } | |
89 | ||
90 | ||
e56d99bf MD |
91 | #define uatomic_cmpxchg(addr, old, _new) \ |
92 | ((__typeof__(*(addr))) _uatomic_cmpxchg((addr), \ | |
93 | caa_cast_long_keep_sign(old), \ | |
94 | caa_cast_long_keep_sign(_new),\ | |
8760d94e PB |
95 | sizeof(*(addr)))) |
96 | ||
97 | ||
bf33aaea PB |
98 | /* uatomic_and */ |
99 | ||
100 | #ifndef uatomic_and | |
101 | static inline __attribute__((always_inline)) | |
102 | void _uatomic_and(void *addr, unsigned long val, | |
103 | int len) | |
104 | { | |
105 | switch (len) { | |
106 | #ifdef UATOMIC_HAS_ATOMIC_BYTE | |
107 | case 1: | |
2917c006 | 108 | __sync_and_and_fetch_1((uint8_t *) addr, val); |
c51d5c6e | 109 | return; |
bf33aaea PB |
110 | #endif |
111 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
112 | case 2: | |
2917c006 | 113 | __sync_and_and_fetch_2((uint16_t *) addr, val); |
c51d5c6e | 114 | return; |
bf33aaea PB |
115 | #endif |
116 | case 4: | |
2917c006 | 117 | __sync_and_and_fetch_4((uint32_t *) addr, val); |
c51d5c6e | 118 | return; |
bf33aaea PB |
119 | #if (CAA_BITS_PER_LONG == 64) |
120 | case 8: | |
2917c006 | 121 | __sync_and_and_fetch_8((uint64_t *) addr, val); |
c51d5c6e | 122 | return; |
bf33aaea PB |
123 | #endif |
124 | } | |
125 | _uatomic_link_error(); | |
bf33aaea PB |
126 | } |
127 | ||
128 | #define uatomic_and(addr, v) \ | |
129 | (_uatomic_and((addr), \ | |
e56d99bf MD |
130 | caa_cast_long_keep_sign(v), \ |
131 | sizeof(*(addr)))) | |
42e83919 MD |
132 | #define cmm_smp_mb__before_uatomic_and() cmm_barrier() |
133 | #define cmm_smp_mb__after_uatomic_and() cmm_barrier() | |
2812a2d2 | 134 | |
bf33aaea PB |
135 | #endif |
136 | ||
985b35b1 PB |
137 | /* uatomic_or */ |
138 | ||
139 | #ifndef uatomic_or | |
140 | static inline __attribute__((always_inline)) | |
141 | void _uatomic_or(void *addr, unsigned long val, | |
142 | int len) | |
143 | { | |
144 | switch (len) { | |
145 | #ifdef UATOMIC_HAS_ATOMIC_BYTE | |
146 | case 1: | |
2917c006 | 147 | __sync_or_and_fetch_1((uint8_t *) addr, val); |
c51d5c6e | 148 | return; |
985b35b1 PB |
149 | #endif |
150 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
151 | case 2: | |
2917c006 | 152 | __sync_or_and_fetch_2((uint16_t *) addr, val); |
c51d5c6e | 153 | return; |
985b35b1 PB |
154 | #endif |
155 | case 4: | |
2917c006 | 156 | __sync_or_and_fetch_4((uint32_t *) addr, val); |
c51d5c6e | 157 | return; |
985b35b1 PB |
158 | #if (CAA_BITS_PER_LONG == 64) |
159 | case 8: | |
2917c006 | 160 | __sync_or_and_fetch_8((uint64_t *) addr, val); |
c51d5c6e | 161 | return; |
985b35b1 PB |
162 | #endif |
163 | } | |
164 | _uatomic_link_error(); | |
c51d5c6e | 165 | return; |
985b35b1 PB |
166 | } |
167 | ||
168 | #define uatomic_or(addr, v) \ | |
169 | (_uatomic_or((addr), \ | |
e56d99bf MD |
170 | caa_cast_long_keep_sign(v), \ |
171 | sizeof(*(addr)))) | |
42e83919 MD |
172 | #define cmm_smp_mb__before_uatomic_or() cmm_barrier() |
173 | #define cmm_smp_mb__after_uatomic_or() cmm_barrier() | |
2812a2d2 | 174 | |
985b35b1 PB |
175 | #endif |
176 | ||
2812a2d2 | 177 | |
8760d94e PB |
178 | /* uatomic_add_return */ |
179 | ||
180 | #ifndef uatomic_add_return | |
181 | static inline __attribute__((always_inline)) | |
182 | unsigned long _uatomic_add_return(void *addr, unsigned long val, | |
183 | int len) | |
184 | { | |
185 | switch (len) { | |
f469d839 PB |
186 | #ifdef UATOMIC_HAS_ATOMIC_BYTE |
187 | case 1: | |
2917c006 | 188 | return __sync_add_and_fetch_1((uint8_t *) addr, val); |
f469d839 PB |
189 | #endif |
190 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
191 | case 2: | |
2917c006 | 192 | return __sync_add_and_fetch_2((uint16_t *) addr, val); |
f469d839 | 193 | #endif |
8760d94e | 194 | case 4: |
2917c006 | 195 | return __sync_add_and_fetch_4((uint32_t *) addr, val); |
b39e1761 | 196 | #if (CAA_BITS_PER_LONG == 64) |
8760d94e | 197 | case 8: |
2917c006 | 198 | return __sync_add_and_fetch_8((uint64_t *) addr, val); |
8760d94e PB |
199 | #endif |
200 | } | |
201 | _uatomic_link_error(); | |
202 | return 0; | |
203 | } | |
204 | ||
205 | ||
e56d99bf MD |
206 | #define uatomic_add_return(addr, v) \ |
207 | ((__typeof__(*(addr))) _uatomic_add_return((addr), \ | |
208 | caa_cast_long_keep_sign(v), \ | |
209 | sizeof(*(addr)))) | |
8760d94e PB |
210 | #endif /* #ifndef uatomic_add_return */ |
211 | ||
212 | #ifndef uatomic_xchg | |
213 | /* xchg */ | |
214 | ||
215 | static inline __attribute__((always_inline)) | |
216 | unsigned long _uatomic_exchange(void *addr, unsigned long val, int len) | |
217 | { | |
218 | switch (len) { | |
f469d839 PB |
219 | #ifdef UATOMIC_HAS_ATOMIC_BYTE |
220 | case 1: | |
221 | { | |
2917c006 | 222 | uint8_t old; |
f469d839 PB |
223 | |
224 | do { | |
2917c006 KR |
225 | old = uatomic_read((uint8_t *) addr); |
226 | } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr, | |
227 | old, val)); | |
f469d839 PB |
228 | |
229 | return old; | |
230 | } | |
231 | #endif | |
232 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
233 | case 2: | |
234 | { | |
2917c006 | 235 | uint16_t old; |
f469d839 PB |
236 | |
237 | do { | |
2917c006 KR |
238 | old = uatomic_read((uint16_t *) addr); |
239 | } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr, | |
240 | old, val)); | |
f469d839 PB |
241 | |
242 | return old; | |
243 | } | |
244 | #endif | |
8760d94e PB |
245 | case 4: |
246 | { | |
2917c006 | 247 | uint32_t old; |
8760d94e PB |
248 | |
249 | do { | |
2917c006 KR |
250 | old = uatomic_read((uint32_t *) addr); |
251 | } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr, | |
252 | old, val)); | |
8760d94e | 253 | |
2f2908d0 | 254 | return old; |
8760d94e | 255 | } |
b39e1761 | 256 | #if (CAA_BITS_PER_LONG == 64) |
8760d94e PB |
257 | case 8: |
258 | { | |
2917c006 | 259 | uint64_t old; |
8760d94e PB |
260 | |
261 | do { | |
2917c006 KR |
262 | old = uatomic_read((uint64_t *) addr); |
263 | } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr, | |
264 | old, val)); | |
8760d94e PB |
265 | |
266 | return old; | |
267 | } | |
268 | #endif | |
269 | } | |
270 | _uatomic_link_error(); | |
271 | return 0; | |
272 | } | |
273 | ||
274 | #define uatomic_xchg(addr, v) \ | |
e56d99bf MD |
275 | ((__typeof__(*(addr))) _uatomic_exchange((addr), \ |
276 | caa_cast_long_keep_sign(v), \ | |
8760d94e PB |
277 | sizeof(*(addr)))) |
278 | #endif /* #ifndef uatomic_xchg */ | |
279 | ||
280 | #else /* #ifndef uatomic_cmpxchg */ | |
281 | ||
bf33aaea PB |
282 | #ifndef uatomic_and |
283 | /* uatomic_and */ | |
284 | ||
285 | static inline __attribute__((always_inline)) | |
286 | void _uatomic_and(void *addr, unsigned long val, int len) | |
287 | { | |
288 | switch (len) { | |
289 | #ifdef UATOMIC_HAS_ATOMIC_BYTE | |
290 | case 1: | |
291 | { | |
2917c006 | 292 | uint8_t old, oldt; |
bf33aaea | 293 | |
2917c006 | 294 | oldt = uatomic_read((uint8_t *) addr); |
bf33aaea PB |
295 | do { |
296 | old = oldt; | |
297 | oldt = _uatomic_cmpxchg(addr, old, old & val, 1); | |
298 | } while (oldt != old); | |
e6e5957d MD |
299 | |
300 | return; | |
bf33aaea PB |
301 | } |
302 | #endif | |
303 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
304 | case 2: | |
305 | { | |
2917c006 | 306 | uint16_t old, oldt; |
bf33aaea | 307 | |
2917c006 | 308 | oldt = uatomic_read((uint16_t *) addr); |
bf33aaea PB |
309 | do { |
310 | old = oldt; | |
311 | oldt = _uatomic_cmpxchg(addr, old, old & val, 2); | |
312 | } while (oldt != old); | |
313 | } | |
314 | #endif | |
315 | case 4: | |
316 | { | |
2917c006 | 317 | uint32_t old, oldt; |
bf33aaea | 318 | |
2917c006 | 319 | oldt = uatomic_read((uint32_t *) addr); |
bf33aaea PB |
320 | do { |
321 | old = oldt; | |
322 | oldt = _uatomic_cmpxchg(addr, old, old & val, 4); | |
323 | } while (oldt != old); | |
e6e5957d MD |
324 | |
325 | return; | |
bf33aaea PB |
326 | } |
327 | #if (CAA_BITS_PER_LONG == 64) | |
328 | case 8: | |
329 | { | |
2917c006 | 330 | uint64_t old, oldt; |
bf33aaea | 331 | |
2917c006 | 332 | oldt = uatomic_read((uint64_t *) addr); |
bf33aaea PB |
333 | do { |
334 | old = oldt; | |
335 | oldt = _uatomic_cmpxchg(addr, old, old & val, 8); | |
336 | } while (oldt != old); | |
e6e5957d MD |
337 | |
338 | return; | |
bf33aaea PB |
339 | } |
340 | #endif | |
341 | } | |
342 | _uatomic_link_error(); | |
bf33aaea PB |
343 | } |
344 | ||
e56d99bf MD |
345 | #define uatomic_and(addr, v) \ |
346 | (_uatomic_and((addr), \ | |
347 | caa_cast_long_keep_sign(v), \ | |
348 | sizeof(*(addr)))) | |
42e83919 MD |
349 | #define cmm_smp_mb__before_uatomic_and() cmm_barrier() |
350 | #define cmm_smp_mb__after_uatomic_and() cmm_barrier() | |
2812a2d2 | 351 | |
bf33aaea PB |
352 | #endif /* #ifndef uatomic_and */ |
353 | ||
985b35b1 PB |
354 | #ifndef uatomic_or |
355 | /* uatomic_or */ | |
356 | ||
357 | static inline __attribute__((always_inline)) | |
358 | void _uatomic_or(void *addr, unsigned long val, int len) | |
359 | { | |
360 | switch (len) { | |
361 | #ifdef UATOMIC_HAS_ATOMIC_BYTE | |
362 | case 1: | |
363 | { | |
2917c006 | 364 | uint8_t old, oldt; |
985b35b1 | 365 | |
2917c006 | 366 | oldt = uatomic_read((uint8_t *) addr); |
985b35b1 PB |
367 | do { |
368 | old = oldt; | |
369 | oldt = _uatomic_cmpxchg(addr, old, old | val, 1); | |
370 | } while (oldt != old); | |
e6e5957d MD |
371 | |
372 | return; | |
985b35b1 PB |
373 | } |
374 | #endif | |
375 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
376 | case 2: | |
377 | { | |
2917c006 | 378 | uint16_t old, oldt; |
985b35b1 | 379 | |
2917c006 | 380 | oldt = uatomic_read((uint16_t *) addr); |
985b35b1 PB |
381 | do { |
382 | old = oldt; | |
383 | oldt = _uatomic_cmpxchg(addr, old, old | val, 2); | |
384 | } while (oldt != old); | |
e6e5957d MD |
385 | |
386 | return; | |
985b35b1 PB |
387 | } |
388 | #endif | |
389 | case 4: | |
390 | { | |
2917c006 | 391 | uint32_t old, oldt; |
985b35b1 | 392 | |
2917c006 | 393 | oldt = uatomic_read((uint32_t *) addr); |
985b35b1 PB |
394 | do { |
395 | old = oldt; | |
396 | oldt = _uatomic_cmpxchg(addr, old, old | val, 4); | |
397 | } while (oldt != old); | |
e6e5957d MD |
398 | |
399 | return; | |
985b35b1 PB |
400 | } |
401 | #if (CAA_BITS_PER_LONG == 64) | |
402 | case 8: | |
403 | { | |
2917c006 | 404 | uint64_t old, oldt; |
985b35b1 | 405 | |
2917c006 | 406 | oldt = uatomic_read((uint64_t *) addr); |
985b35b1 PB |
407 | do { |
408 | old = oldt; | |
409 | oldt = _uatomic_cmpxchg(addr, old, old | val, 8); | |
410 | } while (oldt != old); | |
e6e5957d MD |
411 | |
412 | return; | |
985b35b1 PB |
413 | } |
414 | #endif | |
415 | } | |
416 | _uatomic_link_error(); | |
985b35b1 PB |
417 | } |
418 | ||
e56d99bf MD |
419 | #define uatomic_or(addr, v) \ |
420 | (_uatomic_or((addr), \ | |
421 | caa_cast_long_keep_sign(v), \ | |
422 | sizeof(*(addr)))) | |
42e83919 MD |
423 | #define cmm_smp_mb__before_uatomic_or() cmm_barrier() |
424 | #define cmm_smp_mb__after_uatomic_or() cmm_barrier() | |
2812a2d2 | 425 | |
985b35b1 PB |
426 | #endif /* #ifndef uatomic_or */ |
427 | ||
8760d94e PB |
428 | #ifndef uatomic_add_return |
429 | /* uatomic_add_return */ | |
430 | ||
431 | static inline __attribute__((always_inline)) | |
432 | unsigned long _uatomic_add_return(void *addr, unsigned long val, int len) | |
433 | { | |
434 | switch (len) { | |
f469d839 PB |
435 | #ifdef UATOMIC_HAS_ATOMIC_BYTE |
436 | case 1: | |
437 | { | |
2917c006 | 438 | uint8_t old, oldt; |
f469d839 | 439 | |
2917c006 | 440 | oldt = uatomic_read((uint8_t *) addr); |
f469d839 PB |
441 | do { |
442 | old = oldt; | |
2917c006 | 443 | oldt = uatomic_cmpxchg((uint8_t *) addr, |
b4e6d540 | 444 | old, old + val); |
f469d839 PB |
445 | } while (oldt != old); |
446 | ||
447 | return old + val; | |
448 | } | |
449 | #endif | |
450 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
451 | case 2: | |
452 | { | |
2917c006 | 453 | uint16_t old, oldt; |
f469d839 | 454 | |
2917c006 | 455 | oldt = uatomic_read((uint16_t *) addr); |
f469d839 PB |
456 | do { |
457 | old = oldt; | |
2917c006 | 458 | oldt = uatomic_cmpxchg((uint16_t *) addr, |
b4e6d540 | 459 | old, old + val); |
f469d839 PB |
460 | } while (oldt != old); |
461 | ||
462 | return old + val; | |
463 | } | |
464 | #endif | |
8760d94e PB |
465 | case 4: |
466 | { | |
2917c006 | 467 | uint32_t old, oldt; |
8760d94e | 468 | |
2917c006 | 469 | oldt = uatomic_read((uint32_t *) addr); |
8760d94e PB |
470 | do { |
471 | old = oldt; | |
2917c006 | 472 | oldt = uatomic_cmpxchg((uint32_t *) addr, |
b4e6d540 | 473 | old, old + val); |
8760d94e PB |
474 | } while (oldt != old); |
475 | ||
476 | return old + val; | |
477 | } | |
b39e1761 | 478 | #if (CAA_BITS_PER_LONG == 64) |
8760d94e PB |
479 | case 8: |
480 | { | |
2917c006 | 481 | uint64_t old, oldt; |
8760d94e | 482 | |
2917c006 | 483 | oldt = uatomic_read((uint64_t *) addr); |
8760d94e PB |
484 | do { |
485 | old = oldt; | |
2917c006 | 486 | oldt = uatomic_cmpxchg((uint64_t *) addr, |
b4e6d540 | 487 | old, old + val); |
8760d94e PB |
488 | } while (oldt != old); |
489 | ||
490 | return old + val; | |
491 | } | |
492 | #endif | |
493 | } | |
494 | _uatomic_link_error(); | |
495 | return 0; | |
496 | } | |
497 | ||
e56d99bf MD |
498 | #define uatomic_add_return(addr, v) \ |
499 | ((__typeof__(*(addr))) _uatomic_add_return((addr), \ | |
500 | caa_cast_long_keep_sign(v), \ | |
501 | sizeof(*(addr)))) | |
8760d94e PB |
502 | #endif /* #ifndef uatomic_add_return */ |
503 | ||
504 | #ifndef uatomic_xchg | |
505 | /* xchg */ | |
506 | ||
507 | static inline __attribute__((always_inline)) | |
508 | unsigned long _uatomic_exchange(void *addr, unsigned long val, int len) | |
509 | { | |
510 | switch (len) { | |
f469d839 PB |
511 | #ifdef UATOMIC_HAS_ATOMIC_BYTE |
512 | case 1: | |
513 | { | |
2917c006 | 514 | uint8_t old, oldt; |
f469d839 | 515 | |
2917c006 | 516 | oldt = uatomic_read((uint8_t *) addr); |
f469d839 PB |
517 | do { |
518 | old = oldt; | |
2917c006 | 519 | oldt = uatomic_cmpxchg((uint8_t *) addr, |
b4e6d540 | 520 | old, val); |
f469d839 PB |
521 | } while (oldt != old); |
522 | ||
523 | return old; | |
524 | } | |
525 | #endif | |
526 | #ifdef UATOMIC_HAS_ATOMIC_SHORT | |
527 | case 2: | |
528 | { | |
2917c006 | 529 | uint16_t old, oldt; |
f469d839 | 530 | |
2917c006 | 531 | oldt = uatomic_read((uint16_t *) addr); |
f469d839 PB |
532 | do { |
533 | old = oldt; | |
2917c006 | 534 | oldt = uatomic_cmpxchg((uint16_t *) addr, |
b4e6d540 | 535 | old, val); |
f469d839 PB |
536 | } while (oldt != old); |
537 | ||
538 | return old; | |
539 | } | |
540 | #endif | |
8760d94e PB |
541 | case 4: |
542 | { | |
2917c006 | 543 | uint32_t old, oldt; |
8760d94e | 544 | |
2917c006 | 545 | oldt = uatomic_read((uint32_t *) addr); |
8760d94e PB |
546 | do { |
547 | old = oldt; | |
2917c006 | 548 | oldt = uatomic_cmpxchg((uint32_t *) addr, |
b4e6d540 | 549 | old, val); |
8760d94e PB |
550 | } while (oldt != old); |
551 | ||
552 | return old; | |
553 | } | |
b39e1761 | 554 | #if (CAA_BITS_PER_LONG == 64) |
8760d94e PB |
555 | case 8: |
556 | { | |
2917c006 | 557 | uint64_t old, oldt; |
8760d94e | 558 | |
2917c006 | 559 | oldt = uatomic_read((uint64_t *) addr); |
8760d94e PB |
560 | do { |
561 | old = oldt; | |
2917c006 | 562 | oldt = uatomic_cmpxchg((uint64_t *) addr, |
b4e6d540 | 563 | old, val); |
8760d94e PB |
564 | } while (oldt != old); |
565 | ||
566 | return old; | |
567 | } | |
568 | #endif | |
569 | } | |
570 | _uatomic_link_error(); | |
571 | return 0; | |
572 | } | |
573 | ||
574 | #define uatomic_xchg(addr, v) \ | |
e56d99bf MD |
575 | ((__typeof__(*(addr))) _uatomic_exchange((addr), \ |
576 | caa_cast_long_keep_sign(v), \ | |
8760d94e PB |
577 | sizeof(*(addr)))) |
578 | #endif /* #ifndef uatomic_xchg */ | |
579 | ||
580 | #endif /* #else #ifndef uatomic_cmpxchg */ | |
581 | ||
582 | /* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */ | |
583 | ||
584 | #ifndef uatomic_add | |
585 | #define uatomic_add(addr, v) (void)uatomic_add_return((addr), (v)) | |
42e83919 MD |
586 | #define cmm_smp_mb__before_uatomic_add() cmm_barrier() |
587 | #define cmm_smp_mb__after_uatomic_add() cmm_barrier() | |
8760d94e PB |
588 | #endif |
589 | ||
e56d99bf MD |
590 | #define uatomic_sub_return(addr, v) \ |
591 | uatomic_add_return((addr), -(caa_cast_long_keep_sign(v))) | |
592 | #define uatomic_sub(addr, v) \ | |
593 | uatomic_add((addr), -(caa_cast_long_keep_sign(v))) | |
42e83919 MD |
594 | #define cmm_smp_mb__before_uatomic_sub() cmm_smp_mb__before_uatomic_add() |
595 | #define cmm_smp_mb__after_uatomic_sub() cmm_smp_mb__after_uatomic_add() | |
8760d94e PB |
596 | |
597 | #ifndef uatomic_inc | |
598 | #define uatomic_inc(addr) uatomic_add((addr), 1) | |
42e83919 MD |
599 | #define cmm_smp_mb__before_uatomic_inc() cmm_smp_mb__before_uatomic_add() |
600 | #define cmm_smp_mb__after_uatomic_inc() cmm_smp_mb__after_uatomic_add() | |
8760d94e PB |
601 | #endif |
602 | ||
603 | #ifndef uatomic_dec | |
604 | #define uatomic_dec(addr) uatomic_add((addr), -1) | |
42e83919 MD |
605 | #define cmm_smp_mb__before_uatomic_dec() cmm_smp_mb__before_uatomic_add() |
606 | #define cmm_smp_mb__after_uatomic_dec() cmm_smp_mb__after_uatomic_add() | |
8760d94e PB |
607 | #endif |
608 | ||
609 | #ifdef __cplusplus | |
610 | } | |
611 | #endif | |
612 | ||
613 | #endif /* _URCU_UATOMIC_GENERIC_H */ |