Commit | Line | Data |
---|---|---|
ec4e58a3 MD |
1 | #ifndef _URCU_ARCH_UATOMIC_X86_H |
2 | #define _URCU_ARCH_UATOMIC_X86_H | |
0114ba7f | 3 | |
67ecffc0 | 4 | /* |
0114ba7f MD |
5 | * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. |
6 | * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. | |
7 | * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. | |
8 | * Copyright (c) 2009 Mathieu Desnoyers | |
9 | * | |
10 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED | |
11 | * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. | |
12 | * | |
13 | * Permission is hereby granted to use or copy this program | |
14 | * for any purpose, provided the above notices are retained on all copies. | |
15 | * Permission to modify the code and to distribute modified code is granted, | |
16 | * provided the above notices are retained, and a notice that the code was | |
17 | * modified is included with the above copyright notice. | |
18 | * | |
ec4e58a3 | 19 | * Code inspired from libuatomic_ops-1.2, inherited in part from the |
0114ba7f MD |
20 | * Boehm-Demers-Weiser conservative garbage collector. |
21 | */ | |
22 | ||
ae505838 | 23 | #include <urcu/config.h> |
ec4e58a3 | 24 | #include <urcu/compiler.h> |
bf9de1b7 | 25 | #include <urcu/system.h> |
0fad128b | 26 | |
f469d839 PB |
27 | #define UATOMIC_HAS_ATOMIC_BYTE |
28 | #define UATOMIC_HAS_ATOMIC_SHORT | |
29 | ||
36bc70a8 MD |
30 | #ifdef __cplusplus |
31 | extern "C" { | |
67ecffc0 | 32 | #endif |
36bc70a8 | 33 | |
0114ba7f | 34 | /* |
0114ba7f MD |
35 | * Derived from AO_compare_and_swap() and AO_test_and_set_full(). |
36 | */ | |
37 | ||
03155c0d MD |
38 | /* |
39 | * The __hp() macro casts the void pointer "x" to a pointer to a structure | |
40 | * containing an array of char of the specified size. This allows passing the | |
41 | * @addr arguments of the following inline functions as "m" and "+m" operands | |
42 | * to the assembly. | |
43 | */ | |
44 | ||
45 | #define __hp(size, x) ((struct { char v[size]; } *)(x)) | |
cc1be41b | 46 | |
424d4ed5 | 47 | #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v))) |
0fad128b | 48 | |
cc1be41b MD |
49 | /* cmpxchg */ |
50 | ||
5dba80f9 | 51 | static inline __attribute__((always_inline)) |
bf9de1b7 | 52 | unsigned long __uatomic_cmpxchg(void *addr, unsigned long old, |
0fad128b | 53 | unsigned long _new, int len) |
0114ba7f | 54 | { |
cc1be41b MD |
55 | switch (len) { |
56 | case 1: | |
57 | { | |
58 | unsigned char result = old; | |
0fad128b | 59 | |
cc1be41b MD |
60 | __asm__ __volatile__( |
61 | "lock; cmpxchgb %2, %1" | |
03155c0d | 62 | : "+a"(result), "+m"(*__hp(len, addr)) |
cc1be41b | 63 | : "q"((unsigned char)_new) |
0114ba7f | 64 | : "memory"); |
cc1be41b MD |
65 | return result; |
66 | } | |
67 | case 2: | |
68 | { | |
69 | unsigned short result = old; | |
0fad128b | 70 | |
cc1be41b MD |
71 | __asm__ __volatile__( |
72 | "lock; cmpxchgw %2, %1" | |
03155c0d | 73 | : "+a"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
74 | : "r"((unsigned short)_new) |
75 | : "memory"); | |
76 | return result; | |
77 | } | |
78 | case 4: | |
79 | { | |
80 | unsigned int result = old; | |
0fad128b | 81 | |
cc1be41b MD |
82 | __asm__ __volatile__( |
83 | "lock; cmpxchgl %2, %1" | |
03155c0d | 84 | : "+a"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
85 | : "r"((unsigned int)_new) |
86 | : "memory"); | |
87 | return result; | |
88 | } | |
e040d717 | 89 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
90 | case 8: |
91 | { | |
6edb297e | 92 | unsigned long result = old; |
0fad128b | 93 | |
cc1be41b | 94 | __asm__ __volatile__( |
2c5e5fb3 | 95 | "lock; cmpxchgq %2, %1" |
03155c0d | 96 | : "+a"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
97 | : "r"((unsigned long)_new) |
98 | : "memory"); | |
99 | return result; | |
100 | } | |
101 | #endif | |
102 | } | |
d0bbd9c2 MD |
103 | /* |
104 | * generate an illegal instruction. Cannot catch this with | |
105 | * linker tricks when optimizations are disabled. | |
106 | */ | |
cc1be41b MD |
107 | __asm__ __volatile__("ud2"); |
108 | return 0; | |
0114ba7f MD |
109 | } |
110 | ||
bf9de1b7 | 111 | #define _uatomic_cmpxchg(addr, old, _new) \ |
e56d99bf MD |
112 | ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \ |
113 | caa_cast_long_keep_sign(old), \ | |
114 | caa_cast_long_keep_sign(_new),\ | |
cc1be41b MD |
115 | sizeof(*(addr)))) |
116 | ||
117 | /* xchg */ | |
0114ba7f | 118 | |
5dba80f9 | 119 | static inline __attribute__((always_inline)) |
bf9de1b7 | 120 | unsigned long __uatomic_exchange(void *addr, unsigned long val, int len) |
0114ba7f | 121 | { |
cc1be41b MD |
122 | /* Note: the "xchg" instruction does not need a "lock" prefix. */ |
123 | switch (len) { | |
124 | case 1: | |
125 | { | |
126 | unsigned char result; | |
127 | __asm__ __volatile__( | |
128 | "xchgb %0, %1" | |
03155c0d | 129 | : "=q"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
130 | : "0" ((unsigned char)val) |
131 | : "memory"); | |
132 | return result; | |
133 | } | |
134 | case 2: | |
135 | { | |
136 | unsigned short result; | |
137 | __asm__ __volatile__( | |
138 | "xchgw %0, %1" | |
03155c0d | 139 | : "=r"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
140 | : "0" ((unsigned short)val) |
141 | : "memory"); | |
142 | return result; | |
143 | } | |
144 | case 4: | |
145 | { | |
146 | unsigned int result; | |
147 | __asm__ __volatile__( | |
148 | "xchgl %0, %1" | |
03155c0d | 149 | : "=r"(result), "+m"(*__hp(len, addr)) |
cc1be41b MD |
150 | : "0" ((unsigned int)val) |
151 | : "memory"); | |
152 | return result; | |
153 | } | |
e040d717 | 154 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
155 | case 8: |
156 | { | |
157 | unsigned long result; | |
158 | __asm__ __volatile__( | |
0114ba7f | 159 | "xchgq %0, %1" |
03155c0d | 160 | : "=r"(result), "+m"(*__hp(len, addr)) |
cc1be41b | 161 | : "0" ((unsigned long)val) |
0114ba7f | 162 | : "memory"); |
cc1be41b MD |
163 | return result; |
164 | } | |
165 | #endif | |
166 | } | |
d0bbd9c2 MD |
167 | /* |
168 | * generate an illegal instruction. Cannot catch this with | |
169 | * linker tricks when optimizations are disabled. | |
170 | */ | |
cc1be41b MD |
171 | __asm__ __volatile__("ud2"); |
172 | return 0; | |
0114ba7f MD |
173 | } |
174 | ||
bf9de1b7 | 175 | #define _uatomic_xchg(addr, v) \ |
e56d99bf MD |
176 | ((__typeof__(*(addr))) __uatomic_exchange((addr), \ |
177 | caa_cast_long_keep_sign(v), \ | |
cc1be41b MD |
178 | sizeof(*(addr)))) |
179 | ||
8760d94e | 180 | /* uatomic_add_return */ |
0fad128b MD |
181 | |
182 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 183 | unsigned long __uatomic_add_return(void *addr, unsigned long val, |
0fad128b MD |
184 | int len) |
185 | { | |
186 | switch (len) { | |
187 | case 1: | |
188 | { | |
189 | unsigned char result = val; | |
190 | ||
191 | __asm__ __volatile__( | |
192 | "lock; xaddb %1, %0" | |
03155c0d | 193 | : "+m"(*__hp(len, addr)), "+q" (result) |
0fad128b MD |
194 | : |
195 | : "memory"); | |
196 | return result + (unsigned char)val; | |
197 | } | |
198 | case 2: | |
199 | { | |
200 | unsigned short result = val; | |
201 | ||
202 | __asm__ __volatile__( | |
203 | "lock; xaddw %1, %0" | |
03155c0d | 204 | : "+m"(*__hp(len, addr)), "+r" (result) |
0fad128b MD |
205 | : |
206 | : "memory"); | |
207 | return result + (unsigned short)val; | |
208 | } | |
209 | case 4: | |
210 | { | |
211 | unsigned int result = val; | |
212 | ||
213 | __asm__ __volatile__( | |
214 | "lock; xaddl %1, %0" | |
03155c0d | 215 | : "+m"(*__hp(len, addr)), "+r" (result) |
0fad128b MD |
216 | : |
217 | : "memory"); | |
218 | return result + (unsigned int)val; | |
219 | } | |
e040d717 | 220 | #if (CAA_BITS_PER_LONG == 64) |
0fad128b MD |
221 | case 8: |
222 | { | |
223 | unsigned long result = val; | |
224 | ||
225 | __asm__ __volatile__( | |
226 | "lock; xaddq %1, %0" | |
03155c0d | 227 | : "+m"(*__hp(len, addr)), "+r" (result) |
0fad128b MD |
228 | : |
229 | : "memory"); | |
230 | return result + (unsigned long)val; | |
231 | } | |
232 | #endif | |
233 | } | |
d0bbd9c2 MD |
234 | /* |
235 | * generate an illegal instruction. Cannot catch this with | |
236 | * linker tricks when optimizations are disabled. | |
237 | */ | |
0fad128b MD |
238 | __asm__ __volatile__("ud2"); |
239 | return 0; | |
240 | } | |
241 | ||
e56d99bf MD |
242 | #define _uatomic_add_return(addr, v) \ |
243 | ((__typeof__(*(addr))) __uatomic_add_return((addr), \ | |
244 | caa_cast_long_keep_sign(v), \ | |
245 | sizeof(*(addr)))) | |
0fad128b | 246 | |
bf33aaea PB |
247 | /* uatomic_and */ |
248 | ||
249 | static inline __attribute__((always_inline)) | |
250 | void __uatomic_and(void *addr, unsigned long val, int len) | |
251 | { | |
252 | switch (len) { | |
253 | case 1: | |
254 | { | |
255 | __asm__ __volatile__( | |
256 | "lock; andb %1, %0" | |
03155c0d | 257 | : "=m"(*__hp(len, addr)) |
bf33aaea PB |
258 | : "iq" ((unsigned char)val) |
259 | : "memory"); | |
260 | return; | |
261 | } | |
262 | case 2: | |
263 | { | |
264 | __asm__ __volatile__( | |
265 | "lock; andw %1, %0" | |
03155c0d | 266 | : "=m"(*__hp(len, addr)) |
bf33aaea PB |
267 | : "ir" ((unsigned short)val) |
268 | : "memory"); | |
269 | return; | |
270 | } | |
271 | case 4: | |
272 | { | |
273 | __asm__ __volatile__( | |
274 | "lock; andl %1, %0" | |
03155c0d | 275 | : "=m"(*__hp(len, addr)) |
bf33aaea PB |
276 | : "ir" ((unsigned int)val) |
277 | : "memory"); | |
278 | return; | |
279 | } | |
280 | #if (CAA_BITS_PER_LONG == 64) | |
281 | case 8: | |
282 | { | |
283 | __asm__ __volatile__( | |
284 | "lock; andq %1, %0" | |
03155c0d | 285 | : "=m"(*__hp(len, addr)) |
bf33aaea PB |
286 | : "er" ((unsigned long)val) |
287 | : "memory"); | |
288 | return; | |
289 | } | |
290 | #endif | |
291 | } | |
d0bbd9c2 MD |
292 | /* |
293 | * generate an illegal instruction. Cannot catch this with | |
294 | * linker tricks when optimizations are disabled. | |
295 | */ | |
bf33aaea PB |
296 | __asm__ __volatile__("ud2"); |
297 | return; | |
298 | } | |
299 | ||
300 | #define _uatomic_and(addr, v) \ | |
e56d99bf | 301 | (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
bf33aaea | 302 | |
985b35b1 PB |
303 | /* uatomic_or */ |
304 | ||
305 | static inline __attribute__((always_inline)) | |
306 | void __uatomic_or(void *addr, unsigned long val, int len) | |
307 | { | |
308 | switch (len) { | |
309 | case 1: | |
310 | { | |
311 | __asm__ __volatile__( | |
312 | "lock; orb %1, %0" | |
03155c0d | 313 | : "=m"(*__hp(len, addr)) |
985b35b1 PB |
314 | : "iq" ((unsigned char)val) |
315 | : "memory"); | |
316 | return; | |
317 | } | |
318 | case 2: | |
319 | { | |
320 | __asm__ __volatile__( | |
321 | "lock; orw %1, %0" | |
03155c0d | 322 | : "=m"(*__hp(len, addr)) |
985b35b1 PB |
323 | : "ir" ((unsigned short)val) |
324 | : "memory"); | |
325 | return; | |
326 | } | |
327 | case 4: | |
328 | { | |
329 | __asm__ __volatile__( | |
330 | "lock; orl %1, %0" | |
03155c0d | 331 | : "=m"(*__hp(len, addr)) |
985b35b1 PB |
332 | : "ir" ((unsigned int)val) |
333 | : "memory"); | |
334 | return; | |
335 | } | |
336 | #if (CAA_BITS_PER_LONG == 64) | |
337 | case 8: | |
338 | { | |
339 | __asm__ __volatile__( | |
340 | "lock; orq %1, %0" | |
03155c0d | 341 | : "=m"(*__hp(len, addr)) |
985b35b1 PB |
342 | : "er" ((unsigned long)val) |
343 | : "memory"); | |
344 | return; | |
345 | } | |
346 | #endif | |
347 | } | |
d0bbd9c2 MD |
348 | /* |
349 | * generate an illegal instruction. Cannot catch this with | |
350 | * linker tricks when optimizations are disabled. | |
351 | */ | |
985b35b1 PB |
352 | __asm__ __volatile__("ud2"); |
353 | return; | |
354 | } | |
355 | ||
356 | #define _uatomic_or(addr, v) \ | |
e56d99bf | 357 | (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
985b35b1 | 358 | |
8760d94e | 359 | /* uatomic_add */ |
0114ba7f | 360 | |
5dba80f9 | 361 | static inline __attribute__((always_inline)) |
bf9de1b7 | 362 | void __uatomic_add(void *addr, unsigned long val, int len) |
0114ba7f MD |
363 | { |
364 | switch (len) { | |
cc1be41b MD |
365 | case 1: |
366 | { | |
367 | __asm__ __volatile__( | |
368 | "lock; addb %1, %0" | |
03155c0d | 369 | : "=m"(*__hp(len, addr)) |
87322fe8 MD |
370 | : "iq" ((unsigned char)val) |
371 | : "memory"); | |
cc1be41b MD |
372 | return; |
373 | } | |
374 | case 2: | |
375 | { | |
376 | __asm__ __volatile__( | |
377 | "lock; addw %1, %0" | |
03155c0d | 378 | : "=m"(*__hp(len, addr)) |
87322fe8 MD |
379 | : "ir" ((unsigned short)val) |
380 | : "memory"); | |
cc1be41b MD |
381 | return; |
382 | } | |
383 | case 4: | |
384 | { | |
385 | __asm__ __volatile__( | |
386 | "lock; addl %1, %0" | |
03155c0d | 387 | : "=m"(*__hp(len, addr)) |
87322fe8 MD |
388 | : "ir" ((unsigned int)val) |
389 | : "memory"); | |
cc1be41b MD |
390 | return; |
391 | } | |
e040d717 | 392 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
393 | case 8: |
394 | { | |
395 | __asm__ __volatile__( | |
396 | "lock; addq %1, %0" | |
03155c0d | 397 | : "=m"(*__hp(len, addr)) |
87322fe8 MD |
398 | : "er" ((unsigned long)val) |
399 | : "memory"); | |
cc1be41b MD |
400 | return; |
401 | } | |
0114ba7f MD |
402 | #endif |
403 | } | |
d0bbd9c2 MD |
404 | /* |
405 | * generate an illegal instruction. Cannot catch this with | |
406 | * linker tricks when optimizations are disabled. | |
407 | */ | |
0114ba7f | 408 | __asm__ __volatile__("ud2"); |
a81b8e5e | 409 | return; |
0114ba7f MD |
410 | } |
411 | ||
bf9de1b7 | 412 | #define _uatomic_add(addr, v) \ |
e56d99bf | 413 | (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
0114ba7f | 414 | |
2c5e5fb3 | 415 | |
ec4e58a3 | 416 | /* uatomic_inc */ |
2c5e5fb3 MD |
417 | |
418 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 419 | void __uatomic_inc(void *addr, int len) |
2c5e5fb3 MD |
420 | { |
421 | switch (len) { | |
422 | case 1: | |
423 | { | |
424 | __asm__ __volatile__( | |
425 | "lock; incb %0" | |
03155c0d | 426 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
427 | : |
428 | : "memory"); | |
429 | return; | |
430 | } | |
431 | case 2: | |
432 | { | |
433 | __asm__ __volatile__( | |
434 | "lock; incw %0" | |
03155c0d | 435 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
436 | : |
437 | : "memory"); | |
438 | return; | |
439 | } | |
440 | case 4: | |
441 | { | |
442 | __asm__ __volatile__( | |
443 | "lock; incl %0" | |
03155c0d | 444 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
445 | : |
446 | : "memory"); | |
447 | return; | |
448 | } | |
e040d717 | 449 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
450 | case 8: |
451 | { | |
452 | __asm__ __volatile__( | |
453 | "lock; incq %0" | |
03155c0d | 454 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
455 | : |
456 | : "memory"); | |
457 | return; | |
458 | } | |
459 | #endif | |
460 | } | |
461 | /* generate an illegal instruction. Cannot catch this with linker tricks | |
462 | * when optimizations are disabled. */ | |
463 | __asm__ __volatile__("ud2"); | |
464 | return; | |
465 | } | |
466 | ||
bf9de1b7 | 467 | #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr)))) |
2c5e5fb3 | 468 | |
ec4e58a3 | 469 | /* uatomic_dec */ |
2c5e5fb3 MD |
470 | |
471 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 472 | void __uatomic_dec(void *addr, int len) |
2c5e5fb3 MD |
473 | { |
474 | switch (len) { | |
475 | case 1: | |
476 | { | |
477 | __asm__ __volatile__( | |
478 | "lock; decb %0" | |
03155c0d | 479 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
480 | : |
481 | : "memory"); | |
482 | return; | |
483 | } | |
484 | case 2: | |
485 | { | |
486 | __asm__ __volatile__( | |
487 | "lock; decw %0" | |
03155c0d | 488 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
489 | : |
490 | : "memory"); | |
491 | return; | |
492 | } | |
493 | case 4: | |
494 | { | |
495 | __asm__ __volatile__( | |
496 | "lock; decl %0" | |
03155c0d | 497 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
498 | : |
499 | : "memory"); | |
500 | return; | |
501 | } | |
e040d717 | 502 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
503 | case 8: |
504 | { | |
505 | __asm__ __volatile__( | |
506 | "lock; decq %0" | |
03155c0d | 507 | : "=m"(*__hp(len, addr)) |
2c5e5fb3 MD |
508 | : |
509 | : "memory"); | |
510 | return; | |
511 | } | |
512 | #endif | |
513 | } | |
d0bbd9c2 MD |
514 | /* |
515 | * generate an illegal instruction. Cannot catch this with | |
516 | * linker tricks when optimizations are disabled. | |
517 | */ | |
2c5e5fb3 MD |
518 | __asm__ __volatile__("ud2"); |
519 | return; | |
520 | } | |
521 | ||
bf9de1b7 | 522 | #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr)))) |
0114ba7f | 523 | |
e040d717 | 524 | #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH)) |
02be5561 MD |
525 | extern int __rcu_cas_avail; |
526 | extern int __rcu_cas_init(void); | |
bf9de1b7 MD |
527 | |
528 | #define UATOMIC_COMPAT(insn) \ | |
a0b7f7ea | 529 | ((caa_likely(__rcu_cas_avail > 0)) \ |
bf9de1b7 | 530 | ? (_uatomic_##insn) \ |
a0b7f7ea | 531 | : ((caa_unlikely(__rcu_cas_avail < 0) \ |
02be5561 | 532 | ? ((__rcu_cas_init() > 0) \ |
bf9de1b7 MD |
533 | ? (_uatomic_##insn) \ |
534 | : (compat_uatomic_##insn)) \ | |
535 | : (compat_uatomic_##insn)))) | |
536 | ||
424d4ed5 MD |
537 | /* |
538 | * We leave the return value so we don't break the ABI, but remove the | |
539 | * return value from the API. | |
540 | */ | |
bf9de1b7 MD |
541 | extern unsigned long _compat_uatomic_set(void *addr, |
542 | unsigned long _new, int len); | |
543 | #define compat_uatomic_set(addr, _new) \ | |
424d4ed5 MD |
544 | ((void) _compat_uatomic_set((addr), \ |
545 | caa_cast_long_keep_sign(_new), \ | |
546 | sizeof(*(addr)))) | |
bf9de1b7 MD |
547 | |
548 | ||
549 | extern unsigned long _compat_uatomic_xchg(void *addr, | |
550 | unsigned long _new, int len); | |
551 | #define compat_uatomic_xchg(addr, _new) \ | |
552 | ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \ | |
e56d99bf | 553 | caa_cast_long_keep_sign(_new), \ |
bf9de1b7 | 554 | sizeof(*(addr)))) |
7d413817 MD |
555 | |
556 | extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old, | |
bf9de1b7 MD |
557 | unsigned long _new, int len); |
558 | #define compat_uatomic_cmpxchg(addr, old, _new) \ | |
559 | ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \ | |
e56d99bf MD |
560 | caa_cast_long_keep_sign(old), \ |
561 | caa_cast_long_keep_sign(_new), \ | |
bf9de1b7 | 562 | sizeof(*(addr)))) |
7d413817 | 563 | |
8c43fe72 | 564 | extern void _compat_uatomic_and(void *addr, unsigned long _new, int len); |
bf33aaea | 565 | #define compat_uatomic_and(addr, v) \ |
8c43fe72 | 566 | (_compat_uatomic_and((addr), \ |
e56d99bf | 567 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 568 | sizeof(*(addr)))) |
bf33aaea | 569 | |
8c43fe72 | 570 | extern void _compat_uatomic_or(void *addr, unsigned long _new, int len); |
985b35b1 | 571 | #define compat_uatomic_or(addr, v) \ |
8c43fe72 | 572 | (_compat_uatomic_or((addr), \ |
e56d99bf | 573 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 574 | sizeof(*(addr)))) |
985b35b1 | 575 | |
28ca843d PB |
576 | extern unsigned long _compat_uatomic_add_return(void *addr, |
577 | unsigned long _new, int len); | |
e56d99bf MD |
578 | #define compat_uatomic_add_return(addr, v) \ |
579 | ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \ | |
580 | caa_cast_long_keep_sign(v), \ | |
581 | sizeof(*(addr)))) | |
bf9de1b7 | 582 | |
bf9de1b7 MD |
583 | #define compat_uatomic_add(addr, v) \ |
584 | ((void)compat_uatomic_add_return((addr), (v))) | |
bf9de1b7 MD |
585 | #define compat_uatomic_inc(addr) \ |
586 | (compat_uatomic_add((addr), 1)) | |
587 | #define compat_uatomic_dec(addr) \ | |
8760d94e | 588 | (compat_uatomic_add((addr), -1)) |
bf9de1b7 MD |
589 | |
590 | #else | |
591 | #define UATOMIC_COMPAT(insn) (_uatomic_##insn) | |
7d413817 MD |
592 | #endif |
593 | ||
bf9de1b7 | 594 | /* Read is atomic even in compat mode */ |
bf9de1b7 MD |
595 | #define uatomic_set(addr, v) \ |
596 | UATOMIC_COMPAT(set(addr, v)) | |
8760d94e | 597 | |
bf9de1b7 MD |
598 | #define uatomic_cmpxchg(addr, old, _new) \ |
599 | UATOMIC_COMPAT(cmpxchg(addr, old, _new)) | |
600 | #define uatomic_xchg(addr, v) \ | |
601 | UATOMIC_COMPAT(xchg(addr, v)) | |
2812a2d2 | 602 | |
bf33aaea PB |
603 | #define uatomic_and(addr, v) \ |
604 | UATOMIC_COMPAT(and(addr, v)) | |
42e83919 MD |
605 | #define cmm_smp_mb__before_uatomic_and() cmm_barrier() |
606 | #define cmm_smp_mb__after_uatomic_and() cmm_barrier() | |
2812a2d2 | 607 | |
985b35b1 PB |
608 | #define uatomic_or(addr, v) \ |
609 | UATOMIC_COMPAT(or(addr, v)) | |
42e83919 MD |
610 | #define cmm_smp_mb__before_uatomic_or() cmm_barrier() |
611 | #define cmm_smp_mb__after_uatomic_or() cmm_barrier() | |
2812a2d2 | 612 | |
bf9de1b7 MD |
613 | #define uatomic_add_return(addr, v) \ |
614 | UATOMIC_COMPAT(add_return(addr, v)) | |
8760d94e | 615 | |
bf9de1b7 | 616 | #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v)) |
42e83919 MD |
617 | #define cmm_smp_mb__before_uatomic_add() cmm_barrier() |
618 | #define cmm_smp_mb__after_uatomic_add() cmm_barrier() | |
2812a2d2 | 619 | |
bf9de1b7 | 620 | #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr)) |
42e83919 MD |
621 | #define cmm_smp_mb__before_uatomic_inc() cmm_barrier() |
622 | #define cmm_smp_mb__after_uatomic_inc() cmm_barrier() | |
2812a2d2 | 623 | |
bf9de1b7 | 624 | #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr)) |
42e83919 MD |
625 | #define cmm_smp_mb__before_uatomic_dec() cmm_barrier() |
626 | #define cmm_smp_mb__after_uatomic_dec() cmm_barrier() | |
bf9de1b7 | 627 | |
67ecffc0 | 628 | #ifdef __cplusplus |
36bc70a8 MD |
629 | } |
630 | #endif | |
631 | ||
a2e7bf9c | 632 | #include <urcu/uatomic/generic.h> |
8760d94e | 633 | |
ec4e58a3 | 634 | #endif /* _URCU_ARCH_UATOMIC_X86_H */ |