Commit | Line | Data |
---|---|---|
ec4e58a3 MD |
1 | #ifndef _URCU_ARCH_UATOMIC_X86_H |
2 | #define _URCU_ARCH_UATOMIC_X86_H | |
0114ba7f | 3 | |
67ecffc0 | 4 | /* |
0114ba7f MD |
5 | * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. |
6 | * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. | |
7 | * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. | |
8 | * Copyright (c) 2009 Mathieu Desnoyers | |
9 | * | |
10 | * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED | |
11 | * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. | |
12 | * | |
13 | * Permission is hereby granted to use or copy this program | |
14 | * for any purpose, provided the above notices are retained on all copies. | |
15 | * Permission to modify the code and to distribute modified code is granted, | |
16 | * provided the above notices are retained, and a notice that the code was | |
17 | * modified is included with the above copyright notice. | |
18 | * | |
ec4e58a3 | 19 | * Code inspired from libuatomic_ops-1.2, inherited in part from the |
0114ba7f MD |
20 | * Boehm-Demers-Weiser conservative garbage collector. |
21 | */ | |
22 | ||
c6bc6503 | 23 | #include <urcu/config.h> |
ec4e58a3 | 24 | #include <urcu/compiler.h> |
bf9de1b7 | 25 | #include <urcu/system.h> |
0fad128b | 26 | |
f469d839 PB |
27 | #define UATOMIC_HAS_ATOMIC_BYTE |
28 | #define UATOMIC_HAS_ATOMIC_SHORT | |
29 | ||
36bc70a8 MD |
30 | #ifdef __cplusplus |
31 | extern "C" { | |
67ecffc0 | 32 | #endif |
36bc70a8 | 33 | |
0114ba7f | 34 | /* |
0114ba7f MD |
35 | * Derived from AO_compare_and_swap() and AO_test_and_set_full(). |
36 | */ | |
37 | ||
f97344e0 | 38 | /* |
839eaea5 | 39 | * The __hp() macro casts the void pointer @x to a pointer to a structure |
f97344e0 MD |
40 | * containing an array of char of the specified size. This allows passing the |
41 | * @addr arguments of the following inline functions as "m" and "+m" operands | |
839eaea5 MD |
42 | * to the assembly. The @size parameter should be a constant to support |
43 | * compilers such as clang which do not support VLA. | |
f97344e0 MD |
44 | */ |
45 | ||
46 | #define __hp(size, x) ((struct { char v[size]; } *)(x)) | |
cc1be41b | 47 | |
424d4ed5 | 48 | #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v))) |
0fad128b | 49 | |
cc1be41b MD |
50 | /* cmpxchg */ |
51 | ||
5dba80f9 | 52 | static inline __attribute__((always_inline)) |
bf9de1b7 | 53 | unsigned long __uatomic_cmpxchg(void *addr, unsigned long old, |
0fad128b | 54 | unsigned long _new, int len) |
0114ba7f | 55 | { |
cc1be41b MD |
56 | switch (len) { |
57 | case 1: | |
58 | { | |
59 | unsigned char result = old; | |
0fad128b | 60 | |
cc1be41b MD |
61 | __asm__ __volatile__( |
62 | "lock; cmpxchgb %2, %1" | |
839eaea5 | 63 | : "+a"(result), "+m"(*__hp(1, addr)) |
cc1be41b | 64 | : "q"((unsigned char)_new) |
0114ba7f | 65 | : "memory"); |
cc1be41b MD |
66 | return result; |
67 | } | |
68 | case 2: | |
69 | { | |
70 | unsigned short result = old; | |
0fad128b | 71 | |
cc1be41b MD |
72 | __asm__ __volatile__( |
73 | "lock; cmpxchgw %2, %1" | |
839eaea5 | 74 | : "+a"(result), "+m"(*__hp(2, addr)) |
cc1be41b MD |
75 | : "r"((unsigned short)_new) |
76 | : "memory"); | |
77 | return result; | |
78 | } | |
79 | case 4: | |
80 | { | |
81 | unsigned int result = old; | |
0fad128b | 82 | |
cc1be41b MD |
83 | __asm__ __volatile__( |
84 | "lock; cmpxchgl %2, %1" | |
839eaea5 | 85 | : "+a"(result), "+m"(*__hp(4, addr)) |
cc1be41b MD |
86 | : "r"((unsigned int)_new) |
87 | : "memory"); | |
88 | return result; | |
89 | } | |
e040d717 | 90 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
91 | case 8: |
92 | { | |
6edb297e | 93 | unsigned long result = old; |
0fad128b | 94 | |
cc1be41b | 95 | __asm__ __volatile__( |
2c5e5fb3 | 96 | "lock; cmpxchgq %2, %1" |
839eaea5 | 97 | : "+a"(result), "+m"(*__hp(8, addr)) |
cc1be41b MD |
98 | : "r"((unsigned long)_new) |
99 | : "memory"); | |
100 | return result; | |
101 | } | |
102 | #endif | |
103 | } | |
d0bbd9c2 MD |
104 | /* |
105 | * generate an illegal instruction. Cannot catch this with | |
106 | * linker tricks when optimizations are disabled. | |
107 | */ | |
cc1be41b MD |
108 | __asm__ __volatile__("ud2"); |
109 | return 0; | |
0114ba7f MD |
110 | } |
111 | ||
bf9de1b7 | 112 | #define _uatomic_cmpxchg(addr, old, _new) \ |
e56d99bf MD |
113 | ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \ |
114 | caa_cast_long_keep_sign(old), \ | |
115 | caa_cast_long_keep_sign(_new),\ | |
cc1be41b MD |
116 | sizeof(*(addr)))) |
117 | ||
118 | /* xchg */ | |
0114ba7f | 119 | |
5dba80f9 | 120 | static inline __attribute__((always_inline)) |
bf9de1b7 | 121 | unsigned long __uatomic_exchange(void *addr, unsigned long val, int len) |
0114ba7f | 122 | { |
cc1be41b MD |
123 | /* Note: the "xchg" instruction does not need a "lock" prefix. */ |
124 | switch (len) { | |
125 | case 1: | |
126 | { | |
127 | unsigned char result; | |
128 | __asm__ __volatile__( | |
129 | "xchgb %0, %1" | |
839eaea5 | 130 | : "=q"(result), "+m"(*__hp(1, addr)) |
cc1be41b MD |
131 | : "0" ((unsigned char)val) |
132 | : "memory"); | |
133 | return result; | |
134 | } | |
135 | case 2: | |
136 | { | |
137 | unsigned short result; | |
138 | __asm__ __volatile__( | |
139 | "xchgw %0, %1" | |
839eaea5 | 140 | : "=r"(result), "+m"(*__hp(2, addr)) |
cc1be41b MD |
141 | : "0" ((unsigned short)val) |
142 | : "memory"); | |
143 | return result; | |
144 | } | |
145 | case 4: | |
146 | { | |
147 | unsigned int result; | |
148 | __asm__ __volatile__( | |
149 | "xchgl %0, %1" | |
839eaea5 | 150 | : "=r"(result), "+m"(*__hp(4, addr)) |
cc1be41b MD |
151 | : "0" ((unsigned int)val) |
152 | : "memory"); | |
153 | return result; | |
154 | } | |
e040d717 | 155 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
156 | case 8: |
157 | { | |
158 | unsigned long result; | |
159 | __asm__ __volatile__( | |
0114ba7f | 160 | "xchgq %0, %1" |
839eaea5 | 161 | : "=r"(result), "+m"(*__hp(8, addr)) |
cc1be41b | 162 | : "0" ((unsigned long)val) |
0114ba7f | 163 | : "memory"); |
cc1be41b MD |
164 | return result; |
165 | } | |
166 | #endif | |
167 | } | |
d0bbd9c2 MD |
168 | /* |
169 | * generate an illegal instruction. Cannot catch this with | |
170 | * linker tricks when optimizations are disabled. | |
171 | */ | |
cc1be41b MD |
172 | __asm__ __volatile__("ud2"); |
173 | return 0; | |
0114ba7f MD |
174 | } |
175 | ||
bf9de1b7 | 176 | #define _uatomic_xchg(addr, v) \ |
e56d99bf MD |
177 | ((__typeof__(*(addr))) __uatomic_exchange((addr), \ |
178 | caa_cast_long_keep_sign(v), \ | |
cc1be41b MD |
179 | sizeof(*(addr)))) |
180 | ||
8760d94e | 181 | /* uatomic_add_return */ |
0fad128b MD |
182 | |
183 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 184 | unsigned long __uatomic_add_return(void *addr, unsigned long val, |
0fad128b MD |
185 | int len) |
186 | { | |
187 | switch (len) { | |
188 | case 1: | |
189 | { | |
190 | unsigned char result = val; | |
191 | ||
192 | __asm__ __volatile__( | |
193 | "lock; xaddb %1, %0" | |
839eaea5 | 194 | : "+m"(*__hp(1, addr)), "+q" (result) |
0fad128b MD |
195 | : |
196 | : "memory"); | |
197 | return result + (unsigned char)val; | |
198 | } | |
199 | case 2: | |
200 | { | |
201 | unsigned short result = val; | |
202 | ||
203 | __asm__ __volatile__( | |
204 | "lock; xaddw %1, %0" | |
839eaea5 | 205 | : "+m"(*__hp(2, addr)), "+r" (result) |
0fad128b MD |
206 | : |
207 | : "memory"); | |
208 | return result + (unsigned short)val; | |
209 | } | |
210 | case 4: | |
211 | { | |
212 | unsigned int result = val; | |
213 | ||
214 | __asm__ __volatile__( | |
215 | "lock; xaddl %1, %0" | |
839eaea5 | 216 | : "+m"(*__hp(4, addr)), "+r" (result) |
0fad128b MD |
217 | : |
218 | : "memory"); | |
219 | return result + (unsigned int)val; | |
220 | } | |
e040d717 | 221 | #if (CAA_BITS_PER_LONG == 64) |
0fad128b MD |
222 | case 8: |
223 | { | |
224 | unsigned long result = val; | |
225 | ||
226 | __asm__ __volatile__( | |
227 | "lock; xaddq %1, %0" | |
839eaea5 | 228 | : "+m"(*__hp(8, addr)), "+r" (result) |
0fad128b MD |
229 | : |
230 | : "memory"); | |
231 | return result + (unsigned long)val; | |
232 | } | |
233 | #endif | |
234 | } | |
d0bbd9c2 MD |
235 | /* |
236 | * generate an illegal instruction. Cannot catch this with | |
237 | * linker tricks when optimizations are disabled. | |
238 | */ | |
0fad128b MD |
239 | __asm__ __volatile__("ud2"); |
240 | return 0; | |
241 | } | |
242 | ||
e56d99bf MD |
243 | #define _uatomic_add_return(addr, v) \ |
244 | ((__typeof__(*(addr))) __uatomic_add_return((addr), \ | |
245 | caa_cast_long_keep_sign(v), \ | |
246 | sizeof(*(addr)))) | |
0fad128b | 247 | |
bf33aaea PB |
248 | /* uatomic_and */ |
249 | ||
250 | static inline __attribute__((always_inline)) | |
251 | void __uatomic_and(void *addr, unsigned long val, int len) | |
252 | { | |
253 | switch (len) { | |
254 | case 1: | |
255 | { | |
256 | __asm__ __volatile__( | |
257 | "lock; andb %1, %0" | |
839eaea5 | 258 | : "=m"(*__hp(1, addr)) |
bf33aaea PB |
259 | : "iq" ((unsigned char)val) |
260 | : "memory"); | |
261 | return; | |
262 | } | |
263 | case 2: | |
264 | { | |
265 | __asm__ __volatile__( | |
266 | "lock; andw %1, %0" | |
839eaea5 | 267 | : "=m"(*__hp(2, addr)) |
bf33aaea PB |
268 | : "ir" ((unsigned short)val) |
269 | : "memory"); | |
270 | return; | |
271 | } | |
272 | case 4: | |
273 | { | |
274 | __asm__ __volatile__( | |
275 | "lock; andl %1, %0" | |
839eaea5 | 276 | : "=m"(*__hp(4, addr)) |
bf33aaea PB |
277 | : "ir" ((unsigned int)val) |
278 | : "memory"); | |
279 | return; | |
280 | } | |
281 | #if (CAA_BITS_PER_LONG == 64) | |
282 | case 8: | |
283 | { | |
284 | __asm__ __volatile__( | |
285 | "lock; andq %1, %0" | |
839eaea5 | 286 | : "=m"(*__hp(8, addr)) |
bf33aaea PB |
287 | : "er" ((unsigned long)val) |
288 | : "memory"); | |
289 | return; | |
290 | } | |
291 | #endif | |
292 | } | |
d0bbd9c2 MD |
293 | /* |
294 | * generate an illegal instruction. Cannot catch this with | |
295 | * linker tricks when optimizations are disabled. | |
296 | */ | |
bf33aaea PB |
297 | __asm__ __volatile__("ud2"); |
298 | return; | |
299 | } | |
300 | ||
301 | #define _uatomic_and(addr, v) \ | |
e56d99bf | 302 | (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
bf33aaea | 303 | |
985b35b1 PB |
304 | /* uatomic_or */ |
305 | ||
306 | static inline __attribute__((always_inline)) | |
307 | void __uatomic_or(void *addr, unsigned long val, int len) | |
308 | { | |
309 | switch (len) { | |
310 | case 1: | |
311 | { | |
312 | __asm__ __volatile__( | |
313 | "lock; orb %1, %0" | |
839eaea5 | 314 | : "=m"(*__hp(1, addr)) |
985b35b1 PB |
315 | : "iq" ((unsigned char)val) |
316 | : "memory"); | |
317 | return; | |
318 | } | |
319 | case 2: | |
320 | { | |
321 | __asm__ __volatile__( | |
322 | "lock; orw %1, %0" | |
839eaea5 | 323 | : "=m"(*__hp(2, addr)) |
985b35b1 PB |
324 | : "ir" ((unsigned short)val) |
325 | : "memory"); | |
326 | return; | |
327 | } | |
328 | case 4: | |
329 | { | |
330 | __asm__ __volatile__( | |
331 | "lock; orl %1, %0" | |
839eaea5 | 332 | : "=m"(*__hp(4, addr)) |
985b35b1 PB |
333 | : "ir" ((unsigned int)val) |
334 | : "memory"); | |
335 | return; | |
336 | } | |
337 | #if (CAA_BITS_PER_LONG == 64) | |
338 | case 8: | |
339 | { | |
340 | __asm__ __volatile__( | |
341 | "lock; orq %1, %0" | |
839eaea5 | 342 | : "=m"(*__hp(8, addr)) |
985b35b1 PB |
343 | : "er" ((unsigned long)val) |
344 | : "memory"); | |
345 | return; | |
346 | } | |
347 | #endif | |
348 | } | |
d0bbd9c2 MD |
349 | /* |
350 | * generate an illegal instruction. Cannot catch this with | |
351 | * linker tricks when optimizations are disabled. | |
352 | */ | |
985b35b1 PB |
353 | __asm__ __volatile__("ud2"); |
354 | return; | |
355 | } | |
356 | ||
357 | #define _uatomic_or(addr, v) \ | |
e56d99bf | 358 | (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
985b35b1 | 359 | |
8760d94e | 360 | /* uatomic_add */ |
0114ba7f | 361 | |
5dba80f9 | 362 | static inline __attribute__((always_inline)) |
bf9de1b7 | 363 | void __uatomic_add(void *addr, unsigned long val, int len) |
0114ba7f MD |
364 | { |
365 | switch (len) { | |
cc1be41b MD |
366 | case 1: |
367 | { | |
368 | __asm__ __volatile__( | |
369 | "lock; addb %1, %0" | |
839eaea5 | 370 | : "=m"(*__hp(1, addr)) |
87322fe8 MD |
371 | : "iq" ((unsigned char)val) |
372 | : "memory"); | |
cc1be41b MD |
373 | return; |
374 | } | |
375 | case 2: | |
376 | { | |
377 | __asm__ __volatile__( | |
378 | "lock; addw %1, %0" | |
839eaea5 | 379 | : "=m"(*__hp(2, addr)) |
87322fe8 MD |
380 | : "ir" ((unsigned short)val) |
381 | : "memory"); | |
cc1be41b MD |
382 | return; |
383 | } | |
384 | case 4: | |
385 | { | |
386 | __asm__ __volatile__( | |
387 | "lock; addl %1, %0" | |
839eaea5 | 388 | : "=m"(*__hp(4, addr)) |
87322fe8 MD |
389 | : "ir" ((unsigned int)val) |
390 | : "memory"); | |
cc1be41b MD |
391 | return; |
392 | } | |
e040d717 | 393 | #if (CAA_BITS_PER_LONG == 64) |
cc1be41b MD |
394 | case 8: |
395 | { | |
396 | __asm__ __volatile__( | |
397 | "lock; addq %1, %0" | |
839eaea5 | 398 | : "=m"(*__hp(8, addr)) |
87322fe8 MD |
399 | : "er" ((unsigned long)val) |
400 | : "memory"); | |
cc1be41b MD |
401 | return; |
402 | } | |
0114ba7f MD |
403 | #endif |
404 | } | |
d0bbd9c2 MD |
405 | /* |
406 | * generate an illegal instruction. Cannot catch this with | |
407 | * linker tricks when optimizations are disabled. | |
408 | */ | |
0114ba7f | 409 | __asm__ __volatile__("ud2"); |
a81b8e5e | 410 | return; |
0114ba7f MD |
411 | } |
412 | ||
bf9de1b7 | 413 | #define _uatomic_add(addr, v) \ |
e56d99bf | 414 | (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr)))) |
0114ba7f | 415 | |
2c5e5fb3 | 416 | |
ec4e58a3 | 417 | /* uatomic_inc */ |
2c5e5fb3 MD |
418 | |
419 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 420 | void __uatomic_inc(void *addr, int len) |
2c5e5fb3 MD |
421 | { |
422 | switch (len) { | |
423 | case 1: | |
424 | { | |
425 | __asm__ __volatile__( | |
426 | "lock; incb %0" | |
839eaea5 | 427 | : "=m"(*__hp(1, addr)) |
2c5e5fb3 MD |
428 | : |
429 | : "memory"); | |
430 | return; | |
431 | } | |
432 | case 2: | |
433 | { | |
434 | __asm__ __volatile__( | |
435 | "lock; incw %0" | |
839eaea5 | 436 | : "=m"(*__hp(2, addr)) |
2c5e5fb3 MD |
437 | : |
438 | : "memory"); | |
439 | return; | |
440 | } | |
441 | case 4: | |
442 | { | |
443 | __asm__ __volatile__( | |
444 | "lock; incl %0" | |
839eaea5 | 445 | : "=m"(*__hp(4, addr)) |
2c5e5fb3 MD |
446 | : |
447 | : "memory"); | |
448 | return; | |
449 | } | |
e040d717 | 450 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
451 | case 8: |
452 | { | |
453 | __asm__ __volatile__( | |
454 | "lock; incq %0" | |
839eaea5 | 455 | : "=m"(*__hp(8, addr)) |
2c5e5fb3 MD |
456 | : |
457 | : "memory"); | |
458 | return; | |
459 | } | |
460 | #endif | |
461 | } | |
462 | /* generate an illegal instruction. Cannot catch this with linker tricks | |
463 | * when optimizations are disabled. */ | |
464 | __asm__ __volatile__("ud2"); | |
465 | return; | |
466 | } | |
467 | ||
bf9de1b7 | 468 | #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr)))) |
2c5e5fb3 | 469 | |
ec4e58a3 | 470 | /* uatomic_dec */ |
2c5e5fb3 MD |
471 | |
472 | static inline __attribute__((always_inline)) | |
bf9de1b7 | 473 | void __uatomic_dec(void *addr, int len) |
2c5e5fb3 MD |
474 | { |
475 | switch (len) { | |
476 | case 1: | |
477 | { | |
478 | __asm__ __volatile__( | |
479 | "lock; decb %0" | |
839eaea5 | 480 | : "=m"(*__hp(1, addr)) |
2c5e5fb3 MD |
481 | : |
482 | : "memory"); | |
483 | return; | |
484 | } | |
485 | case 2: | |
486 | { | |
487 | __asm__ __volatile__( | |
488 | "lock; decw %0" | |
839eaea5 | 489 | : "=m"(*__hp(2, addr)) |
2c5e5fb3 MD |
490 | : |
491 | : "memory"); | |
492 | return; | |
493 | } | |
494 | case 4: | |
495 | { | |
496 | __asm__ __volatile__( | |
497 | "lock; decl %0" | |
839eaea5 | 498 | : "=m"(*__hp(4, addr)) |
2c5e5fb3 MD |
499 | : |
500 | : "memory"); | |
501 | return; | |
502 | } | |
e040d717 | 503 | #if (CAA_BITS_PER_LONG == 64) |
2c5e5fb3 MD |
504 | case 8: |
505 | { | |
506 | __asm__ __volatile__( | |
507 | "lock; decq %0" | |
839eaea5 | 508 | : "=m"(*__hp(8, addr)) |
2c5e5fb3 MD |
509 | : |
510 | : "memory"); | |
511 | return; | |
512 | } | |
513 | #endif | |
514 | } | |
d0bbd9c2 MD |
515 | /* |
516 | * generate an illegal instruction. Cannot catch this with | |
517 | * linker tricks when optimizations are disabled. | |
518 | */ | |
2c5e5fb3 MD |
519 | __asm__ __volatile__("ud2"); |
520 | return; | |
521 | } | |
522 | ||
bf9de1b7 | 523 | #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr)))) |
0114ba7f | 524 | |
e040d717 | 525 | #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH)) |
02be5561 MD |
526 | extern int __rcu_cas_avail; |
527 | extern int __rcu_cas_init(void); | |
bf9de1b7 MD |
528 | |
529 | #define UATOMIC_COMPAT(insn) \ | |
a0b7f7ea | 530 | ((caa_likely(__rcu_cas_avail > 0)) \ |
bf9de1b7 | 531 | ? (_uatomic_##insn) \ |
a0b7f7ea | 532 | : ((caa_unlikely(__rcu_cas_avail < 0) \ |
02be5561 | 533 | ? ((__rcu_cas_init() > 0) \ |
bf9de1b7 MD |
534 | ? (_uatomic_##insn) \ |
535 | : (compat_uatomic_##insn)) \ | |
536 | : (compat_uatomic_##insn)))) | |
537 | ||
424d4ed5 MD |
538 | /* |
539 | * We leave the return value so we don't break the ABI, but remove the | |
540 | * return value from the API. | |
541 | */ | |
bf9de1b7 MD |
542 | extern unsigned long _compat_uatomic_set(void *addr, |
543 | unsigned long _new, int len); | |
544 | #define compat_uatomic_set(addr, _new) \ | |
424d4ed5 MD |
545 | ((void) _compat_uatomic_set((addr), \ |
546 | caa_cast_long_keep_sign(_new), \ | |
547 | sizeof(*(addr)))) | |
bf9de1b7 MD |
548 | |
549 | ||
550 | extern unsigned long _compat_uatomic_xchg(void *addr, | |
551 | unsigned long _new, int len); | |
552 | #define compat_uatomic_xchg(addr, _new) \ | |
553 | ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \ | |
e56d99bf | 554 | caa_cast_long_keep_sign(_new), \ |
bf9de1b7 | 555 | sizeof(*(addr)))) |
7d413817 MD |
556 | |
557 | extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old, | |
bf9de1b7 MD |
558 | unsigned long _new, int len); |
559 | #define compat_uatomic_cmpxchg(addr, old, _new) \ | |
560 | ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \ | |
e56d99bf MD |
561 | caa_cast_long_keep_sign(old), \ |
562 | caa_cast_long_keep_sign(_new), \ | |
bf9de1b7 | 563 | sizeof(*(addr)))) |
7d413817 | 564 | |
8c43fe72 | 565 | extern void _compat_uatomic_and(void *addr, unsigned long _new, int len); |
bf33aaea | 566 | #define compat_uatomic_and(addr, v) \ |
8c43fe72 | 567 | (_compat_uatomic_and((addr), \ |
e56d99bf | 568 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 569 | sizeof(*(addr)))) |
bf33aaea | 570 | |
8c43fe72 | 571 | extern void _compat_uatomic_or(void *addr, unsigned long _new, int len); |
985b35b1 | 572 | #define compat_uatomic_or(addr, v) \ |
8c43fe72 | 573 | (_compat_uatomic_or((addr), \ |
e56d99bf | 574 | caa_cast_long_keep_sign(v), \ |
8c43fe72 | 575 | sizeof(*(addr)))) |
985b35b1 | 576 | |
28ca843d PB |
577 | extern unsigned long _compat_uatomic_add_return(void *addr, |
578 | unsigned long _new, int len); | |
e56d99bf MD |
579 | #define compat_uatomic_add_return(addr, v) \ |
580 | ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \ | |
581 | caa_cast_long_keep_sign(v), \ | |
582 | sizeof(*(addr)))) | |
bf9de1b7 | 583 | |
bf9de1b7 MD |
584 | #define compat_uatomic_add(addr, v) \ |
585 | ((void)compat_uatomic_add_return((addr), (v))) | |
bf9de1b7 MD |
586 | #define compat_uatomic_inc(addr) \ |
587 | (compat_uatomic_add((addr), 1)) | |
588 | #define compat_uatomic_dec(addr) \ | |
8760d94e | 589 | (compat_uatomic_add((addr), -1)) |
bf9de1b7 MD |
590 | |
591 | #else | |
592 | #define UATOMIC_COMPAT(insn) (_uatomic_##insn) | |
7d413817 MD |
593 | #endif |
594 | ||
bf9de1b7 | 595 | /* Read is atomic even in compat mode */ |
bf9de1b7 MD |
596 | #define uatomic_set(addr, v) \ |
597 | UATOMIC_COMPAT(set(addr, v)) | |
8760d94e | 598 | |
bf9de1b7 MD |
599 | #define uatomic_cmpxchg(addr, old, _new) \ |
600 | UATOMIC_COMPAT(cmpxchg(addr, old, _new)) | |
601 | #define uatomic_xchg(addr, v) \ | |
602 | UATOMIC_COMPAT(xchg(addr, v)) | |
2812a2d2 | 603 | |
bf33aaea PB |
604 | #define uatomic_and(addr, v) \ |
605 | UATOMIC_COMPAT(and(addr, v)) | |
42e83919 MD |
606 | #define cmm_smp_mb__before_uatomic_and() cmm_barrier() |
607 | #define cmm_smp_mb__after_uatomic_and() cmm_barrier() | |
2812a2d2 | 608 | |
985b35b1 PB |
609 | #define uatomic_or(addr, v) \ |
610 | UATOMIC_COMPAT(or(addr, v)) | |
42e83919 MD |
611 | #define cmm_smp_mb__before_uatomic_or() cmm_barrier() |
612 | #define cmm_smp_mb__after_uatomic_or() cmm_barrier() | |
2812a2d2 | 613 | |
bf9de1b7 MD |
614 | #define uatomic_add_return(addr, v) \ |
615 | UATOMIC_COMPAT(add_return(addr, v)) | |
8760d94e | 616 | |
bf9de1b7 | 617 | #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v)) |
42e83919 MD |
618 | #define cmm_smp_mb__before_uatomic_add() cmm_barrier() |
619 | #define cmm_smp_mb__after_uatomic_add() cmm_barrier() | |
2812a2d2 | 620 | |
bf9de1b7 | 621 | #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr)) |
42e83919 MD |
622 | #define cmm_smp_mb__before_uatomic_inc() cmm_barrier() |
623 | #define cmm_smp_mb__after_uatomic_inc() cmm_barrier() | |
2812a2d2 | 624 | |
bf9de1b7 | 625 | #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr)) |
42e83919 MD |
626 | #define cmm_smp_mb__before_uatomic_dec() cmm_barrier() |
627 | #define cmm_smp_mb__after_uatomic_dec() cmm_barrier() | |
bf9de1b7 | 628 | |
67ecffc0 | 629 | #ifdef __cplusplus |
36bc70a8 MD |
630 | } |
631 | #endif | |
632 | ||
a2e7bf9c | 633 | #include <urcu/uatomic/generic.h> |
8760d94e | 634 | |
ec4e58a3 | 635 | #endif /* _URCU_ARCH_UATOMIC_X86_H */ |