1 #ifndef _URCU_ARCH_UATOMIC_X86_H
2 #define _URCU_ARCH_UATOMIC_X86_H
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
23 #include <urcu/config.h>
24 #include <urcu/compiler.h>
25 #include <urcu/system.h>
27 #define UATOMIC_HAS_ATOMIC_BYTE
28 #define UATOMIC_HAS_ATOMIC_SHORT
35 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
39 * The __hp() macro casts the void pointer @x to a pointer to a structure
40 * containing an array of char of the specified size. This allows passing the
41 * @addr arguments of the following inline functions as "m" and "+m" operands
42 * to the assembly. The @size parameter should be a constant to support
43 * compilers such as clang which do not support VLA. Create typedefs because
44 * C++ does not allow types be defined in casts.
47 typedef struct { char v
[1]; } __hp_1
;
48 typedef struct { char v
[2]; } __hp_2
;
49 typedef struct { char v
[4]; } __hp_4
;
50 typedef struct { char v
[8]; } __hp_8
;
52 #define __hp(size, x) ((__hp_##size *)(x))
54 #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
58 static inline __attribute__((always_inline
))
59 unsigned long __uatomic_cmpxchg(void *addr
, unsigned long old
,
60 unsigned long _new
, int len
)
65 unsigned char result
= old
;
68 "lock; cmpxchgb %2, %1"
69 : "+a"(result
), "+m"(*__hp(1, addr
))
70 : "q"((unsigned char)_new
)
76 unsigned short result
= old
;
79 "lock; cmpxchgw %2, %1"
80 : "+a"(result
), "+m"(*__hp(2, addr
))
81 : "r"((unsigned short)_new
)
87 unsigned int result
= old
;
90 "lock; cmpxchgl %2, %1"
91 : "+a"(result
), "+m"(*__hp(4, addr
))
92 : "r"((unsigned int)_new
)
96 #if (CAA_BITS_PER_LONG == 64)
99 unsigned long result
= old
;
101 __asm__
__volatile__(
102 "lock; cmpxchgq %2, %1"
103 : "+a"(result
), "+m"(*__hp(8, addr
))
104 : "r"((unsigned long)_new
)
111 * generate an illegal instruction. Cannot catch this with
112 * linker tricks when optimizations are disabled.
114 __asm__
__volatile__("ud2");
118 #define _uatomic_cmpxchg(addr, old, _new) \
119 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
120 caa_cast_long_keep_sign(old), \
121 caa_cast_long_keep_sign(_new),\
126 static inline __attribute__((always_inline
))
127 unsigned long __uatomic_exchange(void *addr
, unsigned long val
, int len
)
129 /* Note: the "xchg" instruction does not need a "lock" prefix. */
133 unsigned char result
;
134 __asm__
__volatile__(
136 : "=q"(result
), "+m"(*__hp(1, addr
))
137 : "0" ((unsigned char)val
)
143 unsigned short result
;
144 __asm__
__volatile__(
146 : "=r"(result
), "+m"(*__hp(2, addr
))
147 : "0" ((unsigned short)val
)
154 __asm__
__volatile__(
156 : "=r"(result
), "+m"(*__hp(4, addr
))
157 : "0" ((unsigned int)val
)
161 #if (CAA_BITS_PER_LONG == 64)
164 unsigned long result
;
165 __asm__
__volatile__(
167 : "=r"(result
), "+m"(*__hp(8, addr
))
168 : "0" ((unsigned long)val
)
175 * generate an illegal instruction. Cannot catch this with
176 * linker tricks when optimizations are disabled.
178 __asm__
__volatile__("ud2");
182 #define _uatomic_xchg(addr, v) \
183 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
184 caa_cast_long_keep_sign(v), \
187 /* uatomic_add_return */
189 static inline __attribute__((always_inline
))
190 unsigned long __uatomic_add_return(void *addr
, unsigned long val
,
196 unsigned char result
= val
;
198 __asm__
__volatile__(
200 : "+m"(*__hp(1, addr
)), "+q" (result
)
203 return result
+ (unsigned char)val
;
207 unsigned short result
= val
;
209 __asm__
__volatile__(
211 : "+m"(*__hp(2, addr
)), "+r" (result
)
214 return result
+ (unsigned short)val
;
218 unsigned int result
= val
;
220 __asm__
__volatile__(
222 : "+m"(*__hp(4, addr
)), "+r" (result
)
225 return result
+ (unsigned int)val
;
227 #if (CAA_BITS_PER_LONG == 64)
230 unsigned long result
= val
;
232 __asm__
__volatile__(
234 : "+m"(*__hp(8, addr
)), "+r" (result
)
237 return result
+ (unsigned long)val
;
242 * generate an illegal instruction. Cannot catch this with
243 * linker tricks when optimizations are disabled.
245 __asm__
__volatile__("ud2");
249 #define _uatomic_add_return(addr, v) \
250 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
251 caa_cast_long_keep_sign(v), \
256 static inline __attribute__((always_inline
))
257 void __uatomic_and(void *addr
, unsigned long val
, int len
)
262 __asm__
__volatile__(
264 : "=m"(*__hp(1, addr
))
265 : "iq" ((unsigned char)val
)
271 __asm__
__volatile__(
273 : "=m"(*__hp(2, addr
))
274 : "ir" ((unsigned short)val
)
280 __asm__
__volatile__(
282 : "=m"(*__hp(4, addr
))
283 : "ir" ((unsigned int)val
)
287 #if (CAA_BITS_PER_LONG == 64)
290 __asm__
__volatile__(
292 : "=m"(*__hp(8, addr
))
293 : "er" ((unsigned long)val
)
300 * generate an illegal instruction. Cannot catch this with
301 * linker tricks when optimizations are disabled.
303 __asm__
__volatile__("ud2");
307 #define _uatomic_and(addr, v) \
308 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
312 static inline __attribute__((always_inline
))
313 void __uatomic_or(void *addr
, unsigned long val
, int len
)
318 __asm__
__volatile__(
320 : "=m"(*__hp(1, addr
))
321 : "iq" ((unsigned char)val
)
327 __asm__
__volatile__(
329 : "=m"(*__hp(2, addr
))
330 : "ir" ((unsigned short)val
)
336 __asm__
__volatile__(
338 : "=m"(*__hp(4, addr
))
339 : "ir" ((unsigned int)val
)
343 #if (CAA_BITS_PER_LONG == 64)
346 __asm__
__volatile__(
348 : "=m"(*__hp(8, addr
))
349 : "er" ((unsigned long)val
)
356 * generate an illegal instruction. Cannot catch this with
357 * linker tricks when optimizations are disabled.
359 __asm__
__volatile__("ud2");
363 #define _uatomic_or(addr, v) \
364 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
368 static inline __attribute__((always_inline
))
369 void __uatomic_add(void *addr
, unsigned long val
, int len
)
374 __asm__
__volatile__(
376 : "=m"(*__hp(1, addr
))
377 : "iq" ((unsigned char)val
)
383 __asm__
__volatile__(
385 : "=m"(*__hp(2, addr
))
386 : "ir" ((unsigned short)val
)
392 __asm__
__volatile__(
394 : "=m"(*__hp(4, addr
))
395 : "ir" ((unsigned int)val
)
399 #if (CAA_BITS_PER_LONG == 64)
402 __asm__
__volatile__(
404 : "=m"(*__hp(8, addr
))
405 : "er" ((unsigned long)val
)
412 * generate an illegal instruction. Cannot catch this with
413 * linker tricks when optimizations are disabled.
415 __asm__
__volatile__("ud2");
419 #define _uatomic_add(addr, v) \
420 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
425 static inline __attribute__((always_inline
))
426 void __uatomic_inc(void *addr
, int len
)
431 __asm__
__volatile__(
433 : "=m"(*__hp(1, addr
))
440 __asm__
__volatile__(
442 : "=m"(*__hp(2, addr
))
449 __asm__
__volatile__(
451 : "=m"(*__hp(4, addr
))
456 #if (CAA_BITS_PER_LONG == 64)
459 __asm__
__volatile__(
461 : "=m"(*__hp(8, addr
))
468 /* generate an illegal instruction. Cannot catch this with linker tricks
469 * when optimizations are disabled. */
470 __asm__
__volatile__("ud2");
474 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
478 static inline __attribute__((always_inline
))
479 void __uatomic_dec(void *addr
, int len
)
484 __asm__
__volatile__(
486 : "=m"(*__hp(1, addr
))
493 __asm__
__volatile__(
495 : "=m"(*__hp(2, addr
))
502 __asm__
__volatile__(
504 : "=m"(*__hp(4, addr
))
509 #if (CAA_BITS_PER_LONG == 64)
512 __asm__
__volatile__(
514 : "=m"(*__hp(8, addr
))
522 * generate an illegal instruction. Cannot catch this with
523 * linker tricks when optimizations are disabled.
525 __asm__
__volatile__("ud2");
529 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
531 #if ((CAA_BITS_PER_LONG != 64) && defined(CONFIG_RCU_COMPAT_ARCH))
532 extern int __rcu_cas_avail
;
533 extern int __rcu_cas_init(void);
535 #define UATOMIC_COMPAT(insn) \
536 ((caa_likely(__rcu_cas_avail > 0)) \
537 ? (_uatomic_##insn) \
538 : ((caa_unlikely(__rcu_cas_avail < 0) \
539 ? ((__rcu_cas_init() > 0) \
540 ? (_uatomic_##insn) \
541 : (compat_uatomic_##insn)) \
542 : (compat_uatomic_##insn))))
545 * We leave the return value so we don't break the ABI, but remove the
546 * return value from the API.
548 extern unsigned long _compat_uatomic_set(void *addr
,
549 unsigned long _new
, int len
);
550 #define compat_uatomic_set(addr, _new) \
551 ((void) _compat_uatomic_set((addr), \
552 caa_cast_long_keep_sign(_new), \
556 extern unsigned long _compat_uatomic_xchg(void *addr
,
557 unsigned long _new
, int len
);
558 #define compat_uatomic_xchg(addr, _new) \
559 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
560 caa_cast_long_keep_sign(_new), \
563 extern unsigned long _compat_uatomic_cmpxchg(void *addr
, unsigned long old
,
564 unsigned long _new
, int len
);
565 #define compat_uatomic_cmpxchg(addr, old, _new) \
566 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
567 caa_cast_long_keep_sign(old), \
568 caa_cast_long_keep_sign(_new), \
571 extern void _compat_uatomic_and(void *addr
, unsigned long _new
, int len
);
572 #define compat_uatomic_and(addr, v) \
573 (_compat_uatomic_and((addr), \
574 caa_cast_long_keep_sign(v), \
577 extern void _compat_uatomic_or(void *addr
, unsigned long _new
, int len
);
578 #define compat_uatomic_or(addr, v) \
579 (_compat_uatomic_or((addr), \
580 caa_cast_long_keep_sign(v), \
583 extern unsigned long _compat_uatomic_add_return(void *addr
,
584 unsigned long _new
, int len
);
585 #define compat_uatomic_add_return(addr, v) \
586 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
587 caa_cast_long_keep_sign(v), \
590 #define compat_uatomic_add(addr, v) \
591 ((void)compat_uatomic_add_return((addr), (v)))
592 #define compat_uatomic_inc(addr) \
593 (compat_uatomic_add((addr), 1))
594 #define compat_uatomic_dec(addr) \
595 (compat_uatomic_add((addr), -1))
598 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
601 /* Read is atomic even in compat mode */
602 #define uatomic_set(addr, v) \
603 UATOMIC_COMPAT(set(addr, v))
605 #define uatomic_cmpxchg(addr, old, _new) \
606 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
607 #define uatomic_xchg(addr, v) \
608 UATOMIC_COMPAT(xchg(addr, v))
610 #define uatomic_and(addr, v) \
611 UATOMIC_COMPAT(and(addr, v))
612 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
613 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
615 #define uatomic_or(addr, v) \
616 UATOMIC_COMPAT(or(addr, v))
617 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
618 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
620 #define uatomic_add_return(addr, v) \
621 UATOMIC_COMPAT(add_return(addr, v))
623 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
624 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
625 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
627 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
628 #define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
629 #define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
631 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
632 #define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
633 #define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
639 #include <urcu/uatomic/generic.h>
641 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.04483 seconds and 4 git commands to generate.