c02c96d88af8bcf4beaedb36475daacf6a741a46
1 #ifndef _URCU_ARCH_UATOMIC_X86_H
2 #define _URCU_ARCH_UATOMIC_X86_H
5 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
6 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
7 * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
8 * Copyright (c) 2009 Mathieu Desnoyers
10 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
11 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
13 * Permission is hereby granted to use or copy this program
14 * for any purpose, provided the above notices are retained on all copies.
15 * Permission to modify the code and to distribute modified code is granted,
16 * provided the above notices are retained, and a notice that the code was
17 * modified is included with the above copyright notice.
19 * Code inspired from libuatomic_ops-1.2, inherited in part from the
20 * Boehm-Demers-Weiser conservative garbage collector.
23 #include <urcu/arch.h>
24 #include <urcu/config.h>
25 #include <urcu/compiler.h>
26 #include <urcu/system.h>
28 #define UATOMIC_HAS_ATOMIC_BYTE
29 #define UATOMIC_HAS_ATOMIC_SHORT
36 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
40 * The __hp() macro casts the void pointer @x to a pointer to a structure
41 * containing an array of char of the specified size. This allows passing the
42 * @addr arguments of the following inline functions as "m" and "+m" operands
43 * to the assembly. The @size parameter should be a constant to support
44 * compilers such as clang which do not support VLA.
47 #define __hp(size, x) ((struct { char v[size]; } *)(x))
49 #define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
53 static inline __attribute__((always_inline
))
54 unsigned long __uatomic_cmpxchg(void *addr
, unsigned long old
,
55 unsigned long _new
, int len
)
60 unsigned char result
= old
;
63 "lock; cmpxchgb %2, %1"
64 : "+a"(result
), "+m"(*__hp(1, addr
))
65 : "q"((unsigned char)_new
)
71 unsigned short result
= old
;
74 "lock; cmpxchgw %2, %1"
75 : "+a"(result
), "+m"(*__hp(2, addr
))
76 : "r"((unsigned short)_new
)
82 unsigned int result
= old
;
85 "lock; cmpxchgl %2, %1"
86 : "+a"(result
), "+m"(*__hp(4, addr
))
87 : "r"((unsigned int)_new
)
91 #if (CAA_BITS_PER_LONG == 64)
94 unsigned long result
= old
;
97 "lock; cmpxchgq %2, %1"
98 : "+a"(result
), "+m"(*__hp(8, addr
))
99 : "r"((unsigned long)_new
)
106 * generate an illegal instruction. Cannot catch this with
107 * linker tricks when optimizations are disabled.
109 __asm__
__volatile__("ud2");
113 #define _uatomic_cmpxchg(addr, old, _new) \
114 ((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
115 caa_cast_long_keep_sign(old), \
116 caa_cast_long_keep_sign(_new),\
121 static inline __attribute__((always_inline
))
122 unsigned long __uatomic_exchange(void *addr
, unsigned long val
, int len
)
124 /* Note: the "xchg" instruction does not need a "lock" prefix. */
128 unsigned char result
;
129 __asm__
__volatile__(
131 : "=q"(result
), "+m"(*__hp(1, addr
))
132 : "0" ((unsigned char)val
)
138 unsigned short result
;
139 __asm__
__volatile__(
141 : "=r"(result
), "+m"(*__hp(2, addr
))
142 : "0" ((unsigned short)val
)
149 __asm__
__volatile__(
151 : "=r"(result
), "+m"(*__hp(4, addr
))
152 : "0" ((unsigned int)val
)
156 #if (CAA_BITS_PER_LONG == 64)
159 unsigned long result
;
160 __asm__
__volatile__(
162 : "=r"(result
), "+m"(*__hp(8, addr
))
163 : "0" ((unsigned long)val
)
170 * generate an illegal instruction. Cannot catch this with
171 * linker tricks when optimizations are disabled.
173 __asm__
__volatile__("ud2");
177 #define _uatomic_xchg(addr, v) \
178 ((__typeof__(*(addr))) __uatomic_exchange((addr), \
179 caa_cast_long_keep_sign(v), \
182 /* uatomic_add_return */
184 static inline __attribute__((always_inline
))
185 unsigned long __uatomic_add_return(void *addr
, unsigned long val
,
191 unsigned char result
= val
;
193 __asm__
__volatile__(
195 : "+m"(*__hp(1, addr
)), "+q" (result
)
198 return result
+ (unsigned char)val
;
202 unsigned short result
= val
;
204 __asm__
__volatile__(
206 : "+m"(*__hp(2, addr
)), "+r" (result
)
209 return result
+ (unsigned short)val
;
213 unsigned int result
= val
;
215 __asm__
__volatile__(
217 : "+m"(*__hp(4, addr
)), "+r" (result
)
220 return result
+ (unsigned int)val
;
222 #if (CAA_BITS_PER_LONG == 64)
225 unsigned long result
= val
;
227 __asm__
__volatile__(
229 : "+m"(*__hp(8, addr
)), "+r" (result
)
232 return result
+ (unsigned long)val
;
237 * generate an illegal instruction. Cannot catch this with
238 * linker tricks when optimizations are disabled.
240 __asm__
__volatile__("ud2");
244 #define _uatomic_add_return(addr, v) \
245 ((__typeof__(*(addr))) __uatomic_add_return((addr), \
246 caa_cast_long_keep_sign(v), \
251 static inline __attribute__((always_inline
))
252 void __uatomic_and(void *addr
, unsigned long val
, int len
)
257 __asm__
__volatile__(
259 : "=m"(*__hp(1, addr
))
260 : "iq" ((unsigned char)val
)
266 __asm__
__volatile__(
268 : "=m"(*__hp(2, addr
))
269 : "ir" ((unsigned short)val
)
275 __asm__
__volatile__(
277 : "=m"(*__hp(4, addr
))
278 : "ir" ((unsigned int)val
)
282 #if (CAA_BITS_PER_LONG == 64)
285 __asm__
__volatile__(
287 : "=m"(*__hp(8, addr
))
288 : "er" ((unsigned long)val
)
295 * generate an illegal instruction. Cannot catch this with
296 * linker tricks when optimizations are disabled.
298 __asm__
__volatile__("ud2");
302 #define _uatomic_and(addr, v) \
303 (__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
307 static inline __attribute__((always_inline
))
308 void __uatomic_or(void *addr
, unsigned long val
, int len
)
313 __asm__
__volatile__(
315 : "=m"(*__hp(1, addr
))
316 : "iq" ((unsigned char)val
)
322 __asm__
__volatile__(
324 : "=m"(*__hp(2, addr
))
325 : "ir" ((unsigned short)val
)
331 __asm__
__volatile__(
333 : "=m"(*__hp(4, addr
))
334 : "ir" ((unsigned int)val
)
338 #if (CAA_BITS_PER_LONG == 64)
341 __asm__
__volatile__(
343 : "=m"(*__hp(8, addr
))
344 : "er" ((unsigned long)val
)
351 * generate an illegal instruction. Cannot catch this with
352 * linker tricks when optimizations are disabled.
354 __asm__
__volatile__("ud2");
358 #define _uatomic_or(addr, v) \
359 (__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
363 static inline __attribute__((always_inline
))
364 void __uatomic_add(void *addr
, unsigned long val
, int len
)
369 __asm__
__volatile__(
371 : "=m"(*__hp(1, addr
))
372 : "iq" ((unsigned char)val
)
378 __asm__
__volatile__(
380 : "=m"(*__hp(2, addr
))
381 : "ir" ((unsigned short)val
)
387 __asm__
__volatile__(
389 : "=m"(*__hp(4, addr
))
390 : "ir" ((unsigned int)val
)
394 #if (CAA_BITS_PER_LONG == 64)
397 __asm__
__volatile__(
399 : "=m"(*__hp(8, addr
))
400 : "er" ((unsigned long)val
)
407 * generate an illegal instruction. Cannot catch this with
408 * linker tricks when optimizations are disabled.
410 __asm__
__volatile__("ud2");
414 #define _uatomic_add(addr, v) \
415 (__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
420 static inline __attribute__((always_inline
))
421 void __uatomic_inc(void *addr
, int len
)
426 __asm__
__volatile__(
428 : "=m"(*__hp(1, addr
))
435 __asm__
__volatile__(
437 : "=m"(*__hp(2, addr
))
444 __asm__
__volatile__(
446 : "=m"(*__hp(4, addr
))
451 #if (CAA_BITS_PER_LONG == 64)
454 __asm__
__volatile__(
456 : "=m"(*__hp(8, addr
))
463 /* generate an illegal instruction. Cannot catch this with linker tricks
464 * when optimizations are disabled. */
465 __asm__
__volatile__("ud2");
469 #define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
473 static inline __attribute__((always_inline
))
474 void __uatomic_dec(void *addr
, int len
)
479 __asm__
__volatile__(
481 : "=m"(*__hp(1, addr
))
488 __asm__
__volatile__(
490 : "=m"(*__hp(2, addr
))
497 __asm__
__volatile__(
499 : "=m"(*__hp(4, addr
))
504 #if (CAA_BITS_PER_LONG == 64)
507 __asm__
__volatile__(
509 : "=m"(*__hp(8, addr
))
517 * generate an illegal instruction. Cannot catch this with
518 * linker tricks when optimizations are disabled.
520 __asm__
__volatile__("ud2");
524 #define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
526 #if ((CAA_BITS_PER_LONG != 64) && defined(URCU_ARCH_I386))
528 /* For backwards compat */
529 #define CONFIG_RCU_COMPAT_ARCH 1
531 extern int __rcu_cas_avail
;
532 extern int __rcu_cas_init(void);
534 #define UATOMIC_COMPAT(insn) \
535 ((caa_likely(__rcu_cas_avail > 0)) \
536 ? (_uatomic_##insn) \
537 : ((caa_unlikely(__rcu_cas_avail < 0) \
538 ? ((__rcu_cas_init() > 0) \
539 ? (_uatomic_##insn) \
540 : (compat_uatomic_##insn)) \
541 : (compat_uatomic_##insn))))
544 * We leave the return value so we don't break the ABI, but remove the
545 * return value from the API.
547 extern unsigned long _compat_uatomic_set(void *addr
,
548 unsigned long _new
, int len
);
549 #define compat_uatomic_set(addr, _new) \
550 ((void) _compat_uatomic_set((addr), \
551 caa_cast_long_keep_sign(_new), \
555 extern unsigned long _compat_uatomic_xchg(void *addr
,
556 unsigned long _new
, int len
);
557 #define compat_uatomic_xchg(addr, _new) \
558 ((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
559 caa_cast_long_keep_sign(_new), \
562 extern unsigned long _compat_uatomic_cmpxchg(void *addr
, unsigned long old
,
563 unsigned long _new
, int len
);
564 #define compat_uatomic_cmpxchg(addr, old, _new) \
565 ((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
566 caa_cast_long_keep_sign(old), \
567 caa_cast_long_keep_sign(_new), \
570 extern void _compat_uatomic_and(void *addr
, unsigned long _new
, int len
);
571 #define compat_uatomic_and(addr, v) \
572 (_compat_uatomic_and((addr), \
573 caa_cast_long_keep_sign(v), \
576 extern void _compat_uatomic_or(void *addr
, unsigned long _new
, int len
);
577 #define compat_uatomic_or(addr, v) \
578 (_compat_uatomic_or((addr), \
579 caa_cast_long_keep_sign(v), \
582 extern unsigned long _compat_uatomic_add_return(void *addr
,
583 unsigned long _new
, int len
);
584 #define compat_uatomic_add_return(addr, v) \
585 ((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
586 caa_cast_long_keep_sign(v), \
589 #define compat_uatomic_add(addr, v) \
590 ((void)compat_uatomic_add_return((addr), (v)))
591 #define compat_uatomic_inc(addr) \
592 (compat_uatomic_add((addr), 1))
593 #define compat_uatomic_dec(addr) \
594 (compat_uatomic_add((addr), -1))
597 #define UATOMIC_COMPAT(insn) (_uatomic_##insn)
600 /* Read is atomic even in compat mode */
601 #define uatomic_set(addr, v) \
602 UATOMIC_COMPAT(set(addr, v))
604 #define uatomic_cmpxchg(addr, old, _new) \
605 UATOMIC_COMPAT(cmpxchg(addr, old, _new))
606 #define uatomic_xchg(addr, v) \
607 UATOMIC_COMPAT(xchg(addr, v))
609 #define uatomic_and(addr, v) \
610 UATOMIC_COMPAT(and(addr, v))
611 #define cmm_smp_mb__before_uatomic_and() cmm_barrier()
612 #define cmm_smp_mb__after_uatomic_and() cmm_barrier()
614 #define uatomic_or(addr, v) \
615 UATOMIC_COMPAT(or(addr, v))
616 #define cmm_smp_mb__before_uatomic_or() cmm_barrier()
617 #define cmm_smp_mb__after_uatomic_or() cmm_barrier()
619 #define uatomic_add_return(addr, v) \
620 UATOMIC_COMPAT(add_return(addr, v))
622 #define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
623 #define cmm_smp_mb__before_uatomic_add() cmm_barrier()
624 #define cmm_smp_mb__after_uatomic_add() cmm_barrier()
626 #define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
627 #define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
628 #define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
630 #define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
631 #define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
632 #define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
638 #include <urcu/uatomic/generic.h>
640 #endif /* _URCU_ARCH_UATOMIC_X86_H */
This page took 0.044224 seconds and 4 git commands to generate.