[userspace-rcu.git] / include / urcu / uatomic / x86.h

// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation.  All rights reserved.
// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics.  All rights reserved.
// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
//
// SPDX-License-Identifier: LicenseRef-Boehm-GC

#ifndef _URCU_ARCH_UATOMIC_X86_H
#define _URCU_ARCH_UATOMIC_X86_H

/*
 * Code inspired from libuatomic_ops-1.2, inherited in part from the
 * Boehm-Demers-Weiser conservative garbage collector.
 */

#include <urcu/arch.h>
#include <urcu/config.h>
#include <urcu/compiler.h>
#include <urcu/system.h>

#define UATOMIC_HAS_ATOMIC_BYTE
#define UATOMIC_HAS_ATOMIC_SHORT

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Derived from AO_compare_and_swap() and AO_test_and_set_full().
 */

/*
 * The __hp() macro casts the void pointer @x to a pointer to a structure
 * containing an array of char of the specified size. This allows passing the
 * @addr arguments of the following inline functions as "m" and "+m" operands
 * to the assembly. The @size parameter should be a constant to support
 * compilers such as clang which do not support VLA. Create typedefs because
 * C++ does not allow types be defined in casts.
 */

typedef struct { char v[1]; } __hp_1;
typedef struct { char v[2]; } __hp_2;
typedef struct { char v[4]; } __hp_4;
typedef struct { char v[8]; } __hp_8;

#define __hp(size, x)	((__hp_##size *)(x))

#define _uatomic_set(addr, v)	((void) CMM_STORE_SHARED(*(addr), (v)))

/* cmpxchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
			      unsigned long _new, int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = old;

		__asm__ __volatile__(
		"lock; cmpxchgb %2, %1"
			: "+a"(result), "+m"(*__hp(1, addr))
			: "q"((unsigned char)_new)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result = old;

		__asm__ __volatile__(
		"lock; cmpxchgw %2, %1"
			: "+a"(result), "+m"(*__hp(2, addr))
			: "r"((unsigned short)_new)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result = old;

		__asm__ __volatile__(
		"lock; cmpxchgl %2, %1"
			: "+a"(result), "+m"(*__hp(4, addr))
			: "r"((unsigned int)_new)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = old;

		__asm__ __volatile__(
		"lock; cmpxchgq %2, %1"
			: "+a"(result), "+m"(*__hp(8, addr))
			: "r"((unsigned long)_new)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_cmpxchg(addr, old, _new)				      \
	((__typeof__(*(addr))) __uatomic_cmpxchg((addr),		      \
						caa_cast_long_keep_sign(old), \
						caa_cast_long_keep_sign(_new),\
						sizeof(*(addr))))

/* xchg */

static inline __attribute__((always_inline))
unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
{
	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	switch (len) {
	case 1:
	{
		unsigned char result;
		__asm__ __volatile__(
		"xchgb %0, %1"
			: "=q"(result), "+m"(*__hp(1, addr))
			: "0" ((unsigned char)val)
			: "memory");
		return result;
	}
	case 2:
	{
		unsigned short result;
		__asm__ __volatile__(
		"xchgw %0, %1"
			: "=r"(result), "+m"(*__hp(2, addr))
			: "0" ((unsigned short)val)
			: "memory");
		return result;
	}
	case 4:
	{
		unsigned int result;
		__asm__ __volatile__(
		"xchgl %0, %1"
			: "=r"(result), "+m"(*__hp(4, addr))
			: "0" ((unsigned int)val)
			: "memory");
		return result;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result;
		__asm__ __volatile__(
		"xchgq %0, %1"
			: "=r"(result), "+m"(*__hp(8, addr))
			: "0" ((unsigned long)val)
			: "memory");
		return result;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_xchg(addr, v)						      \
	((__typeof__(*(addr))) __uatomic_exchange((addr),		      \
						caa_cast_long_keep_sign(v),   \
						sizeof(*(addr))))

/* uatomic_add_return */

static inline __attribute__((always_inline))
unsigned long __uatomic_add_return(void *addr, unsigned long val,
				 int len)
{
	switch (len) {
	case 1:
	{
		unsigned char result = val;

		__asm__ __volatile__(
		"lock; xaddb %1, %0"
			: "+m"(*__hp(1, addr)), "+q" (result)
			:
			: "memory");
		return result + (unsigned char)val;
	}
	case 2:
	{
		unsigned short result = val;

		__asm__ __volatile__(
		"lock; xaddw %1, %0"
			: "+m"(*__hp(2, addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned short)val;
	}
	case 4:
	{
		unsigned int result = val;

		__asm__ __volatile__(
		"lock; xaddl %1, %0"
			: "+m"(*__hp(4, addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned int)val;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		unsigned long result = val;

		__asm__ __volatile__(
		"lock; xaddq %1, %0"
			: "+m"(*__hp(8, addr)), "+r" (result)
			:
			: "memory");
		return result + (unsigned long)val;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return 0;
}

#define _uatomic_add_return(addr, v)					    \
	((__typeof__(*(addr))) __uatomic_add_return((addr),		    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

/* uatomic_and */

static inline __attribute__((always_inline))
void __uatomic_and(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; andb %1, %0"
			: "=m"(*__hp(1, addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; andw %1, %0"
			: "=m"(*__hp(2, addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; andl %1, %0"
			: "=m"(*__hp(4, addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; andq %1, %0"
			: "=m"(*__hp(8, addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_and(addr, v)						   \
	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_or */

static inline __attribute__((always_inline))
void __uatomic_or(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; orb %1, %0"
			: "=m"(*__hp(1, addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; orw %1, %0"
			: "=m"(*__hp(2, addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; orl %1, %0"
			: "=m"(*__hp(4, addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; orq %1, %0"
			: "=m"(*__hp(8, addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_or(addr, v)						   \
	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))

/* uatomic_add */

static inline __attribute__((always_inline))
void __uatomic_add(void *addr, unsigned long val, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; addb %1, %0"
			: "=m"(*__hp(1, addr))
			: "iq" ((unsigned char)val)
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; addw %1, %0"
			: "=m"(*__hp(2, addr))
			: "ir" ((unsigned short)val)
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; addl %1, %0"
			: "=m"(*__hp(4, addr))
			: "ir" ((unsigned int)val)
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; addq %1, %0"
			: "=m"(*__hp(8, addr))
			: "er" ((unsigned long)val)
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_add(addr, v)						   \
	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))


/* uatomic_inc */

static inline __attribute__((always_inline))
void __uatomic_inc(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; incb %0"
			: "=m"(*__hp(1, addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; incw %0"
			: "=m"(*__hp(2, addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; incl %0"
			: "=m"(*__hp(4, addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; incq %0"
			: "=m"(*__hp(8, addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/* generate an illegal instruction. Cannot catch this with linker tricks
	 * when optimizations are disabled. */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_inc(addr)	(__uatomic_inc((addr), sizeof(*(addr))))

/* uatomic_dec */

static inline __attribute__((always_inline))
void __uatomic_dec(void *addr, int len)
{
	switch (len) {
	case 1:
	{
		__asm__ __volatile__(
		"lock; decb %0"
			: "=m"(*__hp(1, addr))
			:
			: "memory");
		return;
	}
	case 2:
	{
		__asm__ __volatile__(
		"lock; decw %0"
			: "=m"(*__hp(2, addr))
			:
			: "memory");
		return;
	}
	case 4:
	{
		__asm__ __volatile__(
		"lock; decl %0"
			: "=m"(*__hp(4, addr))
			:
			: "memory");
		return;
	}
#if (CAA_BITS_PER_LONG == 64)
	case 8:
	{
		__asm__ __volatile__(
		"lock; decq %0"
			: "=m"(*__hp(8, addr))
			:
			: "memory");
		return;
	}
#endif
	}
	/*
	 * generate an illegal instruction. Cannot catch this with
	 * linker tricks when optimizations are disabled.
	 */
	__asm__ __volatile__("ud2");
	return;
}

#define _uatomic_dec(addr)	(__uatomic_dec((addr), sizeof(*(addr))))

#ifdef URCU_ARCH_X86_NO_CAS

/* For backwards compat */
#define CONFIG_RCU_COMPAT_ARCH 1

extern int __rcu_cas_avail;
extern int __rcu_cas_init(void);

#define UATOMIC_COMPAT(insn)							\
	((caa_likely(__rcu_cas_avail > 0))						\
	? (_uatomic_##insn)							\
		: ((caa_unlikely(__rcu_cas_avail < 0)				\
			? ((__rcu_cas_init() > 0)				\
				? (_uatomic_##insn)				\
				: (compat_uatomic_##insn))			\
			: (compat_uatomic_##insn))))

/*
 * We leave the return value so we don't break the ABI, but remove the
 * return value from the API.
 */
extern unsigned long _compat_uatomic_set(void *addr,
					 unsigned long _new, int len);
#define compat_uatomic_set(addr, _new)				     	       \
	((void) _compat_uatomic_set((addr),				       \
				caa_cast_long_keep_sign(_new),		       \
				sizeof(*(addr))))


extern unsigned long _compat_uatomic_xchg(void *addr,
					  unsigned long _new, int len);
#define compat_uatomic_xchg(addr, _new)					       \
	((__typeof__(*(addr))) _compat_uatomic_xchg((addr),		       \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
					     unsigned long _new, int len);
#define compat_uatomic_cmpxchg(addr, old, _new)				       \
	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr),		       \
						caa_cast_long_keep_sign(old),  \
						caa_cast_long_keep_sign(_new), \
						sizeof(*(addr))))

extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
#define compat_uatomic_and(addr, v)				       \
	(_compat_uatomic_and((addr),				       \
			caa_cast_long_keep_sign(v),		       \
			sizeof(*(addr))))

extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
#define compat_uatomic_or(addr, v)				       \
	(_compat_uatomic_or((addr),				       \
			  caa_cast_long_keep_sign(v),		       \
			  sizeof(*(addr))))

extern unsigned long _compat_uatomic_add_return(void *addr,
						unsigned long _new, int len);
#define compat_uatomic_add_return(addr, v)			            \
	((__typeof__(*(addr))) _compat_uatomic_add_return((addr),     	    \
						caa_cast_long_keep_sign(v), \
						sizeof(*(addr))))

#define compat_uatomic_add(addr, v)					       \
		((void)compat_uatomic_add_return((addr), (v)))
#define compat_uatomic_inc(addr)					       \
		(compat_uatomic_add((addr), 1))
#define compat_uatomic_dec(addr)					       \
		(compat_uatomic_add((addr), -1))

#else
#define UATOMIC_COMPAT(insn)	(_uatomic_##insn)
#endif

/* Read is atomic even in compat mode */
#define uatomic_set(addr, v)			\
		UATOMIC_COMPAT(set(addr, v))

#define uatomic_cmpxchg(addr, old, _new)	\
		UATOMIC_COMPAT(cmpxchg(addr, old, _new))
#define uatomic_xchg(addr, v)			\
		UATOMIC_COMPAT(xchg(addr, v))

#define uatomic_and(addr, v)		\
		UATOMIC_COMPAT(and(addr, v))
#define cmm_smp_mb__before_uatomic_and()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_and()		cmm_barrier()

#define uatomic_or(addr, v)		\
		UATOMIC_COMPAT(or(addr, v))
#define cmm_smp_mb__before_uatomic_or()		cmm_barrier()
#define cmm_smp_mb__after_uatomic_or()		cmm_barrier()

#define uatomic_add_return(addr, v)		\
		UATOMIC_COMPAT(add_return(addr, v))

#define uatomic_add(addr, v)	UATOMIC_COMPAT(add(addr, v))
#define cmm_smp_mb__before_uatomic_add()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_add()		cmm_barrier()

#define uatomic_inc(addr)	UATOMIC_COMPAT(inc(addr))
#define cmm_smp_mb__before_uatomic_inc()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_inc()		cmm_barrier()

#define uatomic_dec(addr)	UATOMIC_COMPAT(dec(addr))
#define cmm_smp_mb__before_uatomic_dec()	cmm_barrier()
#define cmm_smp_mb__after_uatomic_dec()		cmm_barrier()

#ifdef __cplusplus
}
#endif

#include <urcu/uatomic/generic.h>

#endif /* _URCU_ARCH_UATOMIC_X86_H */
Commit	Line	Data
d3d3857f MJ	1	// SPDX-FileCopyrightText: 1991-1994 by Xerox Corporation. All rights reserved.
	2	// SPDX-FileCopyrightText: 1996-1999 by Silicon Graphics. All rights reserved.
	3	// SPDX-FileCopyrightText: 1999-2004 Hewlett-Packard Development Company, L.P.
	4	// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	5	//
	6	// SPDX-License-Identifier: LicenseRef-Boehm-GC
	7
ec4e58a3 MD	8	#ifndef _URCU_ARCH_UATOMIC_X86_H
ec4e58a3 MD	9	#define _URCU_ARCH_UATOMIC_X86_H
0114ba7f	10
67ecffc0	11	/*
ec4e58a3	12	* Code inspired from libuatomic_ops-1.2, inherited in part from the
0114ba7f MD	13	* Boehm-Demers-Weiser conservative garbage collector.
	14	*/
	15
0b1e236d	16	#include <urcu/arch.h>
375db287	17	#include <urcu/config.h>
ec4e58a3	18	#include <urcu/compiler.h>
bf9de1b7	19	#include <urcu/system.h>
0fad128b	20
f469d839 PB	21	#define UATOMIC_HAS_ATOMIC_BYTE
	22	#define UATOMIC_HAS_ATOMIC_SHORT
	23
36bc70a8 MD	24	#ifdef __cplusplus
36bc70a8 MD	25	extern "C" {
67ecffc0	26	#endif
36bc70a8	27
0114ba7f	28	/*
0114ba7f MD	29	* Derived from AO_compare_and_swap() and AO_test_and_set_full().
	30	*/
	31
835b9ab3	32	/*
71323499	33	* The __hp() macro casts the void pointer @x to a pointer to a structure
835b9ab3 MD	34	* containing an array of char of the specified size. This allows passing the
835b9ab3 MD	35	* @addr arguments of the following inline functions as "m" and "+m" operands
71323499	36	* to the assembly. The @size parameter should be a constant to support
13bf2f57 MD	37	* compilers such as clang which do not support VLA. Create typedefs because
13bf2f57 MD	38	* C++ does not allow types be defined in casts.
835b9ab3 MD	39	*/
835b9ab3 MD	40
13bf2f57 MD	41	typedef struct { char v[1]; } __hp_1;
	42	typedef struct { char v[2]; } __hp_2;
	43	typedef struct { char v[4]; } __hp_4;
	44	typedef struct { char v[8]; } __hp_8;
	45
	46	#define __hp(size, x) ((__hp_##size *)(x))
cc1be41b	47
424d4ed5	48	#define _uatomic_set(addr, v) ((void) CMM_STORE_SHARED(*(addr), (v)))
0fad128b	49
cc1be41b MD	50	/* cmpxchg */
cc1be41b MD	51
5dba80f9	52	static inline __attribute__((always_inline))
bf9de1b7	53	unsigned long __uatomic_cmpxchg(void *addr, unsigned long old,
0fad128b	54	unsigned long _new, int len)
0114ba7f	55	{
cc1be41b MD	56	switch (len) {
	57	case 1:
	58	{
	59	unsigned char result = old;
0fad128b	60
cc1be41b MD	61	__asm__ __volatile__(
cc1be41b MD	62	"lock; cmpxchgb %2, %1"
71323499	63	: "+a"(result), "+m"(*__hp(1, addr))
cc1be41b	64	: "q"((unsigned char)_new)
0114ba7f	65	: "memory");
cc1be41b MD	66	return result;
	67	}
	68	case 2:
	69	{
	70	unsigned short result = old;
0fad128b	71
cc1be41b MD	72	__asm__ __volatile__(
cc1be41b MD	73	"lock; cmpxchgw %2, %1"
71323499	74	: "+a"(result), "+m"(*__hp(2, addr))
cc1be41b MD	75	: "r"((unsigned short)_new)
	76	: "memory");
	77	return result;
	78	}
	79	case 4:
	80	{
	81	unsigned int result = old;
0fad128b	82
cc1be41b MD	83	__asm__ __volatile__(
cc1be41b MD	84	"lock; cmpxchgl %2, %1"
71323499	85	: "+a"(result), "+m"(*__hp(4, addr))
cc1be41b MD	86	: "r"((unsigned int)_new)
	87	: "memory");
	88	return result;
	89	}
e040d717	90	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	91	case 8:
cc1be41b MD	92	{
6edb297e	93	unsigned long result = old;
0fad128b	94
cc1be41b	95	__asm__ __volatile__(
2c5e5fb3	96	"lock; cmpxchgq %2, %1"
71323499	97	: "+a"(result), "+m"(*__hp(8, addr))
cc1be41b MD	98	: "r"((unsigned long)_new)
	99	: "memory");
	100	return result;
	101	}
	102	#endif
	103	}
d0bbd9c2 MD	104	/*
	105	* generate an illegal instruction. Cannot catch this with
	106	* linker tricks when optimizations are disabled.
	107	*/
cc1be41b MD	108	__asm__ __volatile__("ud2");
cc1be41b MD	109	return 0;
0114ba7f MD	110	}
0114ba7f MD	111
bf9de1b7	112	#define _uatomic_cmpxchg(addr, old, _new) \
e56d99bf MD	113	((__typeof__(*(addr))) __uatomic_cmpxchg((addr), \
	114	caa_cast_long_keep_sign(old), \
	115	caa_cast_long_keep_sign(_new),\
cc1be41b MD	116	sizeof(*(addr))))
	117
	118	/* xchg */
0114ba7f	119
5dba80f9	120	static inline __attribute__((always_inline))
bf9de1b7	121	unsigned long __uatomic_exchange(void *addr, unsigned long val, int len)
0114ba7f	122	{
cc1be41b MD	123	/* Note: the "xchg" instruction does not need a "lock" prefix. */
	124	switch (len) {
	125	case 1:
	126	{
	127	unsigned char result;
	128	__asm__ __volatile__(
	129	"xchgb %0, %1"
71323499	130	: "=q"(result), "+m"(*__hp(1, addr))
cc1be41b MD	131	: "0" ((unsigned char)val)
	132	: "memory");
	133	return result;
	134	}
	135	case 2:
	136	{
	137	unsigned short result;
	138	__asm__ __volatile__(
	139	"xchgw %0, %1"
71323499	140	: "=r"(result), "+m"(*__hp(2, addr))
cc1be41b MD	141	: "0" ((unsigned short)val)
	142	: "memory");
	143	return result;
	144	}
	145	case 4:
	146	{
	147	unsigned int result;
	148	__asm__ __volatile__(
	149	"xchgl %0, %1"
71323499	150	: "=r"(result), "+m"(*__hp(4, addr))
cc1be41b MD	151	: "0" ((unsigned int)val)
	152	: "memory");
	153	return result;
	154	}
e040d717	155	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	156	case 8:
	157	{
	158	unsigned long result;
	159	__asm__ __volatile__(
0114ba7f	160	"xchgq %0, %1"
71323499	161	: "=r"(result), "+m"(*__hp(8, addr))
cc1be41b	162	: "0" ((unsigned long)val)
0114ba7f	163	: "memory");
cc1be41b MD	164	return result;
	165	}
	166	#endif
	167	}
d0bbd9c2 MD	168	/*
	169	* generate an illegal instruction. Cannot catch this with
	170	* linker tricks when optimizations are disabled.
	171	*/
cc1be41b MD	172	__asm__ __volatile__("ud2");
cc1be41b MD	173	return 0;
0114ba7f MD	174	}
0114ba7f MD	175
bf9de1b7	176	#define _uatomic_xchg(addr, v) \
e56d99bf MD	177	((__typeof__(*(addr))) __uatomic_exchange((addr), \
e56d99bf MD	178	caa_cast_long_keep_sign(v), \
cc1be41b MD	179	sizeof(*(addr))))
cc1be41b MD	180
8760d94e	181	/* uatomic_add_return */
0fad128b MD	182
0fad128b MD	183	static inline __attribute__((always_inline))
bf9de1b7	184	unsigned long __uatomic_add_return(void *addr, unsigned long val,
0fad128b MD	185	int len)
	186	{
	187	switch (len) {
	188	case 1:
	189	{
	190	unsigned char result = val;
	191
	192	__asm__ __volatile__(
	193	"lock; xaddb %1, %0"
71323499	194	: "+m"(*__hp(1, addr)), "+q" (result)
0fad128b MD	195	:
	196	: "memory");
	197	return result + (unsigned char)val;
	198	}
	199	case 2:
	200	{
	201	unsigned short result = val;
	202
	203	__asm__ __volatile__(
	204	"lock; xaddw %1, %0"
71323499	205	: "+m"(*__hp(2, addr)), "+r" (result)
0fad128b MD	206	:
	207	: "memory");
	208	return result + (unsigned short)val;
	209	}
	210	case 4:
	211	{
	212	unsigned int result = val;
	213
	214	__asm__ __volatile__(
	215	"lock; xaddl %1, %0"
71323499	216	: "+m"(*__hp(4, addr)), "+r" (result)
0fad128b MD	217	:
	218	: "memory");
	219	return result + (unsigned int)val;
	220	}
e040d717	221	#if (CAA_BITS_PER_LONG == 64)
0fad128b MD	222	case 8:
	223	{
	224	unsigned long result = val;
	225
	226	__asm__ __volatile__(
	227	"lock; xaddq %1, %0"
71323499	228	: "+m"(*__hp(8, addr)), "+r" (result)
0fad128b MD	229	:
	230	: "memory");
	231	return result + (unsigned long)val;
	232	}
	233	#endif
	234	}
d0bbd9c2 MD	235	/*
	236	* generate an illegal instruction. Cannot catch this with
	237	* linker tricks when optimizations are disabled.
	238	*/
0fad128b MD	239	__asm__ __volatile__("ud2");
	240	return 0;
	241	}
	242
e56d99bf MD	243	#define _uatomic_add_return(addr, v) \
	244	((__typeof__(*(addr))) __uatomic_add_return((addr), \
	245	caa_cast_long_keep_sign(v), \
	246	sizeof(*(addr))))
0fad128b	247
bf33aaea PB	248	/* uatomic_and */
	249
	250	static inline __attribute__((always_inline))
	251	void __uatomic_and(void *addr, unsigned long val, int len)
	252	{
	253	switch (len) {
	254	case 1:
	255	{
	256	__asm__ __volatile__(
	257	"lock; andb %1, %0"
71323499	258	: "=m"(*__hp(1, addr))
bf33aaea PB	259	: "iq" ((unsigned char)val)
	260	: "memory");
	261	return;
	262	}
	263	case 2:
	264	{
	265	__asm__ __volatile__(
	266	"lock; andw %1, %0"
71323499	267	: "=m"(*__hp(2, addr))
bf33aaea PB	268	: "ir" ((unsigned short)val)
	269	: "memory");
	270	return;
	271	}
	272	case 4:
	273	{
	274	__asm__ __volatile__(
	275	"lock; andl %1, %0"
71323499	276	: "=m"(*__hp(4, addr))
bf33aaea PB	277	: "ir" ((unsigned int)val)
	278	: "memory");
	279	return;
	280	}
	281	#if (CAA_BITS_PER_LONG == 64)
	282	case 8:
	283	{
	284	__asm__ __volatile__(
	285	"lock; andq %1, %0"
71323499	286	: "=m"(*__hp(8, addr))
bf33aaea PB	287	: "er" ((unsigned long)val)
	288	: "memory");
	289	return;
	290	}
	291	#endif
	292	}
d0bbd9c2 MD	293	/*
	294	* generate an illegal instruction. Cannot catch this with
	295	* linker tricks when optimizations are disabled.
	296	*/
bf33aaea PB	297	__asm__ __volatile__("ud2");
	298	return;
	299	}
	300
	301	#define _uatomic_and(addr, v) \
e56d99bf	302	(__uatomic_and((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
bf33aaea	303
985b35b1 PB	304	/* uatomic_or */
	305
	306	static inline __attribute__((always_inline))
	307	void __uatomic_or(void *addr, unsigned long val, int len)
	308	{
	309	switch (len) {
	310	case 1:
	311	{
	312	__asm__ __volatile__(
	313	"lock; orb %1, %0"
71323499	314	: "=m"(*__hp(1, addr))
985b35b1 PB	315	: "iq" ((unsigned char)val)
	316	: "memory");
	317	return;
	318	}
	319	case 2:
	320	{
	321	__asm__ __volatile__(
	322	"lock; orw %1, %0"
71323499	323	: "=m"(*__hp(2, addr))
985b35b1 PB	324	: "ir" ((unsigned short)val)
	325	: "memory");
	326	return;
	327	}
	328	case 4:
	329	{
	330	__asm__ __volatile__(
	331	"lock; orl %1, %0"
71323499	332	: "=m"(*__hp(4, addr))
985b35b1 PB	333	: "ir" ((unsigned int)val)
	334	: "memory");
	335	return;
	336	}
	337	#if (CAA_BITS_PER_LONG == 64)
	338	case 8:
	339	{
	340	__asm__ __volatile__(
	341	"lock; orq %1, %0"
71323499	342	: "=m"(*__hp(8, addr))
985b35b1 PB	343	: "er" ((unsigned long)val)
	344	: "memory");
	345	return;
	346	}
	347	#endif
	348	}
d0bbd9c2 MD	349	/*
	350	* generate an illegal instruction. Cannot catch this with
	351	* linker tricks when optimizations are disabled.
	352	*/
985b35b1 PB	353	__asm__ __volatile__("ud2");
	354	return;
	355	}
	356
	357	#define _uatomic_or(addr, v) \
e56d99bf	358	(__uatomic_or((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
985b35b1	359
8760d94e	360	/* uatomic_add */
0114ba7f	361
5dba80f9	362	static inline __attribute__((always_inline))
bf9de1b7	363	void __uatomic_add(void *addr, unsigned long val, int len)
0114ba7f MD	364	{
0114ba7f MD	365	switch (len) {
cc1be41b MD	366	case 1:
	367	{
	368	__asm__ __volatile__(
	369	"lock; addb %1, %0"
71323499	370	: "=m"(*__hp(1, addr))
87322fe8 MD	371	: "iq" ((unsigned char)val)
87322fe8 MD	372	: "memory");
cc1be41b MD	373	return;
	374	}
	375	case 2:
	376	{
	377	__asm__ __volatile__(
	378	"lock; addw %1, %0"
71323499	379	: "=m"(*__hp(2, addr))
87322fe8 MD	380	: "ir" ((unsigned short)val)
87322fe8 MD	381	: "memory");
cc1be41b MD	382	return;
	383	}
	384	case 4:
	385	{
	386	__asm__ __volatile__(
	387	"lock; addl %1, %0"
71323499	388	: "=m"(*__hp(4, addr))
87322fe8 MD	389	: "ir" ((unsigned int)val)
87322fe8 MD	390	: "memory");
cc1be41b MD	391	return;
cc1be41b MD	392	}
e040d717	393	#if (CAA_BITS_PER_LONG == 64)
cc1be41b MD	394	case 8:
	395	{
	396	__asm__ __volatile__(
	397	"lock; addq %1, %0"
71323499	398	: "=m"(*__hp(8, addr))
87322fe8 MD	399	: "er" ((unsigned long)val)
87322fe8 MD	400	: "memory");
cc1be41b MD	401	return;
cc1be41b MD	402	}
0114ba7f MD	403	#endif
0114ba7f MD	404	}
d0bbd9c2 MD	405	/*
	406	* generate an illegal instruction. Cannot catch this with
	407	* linker tricks when optimizations are disabled.
	408	*/
0114ba7f	409	__asm__ __volatile__("ud2");
a81b8e5e	410	return;
0114ba7f MD	411	}
0114ba7f MD	412
bf9de1b7	413	#define _uatomic_add(addr, v) \
e56d99bf	414	(__uatomic_add((addr), caa_cast_long_keep_sign(v), sizeof(*(addr))))
0114ba7f	415
2c5e5fb3	416
ec4e58a3	417	/* uatomic_inc */
2c5e5fb3 MD	418
2c5e5fb3 MD	419	static inline __attribute__((always_inline))
bf9de1b7	420	void __uatomic_inc(void *addr, int len)
2c5e5fb3 MD	421	{
	422	switch (len) {
	423	case 1:
	424	{
	425	__asm__ __volatile__(
	426	"lock; incb %0"
71323499	427	: "=m"(*__hp(1, addr))
2c5e5fb3 MD	428	:
	429	: "memory");
	430	return;
	431	}
	432	case 2:
	433	{
	434	__asm__ __volatile__(
	435	"lock; incw %0"
71323499	436	: "=m"(*__hp(2, addr))
2c5e5fb3 MD	437	:
	438	: "memory");
	439	return;
	440	}
	441	case 4:
	442	{
	443	__asm__ __volatile__(
	444	"lock; incl %0"
71323499	445	: "=m"(*__hp(4, addr))
2c5e5fb3 MD	446	:
	447	: "memory");
	448	return;
	449	}
e040d717	450	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	451	case 8:
	452	{
	453	__asm__ __volatile__(
	454	"lock; incq %0"
71323499	455	: "=m"(*__hp(8, addr))
2c5e5fb3 MD	456	:
	457	: "memory");
	458	return;
	459	}
	460	#endif
	461	}
	462	/* generate an illegal instruction. Cannot catch this with linker tricks
	463	* when optimizations are disabled. */
	464	__asm__ __volatile__("ud2");
	465	return;
	466	}
	467
bf9de1b7	468	#define _uatomic_inc(addr) (__uatomic_inc((addr), sizeof(*(addr))))
2c5e5fb3	469
ec4e58a3	470	/* uatomic_dec */
2c5e5fb3 MD	471
2c5e5fb3 MD	472	static inline __attribute__((always_inline))
bf9de1b7	473	void __uatomic_dec(void *addr, int len)
2c5e5fb3 MD	474	{
	475	switch (len) {
	476	case 1:
	477	{
	478	__asm__ __volatile__(
	479	"lock; decb %0"
71323499	480	: "=m"(*__hp(1, addr))
2c5e5fb3 MD	481	:
	482	: "memory");
	483	return;
	484	}
	485	case 2:
	486	{
	487	__asm__ __volatile__(
	488	"lock; decw %0"
71323499	489	: "=m"(*__hp(2, addr))
2c5e5fb3 MD	490	:
	491	: "memory");
	492	return;
	493	}
	494	case 4:
	495	{
	496	__asm__ __volatile__(
	497	"lock; decl %0"
71323499	498	: "=m"(*__hp(4, addr))
2c5e5fb3 MD	499	:
	500	: "memory");
	501	return;
	502	}
e040d717	503	#if (CAA_BITS_PER_LONG == 64)
2c5e5fb3 MD	504	case 8:
	505	{
	506	__asm__ __volatile__(
	507	"lock; decq %0"
71323499	508	: "=m"(*__hp(8, addr))
2c5e5fb3 MD	509	:
	510	: "memory");
	511	return;
	512	}
	513	#endif
	514	}
d0bbd9c2 MD	515	/*
	516	* generate an illegal instruction. Cannot catch this with
	517	* linker tricks when optimizations are disabled.
	518	*/
2c5e5fb3 MD	519	__asm__ __volatile__("ud2");
	520	return;
	521	}
	522
bf9de1b7	523	#define _uatomic_dec(addr) (__uatomic_dec((addr), sizeof(*(addr))))
0114ba7f	524
101389e4	525	#ifdef URCU_ARCH_X86_NO_CAS
0b1e236d MJ	526
	527	/* For backwards compat */
	528	#define CONFIG_RCU_COMPAT_ARCH 1
	529
02be5561 MD	530	extern int __rcu_cas_avail;
02be5561 MD	531	extern int __rcu_cas_init(void);
bf9de1b7 MD	532
bf9de1b7 MD	533	#define UATOMIC_COMPAT(insn) \
a0b7f7ea	534	((caa_likely(__rcu_cas_avail > 0)) \
bf9de1b7	535	? (_uatomic_##insn) \
a0b7f7ea	536	: ((caa_unlikely(__rcu_cas_avail < 0) \
02be5561	537	? ((__rcu_cas_init() > 0) \
bf9de1b7 MD	538	? (_uatomic_##insn) \
	539	: (compat_uatomic_##insn)) \
	540	: (compat_uatomic_##insn))))
	541
424d4ed5 MD	542	/*
	543	* We leave the return value so we don't break the ABI, but remove the
	544	* return value from the API.
	545	*/
bf9de1b7 MD	546	extern unsigned long _compat_uatomic_set(void *addr,
	547	unsigned long _new, int len);
	548	#define compat_uatomic_set(addr, _new) \
424d4ed5 MD	549	((void) _compat_uatomic_set((addr), \
	550	caa_cast_long_keep_sign(_new), \
	551	sizeof(*(addr))))
bf9de1b7 MD	552
	553
	554	extern unsigned long _compat_uatomic_xchg(void *addr,
	555	unsigned long _new, int len);
	556	#define compat_uatomic_xchg(addr, _new) \
	557	((__typeof__(*(addr))) _compat_uatomic_xchg((addr), \
e56d99bf	558	caa_cast_long_keep_sign(_new), \
bf9de1b7	559	sizeof(*(addr))))
7d413817 MD	560
7d413817 MD	561	extern unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old,
bf9de1b7 MD	562	unsigned long _new, int len);
	563	#define compat_uatomic_cmpxchg(addr, old, _new) \
	564	((__typeof__(*(addr))) _compat_uatomic_cmpxchg((addr), \
e56d99bf MD	565	caa_cast_long_keep_sign(old), \
e56d99bf MD	566	caa_cast_long_keep_sign(_new), \
bf9de1b7	567	sizeof(*(addr))))
7d413817	568
8c43fe72	569	extern void _compat_uatomic_and(void *addr, unsigned long _new, int len);
bf33aaea	570	#define compat_uatomic_and(addr, v) \
8c43fe72	571	(_compat_uatomic_and((addr), \
e56d99bf	572	caa_cast_long_keep_sign(v), \
8c43fe72	573	sizeof(*(addr))))
bf33aaea	574
8c43fe72	575	extern void _compat_uatomic_or(void *addr, unsigned long _new, int len);
985b35b1	576	#define compat_uatomic_or(addr, v) \
8c43fe72	577	(_compat_uatomic_or((addr), \
e56d99bf	578	caa_cast_long_keep_sign(v), \
8c43fe72	579	sizeof(*(addr))))
985b35b1	580
28ca843d PB	581	extern unsigned long _compat_uatomic_add_return(void *addr,
28ca843d PB	582	unsigned long _new, int len);
e56d99bf MD	583	#define compat_uatomic_add_return(addr, v) \
	584	((__typeof__(*(addr))) _compat_uatomic_add_return((addr), \
	585	caa_cast_long_keep_sign(v), \
	586	sizeof(*(addr))))
bf9de1b7	587
bf9de1b7 MD	588	#define compat_uatomic_add(addr, v) \
bf9de1b7 MD	589	((void)compat_uatomic_add_return((addr), (v)))
bf9de1b7 MD	590	#define compat_uatomic_inc(addr) \
	591	(compat_uatomic_add((addr), 1))
	592	#define compat_uatomic_dec(addr) \
8760d94e	593	(compat_uatomic_add((addr), -1))
bf9de1b7 MD	594
	595	#else
	596	#define UATOMIC_COMPAT(insn) (_uatomic_##insn)
7d413817 MD	597	#endif
7d413817 MD	598
bf9de1b7	599	/* Read is atomic even in compat mode */
bf9de1b7 MD	600	#define uatomic_set(addr, v) \
bf9de1b7 MD	601	UATOMIC_COMPAT(set(addr, v))
8760d94e	602
bf9de1b7 MD	603	#define uatomic_cmpxchg(addr, old, _new) \
	604	UATOMIC_COMPAT(cmpxchg(addr, old, _new))
	605	#define uatomic_xchg(addr, v) \
	606	UATOMIC_COMPAT(xchg(addr, v))
2812a2d2	607
bf33aaea PB	608	#define uatomic_and(addr, v) \
bf33aaea PB	609	UATOMIC_COMPAT(and(addr, v))
42e83919 MD	610	#define cmm_smp_mb__before_uatomic_and() cmm_barrier()
42e83919 MD	611	#define cmm_smp_mb__after_uatomic_and() cmm_barrier()
2812a2d2	612
985b35b1 PB	613	#define uatomic_or(addr, v) \
985b35b1 PB	614	UATOMIC_COMPAT(or(addr, v))
42e83919 MD	615	#define cmm_smp_mb__before_uatomic_or() cmm_barrier()
42e83919 MD	616	#define cmm_smp_mb__after_uatomic_or() cmm_barrier()
2812a2d2	617
bf9de1b7 MD	618	#define uatomic_add_return(addr, v) \
bf9de1b7 MD	619	UATOMIC_COMPAT(add_return(addr, v))
8760d94e	620
bf9de1b7	621	#define uatomic_add(addr, v) UATOMIC_COMPAT(add(addr, v))
42e83919 MD	622	#define cmm_smp_mb__before_uatomic_add() cmm_barrier()
42e83919 MD	623	#define cmm_smp_mb__after_uatomic_add() cmm_barrier()
2812a2d2	624
bf9de1b7	625	#define uatomic_inc(addr) UATOMIC_COMPAT(inc(addr))
42e83919 MD	626	#define cmm_smp_mb__before_uatomic_inc() cmm_barrier()
42e83919 MD	627	#define cmm_smp_mb__after_uatomic_inc() cmm_barrier()
2812a2d2	628
bf9de1b7	629	#define uatomic_dec(addr) UATOMIC_COMPAT(dec(addr))
42e83919 MD	630	#define cmm_smp_mb__before_uatomic_dec() cmm_barrier()
42e83919 MD	631	#define cmm_smp_mb__after_uatomic_dec() cmm_barrier()
bf9de1b7	632
67ecffc0	633	#ifdef __cplusplus
36bc70a8 MD	634	}
	635	#endif
	636
a2e7bf9c	637	#include <urcu/uatomic/generic.h>
8760d94e	638
ec4e58a3	639	#endif /* _URCU_ARCH_UATOMIC_X86_H */