From 89280d020bf064d1055c360fb9974f128051043f Mon Sep 17 00:00:00 2001 From: Olivier Dion Date: Mon, 21 Oct 2024 13:31:28 -0400 Subject: [PATCH] Introduce _CMM_TOOLCHAIN_SUPPORT_C11_MM If the toolchain used supports the C11 memory model, define the private _CMM_TOOLCHAIN_SUPPORT_C11_MM macro. This is used to implement `uatomic_load()/uatomic_store' in term of `__atomic' builtins instead of using volatile accesses, therefore reducing possible redundant memory barriers for some memory orders. Change-Id: I0df3c202daf74ac012338583b761de15687d25f3 Signed-off-by: Olivier Dion Signed-off-by: Mathieu Desnoyers --- include/urcu/compiler.h | 11 ++++++++ include/urcu/uatomic.h | 2 +- include/urcu/uatomic/generic.h | 47 ++++++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/include/urcu/compiler.h b/include/urcu/compiler.h index 31a474c..0e94d2f 100644 --- a/include/urcu/compiler.h +++ b/include/urcu/compiler.h @@ -40,6 +40,17 @@ */ #define CMM_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x)) +/* + * If the toolchain support the C11 memory model, define the private macro + * _CMM_TOOLCHAIN_SUPPORT_C11_MM. + */ +#if ((defined (__cplusplus) && __cplusplus >= 201103L) || \ + (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L)) +# define _CMM_TOOLCHAIN_SUPPORT_C11_MM +#elif defined(CONFIG_RCU_USE_ATOMIC_BUILTINS) +# error "URCU was configured to use atomic builtins, but this toolchain does not support them." +#endif + #ifndef caa_max #define caa_max(a,b) ((a)>(b)?(a):(b)) #endif diff --git a/include/urcu/uatomic.h b/include/urcu/uatomic.h index 69154d3..1af3cd8 100644 --- a/include/urcu/uatomic.h +++ b/include/urcu/uatomic.h @@ -19,7 +19,7 @@ enum cmm_memorder { CMM_SEQ_CST_FENCE = 6, }; -#ifdef CONFIG_RCU_USE_ATOMIC_BUILTINS +#if defined(_CMM_TOOLCHAIN_SUPPORT_C11_MM) /* * Make sure that CMM_SEQ_CST_FENCE is not equivalent to other memory orders. diff --git a/include/urcu/uatomic/generic.h b/include/urcu/uatomic/generic.h index 24a025f..e32f179 100644 --- a/include/urcu/uatomic/generic.h +++ b/include/urcu/uatomic/generic.h @@ -77,21 +77,39 @@ extern "C" { } while(0) #endif /* _cmm_compat_c11_smp_mb__after_mo */ +/* + * If the toolchain supports the C11 memory model, then it is safe to implement + * `uatomic_store_mo()' in term of __atomic builtins. This has the effect of + * reducing the number of emitted memory barriers except for the + * CMM_SEQ_CST_FENCE memory order. + */ +#ifndef uatomic_store_mo +# ifdef _CMM_TOOLCHAIN_SUPPORT_C11_MM +# define uatomic_store_mo(addr, v, mo) \ do { \ - _cmm_compat_c11_smp_mb__before_mo(op, mo); \ - op(addr, v); \ - _cmm_compat_c11_smp_mb__after_mo(op, mo); \ + __atomic_store_n(cmm_cast_volatile(addr), v, \ + cmm_to_c11(mo)); \ + cmm_seq_cst_fence_after_atomic(mo); \ } while (0) - -#define uatomic_store_mo(addr, v, mo) \ +# else +# define uatomic_store_mo(addr, v, mo) \ do { \ - _cmm_compat_c11_smp_mb__before_mo(uatomic_set, mo); \ - uatomic_set(addr, v); \ - _cmm_compat_c11_smp_mb__after_mo(uatomic_set, mo); \ + _cmm_compat_c11_smp_mb__before_mo(uatomic_store, mo); \ + (void) CMM_STORE_SHARED(addr, v); \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_store, mo); \ } while (0) +# endif /* _CMM_TOOLCHAIN_SUPPORT_C11_MM */ +#endif /* !uatomic_store */ /* + * If the toolchain supports the C11 memory model, then it is safe to implement + * `uatomic_load_mo()' in term of __atomic builtins. This has the effect of + * reducing the number of emitted memory barriers except for the + * CMM_SEQ_CST_FENCE memory order. */ +#ifndef uatomic_load_mo +# ifdef _CMM_TOOLCHAIN_SUPPORT_C11_MM +# define uatomic_load_mo(addr, mo) \ __extension__ \ ({ \ __typeof__(*(addr)) _value = \ @@ -101,17 +119,18 @@ extern "C" { \ _value; \ }) - - -#define uatomic_load_mo(addr, mo) \ +# else +# define uatomic_load_mo(addr, mo) \ __extension__ \ ({ \ - _cmm_compat_c11_smp_mb__before_mo(uatomic_read, mo); \ - __typeof__(*(addr)) _rcu_value = uatomic_read(addr); \ - _cmm_compat_c11_smp_mb__after_mo(uatomic_read, mo); \ + _cmm_compat_c11_smp_mb__before_mo(uatomic_load, mo); \ + __typeof__(*(addr)) _rcu_value = CMM_LOAD_SHARED(addr); \ + _cmm_compat_c11_smp_mb__after_mo(uatomic_load, mo); \ \ _rcu_value; \ }) +# endif /* _CMM_TOOLCHAIN_SUPPORT_C11_MM */ +#endif /* !uatomic_load */ #if !defined __OPTIMIZE__ || defined UATOMIC_NO_LINK_ERROR #ifdef ILLEGAL_INSTR -- 2.39.5