extend x86 atomic operations

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

diff --git a/arch_atomic_x86.h b/arch_atomic_x86.h
index f471a393960d009587e2a6d0b0d8eb25f158683f..a762a5ce8b366b74bbff3333607f776c0d18fbcb 100644 (file)
--- a/arch_atomic_x86.h
+++ b/arch_atomic_x86.h
@@ -30,48 +30,171 @@
  * Derived from AO_compare_and_swap() and AO_test_and_set_full().
  */
 
+struct __atomic_dummy {
+       unsigned long v[10];
+};
+#define __hp(x)        ((struct __atomic_dummy *)(x))
+
+/* cmpxchg */
+
 static inline __attribute__((always_inline))
-unsigned int atomic_exchange_32(volatile unsigned int *addr, unsigned int val)
+unsigned long _atomic_cmpxchg(volatile void *addr, unsigned long old,
+               unsigned long _new, int len)
 {
-       unsigned int result;
-
-       __asm__ __volatile__(
-               /* Note: the "xchg" instruction does not need a "lock" prefix */
-               "xchgl %0, %1"
-                       : "=&r"(result), "=m"(*addr)
-                       : "0" (val), "m"(*addr)
+       switch (len) {
+       case 1:
+       {
+               unsigned char result = old;
+               __asm__ __volatile__(
+               "lock; cmpxchgb %2, %1"
+                       : "+a"(result), "+m"(*__hp(addr))
+                       : "q"((unsigned char)_new)
                        : "memory");
-
-       return result;
+               return result;
+       }
+       case 2:
+       {
+               unsigned short result = old;
+               __asm__ __volatile__(
+               "lock; cmpxchgw %2, %1"
+                       : "+a"(result), "+m"(*__hp(addr))
+                       : "r"((unsigned short)_new)
+                       : "memory");
+               return result;
+       }
+       case 4:
+       {
+               unsigned int result = old;
+               __asm__ __volatile__(
+               "lock; cmpxchgl %2, %1"
+                       : "+a"(result), "+m"(*__hp(addr))
+                       : "r"((unsigned int)_new)
+                       : "memory");
+               return result;
+       }
+#if (BITS_PER_LONG == 64)
+       case 8:
+       {
+               unsigned int result = old;
+               __asm__ __volatile__(
+               "lock; cmpxchgl %2, %1"
+                       : "+a"(result), "+m"(*__hp(addr))
+                       : "r"((unsigned long)_new)
+                       : "memory");
+               return result;
+       }
+#endif
+       }
+       /* generate an illegal instruction. Cannot catch this with linker tricks
+        * when optimizations are disabled. */
+       __asm__ __volatile__("ud2");
+       return 0;
 }
 
-#if (BITS_PER_LONG == 64)
+#define cmpxchg(addr, old, _new)                                           \
+       ((__typeof__(*(addr))) _atomic_cmpxchg((addr), (unsigned long)(old),\
+                                               (unsigned long)(_new),      \
+                                               sizeof(*(addr))))
+
+/* xchg */
 
 static inline __attribute__((always_inline))
-unsigned long atomic_exchange_64(volatile unsigned long *addr,
-                                unsigned long val)
+unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
 {
-       unsigned long result;
-
-       __asm__ __volatile__(
-               /* Note: the "xchg" instruction does not need a "lock" prefix */
+       /* Note: the "xchg" instruction does not need a "lock" prefix. */
+       switch (len) {
+       case 1:
+       {
+               unsigned char result;
+               __asm__ __volatile__(
+               "xchgb %0, %1"
+                       : "=q"(result), "+m"(*__hp(addr))
+                       : "0" ((unsigned char)val)
+                       : "memory");
+               return result;
+       }
+       case 2:
+       {
+               unsigned short result;
+               __asm__ __volatile__(
+               "xchgw %0, %1"
+                       : "=r"(result), "+m"(*__hp(addr))
+                       : "0" ((unsigned short)val)
+                       : "memory");
+               return result;
+       }
+       case 4:
+       {
+               unsigned int result;
+               __asm__ __volatile__(
+               "xchgl %0, %1"
+                       : "=r"(result), "+m"(*__hp(addr))
+                       : "0" ((unsigned int)val)
+                       : "memory");
+               return result;
+       }
+#if (BITS_PER_LONG == 64)
+       case 8:
+       {
+               unsigned long result;
+               __asm__ __volatile__(
                "xchgq %0, %1"
-                       : "=&r"(result), "=m"(*addr)
-                       : "0" (val), "m"(*addr)
+                       : "=r"(result), "+m"(*__hp(addr))
+                       : "0" ((unsigned long)val)
                        : "memory");
-
-       return result;
+               return result;
+       }
+#endif
+       }
+       /* generate an illegal instruction. Cannot catch this with linker tricks
+        * when optimizations are disabled. */
+       __asm__ __volatile__("ud2");
+       return 0;
 }
 
-#endif
+#define xchg(addr, v)                                                      \
+       ((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
+                                               sizeof(*(addr))))
+
+/* atomic_add */
 
 static inline __attribute__((always_inline))
-unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
+unsigned long _atomic_add(volatile void *addr, unsigned long val, int len)
 {
        switch (len) {
-       case 4: return atomic_exchange_32(addr, val);
+       case 1:
+       {
+               __asm__ __volatile__(
+               "lock; addb %1, %0"
+                       : "=m"(*__hp(addr))
+                       : "q" ((unsigned char)val));
+               return;
+       }
+       case 2:
+       {
+               __asm__ __volatile__(
+               "lock; addw %1, %0"
+                       : "=m"(*__hp(addr))
+                       : "r" ((unsigned short)val));
+               return;
+       }
+       case 4:
+       {
+               __asm__ __volatile__(
+               "lock; addl %1, %0"
+                       : "=m"(*__hp(addr))
+                       : "r" ((unsigned int)val));
+               return;
+       }
 #if (BITS_PER_LONG == 64)
-       case 8: return atomic_exchange_64(addr, val);
+       case 8:
+       {
+               __asm__ __volatile__(
+               "lock; addq %1, %0"
+                       : "=m"(*__hp(addr))
+                       : "r" ((unsigned long)val));
+               return;
+       }
 #endif
        }
        /* generate an illegal instruction. Cannot catch this with linker tricks
@@ -80,9 +203,8 @@ unsigned long _atomic_exchange(volatile void *addr, unsigned long val, int len)
        return 0;
 }
 
-#define xchg(addr, v)                                                      \
-       ((__typeof__(*(addr))) _atomic_exchange((addr), (unsigned long)(v), \
-                                               sizeof(*(addr))))
+#define atomic_add(addr, v)                                               \
+       (_atomic_add((addr), (unsigned long)(v), sizeof(*(addr))))
 
 #endif /* #ifndef _INCLUDE_API_H */