| /* |
| * Copyright IBM Corp. 1999, 2011 |
| * |
| * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, |
| */ |
| |
| #ifndef __ASM_CMPXCHG_H |
| #define __ASM_CMPXCHG_H |
| |
| #include <linux/mmdebug.h> |
| #include <linux/types.h> |
| #include <linux/bug.h> |
| |
| extern void __xchg_called_with_bad_pointer(void); |
| |
| static inline unsigned long __xchg(unsigned long x, void *ptr, int size) |
| { |
| unsigned long addr, old; |
| int shift; |
| |
| switch (size) { |
| case 1: |
| addr = (unsigned long) ptr; |
| shift = (3 ^ (addr & 3)) << 3; |
| addr ^= addr & 3; |
| asm volatile( |
| " l %0,%4\n" |
| "0: lr 0,%0\n" |
| " nr 0,%3\n" |
| " or 0,%2\n" |
| " cs %0,0,%4\n" |
| " jl 0b\n" |
| : "=&d" (old), "=Q" (*(int *) addr) |
| : "d" ((x & 0xff) << shift), "d" (~(0xff << shift)), |
| "Q" (*(int *) addr) : "memory", "cc", "0"); |
| return old >> shift; |
| case 2: |
| addr = (unsigned long) ptr; |
| shift = (2 ^ (addr & 2)) << 3; |
| addr ^= addr & 2; |
| asm volatile( |
| " l %0,%4\n" |
| "0: lr 0,%0\n" |
| " nr 0,%3\n" |
| " or 0,%2\n" |
| " cs %0,0,%4\n" |
| " jl 0b\n" |
| : "=&d" (old), "=Q" (*(int *) addr) |
| : "d" ((x & 0xffff) << shift), "d" (~(0xffff << shift)), |
| "Q" (*(int *) addr) : "memory", "cc", "0"); |
| return old >> shift; |
| case 4: |
| asm volatile( |
| " l %0,%3\n" |
| "0: cs %0,%2,%3\n" |
| " jl 0b\n" |
| : "=&d" (old), "=Q" (*(int *) ptr) |
| : "d" (x), "Q" (*(int *) ptr) |
| : "memory", "cc"); |
| return old; |
| #ifdef CONFIG_64BIT |
| case 8: |
| asm volatile( |
| " lg %0,%3\n" |
| "0: csg %0,%2,%3\n" |
| " jl 0b\n" |
| : "=&d" (old), "=m" (*(long *) ptr) |
| : "d" (x), "Q" (*(long *) ptr) |
| : "memory", "cc"); |
| return old; |
| #endif /* CONFIG_64BIT */ |
| } |
| __xchg_called_with_bad_pointer(); |
| return x; |
| } |
| |
| #define xchg(ptr, x) \ |
| ({ \ |
| __typeof__(*(ptr)) __ret; \ |
| __ret = (__typeof__(*(ptr))) \ |
| __xchg((unsigned long)(x), (void *)(ptr), sizeof(*(ptr)));\ |
| __ret; \ |
| }) |
| |
| /* |
| * Atomic compare and exchange. Compare OLD with MEM, if identical, |
| * store NEW in MEM. Return the initial value in MEM. Success is |
| * indicated by comparing RETURN with OLD. |
| */ |
| |
| #define __HAVE_ARCH_CMPXCHG |
| |
| extern void __cmpxchg_called_with_bad_pointer(void); |
| |
| static inline unsigned long __cmpxchg(void *ptr, unsigned long old, |
| unsigned long new, int size) |
| { |
| unsigned long addr, prev, tmp; |
| int shift; |
| |
| switch (size) { |
| case 1: |
| addr = (unsigned long) ptr; |
| shift = (3 ^ (addr & 3)) << 3; |
| addr ^= addr & 3; |
| asm volatile( |
| " l %0,%2\n" |
| "0: nr %0,%5\n" |
| " lr %1,%0\n" |
| " or %0,%3\n" |
| " or %1,%4\n" |
| " cs %0,%1,%2\n" |
| " jnl 1f\n" |
| " xr %1,%0\n" |
| " nr %1,%5\n" |
| " jnz 0b\n" |
| "1:" |
| : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) |
| : "d" ((old & 0xff) << shift), |
| "d" ((new & 0xff) << shift), |
| "d" (~(0xff << shift)) |
| : "memory", "cc"); |
| return prev >> shift; |
| case 2: |
| addr = (unsigned long) ptr; |
| shift = (2 ^ (addr & 2)) << 3; |
| addr ^= addr & 2; |
| asm volatile( |
| " l %0,%2\n" |
| "0: nr %0,%5\n" |
| " lr %1,%0\n" |
| " or %0,%3\n" |
| " or %1,%4\n" |
| " cs %0,%1,%2\n" |
| " jnl 1f\n" |
| " xr %1,%0\n" |
| " nr %1,%5\n" |
| " jnz 0b\n" |
| "1:" |
| : "=&d" (prev), "=&d" (tmp), "+Q" (*(int *) addr) |
| : "d" ((old & 0xffff) << shift), |
| "d" ((new & 0xffff) << shift), |
| "d" (~(0xffff << shift)) |
| : "memory", "cc"); |
| return prev >> shift; |
| case 4: |
| asm volatile( |
| " cs %0,%3,%1\n" |
| : "=&d" (prev), "=Q" (*(int *) ptr) |
| : "0" (old), "d" (new), "Q" (*(int *) ptr) |
| : "memory", "cc"); |
| return prev; |
| #ifdef CONFIG_64BIT |
| case 8: |
| asm volatile( |
| " csg %0,%3,%1\n" |
| : "=&d" (prev), "=Q" (*(long *) ptr) |
| : "0" (old), "d" (new), "Q" (*(long *) ptr) |
| : "memory", "cc"); |
| return prev; |
| #endif /* CONFIG_64BIT */ |
| } |
| __cmpxchg_called_with_bad_pointer(); |
| return old; |
| } |
| |
| #define cmpxchg(ptr, o, n) \ |
| ({ \ |
| __typeof__(*(ptr)) __ret; \ |
| __ret = (__typeof__(*(ptr))) \ |
| __cmpxchg((ptr), (unsigned long)(o), (unsigned long)(n), \ |
| sizeof(*(ptr))); \ |
| __ret; \ |
| }) |
| |
| #ifdef CONFIG_64BIT |
| #define cmpxchg64(ptr, o, n) \ |
| ({ \ |
| cmpxchg((ptr), (o), (n)); \ |
| }) |
| #else /* CONFIG_64BIT */ |
| static inline unsigned long long __cmpxchg64(void *ptr, |
| unsigned long long old, |
| unsigned long long new) |
| { |
| register_pair rp_old = {.pair = old}; |
| register_pair rp_new = {.pair = new}; |
| unsigned long long *ullptr = ptr; |
| |
| asm volatile( |
| " cds %0,%2,%1" |
| : "+d" (rp_old), "+Q" (*ullptr) |
| : "d" (rp_new) |
| : "memory", "cc"); |
| return rp_old.pair; |
| } |
| |
| #define cmpxchg64(ptr, o, n) \ |
| ({ \ |
| __typeof__(*(ptr)) __ret; \ |
| __ret = (__typeof__(*(ptr))) \ |
| __cmpxchg64((ptr), \ |
| (unsigned long long)(o), \ |
| (unsigned long long)(n)); \ |
| __ret; \ |
| }) |
| #endif /* CONFIG_64BIT */ |
| |
| #define __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, insn) \ |
| ({ \ |
| register __typeof__(*(p1)) __old1 asm("2") = (o1); \ |
| register __typeof__(*(p2)) __old2 asm("3") = (o2); \ |
| register __typeof__(*(p1)) __new1 asm("4") = (n1); \ |
| register __typeof__(*(p2)) __new2 asm("5") = (n2); \ |
| int cc; \ |
| asm volatile( \ |
| insn " %[old],%[new],%[ptr]\n" \ |
| " ipm %[cc]\n" \ |
| " srl %[cc],28" \ |
| : [cc] "=d" (cc), [old] "+d" (__old1), "+d" (__old2) \ |
| : [new] "d" (__new1), "d" (__new2), \ |
| [ptr] "Q" (*(p1)), "Q" (*(p2)) \ |
| : "memory", "cc"); \ |
| !cc; \ |
| }) |
| |
| #define __cmpxchg_double_4(p1, p2, o1, o2, n1, n2) \ |
| __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cds") |
| |
| #define __cmpxchg_double_8(p1, p2, o1, o2, n1, n2) \ |
| __cmpxchg_double_op(p1, p2, o1, o2, n1, n2, "cdsg") |
| |
| extern void __cmpxchg_double_called_with_bad_pointer(void); |
| |
| #define __cmpxchg_double(p1, p2, o1, o2, n1, n2) \ |
| ({ \ |
| int __ret; \ |
| switch (sizeof(*(p1))) { \ |
| case 4: \ |
| __ret = __cmpxchg_double_4(p1, p2, o1, o2, n1, n2); \ |
| break; \ |
| case 8: \ |
| __ret = __cmpxchg_double_8(p1, p2, o1, o2, n1, n2); \ |
| break; \ |
| default: \ |
| __cmpxchg_double_called_with_bad_pointer(); \ |
| } \ |
| __ret; \ |
| }) |
| |
| #define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ |
| ({ \ |
| __typeof__(p1) __p1 = (p1); \ |
| __typeof__(p2) __p2 = (p2); \ |
| int __ret; \ |
| BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ |
| BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ |
| VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ |
| if (sizeof(long) == 4) \ |
| __ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2); \ |
| else \ |
| __ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \ |
| __ret; \ |
| }) |
| |
| #define system_has_cmpxchg_double() 1 |
| |
| #include <asm-generic/cmpxchg-local.h> |
| |
| static inline unsigned long __cmpxchg_local(void *ptr, |
| unsigned long old, |
| unsigned long new, int size) |
| { |
| switch (size) { |
| case 1: |
| case 2: |
| case 4: |
| #ifdef CONFIG_64BIT |
| case 8: |
| #endif |
| return __cmpxchg(ptr, old, new, size); |
| default: |
| return __cmpxchg_local_generic(ptr, old, new, size); |
| } |
| |
| return old; |
| } |
| |
| /* |
| * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make |
| * them available. |
| */ |
| #define cmpxchg_local(ptr, o, n) \ |
| ({ \ |
| __typeof__(*(ptr)) __ret; \ |
| __ret = (__typeof__(*(ptr))) \ |
| __cmpxchg_local((ptr), (unsigned long)(o), \ |
| (unsigned long)(n), sizeof(*(ptr))); \ |
| __ret; \ |
| }) |
| |
| #define cmpxchg64_local(ptr, o, n) cmpxchg64((ptr), (o), (n)) |
| |
| #endif /* __ASM_CMPXCHG_H */ |
