1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020 Loongson Technology Corporation Limited
4 */
5 #ifndef __ASM_CMPXCHG_H
6 #define __ASM_CMPXCHG_H
7
8 #include <asm/barrier.h>
9 #include <linux/build_bug.h>
10
11 #define __xchg_asm(amswap_db, m, val) \
12 ({ \
13 __typeof(val) __ret; \
14 \
15 __asm__ __volatile__ ( \
16 " "amswap_db" %1, %z2, %0 \n" \
17 : "+ZB" (*m), "=&r" (__ret) \
18 : "Jr" (val) \
19 : "memory"); \
20 \
21 __ret; \
22 })
23
24 extern unsigned long __xchg_small(volatile void *ptr, unsigned long x,
25 unsigned int size);
26
27 static __always_inline unsigned long
__xchg(volatile void * ptr,unsigned long x,int size)28 __xchg(volatile void *ptr, unsigned long x, int size)
29 {
30 switch (size) {
31 case 1:
32 case 2:
33 return __xchg_small(ptr, x, size);
34
35 case 4:
36 return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
37
38 case 8:
39 return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
40
41 default:
42 BUILD_BUG();
43 }
44
45 return 0;
46 }
47
48 #define xchg(ptr, x) \
49 ({ \
50 __typeof__(*(ptr)) __res; \
51 \
52 __res = (__typeof__(*(ptr))) \
53 __xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \
54 \
55 __res; \
56 })
57
58 #define __cmpxchg_asm(ld, st, m, old, new) \
59 ({ \
60 __typeof(old) __ret; \
61 \
62 __asm__ __volatile__( \
63 "1: " ld " %0, %2 # __cmpxchg_asm \n" \
64 " bne %0, %z3, 2f \n" \
65 " or $t0, %z4, $zero \n" \
66 " " st " $t0, %1 \n" \
67 " beq $zero, $t0, 1b \n" \
68 "2: \n" \
69 __WEAK_LLSC_MB \
70 : "=&r" (__ret), "=ZB"(*m) \
71 : "ZB"(*m), "Jr" (old), "Jr" (new) \
72 : "t0", "memory"); \
73 \
74 __ret; \
75 })
76
77 extern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old,
78 unsigned long new, unsigned int size);
79
80 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)81 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size)
82 {
83 switch (size) {
84 case 1:
85 case 2:
86 return __cmpxchg_small(ptr, old, new, size);
87
88 case 4:
89 return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr,
90 (u32)old, new);
91
92 case 8:
93 return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr,
94 (u64)old, new);
95
96 default:
97 BUILD_BUG();
98 }
99
100 return 0;
101 }
102
103 #define cmpxchg_local(ptr, old, new) \
104 ((__typeof__(*(ptr))) \
105 __cmpxchg((ptr), \
106 (unsigned long)(__typeof__(*(ptr)))(old), \
107 (unsigned long)(__typeof__(*(ptr)))(new), \
108 sizeof(*(ptr))))
109
110 #define cmpxchg(ptr, old, new) \
111 ({ \
112 __typeof__(*(ptr)) __res; \
113 \
114 __res = cmpxchg_local((ptr), (old), (new)); \
115 \
116 __res; \
117 })
118
119 #ifdef CONFIG_64BIT
120 #define cmpxchg64_local(ptr, o, n) \
121 ({ \
122 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
123 cmpxchg_local((ptr), (o), (n)); \
124 })
125
126 #define cmpxchg64(ptr, o, n) \
127 ({ \
128 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
129 cmpxchg((ptr), (o), (n)); \
130 })
131 #else
132 #include <asm-generic/cmpxchg-local.h>
133 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
134 #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n))
135 #endif
136
137 #endif /* __ASM_CMPXCHG_H */
138