1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_CMPXCHG_H_
3 #define _ASM_POWERPC_CMPXCHG_H_
4
5 #ifdef __KERNEL__
6 #include <linux/compiler.h>
7 #include <asm/synch.h>
8 #include <linux/bug.h>
9
10 #ifdef __BIG_ENDIAN
11 #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
12 #else
13 #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
14 #endif
15
16 #define XCHG_GEN(type, sfx, cl) \
17 static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
18 { \
19 unsigned int prev, prev_mask, tmp, bitoff, off; \
20 \
21 off = (unsigned long)p % sizeof(u32); \
22 bitoff = BITOFF_CAL(sizeof(type), off); \
23 p -= off; \
24 val <<= bitoff; \
25 prev_mask = (u32)(type)-1 << bitoff; \
26 \
27 __asm__ __volatile__( \
28 "1: lwarx %0,0,%3\n" \
29 " andc %1,%0,%5\n" \
30 " or %1,%1,%4\n" \
31 " stwcx. %1,0,%3\n" \
32 " bne- 1b\n" \
33 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
34 : "r" (p), "r" (val), "r" (prev_mask) \
35 : "cc", cl); \
36 \
37 return prev >> bitoff; \
38 }
39
40 #define CMPXCHG_GEN(type, sfx, br, br2, cl) \
41 static inline \
42 u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
43 { \
44 unsigned int prev, prev_mask, tmp, bitoff, off; \
45 \
46 off = (unsigned long)p % sizeof(u32); \
47 bitoff = BITOFF_CAL(sizeof(type), off); \
48 p -= off; \
49 old <<= bitoff; \
50 new <<= bitoff; \
51 prev_mask = (u32)(type)-1 << bitoff; \
52 \
53 __asm__ __volatile__( \
54 br \
55 "1: lwarx %0,0,%3\n" \
56 " and %1,%0,%6\n" \
57 " cmpw 0,%1,%4\n" \
58 " bne- 2f\n" \
59 " andc %1,%0,%6\n" \
60 " or %1,%1,%5\n" \
61 " stwcx. %1,0,%3\n" \
62 " bne- 1b\n" \
63 br2 \
64 "\n" \
65 "2:" \
66 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
67 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
68 : "cc", cl); \
69 \
70 return prev >> bitoff; \
71 }
72
73 /*
74 * Atomic exchange
75 *
76 * Changes the memory location '*p' to be val and returns
77 * the previous value stored there.
78 */
79
80 XCHG_GEN(u8, _local, "memory");
81 XCHG_GEN(u8, _relaxed, "cc");
82 XCHG_GEN(u16, _local, "memory");
83 XCHG_GEN(u16, _relaxed, "cc");
84
85 static __always_inline unsigned long
__xchg_u32_local(volatile void * p,unsigned long val)86 __xchg_u32_local(volatile void *p, unsigned long val)
87 {
88 unsigned long prev;
89
90 __asm__ __volatile__(
91 "1: lwarx %0,0,%2 \n"
92 " stwcx. %3,0,%2 \n\
93 bne- 1b"
94 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
95 : "r" (p), "r" (val)
96 : "cc", "memory");
97
98 return prev;
99 }
100
101 static __always_inline unsigned long
__xchg_u32_relaxed(u32 * p,unsigned long val)102 __xchg_u32_relaxed(u32 *p, unsigned long val)
103 {
104 unsigned long prev;
105
106 __asm__ __volatile__(
107 "1: lwarx %0,0,%2\n"
108 " stwcx. %3,0,%2\n"
109 " bne- 1b"
110 : "=&r" (prev), "+m" (*p)
111 : "r" (p), "r" (val)
112 : "cc");
113
114 return prev;
115 }
116
117 #ifdef CONFIG_PPC64
118 static __always_inline unsigned long
__xchg_u64_local(volatile void * p,unsigned long val)119 __xchg_u64_local(volatile void *p, unsigned long val)
120 {
121 unsigned long prev;
122
123 __asm__ __volatile__(
124 "1: ldarx %0,0,%2 \n"
125 " stdcx. %3,0,%2 \n\
126 bne- 1b"
127 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
128 : "r" (p), "r" (val)
129 : "cc", "memory");
130
131 return prev;
132 }
133
134 static __always_inline unsigned long
__xchg_u64_relaxed(u64 * p,unsigned long val)135 __xchg_u64_relaxed(u64 *p, unsigned long val)
136 {
137 unsigned long prev;
138
139 __asm__ __volatile__(
140 "1: ldarx %0,0,%2\n"
141 " stdcx. %3,0,%2\n"
142 " bne- 1b"
143 : "=&r" (prev), "+m" (*p)
144 : "r" (p), "r" (val)
145 : "cc");
146
147 return prev;
148 }
149 #endif
150
151 static __always_inline unsigned long
__xchg_local(void * ptr,unsigned long x,unsigned int size)152 __xchg_local(void *ptr, unsigned long x, unsigned int size)
153 {
154 switch (size) {
155 case 1:
156 return __xchg_u8_local(ptr, x);
157 case 2:
158 return __xchg_u16_local(ptr, x);
159 case 4:
160 return __xchg_u32_local(ptr, x);
161 #ifdef CONFIG_PPC64
162 case 8:
163 return __xchg_u64_local(ptr, x);
164 #endif
165 }
166 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
167 return x;
168 }
169
170 static __always_inline unsigned long
__xchg_relaxed(void * ptr,unsigned long x,unsigned int size)171 __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
172 {
173 switch (size) {
174 case 1:
175 return __xchg_u8_relaxed(ptr, x);
176 case 2:
177 return __xchg_u16_relaxed(ptr, x);
178 case 4:
179 return __xchg_u32_relaxed(ptr, x);
180 #ifdef CONFIG_PPC64
181 case 8:
182 return __xchg_u64_relaxed(ptr, x);
183 #endif
184 }
185 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
186 return x;
187 }
188 #define xchg_local(ptr,x) \
189 ({ \
190 __typeof__(*(ptr)) _x_ = (x); \
191 (__typeof__(*(ptr))) __xchg_local((ptr), \
192 (unsigned long)_x_, sizeof(*(ptr))); \
193 })
194
195 #define xchg_relaxed(ptr, x) \
196 ({ \
197 __typeof__(*(ptr)) _x_ = (x); \
198 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
199 (unsigned long)_x_, sizeof(*(ptr))); \
200 })
201 /*
202 * Compare and exchange - if *p == old, set it to new,
203 * and return the old value of *p.
204 */
205
206 CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
207 CMPXCHG_GEN(u8, _local, , , "memory");
208 CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
209 CMPXCHG_GEN(u8, _relaxed, , , "cc");
210 CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
211 CMPXCHG_GEN(u16, _local, , , "memory");
212 CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
213 CMPXCHG_GEN(u16, _relaxed, , , "cc");
214
215 static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int * p,unsigned long old,unsigned long new)216 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
217 {
218 unsigned int prev;
219
220 __asm__ __volatile__ (
221 PPC_ATOMIC_ENTRY_BARRIER
222 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
223 cmpw 0,%0,%3\n\
224 bne- 2f\n"
225 " stwcx. %4,0,%2\n\
226 bne- 1b"
227 PPC_ATOMIC_EXIT_BARRIER
228 "\n\
229 2:"
230 : "=&r" (prev), "+m" (*p)
231 : "r" (p), "r" (old), "r" (new)
232 : "cc", "memory");
233
234 return prev;
235 }
236
237 static __always_inline unsigned long
__cmpxchg_u32_local(volatile unsigned int * p,unsigned long old,unsigned long new)238 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
239 unsigned long new)
240 {
241 unsigned int prev;
242
243 __asm__ __volatile__ (
244 "1: lwarx %0,0,%2 # __cmpxchg_u32\n\
245 cmpw 0,%0,%3\n\
246 bne- 2f\n"
247 " stwcx. %4,0,%2\n\
248 bne- 1b"
249 "\n\
250 2:"
251 : "=&r" (prev), "+m" (*p)
252 : "r" (p), "r" (old), "r" (new)
253 : "cc", "memory");
254
255 return prev;
256 }
257
258 static __always_inline unsigned long
__cmpxchg_u32_relaxed(u32 * p,unsigned long old,unsigned long new)259 __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
260 {
261 unsigned long prev;
262
263 __asm__ __volatile__ (
264 "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
265 " cmpw 0,%0,%3\n"
266 " bne- 2f\n"
267 " stwcx. %4,0,%2\n"
268 " bne- 1b\n"
269 "2:"
270 : "=&r" (prev), "+m" (*p)
271 : "r" (p), "r" (old), "r" (new)
272 : "cc");
273
274 return prev;
275 }
276
277 /*
278 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
279 * can avoid superfluous barriers if we use assembly code to implement
280 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
281 * cmpxchg_release() because that will result in putting a barrier in the
282 * middle of a ll/sc loop, which is probably a bad idea. For example, this
283 * might cause the conditional store more likely to fail.
284 */
285 static __always_inline unsigned long
__cmpxchg_u32_acquire(u32 * p,unsigned long old,unsigned long new)286 __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
287 {
288 unsigned long prev;
289
290 __asm__ __volatile__ (
291 "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
292 " cmpw 0,%0,%3\n"
293 " bne- 2f\n"
294 " stwcx. %4,0,%2\n"
295 " bne- 1b\n"
296 PPC_ACQUIRE_BARRIER
297 "\n"
298 "2:"
299 : "=&r" (prev), "+m" (*p)
300 : "r" (p), "r" (old), "r" (new)
301 : "cc", "memory");
302
303 return prev;
304 }
305
306 #ifdef CONFIG_PPC64
307 static __always_inline unsigned long
__cmpxchg_u64(volatile unsigned long * p,unsigned long old,unsigned long new)308 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
309 {
310 unsigned long prev;
311
312 __asm__ __volatile__ (
313 PPC_ATOMIC_ENTRY_BARRIER
314 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
315 cmpd 0,%0,%3\n\
316 bne- 2f\n\
317 stdcx. %4,0,%2\n\
318 bne- 1b"
319 PPC_ATOMIC_EXIT_BARRIER
320 "\n\
321 2:"
322 : "=&r" (prev), "+m" (*p)
323 : "r" (p), "r" (old), "r" (new)
324 : "cc", "memory");
325
326 return prev;
327 }
328
329 static __always_inline unsigned long
__cmpxchg_u64_local(volatile unsigned long * p,unsigned long old,unsigned long new)330 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
331 unsigned long new)
332 {
333 unsigned long prev;
334
335 __asm__ __volatile__ (
336 "1: ldarx %0,0,%2 # __cmpxchg_u64\n\
337 cmpd 0,%0,%3\n\
338 bne- 2f\n\
339 stdcx. %4,0,%2\n\
340 bne- 1b"
341 "\n\
342 2:"
343 : "=&r" (prev), "+m" (*p)
344 : "r" (p), "r" (old), "r" (new)
345 : "cc", "memory");
346
347 return prev;
348 }
349
350 static __always_inline unsigned long
__cmpxchg_u64_relaxed(u64 * p,unsigned long old,unsigned long new)351 __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
352 {
353 unsigned long prev;
354
355 __asm__ __volatile__ (
356 "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
357 " cmpd 0,%0,%3\n"
358 " bne- 2f\n"
359 " stdcx. %4,0,%2\n"
360 " bne- 1b\n"
361 "2:"
362 : "=&r" (prev), "+m" (*p)
363 : "r" (p), "r" (old), "r" (new)
364 : "cc");
365
366 return prev;
367 }
368
369 static __always_inline unsigned long
__cmpxchg_u64_acquire(u64 * p,unsigned long old,unsigned long new)370 __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
371 {
372 unsigned long prev;
373
374 __asm__ __volatile__ (
375 "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
376 " cmpd 0,%0,%3\n"
377 " bne- 2f\n"
378 " stdcx. %4,0,%2\n"
379 " bne- 1b\n"
380 PPC_ACQUIRE_BARRIER
381 "\n"
382 "2:"
383 : "=&r" (prev), "+m" (*p)
384 : "r" (p), "r" (old), "r" (new)
385 : "cc", "memory");
386
387 return prev;
388 }
389 #endif
390
391 static __always_inline unsigned long
__cmpxchg(volatile void * ptr,unsigned long old,unsigned long new,unsigned int size)392 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
393 unsigned int size)
394 {
395 switch (size) {
396 case 1:
397 return __cmpxchg_u8(ptr, old, new);
398 case 2:
399 return __cmpxchg_u16(ptr, old, new);
400 case 4:
401 return __cmpxchg_u32(ptr, old, new);
402 #ifdef CONFIG_PPC64
403 case 8:
404 return __cmpxchg_u64(ptr, old, new);
405 #endif
406 }
407 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
408 return old;
409 }
410
411 static __always_inline unsigned long
__cmpxchg_local(void * ptr,unsigned long old,unsigned long new,unsigned int size)412 __cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
413 unsigned int size)
414 {
415 switch (size) {
416 case 1:
417 return __cmpxchg_u8_local(ptr, old, new);
418 case 2:
419 return __cmpxchg_u16_local(ptr, old, new);
420 case 4:
421 return __cmpxchg_u32_local(ptr, old, new);
422 #ifdef CONFIG_PPC64
423 case 8:
424 return __cmpxchg_u64_local(ptr, old, new);
425 #endif
426 }
427 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
428 return old;
429 }
430
431 static __always_inline unsigned long
__cmpxchg_relaxed(void * ptr,unsigned long old,unsigned long new,unsigned int size)432 __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
433 unsigned int size)
434 {
435 switch (size) {
436 case 1:
437 return __cmpxchg_u8_relaxed(ptr, old, new);
438 case 2:
439 return __cmpxchg_u16_relaxed(ptr, old, new);
440 case 4:
441 return __cmpxchg_u32_relaxed(ptr, old, new);
442 #ifdef CONFIG_PPC64
443 case 8:
444 return __cmpxchg_u64_relaxed(ptr, old, new);
445 #endif
446 }
447 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
448 return old;
449 }
450
451 static __always_inline unsigned long
__cmpxchg_acquire(void * ptr,unsigned long old,unsigned long new,unsigned int size)452 __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
453 unsigned int size)
454 {
455 switch (size) {
456 case 1:
457 return __cmpxchg_u8_acquire(ptr, old, new);
458 case 2:
459 return __cmpxchg_u16_acquire(ptr, old, new);
460 case 4:
461 return __cmpxchg_u32_acquire(ptr, old, new);
462 #ifdef CONFIG_PPC64
463 case 8:
464 return __cmpxchg_u64_acquire(ptr, old, new);
465 #endif
466 }
467 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
468 return old;
469 }
470 #define cmpxchg(ptr, o, n) \
471 ({ \
472 __typeof__(*(ptr)) _o_ = (o); \
473 __typeof__(*(ptr)) _n_ = (n); \
474 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
475 (unsigned long)_n_, sizeof(*(ptr))); \
476 })
477
478
479 #define cmpxchg_local(ptr, o, n) \
480 ({ \
481 __typeof__(*(ptr)) _o_ = (o); \
482 __typeof__(*(ptr)) _n_ = (n); \
483 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
484 (unsigned long)_n_, sizeof(*(ptr))); \
485 })
486
487 #define cmpxchg_relaxed(ptr, o, n) \
488 ({ \
489 __typeof__(*(ptr)) _o_ = (o); \
490 __typeof__(*(ptr)) _n_ = (n); \
491 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
492 (unsigned long)_o_, (unsigned long)_n_, \
493 sizeof(*(ptr))); \
494 })
495
496 #define cmpxchg_acquire(ptr, o, n) \
497 ({ \
498 __typeof__(*(ptr)) _o_ = (o); \
499 __typeof__(*(ptr)) _n_ = (n); \
500 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
501 (unsigned long)_o_, (unsigned long)_n_, \
502 sizeof(*(ptr))); \
503 })
504 #ifdef CONFIG_PPC64
505 #define cmpxchg64(ptr, o, n) \
506 ({ \
507 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
508 cmpxchg((ptr), (o), (n)); \
509 })
510 #define cmpxchg64_local(ptr, o, n) \
511 ({ \
512 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
513 cmpxchg_local((ptr), (o), (n)); \
514 })
515 #define cmpxchg64_relaxed(ptr, o, n) \
516 ({ \
517 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
518 cmpxchg_relaxed((ptr), (o), (n)); \
519 })
520 #define cmpxchg64_acquire(ptr, o, n) \
521 ({ \
522 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
523 cmpxchg_acquire((ptr), (o), (n)); \
524 })
525 #else
526 #include <asm-generic/cmpxchg-local.h>
527 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
528 #endif
529
530 #endif /* __KERNEL__ */
531 #endif /* _ASM_POWERPC_CMPXCHG_H_ */
532