1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_POWERPC_ATOMIC_H_
3 #define _ASM_POWERPC_ATOMIC_H_
4
5 /*
6 * PowerPC atomic operations
7 */
8
9 #ifdef __KERNEL__
10 #include <linux/types.h>
11 #include <asm/cmpxchg.h>
12 #include <asm/barrier.h>
13
14 /*
15 * Since *_return_relaxed and {cmp}xchg_relaxed are implemented with
16 * a "bne-" instruction at the end, so an isync is enough as a acquire barrier
17 * on the platform without lwsync.
18 */
19 #define __atomic_acquire_fence() \
20 __asm__ __volatile__(PPC_ACQUIRE_BARRIER "" : : : "memory")
21
22 #define __atomic_release_fence() \
23 __asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory")
24
atomic_read(const atomic_t * v)25 static __inline__ int atomic_read(const atomic_t *v)
26 {
27 int t;
28
29 __asm__ __volatile__("lwz%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
30
31 return t;
32 }
33
atomic_set(atomic_t * v,int i)34 static __inline__ void atomic_set(atomic_t *v, int i)
35 {
36 __asm__ __volatile__("stw%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
37 }
38
39 #define ATOMIC_OP(op, asm_op) \
40 static __inline__ void atomic_##op(int a, atomic_t *v) \
41 { \
42 int t; \
43 \
44 __asm__ __volatile__( \
45 "1: lwarx %0,0,%3 # atomic_" #op "\n" \
46 #asm_op " %0,%2,%0\n" \
47 " stwcx. %0,0,%3 \n" \
48 " bne- 1b\n" \
49 : "=&r" (t), "+m" (v->counter) \
50 : "r" (a), "r" (&v->counter) \
51 : "cc"); \
52 } \
53
54 #define ATOMIC_OP_RETURN_RELAXED(op, asm_op) \
55 static inline int atomic_##op##_return_relaxed(int a, atomic_t *v) \
56 { \
57 int t; \
58 \
59 __asm__ __volatile__( \
60 "1: lwarx %0,0,%3 # atomic_" #op "_return_relaxed\n" \
61 #asm_op " %0,%2,%0\n" \
62 " stwcx. %0,0,%3\n" \
63 " bne- 1b\n" \
64 : "=&r" (t), "+m" (v->counter) \
65 : "r" (a), "r" (&v->counter) \
66 : "cc"); \
67 \
68 return t; \
69 }
70
71 #define ATOMIC_FETCH_OP_RELAXED(op, asm_op) \
72 static inline int atomic_fetch_##op##_relaxed(int a, atomic_t *v) \
73 { \
74 int res, t; \
75 \
76 __asm__ __volatile__( \
77 "1: lwarx %0,0,%4 # atomic_fetch_" #op "_relaxed\n" \
78 #asm_op " %1,%3,%0\n" \
79 " stwcx. %1,0,%4\n" \
80 " bne- 1b\n" \
81 : "=&r" (res), "=&r" (t), "+m" (v->counter) \
82 : "r" (a), "r" (&v->counter) \
83 : "cc"); \
84 \
85 return res; \
86 }
87
88 #define ATOMIC_OPS(op, asm_op) \
89 ATOMIC_OP(op, asm_op) \
90 ATOMIC_OP_RETURN_RELAXED(op, asm_op) \
91 ATOMIC_FETCH_OP_RELAXED(op, asm_op)
92
ATOMIC_OPS(add,add)93 ATOMIC_OPS(add, add)
94 ATOMIC_OPS(sub, subf)
95
96 #define atomic_add_return_relaxed atomic_add_return_relaxed
97 #define atomic_sub_return_relaxed atomic_sub_return_relaxed
98
99 #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
100 #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
101
102 #undef ATOMIC_OPS
103 #define ATOMIC_OPS(op, asm_op) \
104 ATOMIC_OP(op, asm_op) \
105 ATOMIC_FETCH_OP_RELAXED(op, asm_op)
106
107 ATOMIC_OPS(and, and)
108 ATOMIC_OPS(or, or)
109 ATOMIC_OPS(xor, xor)
110
111 #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
112 #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
113 #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
114
115 #undef ATOMIC_OPS
116 #undef ATOMIC_FETCH_OP_RELAXED
117 #undef ATOMIC_OP_RETURN_RELAXED
118 #undef ATOMIC_OP
119
120 static __inline__ void atomic_inc(atomic_t *v)
121 {
122 int t;
123
124 __asm__ __volatile__(
125 "1: lwarx %0,0,%2 # atomic_inc\n\
126 addic %0,%0,1\n"
127 " stwcx. %0,0,%2 \n\
128 bne- 1b"
129 : "=&r" (t), "+m" (v->counter)
130 : "r" (&v->counter)
131 : "cc", "xer");
132 }
133 #define atomic_inc atomic_inc
134
atomic_inc_return_relaxed(atomic_t * v)135 static __inline__ int atomic_inc_return_relaxed(atomic_t *v)
136 {
137 int t;
138
139 __asm__ __volatile__(
140 "1: lwarx %0,0,%2 # atomic_inc_return_relaxed\n"
141 " addic %0,%0,1\n"
142 " stwcx. %0,0,%2\n"
143 " bne- 1b"
144 : "=&r" (t), "+m" (v->counter)
145 : "r" (&v->counter)
146 : "cc", "xer");
147
148 return t;
149 }
150
atomic_dec(atomic_t * v)151 static __inline__ void atomic_dec(atomic_t *v)
152 {
153 int t;
154
155 __asm__ __volatile__(
156 "1: lwarx %0,0,%2 # atomic_dec\n\
157 addic %0,%0,-1\n"
158 " stwcx. %0,0,%2\n\
159 bne- 1b"
160 : "=&r" (t), "+m" (v->counter)
161 : "r" (&v->counter)
162 : "cc", "xer");
163 }
164 #define atomic_dec atomic_dec
165
atomic_dec_return_relaxed(atomic_t * v)166 static __inline__ int atomic_dec_return_relaxed(atomic_t *v)
167 {
168 int t;
169
170 __asm__ __volatile__(
171 "1: lwarx %0,0,%2 # atomic_dec_return_relaxed\n"
172 " addic %0,%0,-1\n"
173 " stwcx. %0,0,%2\n"
174 " bne- 1b"
175 : "=&r" (t), "+m" (v->counter)
176 : "r" (&v->counter)
177 : "cc", "xer");
178
179 return t;
180 }
181
182 #define atomic_inc_return_relaxed atomic_inc_return_relaxed
183 #define atomic_dec_return_relaxed atomic_dec_return_relaxed
184
185 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
186 #define atomic_cmpxchg_relaxed(v, o, n) \
187 cmpxchg_relaxed(&((v)->counter), (o), (n))
188 #define atomic_cmpxchg_acquire(v, o, n) \
189 cmpxchg_acquire(&((v)->counter), (o), (n))
190
191 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
192 #define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
193
194 /*
195 * Don't want to override the generic atomic_try_cmpxchg_acquire, because
196 * we add a lock hint to the lwarx, which may not be wanted for the
197 * _acquire case (and is not used by the other _acquire variants so it
198 * would be a surprise).
199 */
200 static __always_inline bool
atomic_try_cmpxchg_lock(atomic_t * v,int * old,int new)201 atomic_try_cmpxchg_lock(atomic_t *v, int *old, int new)
202 {
203 int r, o = *old;
204
205 __asm__ __volatile__ (
206 "1:\t" PPC_LWARX(%0,0,%2,1) " # atomic_try_cmpxchg_acquire \n"
207 " cmpw 0,%0,%3 \n"
208 " bne- 2f \n"
209 " stwcx. %4,0,%2 \n"
210 " bne- 1b \n"
211 "\t" PPC_ACQUIRE_BARRIER " \n"
212 "2: \n"
213 : "=&r" (r), "+m" (v->counter)
214 : "r" (&v->counter), "r" (o), "r" (new)
215 : "cr0", "memory");
216
217 if (unlikely(r != o))
218 *old = r;
219 return likely(r == o);
220 }
221
222 /**
223 * atomic_fetch_add_unless - add unless the number is a given value
224 * @v: pointer of type atomic_t
225 * @a: the amount to add to v...
226 * @u: ...unless v is equal to u.
227 *
228 * Atomically adds @a to @v, so long as it was not @u.
229 * Returns the old value of @v.
230 */
atomic_fetch_add_unless(atomic_t * v,int a,int u)231 static __inline__ int atomic_fetch_add_unless(atomic_t *v, int a, int u)
232 {
233 int t;
234
235 __asm__ __volatile__ (
236 PPC_ATOMIC_ENTRY_BARRIER
237 "1: lwarx %0,0,%1 # atomic_fetch_add_unless\n\
238 cmpw 0,%0,%3 \n\
239 beq 2f \n\
240 add %0,%2,%0 \n"
241 " stwcx. %0,0,%1 \n\
242 bne- 1b \n"
243 PPC_ATOMIC_EXIT_BARRIER
244 " subf %0,%2,%0 \n\
245 2:"
246 : "=&r" (t)
247 : "r" (&v->counter), "r" (a), "r" (u)
248 : "cc", "memory");
249
250 return t;
251 }
252 #define atomic_fetch_add_unless atomic_fetch_add_unless
253
254 /**
255 * atomic_inc_not_zero - increment unless the number is zero
256 * @v: pointer of type atomic_t
257 *
258 * Atomically increments @v by 1, so long as @v is non-zero.
259 * Returns non-zero if @v was non-zero, and zero otherwise.
260 */
atomic_inc_not_zero(atomic_t * v)261 static __inline__ int atomic_inc_not_zero(atomic_t *v)
262 {
263 int t1, t2;
264
265 __asm__ __volatile__ (
266 PPC_ATOMIC_ENTRY_BARRIER
267 "1: lwarx %0,0,%2 # atomic_inc_not_zero\n\
268 cmpwi 0,%0,0\n\
269 beq- 2f\n\
270 addic %1,%0,1\n"
271 " stwcx. %1,0,%2\n\
272 bne- 1b\n"
273 PPC_ATOMIC_EXIT_BARRIER
274 "\n\
275 2:"
276 : "=&r" (t1), "=&r" (t2)
277 : "r" (&v->counter)
278 : "cc", "xer", "memory");
279
280 return t1;
281 }
282 #define atomic_inc_not_zero(v) atomic_inc_not_zero((v))
283
284 /*
285 * Atomically test *v and decrement if it is greater than 0.
286 * The function returns the old value of *v minus 1, even if
287 * the atomic variable, v, was not decremented.
288 */
atomic_dec_if_positive(atomic_t * v)289 static __inline__ int atomic_dec_if_positive(atomic_t *v)
290 {
291 int t;
292
293 __asm__ __volatile__(
294 PPC_ATOMIC_ENTRY_BARRIER
295 "1: lwarx %0,0,%1 # atomic_dec_if_positive\n\
296 cmpwi %0,1\n\
297 addi %0,%0,-1\n\
298 blt- 2f\n"
299 " stwcx. %0,0,%1\n\
300 bne- 1b"
301 PPC_ATOMIC_EXIT_BARRIER
302 "\n\
303 2:" : "=&b" (t)
304 : "r" (&v->counter)
305 : "cc", "memory");
306
307 return t;
308 }
309 #define atomic_dec_if_positive atomic_dec_if_positive
310
311 #ifdef __powerpc64__
312
313 #define ATOMIC64_INIT(i) { (i) }
314
atomic64_read(const atomic64_t * v)315 static __inline__ s64 atomic64_read(const atomic64_t *v)
316 {
317 s64 t;
318
319 __asm__ __volatile__("ld%U1%X1 %0,%1" : "=r"(t) : "m"(v->counter));
320
321 return t;
322 }
323
atomic64_set(atomic64_t * v,s64 i)324 static __inline__ void atomic64_set(atomic64_t *v, s64 i)
325 {
326 __asm__ __volatile__("std%U0%X0 %1,%0" : "=m"(v->counter) : "r"(i));
327 }
328
329 #define ATOMIC64_OP(op, asm_op) \
330 static __inline__ void atomic64_##op(s64 a, atomic64_t *v) \
331 { \
332 s64 t; \
333 \
334 __asm__ __volatile__( \
335 "1: ldarx %0,0,%3 # atomic64_" #op "\n" \
336 #asm_op " %0,%2,%0\n" \
337 " stdcx. %0,0,%3 \n" \
338 " bne- 1b\n" \
339 : "=&r" (t), "+m" (v->counter) \
340 : "r" (a), "r" (&v->counter) \
341 : "cc"); \
342 }
343
344 #define ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \
345 static inline s64 \
346 atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \
347 { \
348 s64 t; \
349 \
350 __asm__ __volatile__( \
351 "1: ldarx %0,0,%3 # atomic64_" #op "_return_relaxed\n" \
352 #asm_op " %0,%2,%0\n" \
353 " stdcx. %0,0,%3\n" \
354 " bne- 1b\n" \
355 : "=&r" (t), "+m" (v->counter) \
356 : "r" (a), "r" (&v->counter) \
357 : "cc"); \
358 \
359 return t; \
360 }
361
362 #define ATOMIC64_FETCH_OP_RELAXED(op, asm_op) \
363 static inline s64 \
364 atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \
365 { \
366 s64 res, t; \
367 \
368 __asm__ __volatile__( \
369 "1: ldarx %0,0,%4 # atomic64_fetch_" #op "_relaxed\n" \
370 #asm_op " %1,%3,%0\n" \
371 " stdcx. %1,0,%4\n" \
372 " bne- 1b\n" \
373 : "=&r" (res), "=&r" (t), "+m" (v->counter) \
374 : "r" (a), "r" (&v->counter) \
375 : "cc"); \
376 \
377 return res; \
378 }
379
380 #define ATOMIC64_OPS(op, asm_op) \
381 ATOMIC64_OP(op, asm_op) \
382 ATOMIC64_OP_RETURN_RELAXED(op, asm_op) \
383 ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
384
ATOMIC64_OPS(add,add)385 ATOMIC64_OPS(add, add)
386 ATOMIC64_OPS(sub, subf)
387
388 #define atomic64_add_return_relaxed atomic64_add_return_relaxed
389 #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
390
391 #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
392 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
393
394 #undef ATOMIC64_OPS
395 #define ATOMIC64_OPS(op, asm_op) \
396 ATOMIC64_OP(op, asm_op) \
397 ATOMIC64_FETCH_OP_RELAXED(op, asm_op)
398
399 ATOMIC64_OPS(and, and)
400 ATOMIC64_OPS(or, or)
401 ATOMIC64_OPS(xor, xor)
402
403 #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
404 #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
405 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
406
407 #undef ATOPIC64_OPS
408 #undef ATOMIC64_FETCH_OP_RELAXED
409 #undef ATOMIC64_OP_RETURN_RELAXED
410 #undef ATOMIC64_OP
411
412 static __inline__ void atomic64_inc(atomic64_t *v)
413 {
414 s64 t;
415
416 __asm__ __volatile__(
417 "1: ldarx %0,0,%2 # atomic64_inc\n\
418 addic %0,%0,1\n\
419 stdcx. %0,0,%2 \n\
420 bne- 1b"
421 : "=&r" (t), "+m" (v->counter)
422 : "r" (&v->counter)
423 : "cc", "xer");
424 }
425 #define atomic64_inc atomic64_inc
426
atomic64_inc_return_relaxed(atomic64_t * v)427 static __inline__ s64 atomic64_inc_return_relaxed(atomic64_t *v)
428 {
429 s64 t;
430
431 __asm__ __volatile__(
432 "1: ldarx %0,0,%2 # atomic64_inc_return_relaxed\n"
433 " addic %0,%0,1\n"
434 " stdcx. %0,0,%2\n"
435 " bne- 1b"
436 : "=&r" (t), "+m" (v->counter)
437 : "r" (&v->counter)
438 : "cc", "xer");
439
440 return t;
441 }
442
atomic64_dec(atomic64_t * v)443 static __inline__ void atomic64_dec(atomic64_t *v)
444 {
445 s64 t;
446
447 __asm__ __volatile__(
448 "1: ldarx %0,0,%2 # atomic64_dec\n\
449 addic %0,%0,-1\n\
450 stdcx. %0,0,%2\n\
451 bne- 1b"
452 : "=&r" (t), "+m" (v->counter)
453 : "r" (&v->counter)
454 : "cc", "xer");
455 }
456 #define atomic64_dec atomic64_dec
457
atomic64_dec_return_relaxed(atomic64_t * v)458 static __inline__ s64 atomic64_dec_return_relaxed(atomic64_t *v)
459 {
460 s64 t;
461
462 __asm__ __volatile__(
463 "1: ldarx %0,0,%2 # atomic64_dec_return_relaxed\n"
464 " addic %0,%0,-1\n"
465 " stdcx. %0,0,%2\n"
466 " bne- 1b"
467 : "=&r" (t), "+m" (v->counter)
468 : "r" (&v->counter)
469 : "cc", "xer");
470
471 return t;
472 }
473
474 #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
475 #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
476
477 /*
478 * Atomically test *v and decrement if it is greater than 0.
479 * The function returns the old value of *v minus 1.
480 */
atomic64_dec_if_positive(atomic64_t * v)481 static __inline__ s64 atomic64_dec_if_positive(atomic64_t *v)
482 {
483 s64 t;
484
485 __asm__ __volatile__(
486 PPC_ATOMIC_ENTRY_BARRIER
487 "1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\
488 addic. %0,%0,-1\n\
489 blt- 2f\n\
490 stdcx. %0,0,%1\n\
491 bne- 1b"
492 PPC_ATOMIC_EXIT_BARRIER
493 "\n\
494 2:" : "=&r" (t)
495 : "r" (&v->counter)
496 : "cc", "xer", "memory");
497
498 return t;
499 }
500 #define atomic64_dec_if_positive atomic64_dec_if_positive
501
502 #define atomic64_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
503 #define atomic64_cmpxchg_relaxed(v, o, n) \
504 cmpxchg_relaxed(&((v)->counter), (o), (n))
505 #define atomic64_cmpxchg_acquire(v, o, n) \
506 cmpxchg_acquire(&((v)->counter), (o), (n))
507
508 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
509 #define atomic64_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
510
511 /**
512 * atomic64_fetch_add_unless - add unless the number is a given value
513 * @v: pointer of type atomic64_t
514 * @a: the amount to add to v...
515 * @u: ...unless v is equal to u.
516 *
517 * Atomically adds @a to @v, so long as it was not @u.
518 * Returns the old value of @v.
519 */
atomic64_fetch_add_unless(atomic64_t * v,s64 a,s64 u)520 static __inline__ s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
521 {
522 s64 t;
523
524 __asm__ __volatile__ (
525 PPC_ATOMIC_ENTRY_BARRIER
526 "1: ldarx %0,0,%1 # atomic64_fetch_add_unless\n\
527 cmpd 0,%0,%3 \n\
528 beq 2f \n\
529 add %0,%2,%0 \n"
530 " stdcx. %0,0,%1 \n\
531 bne- 1b \n"
532 PPC_ATOMIC_EXIT_BARRIER
533 " subf %0,%2,%0 \n\
534 2:"
535 : "=&r" (t)
536 : "r" (&v->counter), "r" (a), "r" (u)
537 : "cc", "memory");
538
539 return t;
540 }
541 #define atomic64_fetch_add_unless atomic64_fetch_add_unless
542
543 /**
544 * atomic_inc64_not_zero - increment unless the number is zero
545 * @v: pointer of type atomic64_t
546 *
547 * Atomically increments @v by 1, so long as @v is non-zero.
548 * Returns non-zero if @v was non-zero, and zero otherwise.
549 */
atomic64_inc_not_zero(atomic64_t * v)550 static __inline__ int atomic64_inc_not_zero(atomic64_t *v)
551 {
552 s64 t1, t2;
553
554 __asm__ __volatile__ (
555 PPC_ATOMIC_ENTRY_BARRIER
556 "1: ldarx %0,0,%2 # atomic64_inc_not_zero\n\
557 cmpdi 0,%0,0\n\
558 beq- 2f\n\
559 addic %1,%0,1\n\
560 stdcx. %1,0,%2\n\
561 bne- 1b\n"
562 PPC_ATOMIC_EXIT_BARRIER
563 "\n\
564 2:"
565 : "=&r" (t1), "=&r" (t2)
566 : "r" (&v->counter)
567 : "cc", "xer", "memory");
568
569 return t1 != 0;
570 }
571 #define atomic64_inc_not_zero(v) atomic64_inc_not_zero((v))
572
573 #endif /* __powerpc64__ */
574
575 #endif /* __KERNEL__ */
576 #endif /* _ASM_POWERPC_ATOMIC_H_ */
577