1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Atomic operations usable in machine independent code */
3 #ifndef _LINUX_ATOMIC_H
4 #define _LINUX_ATOMIC_H
5 #include <asm/atomic.h>
6 #include <asm/barrier.h>
7
8 /*
9 * Relaxed variants of xchg, cmpxchg and some atomic operations.
10 *
11 * We support four variants:
12 *
13 * - Fully ordered: The default implementation, no suffix required.
14 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
15 * - Release: Provides RELEASE semantics, _release suffix.
16 * - Relaxed: No ordering guarantees, _relaxed suffix.
17 *
18 * For compound atomics performing both a load and a store, ACQUIRE
19 * semantics apply only to the load and RELEASE semantics only to the
20 * store portion of the operation. Note that a failed cmpxchg_acquire
21 * does -not- imply any memory ordering constraints.
22 *
23 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
24 */
25
26 #ifndef atomic_read_acquire
27 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
28 #endif
29
30 #ifndef atomic_set_release
31 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
32 #endif
33
34 /*
35 * The idea here is to build acquire/release variants by adding explicit
36 * barriers on top of the relaxed variant. In the case where the relaxed
37 * variant is already fully ordered, no additional barriers are needed.
38 *
39 * Besides, if an arch has a special barrier for acquire/release, it could
40 * implement its own __atomic_op_* and use the same framework for building
41 * variants
42 *
43 * If an architecture overrides __atomic_op_acquire() it will probably want
44 * to define smp_mb__after_spinlock().
45 */
46 #ifndef __atomic_op_acquire
47 #define __atomic_op_acquire(op, args...) \
48 ({ \
49 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
50 smp_mb__after_atomic(); \
51 __ret; \
52 })
53 #endif
54
55 #ifndef __atomic_op_release
56 #define __atomic_op_release(op, args...) \
57 ({ \
58 smp_mb__before_atomic(); \
59 op##_relaxed(args); \
60 })
61 #endif
62
63 #ifndef __atomic_op_fence
64 #define __atomic_op_fence(op, args...) \
65 ({ \
66 typeof(op##_relaxed(args)) __ret; \
67 smp_mb__before_atomic(); \
68 __ret = op##_relaxed(args); \
69 smp_mb__after_atomic(); \
70 __ret; \
71 })
72 #endif
73
74 /* atomic_add_return_relaxed */
75 #ifndef atomic_add_return_relaxed
76 #define atomic_add_return_relaxed atomic_add_return
77 #define atomic_add_return_acquire atomic_add_return
78 #define atomic_add_return_release atomic_add_return
79
80 #else /* atomic_add_return_relaxed */
81
82 #ifndef atomic_add_return_acquire
83 #define atomic_add_return_acquire(...) \
84 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
85 #endif
86
87 #ifndef atomic_add_return_release
88 #define atomic_add_return_release(...) \
89 __atomic_op_release(atomic_add_return, __VA_ARGS__)
90 #endif
91
92 #ifndef atomic_add_return
93 #define atomic_add_return(...) \
94 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
95 #endif
96 #endif /* atomic_add_return_relaxed */
97
98 /* atomic_inc_return_relaxed */
99 #ifndef atomic_inc_return_relaxed
100 #define atomic_inc_return_relaxed atomic_inc_return
101 #define atomic_inc_return_acquire atomic_inc_return
102 #define atomic_inc_return_release atomic_inc_return
103
104 #else /* atomic_inc_return_relaxed */
105
106 #ifndef atomic_inc_return_acquire
107 #define atomic_inc_return_acquire(...) \
108 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
109 #endif
110
111 #ifndef atomic_inc_return_release
112 #define atomic_inc_return_release(...) \
113 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
114 #endif
115
116 #ifndef atomic_inc_return
117 #define atomic_inc_return(...) \
118 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
119 #endif
120 #endif /* atomic_inc_return_relaxed */
121
122 /* atomic_sub_return_relaxed */
123 #ifndef atomic_sub_return_relaxed
124 #define atomic_sub_return_relaxed atomic_sub_return
125 #define atomic_sub_return_acquire atomic_sub_return
126 #define atomic_sub_return_release atomic_sub_return
127
128 #else /* atomic_sub_return_relaxed */
129
130 #ifndef atomic_sub_return_acquire
131 #define atomic_sub_return_acquire(...) \
132 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
133 #endif
134
135 #ifndef atomic_sub_return_release
136 #define atomic_sub_return_release(...) \
137 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
138 #endif
139
140 #ifndef atomic_sub_return
141 #define atomic_sub_return(...) \
142 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
143 #endif
144 #endif /* atomic_sub_return_relaxed */
145
146 /* atomic_dec_return_relaxed */
147 #ifndef atomic_dec_return_relaxed
148 #define atomic_dec_return_relaxed atomic_dec_return
149 #define atomic_dec_return_acquire atomic_dec_return
150 #define atomic_dec_return_release atomic_dec_return
151
152 #else /* atomic_dec_return_relaxed */
153
154 #ifndef atomic_dec_return_acquire
155 #define atomic_dec_return_acquire(...) \
156 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
157 #endif
158
159 #ifndef atomic_dec_return_release
160 #define atomic_dec_return_release(...) \
161 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
162 #endif
163
164 #ifndef atomic_dec_return
165 #define atomic_dec_return(...) \
166 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
167 #endif
168 #endif /* atomic_dec_return_relaxed */
169
170
171 /* atomic_fetch_add_relaxed */
172 #ifndef atomic_fetch_add_relaxed
173 #define atomic_fetch_add_relaxed atomic_fetch_add
174 #define atomic_fetch_add_acquire atomic_fetch_add
175 #define atomic_fetch_add_release atomic_fetch_add
176
177 #else /* atomic_fetch_add_relaxed */
178
179 #ifndef atomic_fetch_add_acquire
180 #define atomic_fetch_add_acquire(...) \
181 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
182 #endif
183
184 #ifndef atomic_fetch_add_release
185 #define atomic_fetch_add_release(...) \
186 __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
187 #endif
188
189 #ifndef atomic_fetch_add
190 #define atomic_fetch_add(...) \
191 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
192 #endif
193 #endif /* atomic_fetch_add_relaxed */
194
195 /* atomic_fetch_inc_relaxed */
196 #ifndef atomic_fetch_inc_relaxed
197
198 #ifndef atomic_fetch_inc
199 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
200 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
201 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
202 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
203 #else /* atomic_fetch_inc */
204 #define atomic_fetch_inc_relaxed atomic_fetch_inc
205 #define atomic_fetch_inc_acquire atomic_fetch_inc
206 #define atomic_fetch_inc_release atomic_fetch_inc
207 #endif /* atomic_fetch_inc */
208
209 #else /* atomic_fetch_inc_relaxed */
210
211 #ifndef atomic_fetch_inc_acquire
212 #define atomic_fetch_inc_acquire(...) \
213 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
214 #endif
215
216 #ifndef atomic_fetch_inc_release
217 #define atomic_fetch_inc_release(...) \
218 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
219 #endif
220
221 #ifndef atomic_fetch_inc
222 #define atomic_fetch_inc(...) \
223 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
224 #endif
225 #endif /* atomic_fetch_inc_relaxed */
226
227 /* atomic_fetch_sub_relaxed */
228 #ifndef atomic_fetch_sub_relaxed
229 #define atomic_fetch_sub_relaxed atomic_fetch_sub
230 #define atomic_fetch_sub_acquire atomic_fetch_sub
231 #define atomic_fetch_sub_release atomic_fetch_sub
232
233 #else /* atomic_fetch_sub_relaxed */
234
235 #ifndef atomic_fetch_sub_acquire
236 #define atomic_fetch_sub_acquire(...) \
237 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
238 #endif
239
240 #ifndef atomic_fetch_sub_release
241 #define atomic_fetch_sub_release(...) \
242 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
243 #endif
244
245 #ifndef atomic_fetch_sub
246 #define atomic_fetch_sub(...) \
247 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
248 #endif
249 #endif /* atomic_fetch_sub_relaxed */
250
251 /* atomic_fetch_dec_relaxed */
252 #ifndef atomic_fetch_dec_relaxed
253
254 #ifndef atomic_fetch_dec
255 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
256 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
257 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
258 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
259 #else /* atomic_fetch_dec */
260 #define atomic_fetch_dec_relaxed atomic_fetch_dec
261 #define atomic_fetch_dec_acquire atomic_fetch_dec
262 #define atomic_fetch_dec_release atomic_fetch_dec
263 #endif /* atomic_fetch_dec */
264
265 #else /* atomic_fetch_dec_relaxed */
266
267 #ifndef atomic_fetch_dec_acquire
268 #define atomic_fetch_dec_acquire(...) \
269 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
270 #endif
271
272 #ifndef atomic_fetch_dec_release
273 #define atomic_fetch_dec_release(...) \
274 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
275 #endif
276
277 #ifndef atomic_fetch_dec
278 #define atomic_fetch_dec(...) \
279 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
280 #endif
281 #endif /* atomic_fetch_dec_relaxed */
282
283 /* atomic_fetch_or_relaxed */
284 #ifndef atomic_fetch_or_relaxed
285 #define atomic_fetch_or_relaxed atomic_fetch_or
286 #define atomic_fetch_or_acquire atomic_fetch_or
287 #define atomic_fetch_or_release atomic_fetch_or
288
289 #else /* atomic_fetch_or_relaxed */
290
291 #ifndef atomic_fetch_or_acquire
292 #define atomic_fetch_or_acquire(...) \
293 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
294 #endif
295
296 #ifndef atomic_fetch_or_release
297 #define atomic_fetch_or_release(...) \
298 __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
299 #endif
300
301 #ifndef atomic_fetch_or
302 #define atomic_fetch_or(...) \
303 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
304 #endif
305 #endif /* atomic_fetch_or_relaxed */
306
307 /* atomic_fetch_and_relaxed */
308 #ifndef atomic_fetch_and_relaxed
309 #define atomic_fetch_and_relaxed atomic_fetch_and
310 #define atomic_fetch_and_acquire atomic_fetch_and
311 #define atomic_fetch_and_release atomic_fetch_and
312
313 #else /* atomic_fetch_and_relaxed */
314
315 #ifndef atomic_fetch_and_acquire
316 #define atomic_fetch_and_acquire(...) \
317 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
318 #endif
319
320 #ifndef atomic_fetch_and_release
321 #define atomic_fetch_and_release(...) \
322 __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
323 #endif
324
325 #ifndef atomic_fetch_and
326 #define atomic_fetch_and(...) \
327 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
328 #endif
329 #endif /* atomic_fetch_and_relaxed */
330
331 #ifdef atomic_andnot
332 /* atomic_fetch_andnot_relaxed */
333 #ifndef atomic_fetch_andnot_relaxed
334 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot
335 #define atomic_fetch_andnot_acquire atomic_fetch_andnot
336 #define atomic_fetch_andnot_release atomic_fetch_andnot
337
338 #else /* atomic_fetch_andnot_relaxed */
339
340 #ifndef atomic_fetch_andnot_acquire
341 #define atomic_fetch_andnot_acquire(...) \
342 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
343 #endif
344
345 #ifndef atomic_fetch_andnot_release
346 #define atomic_fetch_andnot_release(...) \
347 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
348 #endif
349
350 #ifndef atomic_fetch_andnot
351 #define atomic_fetch_andnot(...) \
352 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
353 #endif
354 #endif /* atomic_fetch_andnot_relaxed */
355 #endif /* atomic_andnot */
356
357 /* atomic_fetch_xor_relaxed */
358 #ifndef atomic_fetch_xor_relaxed
359 #define atomic_fetch_xor_relaxed atomic_fetch_xor
360 #define atomic_fetch_xor_acquire atomic_fetch_xor
361 #define atomic_fetch_xor_release atomic_fetch_xor
362
363 #else /* atomic_fetch_xor_relaxed */
364
365 #ifndef atomic_fetch_xor_acquire
366 #define atomic_fetch_xor_acquire(...) \
367 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
368 #endif
369
370 #ifndef atomic_fetch_xor_release
371 #define atomic_fetch_xor_release(...) \
372 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
373 #endif
374
375 #ifndef atomic_fetch_xor
376 #define atomic_fetch_xor(...) \
377 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
378 #endif
379 #endif /* atomic_fetch_xor_relaxed */
380
381
382 /* atomic_xchg_relaxed */
383 #ifndef atomic_xchg_relaxed
384 #define atomic_xchg_relaxed atomic_xchg
385 #define atomic_xchg_acquire atomic_xchg
386 #define atomic_xchg_release atomic_xchg
387
388 #else /* atomic_xchg_relaxed */
389
390 #ifndef atomic_xchg_acquire
391 #define atomic_xchg_acquire(...) \
392 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
393 #endif
394
395 #ifndef atomic_xchg_release
396 #define atomic_xchg_release(...) \
397 __atomic_op_release(atomic_xchg, __VA_ARGS__)
398 #endif
399
400 #ifndef atomic_xchg
401 #define atomic_xchg(...) \
402 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
403 #endif
404 #endif /* atomic_xchg_relaxed */
405
406 /* atomic_cmpxchg_relaxed */
407 #ifndef atomic_cmpxchg_relaxed
408 #define atomic_cmpxchg_relaxed atomic_cmpxchg
409 #define atomic_cmpxchg_acquire atomic_cmpxchg
410 #define atomic_cmpxchg_release atomic_cmpxchg
411
412 #else /* atomic_cmpxchg_relaxed */
413
414 #ifndef atomic_cmpxchg_acquire
415 #define atomic_cmpxchg_acquire(...) \
416 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
417 #endif
418
419 #ifndef atomic_cmpxchg_release
420 #define atomic_cmpxchg_release(...) \
421 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
422 #endif
423
424 #ifndef atomic_cmpxchg
425 #define atomic_cmpxchg(...) \
426 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
427 #endif
428 #endif /* atomic_cmpxchg_relaxed */
429
430 #ifndef atomic_try_cmpxchg
431
432 #define __atomic_try_cmpxchg(type, _p, _po, _n) \
433 ({ \
434 typeof(_po) __po = (_po); \
435 typeof(*(_po)) __r, __o = *__po; \
436 __r = atomic_cmpxchg##type((_p), __o, (_n)); \
437 if (unlikely(__r != __o)) \
438 *__po = __r; \
439 likely(__r == __o); \
440 })
441
442 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n)
443 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
444 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
445 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
446
447 #else /* atomic_try_cmpxchg */
448 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
449 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
450 #define atomic_try_cmpxchg_release atomic_try_cmpxchg
451 #endif /* atomic_try_cmpxchg */
452
453 /* cmpxchg_relaxed */
454 #ifndef cmpxchg_relaxed
455 #define cmpxchg_relaxed cmpxchg
456 #define cmpxchg_acquire cmpxchg
457 #define cmpxchg_release cmpxchg
458
459 #else /* cmpxchg_relaxed */
460
461 #ifndef cmpxchg_acquire
462 #define cmpxchg_acquire(...) \
463 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
464 #endif
465
466 #ifndef cmpxchg_release
467 #define cmpxchg_release(...) \
468 __atomic_op_release(cmpxchg, __VA_ARGS__)
469 #endif
470
471 #ifndef cmpxchg
472 #define cmpxchg(...) \
473 __atomic_op_fence(cmpxchg, __VA_ARGS__)
474 #endif
475 #endif /* cmpxchg_relaxed */
476
477 /* cmpxchg64_relaxed */
478 #ifndef cmpxchg64_relaxed
479 #define cmpxchg64_relaxed cmpxchg64
480 #define cmpxchg64_acquire cmpxchg64
481 #define cmpxchg64_release cmpxchg64
482
483 #else /* cmpxchg64_relaxed */
484
485 #ifndef cmpxchg64_acquire
486 #define cmpxchg64_acquire(...) \
487 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
488 #endif
489
490 #ifndef cmpxchg64_release
491 #define cmpxchg64_release(...) \
492 __atomic_op_release(cmpxchg64, __VA_ARGS__)
493 #endif
494
495 #ifndef cmpxchg64
496 #define cmpxchg64(...) \
497 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
498 #endif
499 #endif /* cmpxchg64_relaxed */
500
501 /* xchg_relaxed */
502 #ifndef xchg_relaxed
503 #define xchg_relaxed xchg
504 #define xchg_acquire xchg
505 #define xchg_release xchg
506
507 #else /* xchg_relaxed */
508
509 #ifndef xchg_acquire
510 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
511 #endif
512
513 #ifndef xchg_release
514 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
515 #endif
516
517 #ifndef xchg
518 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
519 #endif
520 #endif /* xchg_relaxed */
521
522 /**
523 * atomic_add_unless - add unless the number is already a given value
524 * @v: pointer of type atomic_t
525 * @a: the amount to add to v...
526 * @u: ...unless v is equal to u.
527 *
528 * Atomically adds @a to @v, so long as @v was not already @u.
529 * Returns non-zero if @v was not @u, and zero otherwise.
530 */
atomic_add_unless(atomic_t * v,int a,int u)531 static inline int atomic_add_unless(atomic_t *v, int a, int u)
532 {
533 return __atomic_add_unless(v, a, u) != u;
534 }
535
536 /**
537 * atomic_inc_not_zero - increment unless the number is zero
538 * @v: pointer of type atomic_t
539 *
540 * Atomically increments @v by 1, so long as @v is non-zero.
541 * Returns non-zero if @v was non-zero, and zero otherwise.
542 */
543 #ifndef atomic_inc_not_zero
544 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
545 #endif
546
547 #ifndef atomic_andnot
atomic_andnot(int i,atomic_t * v)548 static inline void atomic_andnot(int i, atomic_t *v)
549 {
550 atomic_and(~i, v);
551 }
552
atomic_fetch_andnot(int i,atomic_t * v)553 static inline int atomic_fetch_andnot(int i, atomic_t *v)
554 {
555 return atomic_fetch_and(~i, v);
556 }
557
atomic_fetch_andnot_relaxed(int i,atomic_t * v)558 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
559 {
560 return atomic_fetch_and_relaxed(~i, v);
561 }
562
atomic_fetch_andnot_acquire(int i,atomic_t * v)563 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
564 {
565 return atomic_fetch_and_acquire(~i, v);
566 }
567
atomic_fetch_andnot_release(int i,atomic_t * v)568 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
569 {
570 return atomic_fetch_and_release(~i, v);
571 }
572 #endif
573
574 /**
575 * atomic_inc_not_zero_hint - increment if not null
576 * @v: pointer of type atomic_t
577 * @hint: probable value of the atomic before the increment
578 *
579 * This version of atomic_inc_not_zero() gives a hint of probable
580 * value of the atomic. This helps processor to not read the memory
581 * before doing the atomic read/modify/write cycle, lowering
582 * number of bus transactions on some arches.
583 *
584 * Returns: 0 if increment was not done, 1 otherwise.
585 */
586 #ifndef atomic_inc_not_zero_hint
atomic_inc_not_zero_hint(atomic_t * v,int hint)587 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
588 {
589 int val, c = hint;
590
591 /* sanity test, should be removed by compiler if hint is a constant */
592 if (!hint)
593 return atomic_inc_not_zero(v);
594
595 do {
596 val = atomic_cmpxchg(v, c, c + 1);
597 if (val == c)
598 return 1;
599 c = val;
600 } while (c);
601
602 return 0;
603 }
604 #endif
605
606 #ifndef atomic_inc_unless_negative
atomic_inc_unless_negative(atomic_t * p)607 static inline int atomic_inc_unless_negative(atomic_t *p)
608 {
609 int v, v1;
610 for (v = 0; v >= 0; v = v1) {
611 v1 = atomic_cmpxchg(p, v, v + 1);
612 if (likely(v1 == v))
613 return 1;
614 }
615 return 0;
616 }
617 #endif
618
619 #ifndef atomic_dec_unless_positive
atomic_dec_unless_positive(atomic_t * p)620 static inline int atomic_dec_unless_positive(atomic_t *p)
621 {
622 int v, v1;
623 for (v = 0; v <= 0; v = v1) {
624 v1 = atomic_cmpxchg(p, v, v - 1);
625 if (likely(v1 == v))
626 return 1;
627 }
628 return 0;
629 }
630 #endif
631
632 /*
633 * atomic_dec_if_positive - decrement by 1 if old value positive
634 * @v: pointer of type atomic_t
635 *
636 * The function returns the old value of *v minus 1, even if
637 * the atomic variable, v, was not decremented.
638 */
639 #ifndef atomic_dec_if_positive
atomic_dec_if_positive(atomic_t * v)640 static inline int atomic_dec_if_positive(atomic_t *v)
641 {
642 int c, old, dec;
643 c = atomic_read(v);
644 for (;;) {
645 dec = c - 1;
646 if (unlikely(dec < 0))
647 break;
648 old = atomic_cmpxchg((v), c, dec);
649 if (likely(old == c))
650 break;
651 c = old;
652 }
653 return dec;
654 }
655 #endif
656
657 #ifdef CONFIG_GENERIC_ATOMIC64
658 #include <asm-generic/atomic64.h>
659 #endif
660
661 #ifndef atomic64_read_acquire
662 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
663 #endif
664
665 #ifndef atomic64_set_release
666 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
667 #endif
668
669 /* atomic64_add_return_relaxed */
670 #ifndef atomic64_add_return_relaxed
671 #define atomic64_add_return_relaxed atomic64_add_return
672 #define atomic64_add_return_acquire atomic64_add_return
673 #define atomic64_add_return_release atomic64_add_return
674
675 #else /* atomic64_add_return_relaxed */
676
677 #ifndef atomic64_add_return_acquire
678 #define atomic64_add_return_acquire(...) \
679 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
680 #endif
681
682 #ifndef atomic64_add_return_release
683 #define atomic64_add_return_release(...) \
684 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
685 #endif
686
687 #ifndef atomic64_add_return
688 #define atomic64_add_return(...) \
689 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
690 #endif
691 #endif /* atomic64_add_return_relaxed */
692
693 /* atomic64_inc_return_relaxed */
694 #ifndef atomic64_inc_return_relaxed
695 #define atomic64_inc_return_relaxed atomic64_inc_return
696 #define atomic64_inc_return_acquire atomic64_inc_return
697 #define atomic64_inc_return_release atomic64_inc_return
698
699 #else /* atomic64_inc_return_relaxed */
700
701 #ifndef atomic64_inc_return_acquire
702 #define atomic64_inc_return_acquire(...) \
703 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
704 #endif
705
706 #ifndef atomic64_inc_return_release
707 #define atomic64_inc_return_release(...) \
708 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
709 #endif
710
711 #ifndef atomic64_inc_return
712 #define atomic64_inc_return(...) \
713 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
714 #endif
715 #endif /* atomic64_inc_return_relaxed */
716
717
718 /* atomic64_sub_return_relaxed */
719 #ifndef atomic64_sub_return_relaxed
720 #define atomic64_sub_return_relaxed atomic64_sub_return
721 #define atomic64_sub_return_acquire atomic64_sub_return
722 #define atomic64_sub_return_release atomic64_sub_return
723
724 #else /* atomic64_sub_return_relaxed */
725
726 #ifndef atomic64_sub_return_acquire
727 #define atomic64_sub_return_acquire(...) \
728 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
729 #endif
730
731 #ifndef atomic64_sub_return_release
732 #define atomic64_sub_return_release(...) \
733 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
734 #endif
735
736 #ifndef atomic64_sub_return
737 #define atomic64_sub_return(...) \
738 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
739 #endif
740 #endif /* atomic64_sub_return_relaxed */
741
742 /* atomic64_dec_return_relaxed */
743 #ifndef atomic64_dec_return_relaxed
744 #define atomic64_dec_return_relaxed atomic64_dec_return
745 #define atomic64_dec_return_acquire atomic64_dec_return
746 #define atomic64_dec_return_release atomic64_dec_return
747
748 #else /* atomic64_dec_return_relaxed */
749
750 #ifndef atomic64_dec_return_acquire
751 #define atomic64_dec_return_acquire(...) \
752 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
753 #endif
754
755 #ifndef atomic64_dec_return_release
756 #define atomic64_dec_return_release(...) \
757 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
758 #endif
759
760 #ifndef atomic64_dec_return
761 #define atomic64_dec_return(...) \
762 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
763 #endif
764 #endif /* atomic64_dec_return_relaxed */
765
766
767 /* atomic64_fetch_add_relaxed */
768 #ifndef atomic64_fetch_add_relaxed
769 #define atomic64_fetch_add_relaxed atomic64_fetch_add
770 #define atomic64_fetch_add_acquire atomic64_fetch_add
771 #define atomic64_fetch_add_release atomic64_fetch_add
772
773 #else /* atomic64_fetch_add_relaxed */
774
775 #ifndef atomic64_fetch_add_acquire
776 #define atomic64_fetch_add_acquire(...) \
777 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
778 #endif
779
780 #ifndef atomic64_fetch_add_release
781 #define atomic64_fetch_add_release(...) \
782 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
783 #endif
784
785 #ifndef atomic64_fetch_add
786 #define atomic64_fetch_add(...) \
787 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
788 #endif
789 #endif /* atomic64_fetch_add_relaxed */
790
791 /* atomic64_fetch_inc_relaxed */
792 #ifndef atomic64_fetch_inc_relaxed
793
794 #ifndef atomic64_fetch_inc
795 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
796 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
797 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
798 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
799 #else /* atomic64_fetch_inc */
800 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc
801 #define atomic64_fetch_inc_acquire atomic64_fetch_inc
802 #define atomic64_fetch_inc_release atomic64_fetch_inc
803 #endif /* atomic64_fetch_inc */
804
805 #else /* atomic64_fetch_inc_relaxed */
806
807 #ifndef atomic64_fetch_inc_acquire
808 #define atomic64_fetch_inc_acquire(...) \
809 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
810 #endif
811
812 #ifndef atomic64_fetch_inc_release
813 #define atomic64_fetch_inc_release(...) \
814 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
815 #endif
816
817 #ifndef atomic64_fetch_inc
818 #define atomic64_fetch_inc(...) \
819 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
820 #endif
821 #endif /* atomic64_fetch_inc_relaxed */
822
823 /* atomic64_fetch_sub_relaxed */
824 #ifndef atomic64_fetch_sub_relaxed
825 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub
826 #define atomic64_fetch_sub_acquire atomic64_fetch_sub
827 #define atomic64_fetch_sub_release atomic64_fetch_sub
828
829 #else /* atomic64_fetch_sub_relaxed */
830
831 #ifndef atomic64_fetch_sub_acquire
832 #define atomic64_fetch_sub_acquire(...) \
833 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
834 #endif
835
836 #ifndef atomic64_fetch_sub_release
837 #define atomic64_fetch_sub_release(...) \
838 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
839 #endif
840
841 #ifndef atomic64_fetch_sub
842 #define atomic64_fetch_sub(...) \
843 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
844 #endif
845 #endif /* atomic64_fetch_sub_relaxed */
846
847 /* atomic64_fetch_dec_relaxed */
848 #ifndef atomic64_fetch_dec_relaxed
849
850 #ifndef atomic64_fetch_dec
851 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
852 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
853 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
854 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
855 #else /* atomic64_fetch_dec */
856 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec
857 #define atomic64_fetch_dec_acquire atomic64_fetch_dec
858 #define atomic64_fetch_dec_release atomic64_fetch_dec
859 #endif /* atomic64_fetch_dec */
860
861 #else /* atomic64_fetch_dec_relaxed */
862
863 #ifndef atomic64_fetch_dec_acquire
864 #define atomic64_fetch_dec_acquire(...) \
865 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
866 #endif
867
868 #ifndef atomic64_fetch_dec_release
869 #define atomic64_fetch_dec_release(...) \
870 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
871 #endif
872
873 #ifndef atomic64_fetch_dec
874 #define atomic64_fetch_dec(...) \
875 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
876 #endif
877 #endif /* atomic64_fetch_dec_relaxed */
878
879 /* atomic64_fetch_or_relaxed */
880 #ifndef atomic64_fetch_or_relaxed
881 #define atomic64_fetch_or_relaxed atomic64_fetch_or
882 #define atomic64_fetch_or_acquire atomic64_fetch_or
883 #define atomic64_fetch_or_release atomic64_fetch_or
884
885 #else /* atomic64_fetch_or_relaxed */
886
887 #ifndef atomic64_fetch_or_acquire
888 #define atomic64_fetch_or_acquire(...) \
889 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
890 #endif
891
892 #ifndef atomic64_fetch_or_release
893 #define atomic64_fetch_or_release(...) \
894 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
895 #endif
896
897 #ifndef atomic64_fetch_or
898 #define atomic64_fetch_or(...) \
899 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
900 #endif
901 #endif /* atomic64_fetch_or_relaxed */
902
903 /* atomic64_fetch_and_relaxed */
904 #ifndef atomic64_fetch_and_relaxed
905 #define atomic64_fetch_and_relaxed atomic64_fetch_and
906 #define atomic64_fetch_and_acquire atomic64_fetch_and
907 #define atomic64_fetch_and_release atomic64_fetch_and
908
909 #else /* atomic64_fetch_and_relaxed */
910
911 #ifndef atomic64_fetch_and_acquire
912 #define atomic64_fetch_and_acquire(...) \
913 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
914 #endif
915
916 #ifndef atomic64_fetch_and_release
917 #define atomic64_fetch_and_release(...) \
918 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
919 #endif
920
921 #ifndef atomic64_fetch_and
922 #define atomic64_fetch_and(...) \
923 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
924 #endif
925 #endif /* atomic64_fetch_and_relaxed */
926
927 #ifdef atomic64_andnot
928 /* atomic64_fetch_andnot_relaxed */
929 #ifndef atomic64_fetch_andnot_relaxed
930 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
931 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
932 #define atomic64_fetch_andnot_release atomic64_fetch_andnot
933
934 #else /* atomic64_fetch_andnot_relaxed */
935
936 #ifndef atomic64_fetch_andnot_acquire
937 #define atomic64_fetch_andnot_acquire(...) \
938 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
939 #endif
940
941 #ifndef atomic64_fetch_andnot_release
942 #define atomic64_fetch_andnot_release(...) \
943 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
944 #endif
945
946 #ifndef atomic64_fetch_andnot
947 #define atomic64_fetch_andnot(...) \
948 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
949 #endif
950 #endif /* atomic64_fetch_andnot_relaxed */
951 #endif /* atomic64_andnot */
952
953 /* atomic64_fetch_xor_relaxed */
954 #ifndef atomic64_fetch_xor_relaxed
955 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor
956 #define atomic64_fetch_xor_acquire atomic64_fetch_xor
957 #define atomic64_fetch_xor_release atomic64_fetch_xor
958
959 #else /* atomic64_fetch_xor_relaxed */
960
961 #ifndef atomic64_fetch_xor_acquire
962 #define atomic64_fetch_xor_acquire(...) \
963 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
964 #endif
965
966 #ifndef atomic64_fetch_xor_release
967 #define atomic64_fetch_xor_release(...) \
968 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
969 #endif
970
971 #ifndef atomic64_fetch_xor
972 #define atomic64_fetch_xor(...) \
973 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
974 #endif
975 #endif /* atomic64_fetch_xor_relaxed */
976
977
978 /* atomic64_xchg_relaxed */
979 #ifndef atomic64_xchg_relaxed
980 #define atomic64_xchg_relaxed atomic64_xchg
981 #define atomic64_xchg_acquire atomic64_xchg
982 #define atomic64_xchg_release atomic64_xchg
983
984 #else /* atomic64_xchg_relaxed */
985
986 #ifndef atomic64_xchg_acquire
987 #define atomic64_xchg_acquire(...) \
988 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
989 #endif
990
991 #ifndef atomic64_xchg_release
992 #define atomic64_xchg_release(...) \
993 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
994 #endif
995
996 #ifndef atomic64_xchg
997 #define atomic64_xchg(...) \
998 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
999 #endif
1000 #endif /* atomic64_xchg_relaxed */
1001
1002 /* atomic64_cmpxchg_relaxed */
1003 #ifndef atomic64_cmpxchg_relaxed
1004 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1005 #define atomic64_cmpxchg_acquire atomic64_cmpxchg
1006 #define atomic64_cmpxchg_release atomic64_cmpxchg
1007
1008 #else /* atomic64_cmpxchg_relaxed */
1009
1010 #ifndef atomic64_cmpxchg_acquire
1011 #define atomic64_cmpxchg_acquire(...) \
1012 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1013 #endif
1014
1015 #ifndef atomic64_cmpxchg_release
1016 #define atomic64_cmpxchg_release(...) \
1017 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1018 #endif
1019
1020 #ifndef atomic64_cmpxchg
1021 #define atomic64_cmpxchg(...) \
1022 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1023 #endif
1024 #endif /* atomic64_cmpxchg_relaxed */
1025
1026 #ifndef atomic64_try_cmpxchg
1027
1028 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \
1029 ({ \
1030 typeof(_po) __po = (_po); \
1031 typeof(*(_po)) __r, __o = *__po; \
1032 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \
1033 if (unlikely(__r != __o)) \
1034 *__po = __r; \
1035 likely(__r == __o); \
1036 })
1037
1038 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n)
1039 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1040 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1041 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
1042
1043 #else /* atomic64_try_cmpxchg */
1044 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
1045 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
1046 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
1047 #endif /* atomic64_try_cmpxchg */
1048
1049 #ifndef atomic64_andnot
atomic64_andnot(long long i,atomic64_t * v)1050 static inline void atomic64_andnot(long long i, atomic64_t *v)
1051 {
1052 atomic64_and(~i, v);
1053 }
1054
atomic64_fetch_andnot(long long i,atomic64_t * v)1055 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1056 {
1057 return atomic64_fetch_and(~i, v);
1058 }
1059
atomic64_fetch_andnot_relaxed(long long i,atomic64_t * v)1060 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1061 {
1062 return atomic64_fetch_and_relaxed(~i, v);
1063 }
1064
atomic64_fetch_andnot_acquire(long long i,atomic64_t * v)1065 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1066 {
1067 return atomic64_fetch_and_acquire(~i, v);
1068 }
1069
atomic64_fetch_andnot_release(long long i,atomic64_t * v)1070 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1071 {
1072 return atomic64_fetch_and_release(~i, v);
1073 }
1074 #endif
1075
1076 #include <asm-generic/atomic-long.h>
1077
1078 #endif /* _LINUX_ATOMIC_H */
1079