1 /* Atomic operations usable in machine independent code */
2 #ifndef _LINUX_ATOMIC_H
3 #define _LINUX_ATOMIC_H
4 #include <asm/atomic.h>
5 #include <asm/barrier.h>
6
7 /*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25 #ifndef atomic_read_acquire
26 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27 #endif
28
29 #ifndef atomic_set_release
30 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31 #endif
32
33 /*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 *
38 * Besides, if an arch has a special barrier for acquire/release, it could
39 * implement its own __atomic_op_* and use the same framework for building
40 * variants
41 */
42 #ifndef __atomic_op_acquire
43 #define __atomic_op_acquire(op, args...) \
44 ({ \
45 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
46 smp_mb__after_atomic(); \
47 __ret; \
48 })
49 #endif
50
51 #ifndef __atomic_op_release
52 #define __atomic_op_release(op, args...) \
53 ({ \
54 smp_mb__before_atomic(); \
55 op##_relaxed(args); \
56 })
57 #endif
58
59 #ifndef __atomic_op_fence
60 #define __atomic_op_fence(op, args...) \
61 ({ \
62 typeof(op##_relaxed(args)) __ret; \
63 smp_mb__before_atomic(); \
64 __ret = op##_relaxed(args); \
65 smp_mb__after_atomic(); \
66 __ret; \
67 })
68 #endif
69
70 /* atomic_add_return_relaxed */
71 #ifndef atomic_add_return_relaxed
72 #define atomic_add_return_relaxed atomic_add_return
73 #define atomic_add_return_acquire atomic_add_return
74 #define atomic_add_return_release atomic_add_return
75
76 #else /* atomic_add_return_relaxed */
77
78 #ifndef atomic_add_return_acquire
79 #define atomic_add_return_acquire(...) \
80 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
81 #endif
82
83 #ifndef atomic_add_return_release
84 #define atomic_add_return_release(...) \
85 __atomic_op_release(atomic_add_return, __VA_ARGS__)
86 #endif
87
88 #ifndef atomic_add_return
89 #define atomic_add_return(...) \
90 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
91 #endif
92 #endif /* atomic_add_return_relaxed */
93
94 /* atomic_inc_return_relaxed */
95 #ifndef atomic_inc_return_relaxed
96 #define atomic_inc_return_relaxed atomic_inc_return
97 #define atomic_inc_return_acquire atomic_inc_return
98 #define atomic_inc_return_release atomic_inc_return
99
100 #else /* atomic_inc_return_relaxed */
101
102 #ifndef atomic_inc_return_acquire
103 #define atomic_inc_return_acquire(...) \
104 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
105 #endif
106
107 #ifndef atomic_inc_return_release
108 #define atomic_inc_return_release(...) \
109 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
110 #endif
111
112 #ifndef atomic_inc_return
113 #define atomic_inc_return(...) \
114 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
115 #endif
116 #endif /* atomic_inc_return_relaxed */
117
118 /* atomic_sub_return_relaxed */
119 #ifndef atomic_sub_return_relaxed
120 #define atomic_sub_return_relaxed atomic_sub_return
121 #define atomic_sub_return_acquire atomic_sub_return
122 #define atomic_sub_return_release atomic_sub_return
123
124 #else /* atomic_sub_return_relaxed */
125
126 #ifndef atomic_sub_return_acquire
127 #define atomic_sub_return_acquire(...) \
128 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
129 #endif
130
131 #ifndef atomic_sub_return_release
132 #define atomic_sub_return_release(...) \
133 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
134 #endif
135
136 #ifndef atomic_sub_return
137 #define atomic_sub_return(...) \
138 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
139 #endif
140 #endif /* atomic_sub_return_relaxed */
141
142 /* atomic_dec_return_relaxed */
143 #ifndef atomic_dec_return_relaxed
144 #define atomic_dec_return_relaxed atomic_dec_return
145 #define atomic_dec_return_acquire atomic_dec_return
146 #define atomic_dec_return_release atomic_dec_return
147
148 #else /* atomic_dec_return_relaxed */
149
150 #ifndef atomic_dec_return_acquire
151 #define atomic_dec_return_acquire(...) \
152 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
153 #endif
154
155 #ifndef atomic_dec_return_release
156 #define atomic_dec_return_release(...) \
157 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
158 #endif
159
160 #ifndef atomic_dec_return
161 #define atomic_dec_return(...) \
162 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
163 #endif
164 #endif /* atomic_dec_return_relaxed */
165
166
167 /* atomic_fetch_add_relaxed */
168 #ifndef atomic_fetch_add_relaxed
169 #define atomic_fetch_add_relaxed atomic_fetch_add
170 #define atomic_fetch_add_acquire atomic_fetch_add
171 #define atomic_fetch_add_release atomic_fetch_add
172
173 #else /* atomic_fetch_add_relaxed */
174
175 #ifndef atomic_fetch_add_acquire
176 #define atomic_fetch_add_acquire(...) \
177 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
178 #endif
179
180 #ifndef atomic_fetch_add_release
181 #define atomic_fetch_add_release(...) \
182 __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
183 #endif
184
185 #ifndef atomic_fetch_add
186 #define atomic_fetch_add(...) \
187 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
188 #endif
189 #endif /* atomic_fetch_add_relaxed */
190
191 /* atomic_fetch_inc_relaxed */
192 #ifndef atomic_fetch_inc_relaxed
193
194 #ifndef atomic_fetch_inc
195 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
196 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
197 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
198 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
199 #else /* atomic_fetch_inc */
200 #define atomic_fetch_inc_relaxed atomic_fetch_inc
201 #define atomic_fetch_inc_acquire atomic_fetch_inc
202 #define atomic_fetch_inc_release atomic_fetch_inc
203 #endif /* atomic_fetch_inc */
204
205 #else /* atomic_fetch_inc_relaxed */
206
207 #ifndef atomic_fetch_inc_acquire
208 #define atomic_fetch_inc_acquire(...) \
209 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
210 #endif
211
212 #ifndef atomic_fetch_inc_release
213 #define atomic_fetch_inc_release(...) \
214 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
215 #endif
216
217 #ifndef atomic_fetch_inc
218 #define atomic_fetch_inc(...) \
219 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
220 #endif
221 #endif /* atomic_fetch_inc_relaxed */
222
223 /* atomic_fetch_sub_relaxed */
224 #ifndef atomic_fetch_sub_relaxed
225 #define atomic_fetch_sub_relaxed atomic_fetch_sub
226 #define atomic_fetch_sub_acquire atomic_fetch_sub
227 #define atomic_fetch_sub_release atomic_fetch_sub
228
229 #else /* atomic_fetch_sub_relaxed */
230
231 #ifndef atomic_fetch_sub_acquire
232 #define atomic_fetch_sub_acquire(...) \
233 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
234 #endif
235
236 #ifndef atomic_fetch_sub_release
237 #define atomic_fetch_sub_release(...) \
238 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
239 #endif
240
241 #ifndef atomic_fetch_sub
242 #define atomic_fetch_sub(...) \
243 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
244 #endif
245 #endif /* atomic_fetch_sub_relaxed */
246
247 /* atomic_fetch_dec_relaxed */
248 #ifndef atomic_fetch_dec_relaxed
249
250 #ifndef atomic_fetch_dec
251 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
252 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
253 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
254 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
255 #else /* atomic_fetch_dec */
256 #define atomic_fetch_dec_relaxed atomic_fetch_dec
257 #define atomic_fetch_dec_acquire atomic_fetch_dec
258 #define atomic_fetch_dec_release atomic_fetch_dec
259 #endif /* atomic_fetch_dec */
260
261 #else /* atomic_fetch_dec_relaxed */
262
263 #ifndef atomic_fetch_dec_acquire
264 #define atomic_fetch_dec_acquire(...) \
265 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
266 #endif
267
268 #ifndef atomic_fetch_dec_release
269 #define atomic_fetch_dec_release(...) \
270 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
271 #endif
272
273 #ifndef atomic_fetch_dec
274 #define atomic_fetch_dec(...) \
275 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
276 #endif
277 #endif /* atomic_fetch_dec_relaxed */
278
279 /* atomic_fetch_or_relaxed */
280 #ifndef atomic_fetch_or_relaxed
281 #define atomic_fetch_or_relaxed atomic_fetch_or
282 #define atomic_fetch_or_acquire atomic_fetch_or
283 #define atomic_fetch_or_release atomic_fetch_or
284
285 #else /* atomic_fetch_or_relaxed */
286
287 #ifndef atomic_fetch_or_acquire
288 #define atomic_fetch_or_acquire(...) \
289 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
290 #endif
291
292 #ifndef atomic_fetch_or_release
293 #define atomic_fetch_or_release(...) \
294 __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
295 #endif
296
297 #ifndef atomic_fetch_or
298 #define atomic_fetch_or(...) \
299 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
300 #endif
301 #endif /* atomic_fetch_or_relaxed */
302
303 /* atomic_fetch_and_relaxed */
304 #ifndef atomic_fetch_and_relaxed
305 #define atomic_fetch_and_relaxed atomic_fetch_and
306 #define atomic_fetch_and_acquire atomic_fetch_and
307 #define atomic_fetch_and_release atomic_fetch_and
308
309 #else /* atomic_fetch_and_relaxed */
310
311 #ifndef atomic_fetch_and_acquire
312 #define atomic_fetch_and_acquire(...) \
313 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
314 #endif
315
316 #ifndef atomic_fetch_and_release
317 #define atomic_fetch_and_release(...) \
318 __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
319 #endif
320
321 #ifndef atomic_fetch_and
322 #define atomic_fetch_and(...) \
323 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
324 #endif
325 #endif /* atomic_fetch_and_relaxed */
326
327 #ifdef atomic_andnot
328 /* atomic_fetch_andnot_relaxed */
329 #ifndef atomic_fetch_andnot_relaxed
330 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot
331 #define atomic_fetch_andnot_acquire atomic_fetch_andnot
332 #define atomic_fetch_andnot_release atomic_fetch_andnot
333
334 #else /* atomic_fetch_andnot_relaxed */
335
336 #ifndef atomic_fetch_andnot_acquire
337 #define atomic_fetch_andnot_acquire(...) \
338 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
339 #endif
340
341 #ifndef atomic_fetch_andnot_release
342 #define atomic_fetch_andnot_release(...) \
343 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
344 #endif
345
346 #ifndef atomic_fetch_andnot
347 #define atomic_fetch_andnot(...) \
348 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
349 #endif
350 #endif /* atomic_fetch_andnot_relaxed */
351 #endif /* atomic_andnot */
352
353 /* atomic_fetch_xor_relaxed */
354 #ifndef atomic_fetch_xor_relaxed
355 #define atomic_fetch_xor_relaxed atomic_fetch_xor
356 #define atomic_fetch_xor_acquire atomic_fetch_xor
357 #define atomic_fetch_xor_release atomic_fetch_xor
358
359 #else /* atomic_fetch_xor_relaxed */
360
361 #ifndef atomic_fetch_xor_acquire
362 #define atomic_fetch_xor_acquire(...) \
363 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
364 #endif
365
366 #ifndef atomic_fetch_xor_release
367 #define atomic_fetch_xor_release(...) \
368 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
369 #endif
370
371 #ifndef atomic_fetch_xor
372 #define atomic_fetch_xor(...) \
373 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
374 #endif
375 #endif /* atomic_fetch_xor_relaxed */
376
377
378 /* atomic_xchg_relaxed */
379 #ifndef atomic_xchg_relaxed
380 #define atomic_xchg_relaxed atomic_xchg
381 #define atomic_xchg_acquire atomic_xchg
382 #define atomic_xchg_release atomic_xchg
383
384 #else /* atomic_xchg_relaxed */
385
386 #ifndef atomic_xchg_acquire
387 #define atomic_xchg_acquire(...) \
388 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
389 #endif
390
391 #ifndef atomic_xchg_release
392 #define atomic_xchg_release(...) \
393 __atomic_op_release(atomic_xchg, __VA_ARGS__)
394 #endif
395
396 #ifndef atomic_xchg
397 #define atomic_xchg(...) \
398 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
399 #endif
400 #endif /* atomic_xchg_relaxed */
401
402 /* atomic_cmpxchg_relaxed */
403 #ifndef atomic_cmpxchg_relaxed
404 #define atomic_cmpxchg_relaxed atomic_cmpxchg
405 #define atomic_cmpxchg_acquire atomic_cmpxchg
406 #define atomic_cmpxchg_release atomic_cmpxchg
407
408 #else /* atomic_cmpxchg_relaxed */
409
410 #ifndef atomic_cmpxchg_acquire
411 #define atomic_cmpxchg_acquire(...) \
412 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
413 #endif
414
415 #ifndef atomic_cmpxchg_release
416 #define atomic_cmpxchg_release(...) \
417 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
418 #endif
419
420 #ifndef atomic_cmpxchg
421 #define atomic_cmpxchg(...) \
422 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
423 #endif
424 #endif /* atomic_cmpxchg_relaxed */
425
426 /* cmpxchg_relaxed */
427 #ifndef cmpxchg_relaxed
428 #define cmpxchg_relaxed cmpxchg
429 #define cmpxchg_acquire cmpxchg
430 #define cmpxchg_release cmpxchg
431
432 #else /* cmpxchg_relaxed */
433
434 #ifndef cmpxchg_acquire
435 #define cmpxchg_acquire(...) \
436 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
437 #endif
438
439 #ifndef cmpxchg_release
440 #define cmpxchg_release(...) \
441 __atomic_op_release(cmpxchg, __VA_ARGS__)
442 #endif
443
444 #ifndef cmpxchg
445 #define cmpxchg(...) \
446 __atomic_op_fence(cmpxchg, __VA_ARGS__)
447 #endif
448 #endif /* cmpxchg_relaxed */
449
450 /* cmpxchg64_relaxed */
451 #ifndef cmpxchg64_relaxed
452 #define cmpxchg64_relaxed cmpxchg64
453 #define cmpxchg64_acquire cmpxchg64
454 #define cmpxchg64_release cmpxchg64
455
456 #else /* cmpxchg64_relaxed */
457
458 #ifndef cmpxchg64_acquire
459 #define cmpxchg64_acquire(...) \
460 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
461 #endif
462
463 #ifndef cmpxchg64_release
464 #define cmpxchg64_release(...) \
465 __atomic_op_release(cmpxchg64, __VA_ARGS__)
466 #endif
467
468 #ifndef cmpxchg64
469 #define cmpxchg64(...) \
470 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
471 #endif
472 #endif /* cmpxchg64_relaxed */
473
474 /* xchg_relaxed */
475 #ifndef xchg_relaxed
476 #define xchg_relaxed xchg
477 #define xchg_acquire xchg
478 #define xchg_release xchg
479
480 #else /* xchg_relaxed */
481
482 #ifndef xchg_acquire
483 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
484 #endif
485
486 #ifndef xchg_release
487 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
488 #endif
489
490 #ifndef xchg
491 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
492 #endif
493 #endif /* xchg_relaxed */
494
495 /**
496 * atomic_add_unless - add unless the number is already a given value
497 * @v: pointer of type atomic_t
498 * @a: the amount to add to v...
499 * @u: ...unless v is equal to u.
500 *
501 * Atomically adds @a to @v, so long as @v was not already @u.
502 * Returns non-zero if @v was not @u, and zero otherwise.
503 */
atomic_add_unless(atomic_t * v,int a,int u)504 static inline int atomic_add_unless(atomic_t *v, int a, int u)
505 {
506 return __atomic_add_unless(v, a, u) != u;
507 }
508
509 /**
510 * atomic_inc_not_zero - increment unless the number is zero
511 * @v: pointer of type atomic_t
512 *
513 * Atomically increments @v by 1, so long as @v is non-zero.
514 * Returns non-zero if @v was non-zero, and zero otherwise.
515 */
516 #ifndef atomic_inc_not_zero
517 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
518 #endif
519
520 #ifndef atomic_andnot
atomic_andnot(int i,atomic_t * v)521 static inline void atomic_andnot(int i, atomic_t *v)
522 {
523 atomic_and(~i, v);
524 }
525
atomic_fetch_andnot(int i,atomic_t * v)526 static inline int atomic_fetch_andnot(int i, atomic_t *v)
527 {
528 return atomic_fetch_and(~i, v);
529 }
530
atomic_fetch_andnot_relaxed(int i,atomic_t * v)531 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
532 {
533 return atomic_fetch_and_relaxed(~i, v);
534 }
535
atomic_fetch_andnot_acquire(int i,atomic_t * v)536 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
537 {
538 return atomic_fetch_and_acquire(~i, v);
539 }
540
atomic_fetch_andnot_release(int i,atomic_t * v)541 static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
542 {
543 return atomic_fetch_and_release(~i, v);
544 }
545 #endif
546
547 /**
548 * atomic_inc_not_zero_hint - increment if not null
549 * @v: pointer of type atomic_t
550 * @hint: probable value of the atomic before the increment
551 *
552 * This version of atomic_inc_not_zero() gives a hint of probable
553 * value of the atomic. This helps processor to not read the memory
554 * before doing the atomic read/modify/write cycle, lowering
555 * number of bus transactions on some arches.
556 *
557 * Returns: 0 if increment was not done, 1 otherwise.
558 */
559 #ifndef atomic_inc_not_zero_hint
atomic_inc_not_zero_hint(atomic_t * v,int hint)560 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
561 {
562 int val, c = hint;
563
564 /* sanity test, should be removed by compiler if hint is a constant */
565 if (!hint)
566 return atomic_inc_not_zero(v);
567
568 do {
569 val = atomic_cmpxchg(v, c, c + 1);
570 if (val == c)
571 return 1;
572 c = val;
573 } while (c);
574
575 return 0;
576 }
577 #endif
578
579 #ifndef atomic_inc_unless_negative
atomic_inc_unless_negative(atomic_t * p)580 static inline int atomic_inc_unless_negative(atomic_t *p)
581 {
582 int v, v1;
583 for (v = 0; v >= 0; v = v1) {
584 v1 = atomic_cmpxchg(p, v, v + 1);
585 if (likely(v1 == v))
586 return 1;
587 }
588 return 0;
589 }
590 #endif
591
592 #ifndef atomic_dec_unless_positive
atomic_dec_unless_positive(atomic_t * p)593 static inline int atomic_dec_unless_positive(atomic_t *p)
594 {
595 int v, v1;
596 for (v = 0; v <= 0; v = v1) {
597 v1 = atomic_cmpxchg(p, v, v - 1);
598 if (likely(v1 == v))
599 return 1;
600 }
601 return 0;
602 }
603 #endif
604
605 /*
606 * atomic_dec_if_positive - decrement by 1 if old value positive
607 * @v: pointer of type atomic_t
608 *
609 * The function returns the old value of *v minus 1, even if
610 * the atomic variable, v, was not decremented.
611 */
612 #ifndef atomic_dec_if_positive
atomic_dec_if_positive(atomic_t * v)613 static inline int atomic_dec_if_positive(atomic_t *v)
614 {
615 int c, old, dec;
616 c = atomic_read(v);
617 for (;;) {
618 dec = c - 1;
619 if (unlikely(dec < 0))
620 break;
621 old = atomic_cmpxchg((v), c, dec);
622 if (likely(old == c))
623 break;
624 c = old;
625 }
626 return dec;
627 }
628 #endif
629
630 #ifdef CONFIG_GENERIC_ATOMIC64
631 #include <asm-generic/atomic64.h>
632 #endif
633
634 #ifndef atomic64_read_acquire
635 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
636 #endif
637
638 #ifndef atomic64_set_release
639 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
640 #endif
641
642 /* atomic64_add_return_relaxed */
643 #ifndef atomic64_add_return_relaxed
644 #define atomic64_add_return_relaxed atomic64_add_return
645 #define atomic64_add_return_acquire atomic64_add_return
646 #define atomic64_add_return_release atomic64_add_return
647
648 #else /* atomic64_add_return_relaxed */
649
650 #ifndef atomic64_add_return_acquire
651 #define atomic64_add_return_acquire(...) \
652 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
653 #endif
654
655 #ifndef atomic64_add_return_release
656 #define atomic64_add_return_release(...) \
657 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
658 #endif
659
660 #ifndef atomic64_add_return
661 #define atomic64_add_return(...) \
662 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
663 #endif
664 #endif /* atomic64_add_return_relaxed */
665
666 /* atomic64_inc_return_relaxed */
667 #ifndef atomic64_inc_return_relaxed
668 #define atomic64_inc_return_relaxed atomic64_inc_return
669 #define atomic64_inc_return_acquire atomic64_inc_return
670 #define atomic64_inc_return_release atomic64_inc_return
671
672 #else /* atomic64_inc_return_relaxed */
673
674 #ifndef atomic64_inc_return_acquire
675 #define atomic64_inc_return_acquire(...) \
676 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
677 #endif
678
679 #ifndef atomic64_inc_return_release
680 #define atomic64_inc_return_release(...) \
681 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
682 #endif
683
684 #ifndef atomic64_inc_return
685 #define atomic64_inc_return(...) \
686 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
687 #endif
688 #endif /* atomic64_inc_return_relaxed */
689
690
691 /* atomic64_sub_return_relaxed */
692 #ifndef atomic64_sub_return_relaxed
693 #define atomic64_sub_return_relaxed atomic64_sub_return
694 #define atomic64_sub_return_acquire atomic64_sub_return
695 #define atomic64_sub_return_release atomic64_sub_return
696
697 #else /* atomic64_sub_return_relaxed */
698
699 #ifndef atomic64_sub_return_acquire
700 #define atomic64_sub_return_acquire(...) \
701 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
702 #endif
703
704 #ifndef atomic64_sub_return_release
705 #define atomic64_sub_return_release(...) \
706 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
707 #endif
708
709 #ifndef atomic64_sub_return
710 #define atomic64_sub_return(...) \
711 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
712 #endif
713 #endif /* atomic64_sub_return_relaxed */
714
715 /* atomic64_dec_return_relaxed */
716 #ifndef atomic64_dec_return_relaxed
717 #define atomic64_dec_return_relaxed atomic64_dec_return
718 #define atomic64_dec_return_acquire atomic64_dec_return
719 #define atomic64_dec_return_release atomic64_dec_return
720
721 #else /* atomic64_dec_return_relaxed */
722
723 #ifndef atomic64_dec_return_acquire
724 #define atomic64_dec_return_acquire(...) \
725 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
726 #endif
727
728 #ifndef atomic64_dec_return_release
729 #define atomic64_dec_return_release(...) \
730 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
731 #endif
732
733 #ifndef atomic64_dec_return
734 #define atomic64_dec_return(...) \
735 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
736 #endif
737 #endif /* atomic64_dec_return_relaxed */
738
739
740 /* atomic64_fetch_add_relaxed */
741 #ifndef atomic64_fetch_add_relaxed
742 #define atomic64_fetch_add_relaxed atomic64_fetch_add
743 #define atomic64_fetch_add_acquire atomic64_fetch_add
744 #define atomic64_fetch_add_release atomic64_fetch_add
745
746 #else /* atomic64_fetch_add_relaxed */
747
748 #ifndef atomic64_fetch_add_acquire
749 #define atomic64_fetch_add_acquire(...) \
750 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
751 #endif
752
753 #ifndef atomic64_fetch_add_release
754 #define atomic64_fetch_add_release(...) \
755 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
756 #endif
757
758 #ifndef atomic64_fetch_add
759 #define atomic64_fetch_add(...) \
760 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
761 #endif
762 #endif /* atomic64_fetch_add_relaxed */
763
764 /* atomic64_fetch_inc_relaxed */
765 #ifndef atomic64_fetch_inc_relaxed
766
767 #ifndef atomic64_fetch_inc
768 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
769 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
770 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
771 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
772 #else /* atomic64_fetch_inc */
773 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc
774 #define atomic64_fetch_inc_acquire atomic64_fetch_inc
775 #define atomic64_fetch_inc_release atomic64_fetch_inc
776 #endif /* atomic64_fetch_inc */
777
778 #else /* atomic64_fetch_inc_relaxed */
779
780 #ifndef atomic64_fetch_inc_acquire
781 #define atomic64_fetch_inc_acquire(...) \
782 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
783 #endif
784
785 #ifndef atomic64_fetch_inc_release
786 #define atomic64_fetch_inc_release(...) \
787 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
788 #endif
789
790 #ifndef atomic64_fetch_inc
791 #define atomic64_fetch_inc(...) \
792 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
793 #endif
794 #endif /* atomic64_fetch_inc_relaxed */
795
796 /* atomic64_fetch_sub_relaxed */
797 #ifndef atomic64_fetch_sub_relaxed
798 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub
799 #define atomic64_fetch_sub_acquire atomic64_fetch_sub
800 #define atomic64_fetch_sub_release atomic64_fetch_sub
801
802 #else /* atomic64_fetch_sub_relaxed */
803
804 #ifndef atomic64_fetch_sub_acquire
805 #define atomic64_fetch_sub_acquire(...) \
806 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
807 #endif
808
809 #ifndef atomic64_fetch_sub_release
810 #define atomic64_fetch_sub_release(...) \
811 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
812 #endif
813
814 #ifndef atomic64_fetch_sub
815 #define atomic64_fetch_sub(...) \
816 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
817 #endif
818 #endif /* atomic64_fetch_sub_relaxed */
819
820 /* atomic64_fetch_dec_relaxed */
821 #ifndef atomic64_fetch_dec_relaxed
822
823 #ifndef atomic64_fetch_dec
824 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
825 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
826 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
827 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
828 #else /* atomic64_fetch_dec */
829 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec
830 #define atomic64_fetch_dec_acquire atomic64_fetch_dec
831 #define atomic64_fetch_dec_release atomic64_fetch_dec
832 #endif /* atomic64_fetch_dec */
833
834 #else /* atomic64_fetch_dec_relaxed */
835
836 #ifndef atomic64_fetch_dec_acquire
837 #define atomic64_fetch_dec_acquire(...) \
838 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
839 #endif
840
841 #ifndef atomic64_fetch_dec_release
842 #define atomic64_fetch_dec_release(...) \
843 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
844 #endif
845
846 #ifndef atomic64_fetch_dec
847 #define atomic64_fetch_dec(...) \
848 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
849 #endif
850 #endif /* atomic64_fetch_dec_relaxed */
851
852 /* atomic64_fetch_or_relaxed */
853 #ifndef atomic64_fetch_or_relaxed
854 #define atomic64_fetch_or_relaxed atomic64_fetch_or
855 #define atomic64_fetch_or_acquire atomic64_fetch_or
856 #define atomic64_fetch_or_release atomic64_fetch_or
857
858 #else /* atomic64_fetch_or_relaxed */
859
860 #ifndef atomic64_fetch_or_acquire
861 #define atomic64_fetch_or_acquire(...) \
862 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
863 #endif
864
865 #ifndef atomic64_fetch_or_release
866 #define atomic64_fetch_or_release(...) \
867 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
868 #endif
869
870 #ifndef atomic64_fetch_or
871 #define atomic64_fetch_or(...) \
872 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
873 #endif
874 #endif /* atomic64_fetch_or_relaxed */
875
876 /* atomic64_fetch_and_relaxed */
877 #ifndef atomic64_fetch_and_relaxed
878 #define atomic64_fetch_and_relaxed atomic64_fetch_and
879 #define atomic64_fetch_and_acquire atomic64_fetch_and
880 #define atomic64_fetch_and_release atomic64_fetch_and
881
882 #else /* atomic64_fetch_and_relaxed */
883
884 #ifndef atomic64_fetch_and_acquire
885 #define atomic64_fetch_and_acquire(...) \
886 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
887 #endif
888
889 #ifndef atomic64_fetch_and_release
890 #define atomic64_fetch_and_release(...) \
891 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
892 #endif
893
894 #ifndef atomic64_fetch_and
895 #define atomic64_fetch_and(...) \
896 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
897 #endif
898 #endif /* atomic64_fetch_and_relaxed */
899
900 #ifdef atomic64_andnot
901 /* atomic64_fetch_andnot_relaxed */
902 #ifndef atomic64_fetch_andnot_relaxed
903 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
904 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
905 #define atomic64_fetch_andnot_release atomic64_fetch_andnot
906
907 #else /* atomic64_fetch_andnot_relaxed */
908
909 #ifndef atomic64_fetch_andnot_acquire
910 #define atomic64_fetch_andnot_acquire(...) \
911 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
912 #endif
913
914 #ifndef atomic64_fetch_andnot_release
915 #define atomic64_fetch_andnot_release(...) \
916 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
917 #endif
918
919 #ifndef atomic64_fetch_andnot
920 #define atomic64_fetch_andnot(...) \
921 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
922 #endif
923 #endif /* atomic64_fetch_andnot_relaxed */
924 #endif /* atomic64_andnot */
925
926 /* atomic64_fetch_xor_relaxed */
927 #ifndef atomic64_fetch_xor_relaxed
928 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor
929 #define atomic64_fetch_xor_acquire atomic64_fetch_xor
930 #define atomic64_fetch_xor_release atomic64_fetch_xor
931
932 #else /* atomic64_fetch_xor_relaxed */
933
934 #ifndef atomic64_fetch_xor_acquire
935 #define atomic64_fetch_xor_acquire(...) \
936 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
937 #endif
938
939 #ifndef atomic64_fetch_xor_release
940 #define atomic64_fetch_xor_release(...) \
941 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
942 #endif
943
944 #ifndef atomic64_fetch_xor
945 #define atomic64_fetch_xor(...) \
946 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
947 #endif
948 #endif /* atomic64_fetch_xor_relaxed */
949
950
951 /* atomic64_xchg_relaxed */
952 #ifndef atomic64_xchg_relaxed
953 #define atomic64_xchg_relaxed atomic64_xchg
954 #define atomic64_xchg_acquire atomic64_xchg
955 #define atomic64_xchg_release atomic64_xchg
956
957 #else /* atomic64_xchg_relaxed */
958
959 #ifndef atomic64_xchg_acquire
960 #define atomic64_xchg_acquire(...) \
961 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
962 #endif
963
964 #ifndef atomic64_xchg_release
965 #define atomic64_xchg_release(...) \
966 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
967 #endif
968
969 #ifndef atomic64_xchg
970 #define atomic64_xchg(...) \
971 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
972 #endif
973 #endif /* atomic64_xchg_relaxed */
974
975 /* atomic64_cmpxchg_relaxed */
976 #ifndef atomic64_cmpxchg_relaxed
977 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg
978 #define atomic64_cmpxchg_acquire atomic64_cmpxchg
979 #define atomic64_cmpxchg_release atomic64_cmpxchg
980
981 #else /* atomic64_cmpxchg_relaxed */
982
983 #ifndef atomic64_cmpxchg_acquire
984 #define atomic64_cmpxchg_acquire(...) \
985 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
986 #endif
987
988 #ifndef atomic64_cmpxchg_release
989 #define atomic64_cmpxchg_release(...) \
990 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
991 #endif
992
993 #ifndef atomic64_cmpxchg
994 #define atomic64_cmpxchg(...) \
995 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
996 #endif
997 #endif /* atomic64_cmpxchg_relaxed */
998
999 #ifndef atomic64_andnot
atomic64_andnot(long long i,atomic64_t * v)1000 static inline void atomic64_andnot(long long i, atomic64_t *v)
1001 {
1002 atomic64_and(~i, v);
1003 }
1004
atomic64_fetch_andnot(long long i,atomic64_t * v)1005 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1006 {
1007 return atomic64_fetch_and(~i, v);
1008 }
1009
atomic64_fetch_andnot_relaxed(long long i,atomic64_t * v)1010 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1011 {
1012 return atomic64_fetch_and_relaxed(~i, v);
1013 }
1014
atomic64_fetch_andnot_acquire(long long i,atomic64_t * v)1015 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1016 {
1017 return atomic64_fetch_and_acquire(~i, v);
1018 }
1019
atomic64_fetch_andnot_release(long long i,atomic64_t * v)1020 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1021 {
1022 return atomic64_fetch_and_release(~i, v);
1023 }
1024 #endif
1025
1026 #include <asm-generic/atomic-long.h>
1027
1028 #endif /* _LINUX_ATOMIC_H */
1029