1 // This header provides cross-platform low-level atomic operations 2 // similar to C11 atomics. 3 // 4 // Operations are sequentially consistent unless they have a suffix indicating 5 // otherwise. If in doubt, prefer the sequentially consistent operations. 6 // 7 // The "_relaxed" suffix for load and store operations indicates the "relaxed" 8 // memory order. They don't provide synchronization, but (roughly speaking) 9 // guarantee somewhat sane behavior for races instead of undefined behavior. 10 // In practice, they correspond to "normal" hardware load and store 11 // instructions, so they are almost as inexpensive as plain loads and stores 12 // in C. 13 // 14 // Note that atomic read-modify-write operations like _Py_atomic_add_* return 15 // the previous value of the atomic variable, not the new value. 16 // 17 // See https://en.cppreference.com/w/c/atomic for more information on C11 18 // atomics. 19 // See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf 20 // "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage 21 // or relaxed atomics. 22 // 23 // Functions with pseudo Python code: 24 // 25 // def _Py_atomic_load(obj): 26 // return obj # sequential consistency 27 // 28 // def _Py_atomic_load_relaxed(obj): 29 // return obj # relaxed consistency 30 // 31 // def _Py_atomic_store(obj, value): 32 // obj = value # sequential consistency 33 // 34 // def _Py_atomic_store_relaxed(obj, value): 35 // obj = value # relaxed consistency 36 // 37 // def _Py_atomic_exchange(obj, value): 38 // # sequential consistency 39 // old_obj = obj 40 // obj = value 41 // return old_obj 42 // 43 // def _Py_atomic_compare_exchange(obj, expected, desired): 44 // # sequential consistency 45 // if obj == expected: 46 // obj = desired 47 // return True 48 // else: 49 // expected = obj 50 // return False 51 // 52 // def _Py_atomic_add(obj, value): 53 // # sequential consistency 54 // old_obj = obj 55 // obj += value 56 // return old_obj 57 // 58 // def _Py_atomic_and(obj, value): 59 // # sequential consistency 60 // old_obj = obj 61 // obj &= value 62 // return old_obj 63 // 64 // def _Py_atomic_or(obj, value): 65 // # sequential consistency 66 // old_obj = obj 67 // obj |= value 68 // return old_obj 69 // 70 // Other functions: 71 // 72 // def _Py_atomic_load_ptr_acquire(obj): 73 // return obj # acquire 74 // 75 // def _Py_atomic_store_ptr_release(obj): 76 // return obj # release 77 // 78 // def _Py_atomic_fence_seq_cst(): 79 // # sequential consistency 80 // ... 81 // 82 // def _Py_atomic_fence_release(): 83 // # release 84 // ... 85 86 #ifndef Py_CPYTHON_ATOMIC_H 87 # error "this header file must not be included directly" 88 #endif 89 90 // --- _Py_atomic_add -------------------------------------------------------- 91 // Atomically adds `value` to `obj` and returns the previous value 92 93 static inline int 94 _Py_atomic_add_int(int *obj, int value); 95 96 static inline int8_t 97 _Py_atomic_add_int8(int8_t *obj, int8_t value); 98 99 static inline int16_t 100 _Py_atomic_add_int16(int16_t *obj, int16_t value); 101 102 static inline int32_t 103 _Py_atomic_add_int32(int32_t *obj, int32_t value); 104 105 static inline int64_t 106 _Py_atomic_add_int64(int64_t *obj, int64_t value); 107 108 static inline intptr_t 109 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value); 110 111 static inline unsigned int 112 _Py_atomic_add_uint(unsigned int *obj, unsigned int value); 113 114 static inline uint8_t 115 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value); 116 117 static inline uint16_t 118 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value); 119 120 static inline uint32_t 121 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value); 122 123 static inline uint64_t 124 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value); 125 126 static inline uintptr_t 127 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value); 128 129 static inline Py_ssize_t 130 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); 131 132 133 // --- _Py_atomic_compare_exchange ------------------------------------------- 134 // Performs an atomic compare-and-exchange. 135 // 136 // - If `*obj` and `*expected` are equal, store `desired` into `*obj` 137 // and return 1 (success). 138 // - Otherwise, store the `*obj` current value into `*expected` 139 // and return 0 (failure). 140 // 141 // These correspond to the C11 atomic_compare_exchange_strong() function. 142 143 static inline int 144 _Py_atomic_compare_exchange_int(int *obj, int *expected, int desired); 145 146 static inline int 147 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired); 148 149 static inline int 150 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired); 151 152 static inline int 153 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired); 154 155 static inline int 156 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired); 157 158 static inline int 159 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired); 160 161 static inline int 162 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired); 163 164 static inline int 165 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired); 166 167 static inline int 168 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired); 169 170 static inline int 171 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired); 172 173 static inline int 174 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired); 175 176 static inline int 177 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); 178 179 static inline int 180 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); 181 182 // NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` 183 // so that we can pass types like `PyObject**` without a cast. 184 static inline int 185 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); 186 187 188 // --- _Py_atomic_exchange --------------------------------------------------- 189 // Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. 190 191 static inline int 192 _Py_atomic_exchange_int(int *obj, int value); 193 194 static inline int8_t 195 _Py_atomic_exchange_int8(int8_t *obj, int8_t value); 196 197 static inline int16_t 198 _Py_atomic_exchange_int16(int16_t *obj, int16_t value); 199 200 static inline int32_t 201 _Py_atomic_exchange_int32(int32_t *obj, int32_t value); 202 203 static inline int64_t 204 _Py_atomic_exchange_int64(int64_t *obj, int64_t value); 205 206 static inline intptr_t 207 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value); 208 209 static inline unsigned int 210 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value); 211 212 static inline uint8_t 213 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value); 214 215 static inline uint16_t 216 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value); 217 218 static inline uint32_t 219 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value); 220 221 static inline uint64_t 222 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value); 223 224 static inline uintptr_t 225 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value); 226 227 static inline Py_ssize_t 228 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value); 229 230 static inline void * 231 _Py_atomic_exchange_ptr(void *obj, void *value); 232 233 234 // --- _Py_atomic_and -------------------------------------------------------- 235 // Performs `*obj &= value` atomically and returns the previous value of `*obj`. 236 237 static inline uint8_t 238 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value); 239 240 static inline uint16_t 241 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value); 242 243 static inline uint32_t 244 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value); 245 246 static inline uint64_t 247 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value); 248 249 static inline uintptr_t 250 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value); 251 252 253 // --- _Py_atomic_or --------------------------------------------------------- 254 // Performs `*obj |= value` atomically and returns the previous value of `*obj`. 255 256 static inline uint8_t 257 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value); 258 259 static inline uint16_t 260 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value); 261 262 static inline uint32_t 263 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value); 264 265 static inline uint64_t 266 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value); 267 268 static inline uintptr_t 269 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value); 270 271 272 // --- _Py_atomic_load ------------------------------------------------------- 273 // Atomically loads `*obj` (sequential consistency) 274 275 static inline int 276 _Py_atomic_load_int(const int *obj); 277 278 static inline int8_t 279 _Py_atomic_load_int8(const int8_t *obj); 280 281 static inline int16_t 282 _Py_atomic_load_int16(const int16_t *obj); 283 284 static inline int32_t 285 _Py_atomic_load_int32(const int32_t *obj); 286 287 static inline int64_t 288 _Py_atomic_load_int64(const int64_t *obj); 289 290 static inline intptr_t 291 _Py_atomic_load_intptr(const intptr_t *obj); 292 293 static inline uint8_t 294 _Py_atomic_load_uint8(const uint8_t *obj); 295 296 static inline uint16_t 297 _Py_atomic_load_uint16(const uint16_t *obj); 298 299 static inline uint32_t 300 _Py_atomic_load_uint32(const uint32_t *obj); 301 302 static inline uint64_t 303 _Py_atomic_load_uint64(const uint64_t *obj); 304 305 static inline uintptr_t 306 _Py_atomic_load_uintptr(const uintptr_t *obj); 307 308 static inline unsigned int 309 _Py_atomic_load_uint(const unsigned int *obj); 310 311 static inline Py_ssize_t 312 _Py_atomic_load_ssize(const Py_ssize_t *obj); 313 314 static inline void * 315 _Py_atomic_load_ptr(const void *obj); 316 317 318 // --- _Py_atomic_load_relaxed ----------------------------------------------- 319 // Loads `*obj` (relaxed consistency, i.e., no ordering) 320 321 static inline int 322 _Py_atomic_load_int_relaxed(const int *obj); 323 324 static inline int8_t 325 _Py_atomic_load_int8_relaxed(const int8_t *obj); 326 327 static inline int16_t 328 _Py_atomic_load_int16_relaxed(const int16_t *obj); 329 330 static inline int32_t 331 _Py_atomic_load_int32_relaxed(const int32_t *obj); 332 333 static inline int64_t 334 _Py_atomic_load_int64_relaxed(const int64_t *obj); 335 336 static inline intptr_t 337 _Py_atomic_load_intptr_relaxed(const intptr_t *obj); 338 339 static inline uint8_t 340 _Py_atomic_load_uint8_relaxed(const uint8_t *obj); 341 342 static inline uint16_t 343 _Py_atomic_load_uint16_relaxed(const uint16_t *obj); 344 345 static inline uint32_t 346 _Py_atomic_load_uint32_relaxed(const uint32_t *obj); 347 348 static inline uint64_t 349 _Py_atomic_load_uint64_relaxed(const uint64_t *obj); 350 351 static inline uintptr_t 352 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj); 353 354 static inline unsigned int 355 _Py_atomic_load_uint_relaxed(const unsigned int *obj); 356 357 static inline Py_ssize_t 358 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj); 359 360 static inline void * 361 _Py_atomic_load_ptr_relaxed(const void *obj); 362 363 static inline unsigned long long 364 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj); 365 366 // --- _Py_atomic_store ------------------------------------------------------ 367 // Atomically performs `*obj = value` (sequential consistency) 368 369 static inline void 370 _Py_atomic_store_int(int *obj, int value); 371 372 static inline void 373 _Py_atomic_store_int8(int8_t *obj, int8_t value); 374 375 static inline void 376 _Py_atomic_store_int16(int16_t *obj, int16_t value); 377 378 static inline void 379 _Py_atomic_store_int32(int32_t *obj, int32_t value); 380 381 static inline void 382 _Py_atomic_store_int64(int64_t *obj, int64_t value); 383 384 static inline void 385 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value); 386 387 static inline void 388 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value); 389 390 static inline void 391 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value); 392 393 static inline void 394 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value); 395 396 static inline void 397 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value); 398 399 static inline void 400 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value); 401 402 static inline void 403 _Py_atomic_store_uint(unsigned int *obj, unsigned int value); 404 405 static inline void 406 _Py_atomic_store_ptr(void *obj, void *value); 407 408 static inline void 409 _Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value); 410 411 412 // --- _Py_atomic_store_relaxed ---------------------------------------------- 413 // Stores `*obj = value` (relaxed consistency, i.e., no ordering) 414 415 static inline void 416 _Py_atomic_store_int_relaxed(int *obj, int value); 417 418 static inline void 419 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value); 420 421 static inline void 422 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value); 423 424 static inline void 425 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value); 426 427 static inline void 428 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value); 429 430 static inline void 431 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value); 432 433 static inline void 434 _Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value); 435 436 static inline void 437 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value); 438 439 static inline void 440 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value); 441 442 static inline void 443 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value); 444 445 static inline void 446 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value); 447 448 static inline void 449 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value); 450 451 static inline void 452 _Py_atomic_store_ptr_relaxed(void *obj, void *value); 453 454 static inline void 455 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); 456 457 static inline void 458 _Py_atomic_store_ullong_relaxed(unsigned long long *obj, 459 unsigned long long value); 460 461 462 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ 463 464 // Loads `*obj` (acquire operation) 465 static inline void * 466 _Py_atomic_load_ptr_acquire(const void *obj); 467 468 static inline uintptr_t 469 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj); 470 471 // Stores `*obj = value` (release operation) 472 static inline void 473 _Py_atomic_store_ptr_release(void *obj, void *value); 474 475 static inline void 476 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); 477 478 static inline void 479 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); 480 481 static inline void 482 _Py_atomic_store_int_release(int *obj, int value); 483 484 static inline int 485 _Py_atomic_load_int_acquire(const int *obj); 486 487 static inline void 488 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); 489 490 static inline void 491 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); 492 493 static inline uint64_t 494 _Py_atomic_load_uint64_acquire(const uint64_t *obj); 495 496 static inline uint32_t 497 _Py_atomic_load_uint32_acquire(const uint32_t *obj); 498 499 static inline Py_ssize_t 500 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj); 501 502 503 504 505 // --- _Py_atomic_fence ------------------------------------------------------ 506 507 // Sequential consistency fence. C11 fences have complex semantics. When 508 // possible, use the atomic operations on variables defined above, which 509 // generally do not require explicit use of a fence. 510 // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence 511 static inline void _Py_atomic_fence_seq_cst(void); 512 513 // Acquire fence 514 static inline void _Py_atomic_fence_acquire(void); 515 516 // Release fence 517 static inline void _Py_atomic_fence_release(void); 518 519 520 #ifndef _Py_USE_GCC_BUILTIN_ATOMICS 521 # if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) 522 # define _Py_USE_GCC_BUILTIN_ATOMICS 1 523 # elif defined(__clang__) 524 # if __has_builtin(__atomic_load) 525 # define _Py_USE_GCC_BUILTIN_ATOMICS 1 526 # endif 527 # endif 528 #endif 529 530 #if _Py_USE_GCC_BUILTIN_ATOMICS 531 # define Py_ATOMIC_GCC_H 532 # include "cpython/pyatomic_gcc.h" 533 # undef Py_ATOMIC_GCC_H 534 #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) 535 # define Py_ATOMIC_STD_H 536 # include "cpython/pyatomic_std.h" 537 # undef Py_ATOMIC_STD_H 538 #elif defined(_MSC_VER) 539 # define Py_ATOMIC_MSC_H 540 # include "cpython/pyatomic_msc.h" 541 # undef Py_ATOMIC_MSC_H 542 #else 543 # error "no available pyatomic implementation for this platform/compiler" 544 #endif 545 546 547 // --- aliases --------------------------------------------------------------- 548 549 #if SIZEOF_LONG == 8 550 # define _Py_atomic_load_ulong(p) \ 551 _Py_atomic_load_uint64((uint64_t *)p) 552 # define _Py_atomic_load_ulong_relaxed(p) \ 553 _Py_atomic_load_uint64_relaxed((uint64_t *)p) 554 # define _Py_atomic_store_ulong(p, v) \ 555 _Py_atomic_store_uint64((uint64_t *)p, v) 556 # define _Py_atomic_store_ulong_relaxed(p, v) \ 557 _Py_atomic_store_uint64_relaxed((uint64_t *)p, v) 558 #elif SIZEOF_LONG == 4 559 # define _Py_atomic_load_ulong(p) \ 560 _Py_atomic_load_uint32((uint32_t *)p) 561 # define _Py_atomic_load_ulong_relaxed(p) \ 562 _Py_atomic_load_uint32_relaxed((uint32_t *)p) 563 # define _Py_atomic_store_ulong(p, v) \ 564 _Py_atomic_store_uint32((uint32_t *)p, v) 565 # define _Py_atomic_store_ulong_relaxed(p, v) \ 566 _Py_atomic_store_uint32_relaxed((uint32_t *)p, v) 567 #else 568 # error "long must be 4 or 8 bytes in size" 569 #endif // SIZEOF_LONG 570