1 // Copyright 2020 The Abseil Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 #ifndef ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 15 #define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 16 17 #include "absl/base/config.h" 18 19 #ifdef _WIN32 20 #include <windows.h> 21 #else 22 #include <sys/time.h> 23 #include <unistd.h> 24 #endif 25 26 #ifdef __linux__ 27 #include <linux/futex.h> 28 #include <sys/syscall.h> 29 #endif 30 31 #include <errno.h> 32 #include <stdio.h> 33 #include <time.h> 34 35 #include <atomic> 36 #include <cstdint> 37 38 #include "absl/base/optimization.h" 39 #include "absl/synchronization/internal/kernel_timeout.h" 40 41 #ifdef ABSL_INTERNAL_HAVE_FUTEX 42 #error ABSL_INTERNAL_HAVE_FUTEX may not be set on the command line 43 #elif defined(__BIONIC__) 44 // Bionic supports all the futex operations we need even when some of the futex 45 // definitions are missing. 46 #define ABSL_INTERNAL_HAVE_FUTEX 47 #elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME) 48 // FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28. 49 #define ABSL_INTERNAL_HAVE_FUTEX 50 #endif 51 52 #ifdef ABSL_INTERNAL_HAVE_FUTEX 53 54 namespace absl { 55 ABSL_NAMESPACE_BEGIN 56 namespace synchronization_internal { 57 58 // Some Android headers are missing these definitions even though they 59 // support these futex operations. 60 #ifdef __BIONIC__ 61 #ifndef SYS_futex 62 #define SYS_futex __NR_futex 63 #endif 64 #ifndef FUTEX_WAIT_BITSET 65 #define FUTEX_WAIT_BITSET 9 66 #endif 67 #ifndef FUTEX_PRIVATE_FLAG 68 #define FUTEX_PRIVATE_FLAG 128 69 #endif 70 #ifndef FUTEX_CLOCK_REALTIME 71 #define FUTEX_CLOCK_REALTIME 256 72 #endif 73 #ifndef FUTEX_BITSET_MATCH_ANY 74 #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF 75 #endif 76 #endif 77 78 #if defined(__NR_futex_time64) && !defined(SYS_futex_time64) 79 #define SYS_futex_time64 __NR_futex_time64 80 #endif 81 82 #if defined(SYS_futex_time64) && !defined(SYS_futex) 83 #define SYS_futex SYS_futex_time64 84 #endif 85 86 class FutexImpl { 87 public: WaitUntil(std::atomic<int32_t> * v,int32_t val,KernelTimeout t)88 static int WaitUntil(std::atomic<int32_t> *v, int32_t val, 89 KernelTimeout t) { 90 int err = 0; 91 if (t.has_timeout()) { 92 // https://locklessinc.com/articles/futex_cheat_sheet/ 93 // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. 94 struct timespec abs_timeout = t.MakeAbsTimespec(); 95 // Atomically check that the futex value is still 0, and if it 96 // is, sleep until abs_timeout or until woken by FUTEX_WAKE. 97 err = syscall( 98 SYS_futex, reinterpret_cast<int32_t *>(v), 99 FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, 100 &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); 101 } else { 102 // Atomically check that the futex value is still 0, and if it 103 // is, sleep until woken by FUTEX_WAKE. 104 err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), 105 FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr); 106 } 107 if (ABSL_PREDICT_FALSE(err != 0)) { 108 err = -errno; 109 } 110 return err; 111 } 112 WaitBitsetAbsoluteTimeout(std::atomic<int32_t> * v,int32_t val,int32_t bits,const struct timespec * abstime)113 static int WaitBitsetAbsoluteTimeout(std::atomic<int32_t> *v, int32_t val, 114 int32_t bits, 115 const struct timespec *abstime) { 116 int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), 117 FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG, val, abstime, 118 nullptr, bits); 119 if (ABSL_PREDICT_FALSE(err != 0)) { 120 err = -errno; 121 } 122 return err; 123 } 124 Wake(std::atomic<int32_t> * v,int32_t count)125 static int Wake(std::atomic<int32_t> *v, int32_t count) { 126 int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), 127 FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); 128 if (ABSL_PREDICT_FALSE(err < 0)) { 129 err = -errno; 130 } 131 return err; 132 } 133 134 // FUTEX_WAKE_BITSET WakeBitset(std::atomic<int32_t> * v,int32_t count,int32_t bits)135 static int WakeBitset(std::atomic<int32_t> *v, int32_t count, int32_t bits) { 136 int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), 137 FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, count, nullptr, 138 nullptr, bits); 139 if (ABSL_PREDICT_FALSE(err < 0)) { 140 err = -errno; 141 } 142 return err; 143 } 144 }; 145 146 class Futex : public FutexImpl {}; 147 148 } // namespace synchronization_internal 149 ABSL_NAMESPACE_END 150 } // namespace absl 151 152 #endif // ABSL_INTERNAL_HAVE_FUTEX 153 154 #endif // ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 155