1 //===--- Implementation of a Linux RawMutex class ---------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H 9 #define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H 10 11 #include "src/__support/CPP/optional.h" 12 #include "src/__support/common.h" 13 #include "src/__support/libc_assert.h" 14 #include "src/__support/macros/attributes.h" 15 #include "src/__support/threads/linux/futex_utils.h" 16 #include "src/__support/threads/linux/futex_word.h" 17 #include "src/__support/threads/sleep.h" 18 #include "src/__support/time/linux/abs_timeout.h" 19 20 #ifndef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 21 #define LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 1 22 #warning "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY is not defined, defaulting to 1" 23 #endif 24 25 #if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 26 #include "src/__support/time/linux/monotonicity.h" 27 #endif 28 29 #ifndef LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 30 #define LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 100 31 #warning \ 32 "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT is not defined, defaulting to 100" 33 #endif 34 35 namespace LIBC_NAMESPACE { 36 // Lock is a simple timable lock for internal usage. 37 // This is separated from Mutex because this one does not need to consider 38 // robustness and reentrancy. Also, this one has spin optimization for shorter 39 // critical sections. 40 class RawMutex { 41 protected: 42 Futex futex; 43 LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00; 44 LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01; 45 LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10; 46 47 private: spin(unsigned spin_count)48 LIBC_INLINE FutexWordType spin(unsigned spin_count) { 49 FutexWordType result; 50 for (;;) { 51 result = futex.load(cpp::MemoryOrder::RELAXED); 52 // spin until one of the following conditions is met: 53 // - the mutex is unlocked 54 // - the mutex is in contention 55 // - the spin count reaches 0 56 if (result != LOCKED || spin_count == 0u) 57 return result; 58 // Pause the pipeline to avoid extraneous memory operations due to 59 // speculation. 60 sleep_briefly(); 61 spin_count--; 62 }; 63 } 64 65 // Return true if the lock is acquired. Return false if timeout happens before 66 // the lock is acquired. lock_slow(cpp::optional<Futex::Timeout> timeout,bool is_pshared,unsigned spin_count)67 LIBC_INLINE bool lock_slow(cpp::optional<Futex::Timeout> timeout, 68 bool is_pshared, unsigned spin_count) { 69 FutexWordType state = spin(spin_count); 70 // Before go into contention state, try to grab the lock. 71 if (state == UNLOCKED && 72 futex.compare_exchange_strong(state, LOCKED, cpp::MemoryOrder::ACQUIRE, 73 cpp::MemoryOrder::RELAXED)) 74 return true; 75 #if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 76 /* ADL should kick in */ 77 if (timeout) 78 ensure_monotonicity(*timeout); 79 #endif 80 for (;;) { 81 // Try to grab the lock if it is unlocked. Mark the contention flag if it 82 // is locked. 83 if (state != IN_CONTENTION && 84 futex.exchange(IN_CONTENTION, cpp::MemoryOrder::ACQUIRE) == UNLOCKED) 85 return true; 86 // Contention persists. Park the thread and wait for further notification. 87 if (ETIMEDOUT == -futex.wait(IN_CONTENTION, timeout, is_pshared)) 88 return false; 89 // Continue to spin after waking up. 90 state = spin(spin_count); 91 } 92 } 93 wake(bool is_pshared)94 LIBC_INLINE void wake(bool is_pshared) { futex.notify_one(is_pshared); } 95 96 public: init(RawMutex * mutex)97 LIBC_INLINE static void init(RawMutex *mutex) { mutex->futex = UNLOCKED; } RawMutex()98 LIBC_INLINE constexpr RawMutex() : futex(UNLOCKED) {} try_lock()99 [[nodiscard]] LIBC_INLINE bool try_lock() { 100 FutexWordType expected = UNLOCKED; 101 // Use strong version since this is a one-time operation. 102 return futex.compare_exchange_strong( 103 expected, LOCKED, cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED); 104 } 105 LIBC_INLINE bool 106 lock(cpp::optional<Futex::Timeout> timeout = cpp::nullopt, 107 bool is_shared = false, 108 unsigned spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) { 109 // Timeout will not be checked if immediate lock is possible. 110 if (LIBC_LIKELY(try_lock())) 111 return true; 112 return lock_slow(timeout, is_shared, spin_count); 113 } 114 LIBC_INLINE bool unlock(bool is_pshared = false) { 115 FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE); 116 // if there is someone waiting, wake them up 117 if (LIBC_UNLIKELY(prev == IN_CONTENTION)) 118 wake(is_pshared); 119 // Detect invalid unlock operation. 120 return prev != UNLOCKED; 121 } destroy(RawMutex * lock)122 LIBC_INLINE void static destroy([[maybe_unused]] RawMutex *lock) { 123 LIBC_ASSERT(lock->futex == UNLOCKED && "Mutex destroyed while used."); 124 } get_raw_futex()125 LIBC_INLINE Futex &get_raw_futex() { return futex; } reset()126 LIBC_INLINE void reset() { futex = UNLOCKED; } 127 }; 128 } // namespace LIBC_NAMESPACE 129 130 #endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H 131