// Copyright 2018 Amanieu d'Antras // // Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. use crate::{ mutex::{RawMutex, RawMutexFair, RawMutexTimed}, GuardNoSend, }; use core::{ cell::{Cell, UnsafeCell}, fmt, marker::PhantomData, mem, num::NonZeroUsize, ops::Deref, sync::atomic::{AtomicUsize, Ordering}, }; #[cfg(feature = "owning_ref")] use owning_ref::StableAddress; #[cfg(feature = "serde")] use serde::{Deserialize, Deserializer, Serialize, Serializer}; /// Helper trait which returns a non-zero thread ID. /// /// The simplest way to implement this trait is to return the address of a /// thread-local variable. /// /// # Safety /// /// Implementations of this trait must ensure that no two active threads share /// the same thread ID. However the ID of a thread that has exited can be /// re-used since that thread is no longer active. pub unsafe trait GetThreadId { /// Initial value. // A “non-constant” const item is a legacy way to supply an initialized value to downstream // static items. Can hopefully be replaced with `const fn new() -> Self` at some point. #[allow(clippy::declare_interior_mutable_const)] const INIT: Self; /// Returns a non-zero thread ID which identifies the current thread of /// execution. fn nonzero_thread_id(&self) -> NonZeroUsize; } /// A raw mutex type that wraps another raw mutex to provide reentrancy. /// /// Although this has the same methods as the [`RawMutex`] trait, it does /// not implement it, and should not be used in the same way, since this /// mutex can successfully acquire a lock multiple times in the same thread. /// Only use this when you know you want a raw mutex that can be locked /// reentrantly; you probably want [`ReentrantMutex`] instead. /// /// [`RawMutex`]: trait.RawMutex.html /// [`ReentrantMutex`]: struct.ReentrantMutex.html pub struct RawReentrantMutex { owner: AtomicUsize, lock_count: Cell, mutex: R, get_thread_id: G, } unsafe impl Send for RawReentrantMutex {} unsafe impl Sync for RawReentrantMutex {} impl RawReentrantMutex { /// Initial value for an unlocked mutex. #[allow(clippy::declare_interior_mutable_const)] pub const INIT: Self = RawReentrantMutex { owner: AtomicUsize::new(0), lock_count: Cell::new(0), mutex: R::INIT, get_thread_id: G::INIT, }; #[inline] fn lock_internal bool>(&self, try_lock: F) -> bool { let id = self.get_thread_id.nonzero_thread_id().get(); if self.owner.load(Ordering::Relaxed) == id { self.lock_count.set( self.lock_count .get() .checked_add(1) .expect("ReentrantMutex lock count overflow"), ); } else { if !try_lock() { return false; } self.owner.store(id, Ordering::Relaxed); debug_assert_eq!(self.lock_count.get(), 0); self.lock_count.set(1); } true } /// Acquires this mutex, blocking if it's held by another thread. #[inline] pub fn lock(&self) { self.lock_internal(|| { self.mutex.lock(); true }); } /// Attempts to acquire this mutex without blocking. Returns `true` /// if the lock was successfully acquired and `false` otherwise. #[inline] pub fn try_lock(&self) -> bool { self.lock_internal(|| self.mutex.try_lock()) } /// Unlocks this mutex. The inner mutex may not be unlocked if /// this mutex was acquired previously in the current thread. /// /// # Safety /// /// This method may only be called if the mutex is held by the current thread. #[inline] pub unsafe fn unlock(&self) { let lock_count = self.lock_count.get() - 1; self.lock_count.set(lock_count); if lock_count == 0 { self.owner.store(0, Ordering::Relaxed); self.mutex.unlock(); } } /// Checks whether the mutex is currently locked. #[inline] pub fn is_locked(&self) -> bool { self.mutex.is_locked() } } impl RawReentrantMutex { /// Unlocks this mutex using a fair unlock protocol. The inner mutex /// may not be unlocked if this mutex was acquired previously in the /// current thread. /// /// # Safety /// /// This method may only be called if the mutex is held by the current thread. #[inline] pub unsafe fn unlock_fair(&self) { let lock_count = self.lock_count.get() - 1; self.lock_count.set(lock_count); if lock_count == 0 { self.owner.store(0, Ordering::Relaxed); self.mutex.unlock_fair(); } } /// Temporarily yields the mutex to a waiting thread if there is one. /// /// This method is functionally equivalent to calling `unlock_fair` followed /// by `lock`, however it can be much more efficient in the case where there /// are no waiting threads. /// /// # Safety /// /// This method may only be called if the mutex is held by the current thread. #[inline] pub unsafe fn bump(&self) { if self.lock_count.get() == 1 { let id = self.owner.load(Ordering::Relaxed); self.owner.store(0, Ordering::Relaxed); self.mutex.bump(); self.owner.store(id, Ordering::Relaxed); } } } impl RawReentrantMutex { /// Attempts to acquire this lock until a timeout is reached. #[inline] pub fn try_lock_until(&self, timeout: R::Instant) -> bool { self.lock_internal(|| self.mutex.try_lock_until(timeout)) } /// Attempts to acquire this lock until a timeout is reached. #[inline] pub fn try_lock_for(&self, timeout: R::Duration) -> bool { self.lock_internal(|| self.mutex.try_lock_for(timeout)) } } /// A mutex which can be recursively locked by a single thread. /// /// This type is identical to `Mutex` except for the following points: /// /// - Locking multiple times from the same thread will work correctly instead of /// deadlocking. /// - `ReentrantMutexGuard` does not give mutable references to the locked data. /// Use a `RefCell` if you need this. /// /// See [`Mutex`](struct.Mutex.html) for more details about the underlying mutex /// primitive. pub struct ReentrantMutex { raw: RawReentrantMutex, data: UnsafeCell, } unsafe impl Send for ReentrantMutex { } unsafe impl Sync for ReentrantMutex { } impl ReentrantMutex { /// Creates a new reentrant mutex in an unlocked state ready for use. #[cfg(feature = "nightly")] #[inline] pub const fn new(val: T) -> ReentrantMutex { ReentrantMutex { data: UnsafeCell::new(val), raw: RawReentrantMutex { owner: AtomicUsize::new(0), lock_count: Cell::new(0), mutex: R::INIT, get_thread_id: G::INIT, }, } } /// Creates a new reentrant mutex in an unlocked state ready for use. #[cfg(not(feature = "nightly"))] #[inline] pub fn new(val: T) -> ReentrantMutex { ReentrantMutex { data: UnsafeCell::new(val), raw: RawReentrantMutex { owner: AtomicUsize::new(0), lock_count: Cell::new(0), mutex: R::INIT, get_thread_id: G::INIT, }, } } /// Consumes this mutex, returning the underlying data. #[inline] pub fn into_inner(self) -> T { self.data.into_inner() } } impl ReentrantMutex { /// Creates a new reentrant mutex based on a pre-existing raw mutex and a /// helper to get the thread ID. /// /// This allows creating a reentrant mutex in a constant context on stable /// Rust. #[inline] pub const fn const_new(raw_mutex: R, get_thread_id: G, val: T) -> ReentrantMutex { ReentrantMutex { data: UnsafeCell::new(val), raw: RawReentrantMutex { owner: AtomicUsize::new(0), lock_count: Cell::new(0), mutex: raw_mutex, get_thread_id, }, } } } impl ReentrantMutex { /// # Safety /// /// The lock must be held when calling this method. #[inline] unsafe fn guard(&self) -> ReentrantMutexGuard<'_, R, G, T> { ReentrantMutexGuard { remutex: &self, marker: PhantomData, } } /// Acquires a reentrant mutex, blocking the current thread until it is able /// to do so. /// /// If the mutex is held by another thread then this function will block the /// local thread until it is available to acquire the mutex. If the mutex is /// already held by the current thread then this function will increment the /// lock reference count and return immediately. Upon returning, /// the thread is the only thread with the mutex held. An RAII guard is /// returned to allow scoped unlock of the lock. When the guard goes out of /// scope, the mutex will be unlocked. #[inline] pub fn lock(&self) -> ReentrantMutexGuard<'_, R, G, T> { self.raw.lock(); // SAFETY: The lock is held, as required. unsafe { self.guard() } } /// Attempts to acquire this lock. /// /// If the lock could not be acquired at this time, then `None` is returned. /// Otherwise, an RAII guard is returned. The lock will be unlocked when the /// guard is dropped. /// /// This function does not block. #[inline] pub fn try_lock(&self) -> Option> { if self.raw.try_lock() { // SAFETY: The lock is held, as required. Some(unsafe { self.guard() }) } else { None } } /// Returns a mutable reference to the underlying data. /// /// Since this call borrows the `ReentrantMutex` mutably, no actual locking needs to /// take place---the mutable borrow statically guarantees no locks exist. #[inline] pub fn get_mut(&mut self) -> &mut T { unsafe { &mut *self.data.get() } } /// Checks whether the mutex is currently locked. #[inline] pub fn is_locked(&self) -> bool { self.raw.is_locked() } /// Forcibly unlocks the mutex. /// /// This is useful when combined with `mem::forget` to hold a lock without /// the need to maintain a `ReentrantMutexGuard` object alive, for example when /// dealing with FFI. /// /// # Safety /// /// This method must only be called if the current thread logically owns a /// `ReentrantMutexGuard` but that guard has be discarded using `mem::forget`. /// Behavior is undefined if a mutex is unlocked when not locked. #[inline] pub unsafe fn force_unlock(&self) { self.raw.unlock(); } /// Returns the underlying raw mutex object. /// /// Note that you will most likely need to import the `RawMutex` trait from /// `lock_api` to be able to call functions on the raw mutex. /// /// # Safety /// /// This method is unsafe because it allows unlocking a mutex while /// still holding a reference to a `ReentrantMutexGuard`. #[inline] pub unsafe fn raw(&self) -> &R { &self.raw.mutex } /// Returns a raw pointer to the underlying data. /// /// This is useful when combined with `mem::forget` to hold a lock without /// the need to maintain a `ReentrantMutexGuard` object alive, for example /// when dealing with FFI. /// /// # Safety /// /// You must ensure that there are no data races when dereferencing the /// returned pointer, for example if the current thread logically owns a /// `ReentrantMutexGuard` but that guard has been discarded using /// `mem::forget`. #[inline] pub fn data_ptr(&self) -> *mut T { self.data.get() } } impl ReentrantMutex { /// Forcibly unlocks the mutex using a fair unlock protocol. /// /// This is useful when combined with `mem::forget` to hold a lock without /// the need to maintain a `ReentrantMutexGuard` object alive, for example when /// dealing with FFI. /// /// # Safety /// /// This method must only be called if the current thread logically owns a /// `ReentrantMutexGuard` but that guard has be discarded using `mem::forget`. /// Behavior is undefined if a mutex is unlocked when not locked. #[inline] pub unsafe fn force_unlock_fair(&self) { self.raw.unlock_fair(); } } impl ReentrantMutex { /// Attempts to acquire this lock until a timeout is reached. /// /// If the lock could not be acquired before the timeout expired, then /// `None` is returned. Otherwise, an RAII guard is returned. The lock will /// be unlocked when the guard is dropped. #[inline] pub fn try_lock_for(&self, timeout: R::Duration) -> Option> { if self.raw.try_lock_for(timeout) { // SAFETY: The lock is held, as required. Some(unsafe { self.guard() }) } else { None } } /// Attempts to acquire this lock until a timeout is reached. /// /// If the lock could not be acquired before the timeout expired, then /// `None` is returned. Otherwise, an RAII guard is returned. The lock will /// be unlocked when the guard is dropped. #[inline] pub fn try_lock_until(&self, timeout: R::Instant) -> Option> { if self.raw.try_lock_until(timeout) { // SAFETY: The lock is held, as required. Some(unsafe { self.guard() }) } else { None } } } impl Default for ReentrantMutex { #[inline] fn default() -> ReentrantMutex { ReentrantMutex::new(Default::default()) } } impl From for ReentrantMutex { #[inline] fn from(t: T) -> ReentrantMutex { ReentrantMutex::new(t) } } impl fmt::Debug for ReentrantMutex { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.try_lock() { Some(guard) => f .debug_struct("ReentrantMutex") .field("data", &&*guard) .finish(), None => { struct LockedPlaceholder; impl fmt::Debug for LockedPlaceholder { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str("") } } f.debug_struct("ReentrantMutex") .field("data", &LockedPlaceholder) .finish() } } } } // Copied and modified from serde #[cfg(feature = "serde")] impl Serialize for ReentrantMutex where R: RawMutex, G: GetThreadId, T: Serialize + ?Sized, { fn serialize(&self, serializer: S) -> Result where S: Serializer, { self.lock().serialize(serializer) } } #[cfg(feature = "serde")] impl<'de, R, G, T> Deserialize<'de> for ReentrantMutex where R: RawMutex, G: GetThreadId, T: Deserialize<'de> + ?Sized, { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { Deserialize::deserialize(deserializer).map(ReentrantMutex::new) } } /// An RAII implementation of a "scoped lock" of a reentrant mutex. When this structure /// is dropped (falls out of scope), the lock will be unlocked. /// /// The data protected by the mutex can be accessed through this guard via its /// `Deref` implementation. #[must_use = "if unused the ReentrantMutex will immediately unlock"] pub struct ReentrantMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { remutex: &'a ReentrantMutex, marker: PhantomData<(&'a T, GuardNoSend)>, } unsafe impl<'a, R: RawMutex + Sync + 'a, G: GetThreadId + Sync + 'a, T: ?Sized + Sync + 'a> Sync for ReentrantMutexGuard<'a, R, G, T> { } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> ReentrantMutexGuard<'a, R, G, T> { /// Returns a reference to the original `ReentrantMutex` object. pub fn remutex(s: &Self) -> &'a ReentrantMutex { s.remutex } /// Makes a new `MappedReentrantMutexGuard` for a component of the locked data. /// /// This operation cannot fail as the `ReentrantMutexGuard` passed /// in already locked the mutex. /// /// This is an associated function that needs to be /// used as `ReentrantMutexGuard::map(...)`. A method would interfere with methods of /// the same name on the contents of the locked data. #[inline] pub fn map(s: Self, f: F) -> MappedReentrantMutexGuard<'a, R, G, U> where F: FnOnce(&T) -> &U, { let raw = &s.remutex.raw; let data = f(unsafe { &*s.remutex.data.get() }); mem::forget(s); MappedReentrantMutexGuard { raw, data, marker: PhantomData, } } /// Attempts to make a new `MappedReentrantMutexGuard` for a component of the /// locked data. The original guard is return if the closure returns `None`. /// /// This operation cannot fail as the `ReentrantMutexGuard` passed /// in already locked the mutex. /// /// This is an associated function that needs to be /// used as `ReentrantMutexGuard::map(...)`. A method would interfere with methods of /// the same name on the contents of the locked data. #[inline] pub fn try_map( s: Self, f: F, ) -> Result, Self> where F: FnOnce(&mut T) -> Option<&mut U>, { let raw = &s.remutex.raw; let data = match f(unsafe { &mut *s.remutex.data.get() }) { Some(data) => data, None => return Err(s), }; mem::forget(s); Ok(MappedReentrantMutexGuard { raw, data, marker: PhantomData, }) } /// Temporarily unlocks the mutex to execute the given function. /// /// This is safe because `&mut` guarantees that there exist no other /// references to the data protected by the mutex. #[inline] pub fn unlocked(s: &mut Self, f: F) -> U where F: FnOnce() -> U, { // Safety: A ReentrantMutexGuard always holds the lock. unsafe { s.remutex.raw.unlock(); } defer!(s.remutex.raw.lock()); f() } } impl<'a, R: RawMutexFair + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> ReentrantMutexGuard<'a, R, G, T> { /// Unlocks the mutex using a fair unlock protocol. /// /// By default, mutexes are unfair and allow the current thread to re-lock /// the mutex before another has the chance to acquire the lock, even if /// that thread has been blocked on the mutex for a long time. This is the /// default because it allows much higher throughput as it avoids forcing a /// context switch on every mutex unlock. This can result in one thread /// acquiring a mutex many more times than other threads. /// /// However in some cases it can be beneficial to ensure fairness by forcing /// the lock to pass on to a waiting thread if there is one. This is done by /// using this method instead of dropping the `ReentrantMutexGuard` normally. #[inline] pub fn unlock_fair(s: Self) { // Safety: A ReentrantMutexGuard always holds the lock unsafe { s.remutex.raw.unlock_fair(); } mem::forget(s); } /// Temporarily unlocks the mutex to execute the given function. /// /// The mutex is unlocked a fair unlock protocol. /// /// This is safe because `&mut` guarantees that there exist no other /// references to the data protected by the mutex. #[inline] pub fn unlocked_fair(s: &mut Self, f: F) -> U where F: FnOnce() -> U, { // Safety: A ReentrantMutexGuard always holds the lock unsafe { s.remutex.raw.unlock_fair(); } defer!(s.remutex.raw.lock()); f() } /// Temporarily yields the mutex to a waiting thread if there is one. /// /// This method is functionally equivalent to calling `unlock_fair` followed /// by `lock`, however it can be much more efficient in the case where there /// are no waiting threads. #[inline] pub fn bump(s: &mut Self) { // Safety: A ReentrantMutexGuard always holds the lock unsafe { s.remutex.raw.bump(); } } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> Deref for ReentrantMutexGuard<'a, R, G, T> { type Target = T; #[inline] fn deref(&self) -> &T { unsafe { &*self.remutex.data.get() } } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> Drop for ReentrantMutexGuard<'a, R, G, T> { #[inline] fn drop(&mut self) { // Safety: A ReentrantMutexGuard always holds the lock. unsafe { self.remutex.raw.unlock(); } } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for ReentrantMutexGuard<'a, R, G, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display for ReentrantMutexGuard<'a, R, G, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } #[cfg(feature = "owning_ref")] unsafe impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> StableAddress for ReentrantMutexGuard<'a, R, G, T> { } /// An RAII mutex guard returned by `ReentrantMutexGuard::map`, which can point to a /// subfield of the protected data. /// /// The main difference between `MappedReentrantMutexGuard` and `ReentrantMutexGuard` is that the /// former doesn't support temporarily unlocking and re-locking, since that /// could introduce soundness issues if the locked object is modified by another /// thread. #[must_use = "if unused the ReentrantMutex will immediately unlock"] pub struct MappedReentrantMutexGuard<'a, R: RawMutex, G: GetThreadId, T: ?Sized> { raw: &'a RawReentrantMutex, data: *const T, marker: PhantomData<&'a T>, } unsafe impl<'a, R: RawMutex + Sync + 'a, G: GetThreadId + Sync + 'a, T: ?Sized + Sync + 'a> Sync for MappedReentrantMutexGuard<'a, R, G, T> { } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> MappedReentrantMutexGuard<'a, R, G, T> { /// Makes a new `MappedReentrantMutexGuard` for a component of the locked data. /// /// This operation cannot fail as the `MappedReentrantMutexGuard` passed /// in already locked the mutex. /// /// This is an associated function that needs to be /// used as `MappedReentrantMutexGuard::map(...)`. A method would interfere with methods of /// the same name on the contents of the locked data. #[inline] pub fn map(s: Self, f: F) -> MappedReentrantMutexGuard<'a, R, G, U> where F: FnOnce(&T) -> &U, { let raw = s.raw; let data = f(unsafe { &*s.data }); mem::forget(s); MappedReentrantMutexGuard { raw, data, marker: PhantomData, } } /// Attempts to make a new `MappedReentrantMutexGuard` for a component of the /// locked data. The original guard is return if the closure returns `None`. /// /// This operation cannot fail as the `MappedReentrantMutexGuard` passed /// in already locked the mutex. /// /// This is an associated function that needs to be /// used as `MappedReentrantMutexGuard::map(...)`. A method would interfere with methods of /// the same name on the contents of the locked data. #[inline] pub fn try_map( s: Self, f: F, ) -> Result, Self> where F: FnOnce(&T) -> Option<&U>, { let raw = s.raw; let data = match f(unsafe { &*s.data }) { Some(data) => data, None => return Err(s), }; mem::forget(s); Ok(MappedReentrantMutexGuard { raw, data, marker: PhantomData, }) } } impl<'a, R: RawMutexFair + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> MappedReentrantMutexGuard<'a, R, G, T> { /// Unlocks the mutex using a fair unlock protocol. /// /// By default, mutexes are unfair and allow the current thread to re-lock /// the mutex before another has the chance to acquire the lock, even if /// that thread has been blocked on the mutex for a long time. This is the /// default because it allows much higher throughput as it avoids forcing a /// context switch on every mutex unlock. This can result in one thread /// acquiring a mutex many more times than other threads. /// /// However in some cases it can be beneficial to ensure fairness by forcing /// the lock to pass on to a waiting thread if there is one. This is done by /// using this method instead of dropping the `ReentrantMutexGuard` normally. #[inline] pub fn unlock_fair(s: Self) { // Safety: A MappedReentrantMutexGuard always holds the lock unsafe { s.raw.unlock_fair(); } mem::forget(s); } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> Deref for MappedReentrantMutexGuard<'a, R, G, T> { type Target = T; #[inline] fn deref(&self) -> &T { unsafe { &*self.data } } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> Drop for MappedReentrantMutexGuard<'a, R, G, T> { #[inline] fn drop(&mut self) { // Safety: A MappedReentrantMutexGuard always holds the lock. unsafe { self.raw.unlock(); } } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for MappedReentrantMutexGuard<'a, R, G, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display for MappedReentrantMutexGuard<'a, R, G, T> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { (**self).fmt(f) } } #[cfg(feature = "owning_ref")] unsafe impl<'a, R: RawMutex + 'a, G: GetThreadId + 'a, T: ?Sized + 'a> StableAddress for MappedReentrantMutexGuard<'a, R, G, T> { }