1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 /* 4 AtomicF{32,64} implementation based on AtomicU{32,64}. 5 6 This module provides atomic float implementations using atomic integer. 7 8 Note that most of `fetch_*` operations of atomic floats are implemented using 9 CAS loops, which can be slower than equivalent operations of atomic integers. 10 11 GPU targets have atomic instructions for float, so GPU targets will use 12 architecture-specific implementations instead of this implementation in the 13 future: https://github.com/taiki-e/portable-atomic/issues/34 / https://github.com/taiki-e/portable-atomic/pull/45 14 */ 15 16 // TODO: fetch_{minimum,maximum}* https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p3008r2.html 17 18 #![cfg_attr( 19 all(target_pointer_width = "16", not(feature = "fallback")), 20 allow(unused_imports, unused_macros) 21 )] 22 23 use core::{cell::UnsafeCell, sync::atomic::Ordering}; 24 25 macro_rules! atomic_float { 26 ( 27 $atomic_type:ident, $float_type:ident, $atomic_int_type:ident, $int_type:ident, 28 $align:literal 29 ) => { 30 #[repr(C, align($align))] 31 pub(crate) struct $atomic_type { 32 v: UnsafeCell<$float_type>, 33 } 34 35 // Send is implicitly implemented. 36 // SAFETY: any data races are prevented by atomic operations. 37 unsafe impl Sync for $atomic_type {} 38 39 impl $atomic_type { 40 #[inline] 41 pub(crate) const fn new(v: $float_type) -> Self { 42 Self { v: UnsafeCell::new(v) } 43 } 44 45 #[inline] 46 pub(crate) fn is_lock_free() -> bool { 47 crate::$atomic_int_type::is_lock_free() 48 } 49 pub(crate) const IS_ALWAYS_LOCK_FREE: bool = 50 crate::$atomic_int_type::is_always_lock_free(); 51 52 #[inline] 53 pub(crate) fn get_mut(&mut self) -> &mut $float_type { 54 // SAFETY: the mutable reference guarantees unique ownership. 55 // (UnsafeCell::get_mut requires Rust 1.50) 56 unsafe { &mut *self.v.get() } 57 } 58 59 #[inline] 60 #[cfg_attr( 61 any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), 62 track_caller 63 )] 64 pub(crate) fn load(&self, order: Ordering) -> $float_type { 65 $float_type::from_bits(self.as_bits().load(order)) 66 } 67 68 #[inline] 69 #[cfg_attr( 70 any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), 71 track_caller 72 )] 73 pub(crate) fn store(&self, val: $float_type, order: Ordering) { 74 self.as_bits().store(val.to_bits(), order) 75 } 76 77 const_fn! { 78 const_if: #[cfg(not(portable_atomic_no_const_raw_ptr_deref))]; 79 #[inline(always)] 80 pub(crate) const fn as_bits(&self) -> &crate::$atomic_int_type { 81 // SAFETY: $atomic_type and $atomic_int_type have the same layout, 82 // and there is no concurrent access to the value that does not go through this method. 83 unsafe { &*(self as *const Self as *const crate::$atomic_int_type) } 84 } 85 } 86 87 #[inline] 88 pub(crate) const fn as_ptr(&self) -> *mut $float_type { 89 self.v.get() 90 } 91 } 92 93 cfg_has_atomic_cas_or_amo32! { 94 impl $atomic_type { 95 #[inline] 96 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 97 pub(crate) fn swap(&self, val: $float_type, order: Ordering) -> $float_type { 98 $float_type::from_bits(self.as_bits().swap(val.to_bits(), order)) 99 } 100 101 cfg_has_atomic_cas! { 102 #[inline] 103 #[cfg_attr( 104 any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), 105 track_caller 106 )] 107 pub(crate) fn compare_exchange( 108 &self, 109 current: $float_type, 110 new: $float_type, 111 success: Ordering, 112 failure: Ordering, 113 ) -> Result<$float_type, $float_type> { 114 match self.as_bits().compare_exchange( 115 current.to_bits(), 116 new.to_bits(), 117 success, 118 failure, 119 ) { 120 Ok(v) => Ok($float_type::from_bits(v)), 121 Err(v) => Err($float_type::from_bits(v)), 122 } 123 } 124 125 #[inline] 126 #[cfg_attr( 127 any(all(debug_assertions, not(portable_atomic_no_track_caller)), miri), 128 track_caller 129 )] 130 pub(crate) fn compare_exchange_weak( 131 &self, 132 current: $float_type, 133 new: $float_type, 134 success: Ordering, 135 failure: Ordering, 136 ) -> Result<$float_type, $float_type> { 137 match self.as_bits().compare_exchange_weak( 138 current.to_bits(), 139 new.to_bits(), 140 success, 141 failure, 142 ) { 143 Ok(v) => Ok($float_type::from_bits(v)), 144 Err(v) => Err($float_type::from_bits(v)), 145 } 146 } 147 148 #[inline] 149 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 150 pub(crate) fn fetch_add(&self, val: $float_type, order: Ordering) -> $float_type { 151 self.fetch_update_(order, |x| x + val) 152 } 153 154 #[inline] 155 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 156 pub(crate) fn fetch_sub(&self, val: $float_type, order: Ordering) -> $float_type { 157 self.fetch_update_(order, |x| x - val) 158 } 159 160 #[inline] 161 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 162 fn fetch_update_<F>(&self, order: Ordering, mut f: F) -> $float_type 163 where 164 F: FnMut($float_type) -> $float_type, 165 { 166 // This is a private function and all instances of `f` only operate on the value 167 // loaded, so there is no need to synchronize the first load/failed CAS. 168 let mut prev = self.load(Ordering::Relaxed); 169 loop { 170 let next = f(prev); 171 match self.compare_exchange_weak(prev, next, order, Ordering::Relaxed) { 172 Ok(x) => return x, 173 Err(next_prev) => prev = next_prev, 174 } 175 } 176 } 177 178 #[inline] 179 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 180 pub(crate) fn fetch_max(&self, val: $float_type, order: Ordering) -> $float_type { 181 self.fetch_update_(order, |x| x.max(val)) 182 } 183 184 #[inline] 185 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 186 pub(crate) fn fetch_min(&self, val: $float_type, order: Ordering) -> $float_type { 187 self.fetch_update_(order, |x| x.min(val)) 188 } 189 } // cfg_has_atomic_cas! 190 #[inline] 191 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 192 pub(crate) fn fetch_neg(&self, order: Ordering) -> $float_type { 193 const NEG_MASK: $int_type = !0 / 2 + 1; 194 $float_type::from_bits(self.as_bits().fetch_xor(NEG_MASK, order)) 195 } 196 197 #[inline] 198 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces 199 pub(crate) fn fetch_abs(&self, order: Ordering) -> $float_type { 200 const ABS_MASK: $int_type = !0 / 2; 201 $float_type::from_bits(self.as_bits().fetch_and(ABS_MASK, order)) 202 } 203 } 204 } // cfg_has_atomic_cas_or_amo32! 205 }; 206 } 207 208 cfg_has_atomic_32! { 209 atomic_float!(AtomicF32, f32, AtomicU32, u32, 4); 210 } 211 cfg_has_atomic_64! { 212 atomic_float!(AtomicF64, f64, AtomicU64, u64, 8); 213 } 214