// Copyright 2016 Amanieu d'Antras // // Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. use bytemuck::NoUninit; #[cfg(feature = "fallback")] use crate::fallback; use core::cmp; use core::mem; use core::num::Wrapping; use core::ops; use core::sync::atomic::Ordering; macro_rules! match_atomic { ($type:ident, $atomic:ident, $impl:expr, $fallback_impl:expr) => { match mem::size_of::<$type>() { #[cfg(target_has_atomic = "8")] 1 if mem::align_of::<$type>() >= 1 => { type $atomic = core::sync::atomic::AtomicU8; $impl } #[cfg(target_has_atomic = "16")] 2 if mem::align_of::<$type>() >= 2 => { type $atomic = core::sync::atomic::AtomicU16; $impl } #[cfg(target_has_atomic = "32")] 4 if mem::align_of::<$type>() >= 4 => { type $atomic = core::sync::atomic::AtomicU32; $impl } #[cfg(target_has_atomic = "64")] 8 if mem::align_of::<$type>() >= 8 => { type $atomic = core::sync::atomic::AtomicU64; $impl } #[cfg(all(feature = "nightly", target_has_atomic = "128"))] 16 if mem::align_of::<$type>() >= 16 => { type $atomic = core::sync::atomic::AtomicU128; $impl } #[cfg(feature = "fallback")] _ => $fallback_impl, #[cfg(not(feature = "fallback"))] _ => panic!("Atomic operations for type `{}` are not available as the `fallback` feature of the `atomic` crate is disabled.", core::any::type_name::<$type>()), } }; } macro_rules! match_signed_atomic { ($type:ident, $atomic:ident, $impl:expr, $fallback_impl:expr) => { match mem::size_of::<$type>() { #[cfg(target_has_atomic = "8")] 1 if mem::align_of::<$type>() >= 1 => { type $atomic = core::sync::atomic::AtomicI8; $impl } #[cfg(target_has_atomic = "16")] 2 if mem::align_of::<$type>() >= 2 => { type $atomic = core::sync::atomic::AtomicI16; $impl } #[cfg(target_has_atomic = "32")] 4 if mem::align_of::<$type>() >= 4 => { type $atomic = core::sync::atomic::AtomicI32; $impl } #[cfg(target_has_atomic = "64")] 8 if mem::align_of::<$type>() >= 8 => { type $atomic = core::sync::atomic::AtomicI64; $impl } #[cfg(all(feature = "nightly", target_has_atomic = "128"))] 16 if mem::align_of::<$type>() >= 16 => { type $atomic = core::sync::atomic::AtomicI128; $impl } #[cfg(feature = "fallback")] _ => $fallback_impl, #[cfg(not(feature = "fallback"))] _ => panic!("Atomic operations for type `{}` are not available as the `fallback` feature of the `atomic` crate is disabled.", core::any::type_name::<$type>()), } }; } #[inline] pub const fn atomic_is_lock_free() -> bool { let size = mem::size_of::(); let align = mem::align_of::(); (cfg!(target_has_atomic = "8") & (size == 1) & (align >= 1)) | (cfg!(target_has_atomic = "16") & (size == 2) & (align >= 2)) | (cfg!(target_has_atomic = "32") & (size == 4) & (align >= 4)) | (cfg!(target_has_atomic = "64") & (size == 8) & (align >= 8)) | (cfg!(feature = "nightly") & cfg!(target_has_atomic = "128") & (size == 16) & (align >= 16)) } #[inline] pub unsafe fn atomic_load(dst: *mut T, order: Ordering) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).load(order)), fallback::atomic_load(dst) ) } #[inline] pub unsafe fn atomic_store(dst: *mut T, val: T, order: Ordering) { match_atomic!( T, A, (*(dst as *const A)).store(mem::transmute_copy(&val), order), fallback::atomic_store(dst, val) ) } #[inline] pub unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).swap(mem::transmute_copy(&val), order)), fallback::atomic_swap(dst, val) ) } #[inline] unsafe fn map_result(r: Result) -> Result { match r { Ok(x) => Ok(mem::transmute_copy(&x)), Err(x) => Err(mem::transmute_copy(&x)), } } #[inline] pub unsafe fn atomic_compare_exchange( dst: *mut T, current: T, new: T, success: Ordering, failure: Ordering, ) -> Result { match_atomic!( T, A, map_result((*(dst as *const A)).compare_exchange( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )), fallback::atomic_compare_exchange(dst, current, new) ) } #[inline] pub unsafe fn atomic_compare_exchange_weak( dst: *mut T, current: T, new: T, success: Ordering, failure: Ordering, ) -> Result { match_atomic!( T, A, map_result((*(dst as *const A)).compare_exchange_weak( mem::transmute_copy(¤t), mem::transmute_copy(&new), success, failure, )), fallback::atomic_compare_exchange(dst, current, new) ) } #[inline] pub unsafe fn atomic_add(dst: *mut T, val: T, order: Ordering) -> T where Wrapping: ops::Add>, { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_add(mem::transmute_copy(&val), order),), fallback::atomic_add(dst, val) ) } #[inline] pub unsafe fn atomic_sub(dst: *mut T, val: T, order: Ordering) -> T where Wrapping: ops::Sub>, { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_sub(mem::transmute_copy(&val), order),), fallback::atomic_sub(dst, val) ) } #[inline] pub unsafe fn atomic_and>( dst: *mut T, val: T, order: Ordering, ) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_and(mem::transmute_copy(&val), order),), fallback::atomic_and(dst, val) ) } #[inline] pub unsafe fn atomic_or>( dst: *mut T, val: T, order: Ordering, ) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_or(mem::transmute_copy(&val), order),), fallback::atomic_or(dst, val) ) } #[inline] pub unsafe fn atomic_xor>( dst: *mut T, val: T, order: Ordering, ) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_xor(mem::transmute_copy(&val), order),), fallback::atomic_xor(dst, val) ) } #[inline] pub unsafe fn atomic_min(dst: *mut T, val: T, order: Ordering) -> T { match_signed_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_min(mem::transmute_copy(&val), order),), fallback::atomic_min(dst, val) ) } #[inline] pub unsafe fn atomic_max(dst: *mut T, val: T, order: Ordering) -> T { match_signed_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_max(mem::transmute_copy(&val), order),), fallback::atomic_max(dst, val) ) } #[inline] pub unsafe fn atomic_umin(dst: *mut T, val: T, order: Ordering) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_min(mem::transmute_copy(&val), order),), fallback::atomic_min(dst, val) ) } #[inline] pub unsafe fn atomic_umax(dst: *mut T, val: T, order: Ordering) -> T { match_atomic!( T, A, mem::transmute_copy(&(*(dst as *const A)).fetch_max(mem::transmute_copy(&val), order),), fallback::atomic_max(dst, val) ) }