1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2
3 /*
4 Run-time CPU feature detection on x86_64 by using CPUID.
5
6 Adapted from https://github.com/rust-lang/stdarch.
7 */
8
9 #![cfg_attr(portable_atomic_sanitize_thread, allow(dead_code))]
10
11 // Miri doesn't support inline assembly used in __cpuid: https://github.com/rust-lang/miri/issues/932
12 // SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
13 #[cfg(any(target_env = "sgx", miri))]
14 compile_error!("internal error: this module is not supported on this environment");
15
16 include!("common.rs");
17
18 #[cfg(not(portable_atomic_no_asm))]
19 use core::arch::asm;
20 use core::arch::x86_64::CpuidResult;
21
22 // Workaround for https://github.com/rust-lang/rust/issues/101346
23 // It is not clear if our use cases are affected, but we implement this just in case.
24 //
25 // Refs:
26 // - https://www.felixcloutier.com/x86/cpuid
27 // - https://en.wikipedia.org/wiki/CPUID
28 // - https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs
29 #[cfg(not(target_env = "sgx"))]
__cpuid(leaf: u32) -> CpuidResult30 fn __cpuid(leaf: u32) -> CpuidResult {
31 let eax;
32 let mut ebx;
33 let ecx;
34 let edx;
35 // SAFETY: Calling `__cpuid`` is safe on all x86_64 CPUs except for SGX,
36 // which doesn't support `cpuid`.
37 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L109
38 unsafe {
39 asm!(
40 "mov {ebx_tmp:r}, rbx", // save rbx which is reserved by LLVM
41 "cpuid",
42 "xchg {ebx_tmp:r}, rbx", // restore rbx
43 ebx_tmp = out(reg) ebx,
44 inout("eax") leaf => eax,
45 inout("ecx") 0 => ecx,
46 out("edx") edx,
47 options(nostack, preserves_flags),
48 );
49 }
50 CpuidResult { eax, ebx, ecx, edx }
51 }
52
53 // https://en.wikipedia.org/wiki/CPUID
54 const _VENDOR_ID_INTEL: [u8; 12] = *b"GenuineIntel"; // Intel
55 const _VENDOR_ID_INTEL2: [u8; 12] = *b"GenuineIotel"; // Intel https://github.com/InstLatx64/InstLatx64/commit/8fdd319884c67d2c6ec1ca0c595b42c1c4b8d803
56 const _VENDOR_ID_AMD: [u8; 12] = *b"AuthenticAMD"; // AMD
57 const _VENDOR_ID_ZHAOXIN: [u8; 12] = *b" Shanghai "; // Zhaoxin
_vendor_id() -> [u8; 12]58 fn _vendor_id() -> [u8; 12] {
59 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L40-L59
60 let CpuidResult { ebx, ecx, edx, .. } = __cpuid(0);
61 let vendor_id: [[u8; 4]; 3] = [ebx.to_ne_bytes(), edx.to_ne_bytes(), ecx.to_ne_bytes()];
62 // SAFETY: transmute is safe because `[u8; 12]` and `[[u8; 4]; 3]` has the same layout.
63 unsafe { core::mem::transmute(vendor_id) }
64 }
_vendor_has_vmovdqa_atomic(vendor_id: [u8; 12]) -> bool65 fn _vendor_has_vmovdqa_atomic(vendor_id: [u8; 12]) -> bool {
66 // VMOVDQA is atomic on Intel, AMD, and Zhaoxin CPUs with AVX.
67 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104688 for details.
68 vendor_id == _VENDOR_ID_INTEL
69 || vendor_id == _VENDOR_ID_INTEL2
70 || vendor_id == _VENDOR_ID_AMD
71 || vendor_id == _VENDOR_ID_ZHAOXIN
72 }
73
74 #[cold]
_detect(info: &mut CpuInfo)75 fn _detect(info: &mut CpuInfo) {
76 let proc_info_ecx = __cpuid(0x0000_0001_u32).ecx;
77
78 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L111
79 if test(proc_info_ecx, 13) {
80 info.set(CpuInfo::HAS_CMPXCHG16B);
81 }
82
83 // We only use VMOVDQA when SSE is enabled. See atomic_load_vmovdqa() in atomic128/x86_64.rs for more.
84 #[cfg(target_feature = "sse")]
85 {
86 use core::arch::x86_64::_xgetbv;
87
88 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/x86.rs#L131-L224
89 let cpu_xsave = test(proc_info_ecx, 26);
90 if cpu_xsave {
91 let cpu_osxsave = test(proc_info_ecx, 27);
92 if cpu_osxsave {
93 // SAFETY: Calling `_xgetbv`` is safe because the CPU has `xsave` support
94 // and OS has set `osxsave`.
95 let xcr0 = unsafe { _xgetbv(0) };
96 let os_avx_support = xcr0 & 6 == 6;
97 if os_avx_support && test(proc_info_ecx, 28) {
98 let vendor_id = _vendor_id();
99 if _vendor_has_vmovdqa_atomic(vendor_id) {
100 info.set(CpuInfo::HAS_VMOVDQA_ATOMIC);
101 }
102 }
103 }
104 }
105 }
106 }
107
108 #[allow(
109 clippy::alloc_instead_of_core,
110 clippy::std_instead_of_alloc,
111 clippy::std_instead_of_core,
112 clippy::undocumented_unsafe_blocks,
113 clippy::wildcard_imports
114 )]
115 #[cfg(test)]
116 mod tests {
117 use std::io::{self, Write};
118
119 use super::*;
120
121 #[test]
122 #[cfg_attr(portable_atomic_test_outline_atomics_detect_false, ignore)]
test_cpuid()123 fn test_cpuid() {
124 assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), detect().has_cmpxchg16b());
125 let vendor_id = _vendor_id();
126 {
127 let stdout = io::stderr();
128 let mut stdout = stdout.lock();
129 let _ = writeln!(stdout, "\n vendor_id: {}", std::str::from_utf8(&vendor_id).unwrap());
130 }
131 if _vendor_has_vmovdqa_atomic(vendor_id) {
132 assert_eq!(std::is_x86_feature_detected!("avx"), detect().has_vmovdqa_atomic());
133 } else {
134 assert!(!detect().has_vmovdqa_atomic());
135 }
136 }
137 }
138