1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 /* 4 Run-time CPU feature detection on AArch64/PowerPC64 Linux/Android/FreeBSD/OpenBSD by parsing ELF auxiliary vectors. 5 6 Supported platforms: 7 - Linux 6.4+ (through prctl) 8 https://github.com/torvalds/linux/commit/ddc65971bb677aa9f6a4c21f76d3133e106f88eb 9 prctl returns an unsupported error if operation is not supported, 10 so we can safely use this on older versions. 11 - glibc 2.16+ (through getauxval) 12 https://github.com/bminor/glibc/commit/c7683a6d02f3ed59f5cd119b3e8547f45a15912f 13 Always available on: 14 - aarch64 (glibc 2.17+ https://github.com/bminor/glibc/blob/glibc-2.17/NEWS#L35) 15 Not always available on: 16 - powerpc64 (glibc 2.3+ https://github.com/bminor/glibc/blob/glibc-2.3/NEWS#L55) 17 Since Rust 1.64, std requires glibc 2.17+ https://blog.rust-lang.org/2022/08/01/Increasing-glibc-kernel-requirements.html 18 - musl 1.1.0+ (through getauxval) 19 https://github.com/bminor/musl/commit/21ada94c4b8c01589367cea300916d7db8461ae7 20 Always available on: 21 - aarch64 (musl 1.1.7+ https://github.com/bminor/musl/blob/v1.1.7/WHATSNEW#L1422) 22 - powerpc64 (musl 1.1.15+ https://github.com/bminor/musl/blob/v1.1.15/WHATSNEW#L1702) 23 Since Rust 1.31, std requires musl 1.1.20+ https://github.com/rust-lang/rust/pull/54430 24 Since Rust 1.37, std requires musl 1.1.22+ https://github.com/rust-lang/rust/pull/61252 25 Since Rust 1.46, std requires musl 1.1.24+ https://github.com/rust-lang/rust/pull/73089 26 Since Rust 1.71, std requires musl 1.2.3+ https://blog.rust-lang.org/2023/05/09/Updating-musl-targets.html 27 - uClibc-ng 1.0.43+ (through getauxval) 28 https://github.com/wbx-github/uclibc-ng/commit/d869bb1600942c01a77539128f9ba5b5b55ad647 29 - Picolibc 1.4.6+ (through getauxval) 30 https://github.com/picolibc/picolibc/commit/19bfe51d62ad7e32533c7f664b5bca8e26286e31 31 - Android 4.3+ (API level 18+) (through getauxval) 32 https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/auxv.h#L49 33 Always available on 64-bit architectures, which is supported on Android 5.0+ (API level 21+) https://android-developers.googleblog.com/2014/10/whats-new-in-android-50-lollipop.html 34 - FreeBSD 12.0+ and 11.4+ (through elf_aux_info) 35 https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470 36 https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h 37 Not always available on: 38 - aarch64 (FreeBSD 11.0+ https://www.freebsd.org/releases/11.0R/announce) 39 - powerpc64 (FreeBSD 9.0+ https://www.freebsd.org/releases/9.0R/announce) 40 Since Rust 1.75, std requires FreeBSD 12+ https://github.com/rust-lang/rust/pull/114521 41 - OpenBSD 7.6+ (through elf_aux_info) 42 https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d 43 44 # Linux/Android 45 46 As of Rust 1.69, is_aarch64_feature_detected always uses dlsym by default 47 on AArch64 Linux/Android, but on the following platforms, we can safely assume 48 getauxval is linked to the binary. 49 50 - On glibc (*-linux-gnu*), [AArch64 support is available on glibc 2.17+](https://github.com/bminor/glibc/blob/glibc-2.17/NEWS#L35) 51 - On musl (*-linux-musl*, *-linux-ohos*), [AArch64 support is available on musl 1.1.7+](https://github.com/bminor/musl/blob/v1.1.7/WHATSNEW#L1422) 52 - On bionic (*-android*), [64-bit architecture support is available on Android 5.0+ (API level 21+)](https://android-developers.googleblog.com/2014/10/whats-new-in-android-50-lollipop.html) 53 54 However, on musl with static linking, it seems that getauxval is not always available, independent of version requirements: https://github.com/rust-lang/rust/issues/89626 55 (That problem may have been fixed in https://github.com/rust-lang/rust/commit/9a04ae4997493e9260352064163285cddc43de3c, 56 but even in the version containing that patch, [there is report](https://github.com/rust-lang/rust/issues/89626#issuecomment-1242636038) 57 of the same error.) 58 59 On other Linux targets, we cannot assume that getauxval is always available, so we don't enable 60 run-time detection by default (can be enabled by `--cfg portable_atomic_outline_atomics`). 61 62 - musl with static linking. See the above for more. 63 Also, dlsym(getauxval) always returns null when statically linked. 64 - uClibc-ng (*-linux-uclibc*, *-l4re-uclibc*). getauxval was recently added (See the above list). 65 - Picolibc. getauxval was recently added (See the above list). 66 67 See also https://github.com/rust-lang/stdarch/pull/1375 68 69 See tests::test_linux_like and aarch64_aa64reg.rs for (test-only) alternative implementations. 70 71 # FreeBSD 72 73 As of nightly-2024-09-07, is_aarch64_feature_detected always uses mrs on 74 AArch64 FreeBSD. However, they do not work on FreeBSD 12 on QEMU (confirmed 75 on FreeBSD 12.{2,3,4}), and we got SIGILL (worked on FreeBSD 13 and 14). 76 77 So use elf_aux_info instead of mrs like compiler-rt does. 78 https://reviews.llvm.org/D109330 79 80 elf_aux_info is available on FreeBSD 12.0+ and 11.4+: 81 https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470 82 https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h 83 On FreeBSD, [AArch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/announce), 84 but FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30: 85 https://www.freebsd.org/security/unsupported 86 See also https://github.com/rust-lang/stdarch/pull/611#issuecomment-445464613 87 88 See tests::test_freebsd and aarch64_aa64reg.rs for (test-only) alternative implementations. 89 90 # OpenBSD 91 92 elf_aux_info is available on OpenBSD 7.6+: 93 https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d 94 95 On AArch64, there is an alternative that available on older version, 96 so we use it (see aarch64_aa64reg.rs). 97 98 # PowerPC64 99 100 On PowerPC64, run-time detection is currently disabled by default mainly for 101 compatibility with older versions of operating systems 102 (can be enabled by `--cfg portable_atomic_outline_atomics`). 103 104 - On glibc, [powerpc64 support is available on glibc 2.3+](https://github.com/bminor/glibc/blob/glibc-2.3/NEWS#L55) 105 - On musl, [powerpc64 support is available on musl 1.1.15+](https://github.com/bminor/musl/blob/v1.1.15/WHATSNEW#L1702) 106 - On FreeBSD, [powerpc64 support is available on FreeBSD 9.0+](https://www.freebsd.org/releases/9.0R/announce) 107 108 (On uClibc-ng, [powerpc64 is not supported](https://github.com/wbx-github/uclibc-ng/commit/d4d4f37fda7fa57e57132ff2f0d735ce7cc2178e)) 109 */ 110 111 include!("common.rs"); 112 113 use os::ffi; 114 #[cfg(any(target_os = "linux", target_os = "android"))] 115 mod os { 116 // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 117 #[cfg_attr(test, allow(dead_code))] 118 pub(super) mod ffi { 119 pub(crate) use super::super::c_types::c_ulong; 120 #[cfg(all(target_arch = "aarch64", target_os = "android"))] 121 pub(crate) use super::super::c_types::{c_char, c_int}; 122 123 // https://github.com/torvalds/linux/blob/v6.11/include/uapi/linux/auxvec.h 124 #[cfg(any(test, target_arch = "aarch64"))] 125 pub(crate) const AT_HWCAP: c_ulong = 16; 126 #[cfg(any( 127 test, 128 all(target_arch = "aarch64", target_pointer_width = "64"), 129 target_arch = "powerpc64", 130 ))] 131 pub(crate) const AT_HWCAP2: c_ulong = 26; 132 133 // Defined in sys/system_properties.h. 134 // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/system_properties.h 135 #[cfg(all(target_arch = "aarch64", target_os = "android"))] 136 pub(crate) const PROP_VALUE_MAX: c_int = 92; 137 138 extern "C" { 139 // Defined in sys/auxv.h. 140 // https://man7.org/linux/man-pages/man3/getauxval.3.html 141 // https://github.com/bminor/glibc/blob/glibc-2.40/misc/sys/auxv.h 142 // https://github.com/bminor/musl/blob/v1.2.5/include/sys/auxv.h 143 // https://github.com/wbx-github/uclibc-ng/blob/v1.0.47/include/sys/auxv.h 144 // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/auxv.h 145 // https://github.com/picolibc/picolibc/blob/1.8.6/newlib/libc/include/sys/auxv.h getauxval(type_: c_ulong) -> c_ulong146 pub(crate) fn getauxval(type_: c_ulong) -> c_ulong; 147 148 // Defined in sys/system_properties.h. 149 // https://github.com/aosp-mirror/platform_bionic/blob/d3ebc2f7c49a9893b114124d4a6b315f3a328764/libc/include/sys/system_properties.h 150 #[cfg(all(target_arch = "aarch64", target_os = "android"))] __system_property_get(name: *const c_char, value: *mut c_char) -> c_int151 pub(crate) fn __system_property_get(name: *const c_char, value: *mut c_char) -> c_int; 152 } 153 } 154 getauxval(type_: ffi::c_ulong) -> ffi::c_ulong155 pub(super) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong { 156 #[cfg(all(target_arch = "aarch64", target_os = "android"))] 157 { 158 // Samsung Exynos 9810 has a bug that big and little cores have different 159 // ISAs. And on older Android (pre-9), the kernel incorrectly reports 160 // that features available only on some cores are available on all cores. 161 // https://reviews.llvm.org/D114523 162 let mut arch = [0_u8; ffi::PROP_VALUE_MAX as usize]; 163 // SAFETY: we've passed a valid C string and a buffer with max length. 164 let len = unsafe { 165 ffi::__system_property_get( 166 b"ro.arch\0".as_ptr().cast::<ffi::c_char>(), 167 arch.as_mut_ptr().cast::<ffi::c_char>(), 168 ) 169 }; 170 // On Exynos, ro.arch is not available on Android 12+, but it is fine 171 // because Android 9+ includes the fix. 172 if len > 0 && arch.starts_with(b"exynos9810") { 173 return 0; 174 } 175 } 176 177 // SAFETY: `getauxval` is thread-safe. See also the module level docs. 178 unsafe { ffi::getauxval(type_) } 179 } 180 } 181 #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] 182 mod os { 183 // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47 184 #[cfg_attr(test, allow(dead_code))] 185 pub(super) mod ffi { 186 pub(crate) use super::super::c_types::{c_int, c_ulong, c_void}; 187 188 // FreeBSD 189 // Defined in sys/elf_common.h. 190 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/sys/sys/elf_common.h 191 // OpenBSD 192 // Defined in sys/auxv.h. 193 // https://github.com/openbsd/src/blob/ed8f5e8d82ace15e4cefca2c82941b15cb1a7830/sys/sys/auxv.h 194 #[cfg(any(test, target_arch = "aarch64"))] 195 pub(crate) const AT_HWCAP: c_int = 25; 196 #[cfg(any(test, target_arch = "powerpc64"))] 197 pub(crate) const AT_HWCAP2: c_int = 26; 198 199 extern "C" { 200 // FreeBSD 201 // Defined in sys/auxv.h. 202 // https://man.freebsd.org/elf_aux_info(3) 203 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/sys/sys/auxv.h 204 // OpenBSD 205 // Defined in sys/auxv.h. 206 // https://man.openbsd.org/elf_aux_info.3 207 // https://github.com/openbsd/src/blob/ed8f5e8d82ace15e4cefca2c82941b15cb1a7830/sys/sys/auxv.h elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int208 pub(crate) fn elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int; 209 } 210 } 211 getauxval(aux: ffi::c_int) -> ffi::c_ulong212 pub(super) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong { 213 #[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)] 214 const OUT_LEN: ffi::c_int = core::mem::size_of::<ffi::c_ulong>() as ffi::c_int; 215 let mut out: ffi::c_ulong = 0; 216 // SAFETY: 217 // - the pointer is valid because we got it from a reference. 218 // - `OUT_LEN` is the same as the size of `out`. 219 // - `elf_aux_info` is thread-safe. 220 unsafe { 221 let res = ffi::elf_aux_info( 222 aux, 223 (&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(), 224 OUT_LEN, 225 ); 226 // If elf_aux_info fails, `out` will be left at zero (which is the proper default value). 227 debug_assert!(res == 0 || out == 0); 228 } 229 out 230 } 231 } 232 233 // Basically, Linux/FreeBSD/OpenBSD use the same hwcap values. 234 // FreeBSD/OpenBSD supports a subset of the hwcap values supported by Linux. 235 use arch::_detect; 236 #[cfg(target_arch = "aarch64")] 237 mod arch { 238 use super::{ffi, os, CpuInfo}; 239 240 // Linux 241 // https://github.com/torvalds/linux/blob/v6.11/arch/arm64/include/uapi/asm/hwcap.h 242 // https://github.com/torvalds/linux/blob/v6.11/Documentation/arch/arm64/elf_hwcaps.rst 243 // FreeBSD 244 // Defined in machine/elf.h. 245 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/sys/arm64/include/elf.h 246 // OpenBSD 247 // Defined in machine/elf.h. 248 // https://github.com/openbsd/src/blob/ed8f5e8d82ace15e4cefca2c82941b15cb1a7830/sys/arch/arm64/include/elf.h 249 // Linux 4.3+ 250 // https://github.com/torvalds/linux/commit/40a1db2434a1b62332b1af25cfa14d7b8c0301fe 251 // FreeBSD 13.0+/12.2+ 252 // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h 253 // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h 254 // OpenBSD 7.6+ 255 // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d 256 pub(super) const HWCAP_ATOMICS: ffi::c_ulong = 1 << 8; 257 // Linux 4.17+ 258 // https://github.com/torvalds/linux/commit/7206dc93a58fb76421c4411eefa3c003337bcb2d 259 // FreeBSD 13.0+/12.2+ 260 // https://github.com/freebsd/freebsd-src/blob/release/13.0.0/sys/arm64/include/elf.h 261 // https://github.com/freebsd/freebsd-src/blob/release/12.2.0/sys/arm64/include/elf.h 262 // OpenBSD 7.6+ 263 // https://github.com/openbsd/src/commit/ef873df06dac50249b2dd380dc6100eee3b0d23d 264 pub(super) const HWCAP_USCAT: ffi::c_ulong = 1 << 25; 265 // Linux 6.7+ 266 // https://github.com/torvalds/linux/commit/338a835f40a849cd89b993e342bd9fbd5684825c 267 #[cfg(any(target_os = "linux", target_os = "android"))] 268 #[cfg(target_pointer_width = "64")] 269 pub(super) const HWCAP2_LRCPC3: ffi::c_ulong = 1 << 46; 270 // Linux 6.7+ 271 // https://github.com/torvalds/linux/commit/94d0657f9f0d311489606589133ebf49e28104d8 272 #[cfg(any(target_os = "linux", target_os = "android"))] 273 #[cfg(target_pointer_width = "64")] 274 pub(super) const HWCAP2_LSE128: ffi::c_ulong = 1 << 47; 275 276 #[cold] _detect(info: &mut CpuInfo)277 pub(super) fn _detect(info: &mut CpuInfo) { 278 let hwcap = os::getauxval(ffi::AT_HWCAP); 279 280 if hwcap & HWCAP_ATOMICS != 0 { 281 info.set(CpuInfo::HAS_LSE); 282 } 283 if hwcap & HWCAP_USCAT != 0 { 284 info.set(CpuInfo::HAS_LSE2); 285 } 286 #[cfg(any(target_os = "linux", target_os = "android"))] 287 // HWCAP2 is not yet available on ILP32: https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git/tree/arch/arm64/include/uapi/asm/hwcap.h?h=staging/ilp32-5.1 288 #[cfg(target_pointer_width = "64")] 289 { 290 let hwcap2 = os::getauxval(ffi::AT_HWCAP2); 291 if hwcap2 & HWCAP2_LRCPC3 != 0 { 292 info.set(CpuInfo::HAS_RCPC3); 293 } 294 if hwcap2 & HWCAP2_LSE128 != 0 { 295 info.set(CpuInfo::HAS_LSE128); 296 } 297 } 298 } 299 } 300 #[cfg(target_arch = "powerpc64")] 301 mod arch { 302 use super::{ffi, os, CpuInfo}; 303 304 // Linux 305 // https://github.com/torvalds/linux/blob/v6.11/arch/powerpc/include/uapi/asm/cputable.h 306 // https://github.com/torvalds/linux/blob/v6.11/Documentation/arch/powerpc/elf_hwcaps.rst 307 // FreeBSD 308 // Defined in machine/cpu.h. 309 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/sys/powerpc/include/cpu.h 310 // OpenBSD 311 // Defined in machine/elf.h. 312 // https://github.com/openbsd/src/blob/ed8f5e8d82ace15e4cefca2c82941b15cb1a7830/sys/arch/powerpc64/include/elf.h 313 // Linux 3.10+ 314 // https://github.com/torvalds/linux/commit/cbbc6f1b1433ef553d57826eee87a84ca49645ce 315 // FreeBSD 11.0+ 316 // https://github.com/freebsd/freebsd-src/commit/b0bf7fcd298133457991b27625bbed766e612730 317 // OpenBSD 7.6+ 318 // https://github.com/openbsd/src/commit/0b0568a19fc4c197871ceafbabc91fabf17ca152 319 pub(super) const PPC_FEATURE2_ARCH_2_07: ffi::c_ulong = 0x80000000; 320 // Linux 4.5+ 321 // https://github.com/torvalds/linux/commit/e708c24cd01ce80b1609d8baccee40ccc3608a01 322 // FreeBSD 12.0+ 323 // https://github.com/freebsd/freebsd-src/commit/18f48e0c72f91bc2d4373078a3f1ab1bcab4d8b3 324 // OpenBSD 7.6+ 325 // https://github.com/openbsd/src/commit/0b0568a19fc4c197871ceafbabc91fabf17ca152 326 pub(super) const PPC_FEATURE2_ARCH_3_00: ffi::c_ulong = 0x00800000; 327 // Linux 5.8+ 328 // https://github.com/torvalds/linux/commit/ee988c11acf6f9464b7b44e9a091bf6afb3b3a49 329 #[cfg(any(target_os = "linux", target_os = "android"))] 330 pub(super) const PPC_FEATURE2_ARCH_3_1: ffi::c_ulong = 0x00040000; 331 332 #[cold] _detect(info: &mut CpuInfo)333 pub(super) fn _detect(info: &mut CpuInfo) { 334 let hwcap2 = os::getauxval(ffi::AT_HWCAP2); 335 336 // power8 337 // Check both 2_07 (power8) and later ISAs (which are superset of 2_07) because 338 // OpenBSD currently doesn't set 2_07 even when 3_00 (power9) is set. 339 // https://github.com/openbsd/src/blob/ed8f5e8d82ace15e4cefca2c82941b15cb1a7830/sys/arch/powerpc64/powerpc64/cpu.c#L224-L243 340 // Other OSes should be fine, but check all OSs in the same way just in case. 341 #[cfg(any(target_os = "linux", target_os = "android"))] 342 let power8_or_later = 343 PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_ARCH_3_00 | PPC_FEATURE2_ARCH_3_1; 344 #[cfg(not(any(target_os = "linux", target_os = "android")))] 345 let power8_or_later = PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_ARCH_3_00; 346 if hwcap2 & power8_or_later != 0 { 347 info.set(CpuInfo::HAS_QUADWORD_ATOMICS); 348 } 349 } 350 } 351 352 #[allow( 353 clippy::alloc_instead_of_core, 354 clippy::std_instead_of_alloc, 355 clippy::std_instead_of_core, 356 clippy::undocumented_unsafe_blocks, 357 clippy::wildcard_imports 358 )] 359 #[cfg(test)] 360 mod tests { 361 use super::*; 362 363 #[cfg(any(target_os = "linux", target_os = "android"))] 364 #[cfg(target_pointer_width = "64")] 365 #[test] test_linux_like()366 fn test_linux_like() { 367 use c_types::*; 368 #[cfg(not(portable_atomic_no_asm))] 369 use std::arch::asm; 370 use std::{mem, vec}; 371 use test_helper::{libc, sys}; 372 373 // Linux kernel 6.4 has added a way to read auxv without depending on either libc or mrs trap. 374 // https://github.com/torvalds/linux/commit/ddc65971bb677aa9f6a4c21f76d3133e106f88eb 375 // 376 // This is currently used only for testing. 377 fn getauxval_pr_get_auxv(type_: ffi::c_ulong) -> Result<ffi::c_ulong, c_int> { 378 #[cfg(target_arch = "aarch64")] 379 unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> { 380 let r: i64; 381 unsafe { 382 asm!( 383 "svc 0", 384 in("x8") sys::__NR_prctl as u64, 385 inout("x0") sys::PR_GET_AUXV as u64 => r, 386 in("x1") ptr_reg!(out), 387 in("x2") len as u64, 388 // arg4 and arg5 must be zero. 389 in("x3") 0_u64, 390 in("x4") 0_u64, 391 options(nostack, preserves_flags), 392 ); 393 } 394 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] 395 if (r as c_int) < 0 { 396 Err(r as c_int) 397 } else { 398 Ok(r as usize) 399 } 400 } 401 #[cfg(target_arch = "powerpc64")] 402 unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> { 403 let r: i64; 404 unsafe { 405 asm!( 406 "sc", 407 "bns+ 2f", 408 "neg %r3, %r3", 409 "2:", 410 inout("r0") sys::__NR_prctl as u64 => _, 411 inout("r3") sys::PR_GET_AUXV as u64 => r, 412 inout("r4") ptr_reg!(out) => _, 413 inout("r5") len as u64 => _, 414 // arg4 and arg5 must be zero. 415 inout("r6") 0_u64 => _, 416 inout("r7") 0_u64 => _, 417 out("r8") _, 418 out("r9") _, 419 out("r10") _, 420 out("r11") _, 421 out("r12") _, 422 out("cr0") _, 423 options(nostack, preserves_flags), 424 ); 425 } 426 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] 427 if (r as c_int) < 0 { 428 Err(r as c_int) 429 } else { 430 Ok(r as usize) 431 } 432 } 433 434 let mut auxv = vec![unsafe { mem::zeroed::<sys::Elf64_auxv_t>() }; 38]; 435 436 let old_len = auxv.len() * mem::size_of::<sys::Elf64_auxv_t>(); 437 438 // SAFETY: 439 // - `out_len` does not exceed the size of `auxv`. 440 let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::<c_void>(), old_len)? }; 441 442 for aux in &auxv { 443 if aux.a_type == type_ { 444 // SAFETY: aux.a_un is #[repr(C)] union and all fields have 445 // the same size and can be safely transmuted to integers. 446 return Ok(unsafe { aux.a_un.a_val }); 447 } 448 } 449 Err(0) 450 } 451 452 unsafe { 453 let mut u = mem::zeroed(); 454 assert_eq!(libc::uname(&mut u), 0); 455 let release = std::ffi::CStr::from_ptr(u.release.as_ptr()); 456 let release = core::str::from_utf8(release.to_bytes()).unwrap(); 457 let mut digits = release.split('.'); 458 let major = digits.next().unwrap().parse::<u32>().unwrap(); 459 let minor = digits.next().unwrap().parse::<u32>().unwrap(); 460 // TODO: qemu-user bug? 461 if (major, minor) < (6, 4) || cfg!(qemu) { 462 std::eprintln!("kernel version: {}.{} (no pr_get_auxv)", major, minor); 463 assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap_err(), -22); 464 assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap_err(), -22); 465 } else { 466 std::eprintln!("kernel version: {}.{} (has pr_get_auxv)", major, minor); 467 assert_eq!( 468 os::getauxval(ffi::AT_HWCAP), 469 getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap() 470 ); 471 assert_eq!( 472 os::getauxval(ffi::AT_HWCAP2), 473 getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap() 474 ); 475 } 476 } 477 } 478 479 #[allow(clippy::cast_sign_loss)] 480 #[cfg(all(target_arch = "aarch64", target_os = "android"))] 481 #[test] test_android()482 fn test_android() { 483 unsafe { 484 let mut arch = [1; ffi::PROP_VALUE_MAX as usize]; 485 let len = ffi::__system_property_get( 486 b"ro.arch\0".as_ptr().cast::<ffi::c_char>(), 487 arch.as_mut_ptr().cast::<ffi::c_char>(), 488 ); 489 assert!(len >= 0); 490 std::eprintln!("len={}", len); 491 std::eprintln!("arch={:?}", arch); 492 std::eprintln!( 493 "arch={:?}", 494 core::str::from_utf8(core::slice::from_raw_parts(arch.as_ptr(), len as usize)) 495 .unwrap() 496 ); 497 } 498 } 499 500 #[allow(clippy::cast_possible_wrap)] 501 #[cfg(target_os = "freebsd")] 502 #[test] test_freebsd()503 fn test_freebsd() { 504 use c_types::*; 505 #[cfg(not(portable_atomic_no_asm))] 506 use std::arch::asm; 507 use std::{mem, ptr}; 508 use test_helper::sys; 509 510 // This is almost equivalent to what elf_aux_info does. 511 // https://man.freebsd.org/elf_aux_info(3) 512 // On FreeBSD, [AArch64 support is available on FreeBSD 11.0+](https://www.freebsd.org/releases/11.0R/announce), 513 // but elf_aux_info is available on FreeBSD 12.0+ and 11.4+: 514 // https://github.com/freebsd/freebsd-src/commit/0b08ae2120cdd08c20a2b806e2fcef4d0a36c470 515 // https://github.com/freebsd/freebsd-src/blob/release/11.4.0/sys/sys/auxv.h 516 // so use sysctl instead of elf_aux_info. 517 // Note that FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30: 518 // https://www.freebsd.org/security/unsupported 519 // 520 // std_detect uses this way, but it appears to be somewhat incorrect 521 // (the type of arg4 of sysctl, auxv is smaller than AT_COUNT, etc.). 522 // https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/std_detect/src/detect/os/freebsd/auxvec.rs#L52 523 // 524 // This is currently used only for testing. 525 // If you want us to use this implementation for compatibility with the older FreeBSD 526 // version that came to EoL a few years ago, please open an issue. 527 fn getauxval_sysctl_libc(type_: ffi::c_int) -> ffi::c_ulong { 528 let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() }; 529 530 let mut len = core::mem::size_of_val(&auxv) as c_size_t; 531 532 // SAFETY: calling getpid is safe. 533 let pid = unsafe { sys::getpid() }; 534 let mib = [ 535 sys::CTL_KERN as c_int, 536 sys::KERN_PROC as c_int, 537 sys::KERN_PROC_AUXV as c_int, 538 pid, 539 ]; 540 541 #[allow(clippy::cast_possible_truncation)] 542 // SAFETY: 543 // - `mib.len()` does not exceed the size of `mib`. 544 // - `len` does not exceed the size of `auxv`. 545 // - `sysctl` is thread-safe. 546 let res = unsafe { 547 sys::sysctl( 548 mib.as_ptr(), 549 mib.len() as c_uint, 550 auxv.as_mut_ptr().cast::<c_void>(), 551 &mut len, 552 ptr::null_mut(), 553 0, 554 ) 555 }; 556 557 if res != -1 { 558 for aux in &auxv { 559 if aux.a_type == type_ as c_long { 560 // SAFETY: aux.a_un is #[repr(C)] union and all fields have 561 // the same size and can be safely transmuted to integers. 562 return unsafe { aux.a_un.a_val as c_ulong }; 563 } 564 } 565 } 566 0 567 } 568 // Similar to the above, but call syscall using asm instead of libc. 569 // Note that FreeBSD does not guarantee the stability of raw syscall as 570 // much as Linux does (It may actually be stable enough, though: 571 // https://lists.llvm.org/pipermail/llvm-dev/2019-June/133393.html, 572 // https://github.com/ziglang/zig/issues/16590). 573 // 574 // This is currently used only for testing. 575 fn getauxval_sysctl_asm_syscall(type_: ffi::c_int) -> Result<ffi::c_ulong, c_int> { 576 #[allow(non_camel_case_types)] 577 type pid_t = c_int; 578 579 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/lib/libc/aarch64/SYS.h 580 // https://github.com/golang/go/blob/4badad8d477ffd7a6b762c35bc69aed82faface7/src/syscall/asm_freebsd_arm64.s 581 #[cfg(target_arch = "aarch64")] 582 #[inline] 583 fn getpid() -> pid_t { 584 #[allow(clippy::cast_possible_truncation)] 585 // SAFETY: calling getpid is safe. 586 unsafe { 587 let n = sys::SYS_getpid; 588 let r: i64; 589 asm!( 590 "svc 0", 591 in("x8") n as u64, 592 out("x0") r, 593 options(nostack, readonly), 594 ); 595 r as pid_t 596 } 597 } 598 #[cfg(target_arch = "aarch64")] 599 #[inline] 600 unsafe fn sysctl( 601 name: *const c_int, 602 name_len: c_uint, 603 old_p: *mut c_void, 604 old_len_p: *mut c_size_t, 605 new_p: *const c_void, 606 new_len: c_size_t, 607 ) -> Result<c_int, c_int> { 608 #[allow(clippy::cast_possible_truncation)] 609 // SAFETY: the caller must uphold the safety contract. 610 unsafe { 611 let mut n = sys::SYS___sysctl as u64; 612 let r: i64; 613 asm!( 614 "svc 0", 615 "b.cc 2f", 616 "mov x8, x0", 617 "mov x0, #-1", 618 "2:", 619 inout("x8") n, 620 inout("x0") ptr_reg!(name) => r, 621 inout("x1") name_len as u64 => _, 622 in("x2") ptr_reg!(old_p), 623 in("x3") ptr_reg!(old_len_p), 624 in("x4") ptr_reg!(new_p), 625 in("x5") new_len as u64, 626 options(nostack), 627 ); 628 if r as c_int == -1 { 629 Err(n as c_int) 630 } else { 631 Ok(r as c_int) 632 } 633 } 634 } 635 636 // https://github.com/freebsd/freebsd-src/blob/release/14.1.0/lib/libc/powerpc64/SYS.h 637 #[cfg(target_arch = "powerpc64")] 638 #[inline] 639 fn getpid() -> pid_t { 640 #[allow(clippy::cast_possible_truncation)] 641 // SAFETY: calling getpid is safe. 642 unsafe { 643 let n = sys::SYS_getpid; 644 let r: i64; 645 asm!( 646 "sc", 647 inout("r0") n as u64 => _, 648 out("r3") r, 649 out("r4") _, 650 out("r5") _, 651 out("r6") _, 652 out("r7") _, 653 out("r8") _, 654 out("r9") _, 655 out("r10") _, 656 out("r11") _, 657 out("r12") _, 658 out("cr0") _, 659 options(nostack, preserves_flags, readonly), 660 ); 661 r as pid_t 662 } 663 } 664 #[cfg(target_arch = "powerpc64")] 665 #[inline] 666 unsafe fn sysctl( 667 name: *const c_int, 668 name_len: c_uint, 669 old_p: *mut c_void, 670 old_len_p: *mut c_size_t, 671 new_p: *const c_void, 672 new_len: c_size_t, 673 ) -> Result<c_int, c_int> { 674 #[allow(clippy::cast_possible_truncation)] 675 // SAFETY: the caller must uphold the safety contract. 676 unsafe { 677 let mut n = sys::SYS___sysctl as u64; 678 let r: i64; 679 asm!( 680 "sc", 681 "bns+ 2f", 682 "mr %r0, %r3", 683 "li %r3, -1", 684 "2:", 685 inout("r0") n, 686 inout("r3") ptr_reg!(name) => r, 687 inout("r4") name_len as u64 => _, 688 inout("r5") ptr_reg!(old_p) => _, 689 inout("r6") ptr_reg!(old_len_p) => _, 690 inout("r7") ptr_reg!(new_p) => _, 691 inout("r8") new_len as u64 => _, 692 out("r9") _, 693 out("r10") _, 694 out("r11") _, 695 out("r12") _, 696 out("cr0") _, 697 options(nostack, preserves_flags), 698 ); 699 if r as c_int == -1 { 700 Err(n as c_int) 701 } else { 702 Ok(r as c_int) 703 } 704 } 705 } 706 707 let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() }; 708 709 let mut len = core::mem::size_of_val(&auxv) as c_size_t; 710 711 let pid = getpid(); 712 let mib = [ 713 sys::CTL_KERN as c_int, 714 sys::KERN_PROC as c_int, 715 sys::KERN_PROC_AUXV as c_int, 716 pid, 717 ]; 718 719 #[allow(clippy::cast_possible_truncation)] 720 // SAFETY: 721 // - `mib.len()` does not exceed the size of `mib`. 722 // - `len` does not exceed the size of `auxv`. 723 // - `sysctl` is thread-safe. 724 unsafe { 725 sysctl( 726 mib.as_ptr(), 727 mib.len() as c_uint, 728 auxv.as_mut_ptr().cast::<c_void>(), 729 &mut len, 730 ptr::null_mut(), 731 0, 732 )?; 733 } 734 735 for aux in &auxv { 736 if aux.a_type == type_ as c_long { 737 // SAFETY: aux.a_un is #[repr(C)] union and all fields have 738 // the same size and can be safely transmuted to integers. 739 return Ok(unsafe { aux.a_un.a_val as c_ulong }); 740 } 741 } 742 Err(0) 743 } 744 745 assert_eq!(os::getauxval(ffi::AT_HWCAP), getauxval_sysctl_libc(ffi::AT_HWCAP)); 746 assert_eq!(os::getauxval(ffi::AT_HWCAP2), getauxval_sysctl_libc(ffi::AT_HWCAP2)); 747 assert_eq!( 748 os::getauxval(ffi::AT_HWCAP), 749 getauxval_sysctl_asm_syscall(ffi::AT_HWCAP).unwrap() 750 ); 751 assert_eq!( 752 os::getauxval(ffi::AT_HWCAP2), 753 // AT_HWCAP2 is only available on FreeBSD 13+, at least on AArch64. 754 getauxval_sysctl_asm_syscall(ffi::AT_HWCAP2).unwrap_or(0) 755 ); 756 } 757 758 // Static assertions for FFI bindings. 759 // This checks that FFI bindings defined in this crate, FFI bindings defined 760 // in libc, and FFI bindings generated for the platform's latest header file 761 // using bindgen have compatible signatures (or the same values if constants). 762 // Since this is static assertion, we can detect problems with 763 // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh) 764 // without actually running tests on these platforms. 765 // See also tools/codegen/src/ffi.rs. 766 // TODO(codegen): auto-generate this test 767 #[allow( 768 clippy::cast_possible_wrap, 769 clippy::cast_sign_loss, 770 clippy::cast_possible_truncation, 771 clippy::no_effect_underscore_binding 772 )] 773 const _: fn() = || { 774 #[cfg(not(target_os = "openbsd"))] 775 use test_helper::libc; 776 use test_helper::sys; 777 #[cfg(not(any(target_os = "freebsd", target_os = "openbsd")))] 778 type AtType = ffi::c_ulong; 779 #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] 780 type AtType = ffi::c_int; 781 #[cfg(any(target_os = "linux", target_os = "android"))] 782 { 783 let mut _getauxval: unsafe extern "C" fn(ffi::c_ulong) -> ffi::c_ulong = ffi::getauxval; 784 _getauxval = libc::getauxval; 785 _getauxval = sys::getauxval; 786 } 787 #[cfg(all(target_arch = "aarch64", target_os = "android"))] 788 { 789 let mut ___system_property_get: unsafe extern "C" fn( 790 *const ffi::c_char, 791 *mut ffi::c_char, 792 ) -> ffi::c_int = ffi::__system_property_get; 793 ___system_property_get = libc::__system_property_get; 794 ___system_property_get = sys::__system_property_get; 795 static_assert!(ffi::PROP_VALUE_MAX == libc::PROP_VALUE_MAX); 796 static_assert!(ffi::PROP_VALUE_MAX == sys::PROP_VALUE_MAX as ffi::c_int); 797 } 798 #[cfg(any(target_os = "freebsd", target_os = "openbsd"))] 799 { 800 let mut _elf_aux_info: unsafe extern "C" fn( 801 ffi::c_int, 802 *mut ffi::c_void, 803 ffi::c_int, 804 ) -> ffi::c_int = ffi::elf_aux_info; 805 #[cfg(not(target_os = "openbsd"))] // libc doesn't have this on OpenBSD 806 { 807 _elf_aux_info = libc::elf_aux_info; 808 } 809 _elf_aux_info = sys::elf_aux_info; 810 } 811 #[cfg(not(target_os = "openbsd"))] // libc doesn't have this on OpenBSD 812 static_assert!(ffi::AT_HWCAP == libc::AT_HWCAP); 813 static_assert!(ffi::AT_HWCAP == sys::AT_HWCAP as AtType); 814 #[cfg(not(target_os = "openbsd"))] // libc doesn't have this on OpenBSD 815 static_assert!(ffi::AT_HWCAP2 == libc::AT_HWCAP2); 816 static_assert!(ffi::AT_HWCAP2 == sys::AT_HWCAP2 as AtType); 817 #[cfg(target_arch = "aarch64")] 818 { 819 #[cfg(any(target_os = "linux", target_os = "android"))] // libc doesn't have this on BSDs 820 static_assert!(arch::HWCAP_ATOMICS == libc::HWCAP_ATOMICS); 821 static_assert!(arch::HWCAP_ATOMICS == sys::HWCAP_ATOMICS as ffi::c_ulong); 822 #[cfg(any(target_os = "linux", target_os = "android"))] // libc doesn't have this on BSDs 823 static_assert!(arch::HWCAP_USCAT == libc::HWCAP_USCAT); 824 static_assert!(arch::HWCAP_USCAT == sys::HWCAP_USCAT as ffi::c_ulong); 825 #[cfg(any(target_os = "linux", target_os = "android"))] 826 #[cfg(target_pointer_width = "64")] 827 { 828 // static_assert!(HWCAP2_LRCPC3 == libc::HWCAP2_LRCPC3); // libc doesn't have this 829 static_assert!(arch::HWCAP2_LRCPC3 == sys::HWCAP2_LRCPC3 as ffi::c_ulong); 830 // static_assert!(HWCAP2_LSE128 == libc::HWCAP2_LSE128); // libc doesn't have this 831 static_assert!(arch::HWCAP2_LSE128 == sys::HWCAP2_LSE128 as ffi::c_ulong); 832 } 833 } 834 #[cfg(target_arch = "powerpc64")] 835 { 836 // static_assert!(arch::PPC_FEATURE2_ARCH_2_07 == libc::PPC_FEATURE2_ARCH_2_07); // libc doesn't have this 837 static_assert!( 838 arch::PPC_FEATURE2_ARCH_2_07 == sys::PPC_FEATURE2_ARCH_2_07 as ffi::c_ulong 839 ); 840 // static_assert!(arch::PPC_FEATURE2_ARCH_3_00 == libc::PPC_FEATURE2_ARCH_3_00); // libc doesn't have this 841 static_assert!( 842 arch::PPC_FEATURE2_ARCH_3_00 == sys::PPC_FEATURE2_ARCH_3_00 as ffi::c_ulong 843 ); 844 #[cfg(any(target_os = "linux", target_os = "android"))] 845 { 846 // static_assert!(arch::PPC_FEATURE2_ARCH_3_1 == libc::PPC_FEATURE2_ARCH_3_1); // libc doesn't have this 847 static_assert!( 848 arch::PPC_FEATURE2_ARCH_3_1 == sys::PPC_FEATURE2_ARCH_3_1 as ffi::c_ulong 849 ); 850 } 851 } 852 }; 853 } 854