1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2
3 /*
4 Run-time CPU feature detection on RISC-V Linux/Android by using riscv_hwprobe.
5
6 On RISC-V, detection using auxv only supports single-letter extensions.
7 So, we use riscv_hwprobe that supports multi-letter extensions.
8
9 Refs: https://github.com/torvalds/linux/blob/v6.11/Documentation/arch/riscv/hwprobe.rst
10 */
11
12 include!("common.rs");
13
14 use core::ptr;
15
16 // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
17 #[allow(non_camel_case_types, non_upper_case_globals)]
18 mod ffi {
19 pub(crate) use super::c_types::{c_long, c_size_t, c_uint, c_ulong};
20
21 // https://github.com/torvalds/linux/blob/v6.11/arch/riscv/include/uapi/asm/hwprobe.h
22 #[derive(Copy, Clone)]
23 #[cfg_attr(test, derive(Debug, PartialEq))]
24 #[repr(C)]
25 pub(crate) struct riscv_hwprobe {
26 pub(crate) key: i64,
27 pub(crate) value: u64,
28 }
29
30 pub(crate) const __NR_riscv_hwprobe: c_long = 258;
31
32 // https://github.com/torvalds/linux/blob/v6.11/arch/riscv/include/uapi/asm/hwprobe.h
33 pub(crate) const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
34 // Linux 6.8+
35 // https://github.com/torvalds/linux/commit/154a3706122978eeb34d8223d49285ed4f3c61fa
36 pub(crate) const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
37
38 #[cfg(not(all(
39 target_os = "linux",
40 any(target_arch = "riscv32", all(target_arch = "riscv64", target_pointer_width = "64")),
41 )))]
42 extern "C" {
43 // https://man7.org/linux/man-pages/man2/syscall.2.html
syscall(number: c_long, ...) -> c_long44 pub(crate) fn syscall(number: c_long, ...) -> c_long;
45 }
46 // Use asm-based syscall for compatibility with non-libc targets if possible.
47 #[cfg(all(
48 target_os = "linux", // https://github.com/bytecodealliance/rustix/issues/1095
49 any(target_arch = "riscv32", all(target_arch = "riscv64", target_pointer_width = "64")),
50 ))]
51 #[inline]
syscall( number: c_long, a0: *mut riscv_hwprobe, a1: c_size_t, a2: c_size_t, a3: *mut c_ulong, a4: c_uint, ) -> c_long52 pub(crate) unsafe fn syscall(
53 number: c_long,
54 a0: *mut riscv_hwprobe,
55 a1: c_size_t,
56 a2: c_size_t,
57 a3: *mut c_ulong,
58 a4: c_uint,
59 ) -> c_long {
60 // arguments must be extended to 64-bit if RV64
61 let a4 = a4 as usize;
62 let r;
63 // SAFETY: the caller must uphold the safety contract.
64 // Refs:
65 // - https://github.com/bminor/musl/blob/v1.2.5/arch/riscv32/syscall_arch.h
66 // - https://github.com/bminor/musl/blob/v1.2.5/arch/riscv64/syscall_arch.h
67 unsafe {
68 core::arch::asm!(
69 "ecall",
70 in("a7") number,
71 inout("a0") a0 => r,
72 in("a1") a1,
73 in("a2") a2,
74 in("a3") a3,
75 in("a4") a4,
76 options(nostack, preserves_flags)
77 );
78 }
79 r
80 }
81
82 // https://github.com/torvalds/linux/blob/v6.11/Documentation/arch/riscv/hwprobe.rst
__riscv_hwprobe( pairs: *mut riscv_hwprobe, pair_count: c_size_t, cpu_set_size: c_size_t, cpus: *mut c_ulong, flags: c_uint, ) -> c_long83 pub(crate) unsafe fn __riscv_hwprobe(
84 pairs: *mut riscv_hwprobe,
85 pair_count: c_size_t,
86 cpu_set_size: c_size_t,
87 cpus: *mut c_ulong,
88 flags: c_uint,
89 ) -> c_long {
90 // SAFETY: the caller must uphold the safety contract.
91 unsafe { syscall(__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags) }
92 }
93 }
94
95 // syscall returns an unsupported error if riscv_hwprobe is not supported,
96 // so we can safely use this function on older versions of Linux.
riscv_hwprobe(out: &mut ffi::riscv_hwprobe) -> bool97 fn riscv_hwprobe(out: &mut ffi::riscv_hwprobe) -> bool {
98 // SAFETY: We've passed the valid pointer and length,
99 // passing null ptr for cpus is safe because cpu_set_size is zero.
100 unsafe { ffi::__riscv_hwprobe(out, 1, 0, ptr::null_mut(), 0) == 0 }
101 }
102
103 #[cold]
_detect(info: &mut CpuInfo)104 fn _detect(info: &mut CpuInfo) {
105 let mut out = ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 };
106 if riscv_hwprobe(&mut out) && out.key != -1 {
107 let value = out.value;
108 if value & ffi::RISCV_HWPROBE_EXT_ZACAS != 0 {
109 info.set(CpuInfo::HAS_ZACAS);
110 }
111 }
112 }
113
114 #[allow(
115 clippy::alloc_instead_of_core,
116 clippy::std_instead_of_alloc,
117 clippy::std_instead_of_core,
118 clippy::undocumented_unsafe_blocks,
119 clippy::wildcard_imports
120 )]
121 #[cfg(test)]
122 mod tests {
123 use super::*;
124
125 // We use asm-based syscall for compatibility with non-libc targets.
126 // This test tests that our ones and libc::syscall returns the same result.
127 #[test]
test_linux_like()128 fn test_linux_like() {
129 use test_helper::libc;
130 unsafe fn __libc_riscv_hwprobe(
131 pairs: *mut ffi::riscv_hwprobe,
132 pair_count: ffi::c_size_t,
133 cpu_set_size: ffi::c_size_t,
134 cpus: *mut ffi::c_ulong,
135 flags: ffi::c_uint,
136 ) -> ffi::c_long {
137 // SAFETY: the caller must uphold the safety contract.
138 unsafe {
139 libc::syscall(ffi::__NR_riscv_hwprobe, pairs, pair_count, cpu_set_size, cpus, flags)
140 }
141 }
142 fn libc_riscv_hwprobe(out: &mut ffi::riscv_hwprobe) -> bool {
143 unsafe { __libc_riscv_hwprobe(out, 1, 0, ptr::null_mut(), 0) == 0 }
144 }
145 let mut out = ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 };
146 let mut libc_out = ffi::riscv_hwprobe { key: ffi::RISCV_HWPROBE_KEY_IMA_EXT_0, value: 0 };
147 assert_eq!(riscv_hwprobe(&mut out), libc_riscv_hwprobe(&mut libc_out));
148 assert_eq!(out, libc_out);
149 }
150
151 // Static assertions for FFI bindings.
152 // This checks that FFI bindings defined in this crate, FFI bindings defined
153 // in libc, and FFI bindings generated for the platform's latest header file
154 // using bindgen have compatible signatures (or the same values if constants).
155 // Since this is static assertion, we can detect problems with
156 // `cargo check --tests --target <target>` run in CI (via TESTS=1 build.sh)
157 // without actually running tests on these platforms.
158 // See also tools/codegen/src/ffi.rs.
159 // TODO(codegen): auto-generate this test
160 #[allow(
161 clippy::cast_possible_wrap,
162 clippy::cast_sign_loss,
163 clippy::no_effect_underscore_binding
164 )]
165 const _: fn() = || {
166 use std::mem;
167 use test_helper::sys;
168 // TODO: syscall
169 // static_assert!(ffi::__NR_riscv_hwprobe == libc::__NR_riscv_hwprobe); // libc doesn't have this
170 static_assert!(ffi::__NR_riscv_hwprobe == sys::__NR_riscv_hwprobe as ffi::c_long);
171 // static_assert!(ffi::RISCV_HWPROBE_KEY_IMA_EXT_0 == libc::RISCV_HWPROBE_KEY_IMA_EXT_0); // libc doesn't have this
172 static_assert!(ffi::RISCV_HWPROBE_KEY_IMA_EXT_0 == sys::RISCV_HWPROBE_KEY_IMA_EXT_0 as i64);
173 // static_assert!(ffi::RISCV_HWPROBE_EXT_ZACAS == libc::RISCV_HWPROBE_EXT_ZACAS); // libc doesn't have this
174 static_assert!(ffi::RISCV_HWPROBE_EXT_ZACAS == sys::RISCV_HWPROBE_EXT_ZACAS);
175 // libc doesn't have this
176 // static_assert!(
177 // mem::size_of::<ffi::riscv_hwprobe>()
178 // == mem::size_of::<libc::riscv_hwprobe>()
179 // );
180 static_assert!(
181 mem::size_of::<ffi::riscv_hwprobe>() == mem::size_of::<sys::riscv_hwprobe>()
182 );
183 let ffi: ffi::riscv_hwprobe = unsafe { mem::zeroed() };
184 let _ = sys::riscv_hwprobe { key: ffi.key, value: ffi.value };
185 };
186 }
187