1 /*
2 * Copyright (c) 2025 Google Inc. All rights reserved
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining
5 * a copy of this software and associated documentation files
6 * (the "Software"), to deal in the Software without restriction,
7 * including without limitation the rights to use, copy, modify, merge,
8 * publish, distribute, sublicense, and/or sell copies of the Software,
9 * and to permit persons to whom the Software is furnished to do so,
10 * subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be
13 * included in all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
18 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
20 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
21 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 use bitflags::bitflags;
25
26 use crate::sys::lk_interrupt_restore;
27 use crate::sys::lk_interrupt_save;
28 use crate::sys::lk_ints_disabled;
29 use crate::sys::spin_lock_save_flags_t;
30 use crate::sys::spin_lock_saved_state_t;
31 use crate::sys::SPIN_LOCK_FLAG_INTERRUPTS;
32 use crate::sys::SPIN_LOCK_FLAG_IRQ;
33 use crate::sys::SPIN_LOCK_FLAG_IRQ_FIQ;
34
35 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
36 use crate::sys::{lk_fiqs_disabled, SPIN_LOCK_FLAG_FIQ};
37
interrupt_save(state: &mut spin_lock_saved_state_t, flags: spin_lock_save_flags_t)38 fn interrupt_save(state: &mut spin_lock_saved_state_t, flags: spin_lock_save_flags_t) {
39 // SAFETY: `lk_interrupt_save` can be called in any context,
40 // and the `statep` arg is from a `&mut`, so it can be written to.
41 unsafe { lk_interrupt_save(state, flags) }
42 }
43
44 /// `state` should be from the corresponding call to [`interrupt_save`],
45 /// and `flags` should be the same `flags` from that [`interrupt_save`] call.
interrupt_restore(state: spin_lock_saved_state_t, flags: spin_lock_save_flags_t)46 fn interrupt_restore(state: spin_lock_saved_state_t, flags: spin_lock_save_flags_t) {
47 // SAFETY: `lk_interrupt_restore` is safe and can be called in any context.
48 unsafe { lk_interrupt_restore(state, flags) }
49 }
50
irqs_disabled() -> bool51 pub fn irqs_disabled() -> bool {
52 // SAFETY: `lk_ints_disabled` can be called in any context.
53 unsafe { lk_ints_disabled() }
54 }
55
56 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
fiqs_disabled() -> bool57 fn fiqs_disabled() -> bool {
58 // SAFETY: `lk_fiqs_disabled` can be called in any context.
59 unsafe { lk_fiqs_disabled() }
60 }
61
62 /// See [`InterruptFlags::check`] for what combinations of [`InterruptFlags`]
63 /// are allowed to be disabled compared to what is already disabled.
64 #[derive(Clone, Copy, PartialEq, Eq)]
65 pub struct InterruptFlags(u32);
66
67 bitflags! {
68 impl InterruptFlags: u32 {
69 /// Disable IRQs (interrupt requests).
70 ///
71 /// IRQs (without FIQs) cannot safely (risking deadlocks)
72 /// be disabled if only FIQs are currently disabled.
73 /// (see [`Self::check`] for more).
74 const IRQ = SPIN_LOCK_FLAG_IRQ;
75
76 /// Disable FIQs (fast interrupt requests).
77 ///
78 /// These are generally lower latency and higher priority than IRQs.
79 ///
80 /// FIQs (without IRQs) cannot safely (risking deadlocks)
81 /// be disabled if only IRQs are currently disabled
82 /// (see [`Self::check`] for more).
83 ///
84 /// Currently, FIQs do not exist on `x86`/`x86_64` `target_arch`es,
85 /// while they always exist on `arm`/`aarch64` `target_arch`es.
86 /// However, in the latter, they are sometimes merged with IRQs.
87 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
88 const FIQ = SPIN_LOCK_FLAG_FIQ;
89
90 /// Disable both IRQs and FIQs.
91 ///
92 /// This is still defined for convenience on platforms with no FIQs,
93 /// where this does not disable FIQs, but FIQs just don't exist at all.
94 const IRQ_FIQ = SPIN_LOCK_FLAG_IRQ_FIQ;
95
96 /// Disable interrupts in general in a platform-agnostic way.
97 ///
98 /// On `arm`/`aarch64` `target_arch`es, this only disables IRQs.
99 const INTERRUPTS = SPIN_LOCK_FLAG_INTERRUPTS;
100 }
101 }
102
103 impl Default for InterruptFlags {
default() -> Self104 fn default() -> Self {
105 Self::INTERRUPTS
106 }
107 }
108
109 impl InterruptFlags {
as_raw(&self) -> spin_lock_save_flags_t110 const fn as_raw(&self) -> spin_lock_save_flags_t {
111 self.bits()
112 }
113
114 /// Returns `self` if `select` is `true`,
115 /// or [`Self::empty`] if `select` is `false`.
select(self, select: bool) -> Self116 pub const fn select(self, select: bool) -> Self {
117 if select {
118 self
119 } else {
120 Self::empty()
121 }
122 }
123
124 /// Checks if these [`InterruptFlags`] can safely (risking potential deadlocks) be disabled
125 /// depending on what is currently already disabled.
126 ///
127 /// More specifically, considering only [IRQ]s and [FIQ]s,
128 /// if only [IRQ]s are requested, then only [FIQ]s being disabled already is not allowed,
129 /// and if only [FIQ]s are requested, then only [IRQ]s being disabled already is not allowed.
130 ///
131 /// Panics on error.
132 ///
133 /// [IRQ]: Self::IRQ
134 /// [FIQ]: Self::FIQ
135 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
136 #[track_caller]
check(&self)137 pub fn check(&self) {
138 let irq = Self::IRQ;
139 let fiq = Self::FIQ;
140
141 let this = *self & (irq | fiq);
142 let disabled = irq.select(irqs_disabled()) | fiq.select(fiqs_disabled());
143
144 if (this == irq && disabled == fiq) || (this == fiq && disabled == irq) {
145 error(this, disabled);
146 }
147
148 #[track_caller]
149 // This function is cold, so don't inline it to save on code size,
150 // which helps the rest of the function be inlined more easily.
151 #[cold]
152 #[inline(never)]
153 fn error(this: InterruptFlags, disabled: InterruptFlags) {
154 let [this, disabled] =
155 [this, disabled]
156 .map(|flags| if flags == InterruptFlags::IRQ { "IRQ" } else { "FIQ" });
157 panic!("can't disable only {this}s for a SpinLock when only {disabled}s are currently disabled");
158 }
159 }
160
161 #[cfg(not(any(target_arch = "arm", target_arch = "aarch64")))]
check(&self)162 pub fn check(&self) {
163 // FIQs don't exist.
164 }
165 }
166
167 /// Go through this `trait` instead of just [`InterruptFlags`]
168 /// so that when the [`InterruptFlags`] are known at compile time,
169 /// we don't need to store them (see [`IRQInterruptFlags`]).
170 pub trait GetInterruptFlags: Clone + Copy + Send + Sync {
171 /// This should always return the same flags for the same `self`.
get(&self) -> InterruptFlags172 fn get(&self) -> InterruptFlags;
173 }
174
175 impl GetInterruptFlags for InterruptFlags {
get(&self) -> InterruptFlags176 fn get(&self) -> InterruptFlags {
177 *self
178 }
179 }
180
181 /// A ZST implementation of [`GetInterruptFlags`] that stores the [`InterruptFlags`] at compile time.
182 /// Most uses should use this (with convenient type aliases in [`interrupt::flags`])
183 /// instead of [`InterruptFlags`] directly
184 ///
185 /// [`interrupt::flags`]: crate::interrupt::flags
186 // TODO use `#![feature(adt_const_params)]` once stabilized,
187 // with `const FLAGS: InterruptFlags` instead of `u32`.
188 #[derive(Clone, Copy)]
189 pub struct ConstInterruptFlags<const FLAGS: u32>;
190
191 impl<const FLAGS: u32> GetInterruptFlags for ConstInterruptFlags<FLAGS> {
get(&self) -> InterruptFlags192 fn get(&self) -> InterruptFlags {
193 // Truncate instead of retain since the `FLAGS: u32` is unchecked.
194 InterruptFlags::from_bits_truncate(FLAGS)
195 }
196 }
197
198 pub mod flags {
199 use super::*;
200
201 // Type aliases can't be used as constructors,
202 // so we define separate `const`s/constructors.
203 // This makes initialization much simpler.
204
205 pub type None = ConstInterruptFlags<{ InterruptFlags::empty().bits() }>;
206 pub const NONE: None = ConstInterruptFlags;
207
208 pub type Irq = ConstInterruptFlags<{ InterruptFlags::IRQ.bits() }>;
209 pub const IRQ: Irq = ConstInterruptFlags;
210
211 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
212 pub type Fiq = ConstInterruptFlags<{ InterruptFlags::FIQ.bits() }>;
213 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
214 pub const FIQ: Fiq = ConstInterruptFlags;
215
216 pub type IrqFiq = ConstInterruptFlags<{ InterruptFlags::IRQ_FIQ.bits() }>;
217 pub const IRQ_FIQ: IrqFiq = ConstInterruptFlags;
218
219 pub type Interrupts = ConstInterruptFlags<{ InterruptFlags::INTERRUPTS.bits() }>;
220 pub const INTERRUPTS: Interrupts = ConstInterruptFlags;
221
222 pub type All = ConstInterruptFlags<{ InterruptFlags::all().bits() }>;
223 pub const ALL: All = ConstInterruptFlags;
224 }
225
226 pub struct InterruptState<F: GetInterruptFlags> {
227 state: spin_lock_saved_state_t,
228 flags: F,
229 }
230
231 impl<F: GetInterruptFlags> InterruptState<F> {
232 /// Disables interrupts and saves the interrupt state.
save(flags: F) -> Self233 pub fn save(flags: F) -> Self {
234 let mut state = Default::default();
235 let raw_flags = flags.get().as_raw();
236 interrupt_save(&mut state, raw_flags);
237 Self { state, flags }
238 }
239 }
240
241 impl<F: GetInterruptFlags> Drop for InterruptState<F> {
drop(&mut self)242 fn drop(&mut self) {
243 let raw_flags = self.flags.get().as_raw();
244 interrupt_restore(self.state, raw_flags);
245 }
246 }
247
248 impl<F: GetInterruptFlags> InterruptState<F> {
249 /// A more explicit restore, equivalent to [`drop`].
restore(self)250 pub fn restore(self) {
251 drop(self)
252 }
253 }
254