1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_IRQ_STACK_H
3 #define _ASM_X86_IRQ_STACK_H
4
5 #include <linux/ptrace.h>
6
7 #include <asm/processor.h>
8
9 #ifdef CONFIG_X86_64
irqstack_active(void)10 static __always_inline bool irqstack_active(void)
11 {
12 return __this_cpu_read(irq_count) != -1;
13 }
14
15 void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
16 void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
17 struct pt_regs *regs);
18 void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
19 struct irq_desc *desc);
20
__run_on_irqstack(void (* func)(void))21 static __always_inline void __run_on_irqstack(void (*func)(void))
22 {
23 void *tos = __this_cpu_read(hardirq_stack_ptr);
24
25 __this_cpu_add(irq_count, 1);
26 asm_call_on_stack(tos - 8, func, NULL);
27 __this_cpu_sub(irq_count, 1);
28 }
29
30 static __always_inline void
__run_sysvec_on_irqstack(void (* func)(struct pt_regs * regs),struct pt_regs * regs)31 __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
32 struct pt_regs *regs)
33 {
34 void *tos = __this_cpu_read(hardirq_stack_ptr);
35
36 __this_cpu_add(irq_count, 1);
37 asm_call_sysvec_on_stack(tos - 8, func, regs);
38 __this_cpu_sub(irq_count, 1);
39 }
40
41 static __always_inline void
__run_irq_on_irqstack(void (* func)(struct irq_desc * desc),struct irq_desc * desc)42 __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
43 struct irq_desc *desc)
44 {
45 void *tos = __this_cpu_read(hardirq_stack_ptr);
46
47 __this_cpu_add(irq_count, 1);
48 asm_call_irq_on_stack(tos - 8, func, desc);
49 __this_cpu_sub(irq_count, 1);
50 }
51
52 #else /* CONFIG_X86_64 */
irqstack_active(void)53 static inline bool irqstack_active(void) { return false; }
__run_on_irqstack(void (* func)(void))54 static inline void __run_on_irqstack(void (*func)(void)) { }
__run_sysvec_on_irqstack(void (* func)(struct pt_regs * regs),struct pt_regs * regs)55 static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
56 struct pt_regs *regs) { }
__run_irq_on_irqstack(void (* func)(struct irq_desc * desc),struct irq_desc * desc)57 static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
58 struct irq_desc *desc) { }
59 #endif /* !CONFIG_X86_64 */
60
irq_needs_irq_stack(struct pt_regs * regs)61 static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
62 {
63 if (IS_ENABLED(CONFIG_X86_32))
64 return false;
65 if (!regs)
66 return !irqstack_active();
67 return !user_mode(regs) && !irqstack_active();
68 }
69
70
run_on_irqstack_cond(void (* func)(void),struct pt_regs * regs)71 static __always_inline void run_on_irqstack_cond(void (*func)(void),
72 struct pt_regs *regs)
73 {
74 lockdep_assert_irqs_disabled();
75
76 if (irq_needs_irq_stack(regs))
77 __run_on_irqstack(func);
78 else
79 func();
80 }
81
82 static __always_inline void
run_sysvec_on_irqstack_cond(void (* func)(struct pt_regs * regs),struct pt_regs * regs)83 run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
84 struct pt_regs *regs)
85 {
86 lockdep_assert_irqs_disabled();
87
88 if (irq_needs_irq_stack(regs))
89 __run_sysvec_on_irqstack(func, regs);
90 else
91 func(regs);
92 }
93
94 static __always_inline void
run_irq_on_irqstack_cond(void (* func)(struct irq_desc * desc),struct irq_desc * desc,struct pt_regs * regs)95 run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
96 struct pt_regs *regs)
97 {
98 lockdep_assert_irqs_disabled();
99
100 if (irq_needs_irq_stack(regs))
101 __run_irq_on_irqstack(func, desc);
102 else
103 func(desc);
104 }
105
106 #endif
107