1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _X86_IRQFLAGS_H_ 3 #define _X86_IRQFLAGS_H_ 4 5 #include <asm/processor-flags.h> 6 7 #ifndef __ASSEMBLY__ 8 9 #include <asm/nospec-branch.h> 10 11 /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ 12 #define __cpuidle __attribute__((__section__(".cpuidle.text"))) 13 14 /* 15 * Interrupt control: 16 */ 17 18 /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ 19 extern inline unsigned long native_save_fl(void); native_save_fl(void)20extern inline unsigned long native_save_fl(void) 21 { 22 unsigned long flags; 23 24 /* 25 * "=rm" is safe here, because "pop" adjusts the stack before 26 * it evaluates its effective address -- this is part of the 27 * documented behavior of the "pop" instruction. 28 */ 29 asm volatile("# __raw_save_flags\n\t" 30 "pushf ; pop %0" 31 : "=rm" (flags) 32 : /* no input */ 33 : "memory"); 34 35 return flags; 36 } 37 38 extern inline void native_restore_fl(unsigned long flags); native_restore_fl(unsigned long flags)39extern inline void native_restore_fl(unsigned long flags) 40 { 41 asm volatile("push %0 ; popf" 42 : /* no output */ 43 :"g" (flags) 44 :"memory", "cc"); 45 } 46 native_irq_disable(void)47static inline void native_irq_disable(void) 48 { 49 asm volatile("cli": : :"memory"); 50 } 51 native_irq_enable(void)52static inline void native_irq_enable(void) 53 { 54 asm volatile("sti": : :"memory"); 55 } 56 native_safe_halt(void)57static inline __cpuidle void native_safe_halt(void) 58 { 59 mds_idle_clear_cpu_buffers(); 60 asm volatile("sti; hlt": : :"memory"); 61 } 62 native_halt(void)63static inline __cpuidle void native_halt(void) 64 { 65 mds_idle_clear_cpu_buffers(); 66 asm volatile("hlt": : :"memory"); 67 } 68 69 #endif 70 71 #ifdef CONFIG_PARAVIRT 72 #include <asm/paravirt.h> 73 #else 74 #ifndef __ASSEMBLY__ 75 #include <linux/types.h> 76 arch_local_save_flags(void)77static inline notrace unsigned long arch_local_save_flags(void) 78 { 79 return native_save_fl(); 80 } 81 arch_local_irq_restore(unsigned long flags)82static inline notrace void arch_local_irq_restore(unsigned long flags) 83 { 84 native_restore_fl(flags); 85 } 86 arch_local_irq_disable(void)87static inline notrace void arch_local_irq_disable(void) 88 { 89 native_irq_disable(); 90 } 91 arch_local_irq_enable(void)92static inline notrace void arch_local_irq_enable(void) 93 { 94 native_irq_enable(); 95 } 96 97 /* 98 * Used in the idle loop; sti takes one instruction cycle 99 * to complete: 100 */ arch_safe_halt(void)101static inline __cpuidle void arch_safe_halt(void) 102 { 103 native_safe_halt(); 104 } 105 106 /* 107 * Used when interrupts are already enabled or to 108 * shutdown the processor: 109 */ halt(void)110static inline __cpuidle void halt(void) 111 { 112 native_halt(); 113 } 114 115 /* 116 * For spinlocks, etc: 117 */ arch_local_irq_save(void)118static inline notrace unsigned long arch_local_irq_save(void) 119 { 120 unsigned long flags = arch_local_save_flags(); 121 arch_local_irq_disable(); 122 return flags; 123 } 124 #else 125 126 #define ENABLE_INTERRUPTS(x) sti 127 #define DISABLE_INTERRUPTS(x) cli 128 129 #ifdef CONFIG_X86_64 130 #define SWAPGS swapgs 131 /* 132 * Currently paravirt can't handle swapgs nicely when we 133 * don't have a stack we can rely on (such as a user space 134 * stack). So we either find a way around these or just fault 135 * and emulate if a guest tries to call swapgs directly. 136 * 137 * Either way, this is a good way to document that we don't 138 * have a reliable stack. x86_64 only. 139 */ 140 #define SWAPGS_UNSAFE_STACK swapgs 141 142 #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ 143 144 #define INTERRUPT_RETURN jmp native_iret 145 #define USERGS_SYSRET64 \ 146 swapgs; \ 147 sysretq; 148 #define USERGS_SYSRET32 \ 149 swapgs; \ 150 sysretl 151 152 #ifdef CONFIG_DEBUG_ENTRY 153 #define SAVE_FLAGS(x) pushfq; popq %rax 154 #endif 155 #else 156 #define INTERRUPT_RETURN iret 157 #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit 158 #define GET_CR0_INTO_EAX movl %cr0, %eax 159 #endif 160 161 162 #endif /* __ASSEMBLY__ */ 163 #endif /* CONFIG_PARAVIRT */ 164 165 #ifndef __ASSEMBLY__ arch_irqs_disabled_flags(unsigned long flags)166static inline int arch_irqs_disabled_flags(unsigned long flags) 167 { 168 return !(flags & X86_EFLAGS_IF); 169 } 170 arch_irqs_disabled(void)171static inline int arch_irqs_disabled(void) 172 { 173 unsigned long flags = arch_local_save_flags(); 174 175 return arch_irqs_disabled_flags(flags); 176 } 177 #endif /* !__ASSEMBLY__ */ 178 179 #ifdef __ASSEMBLY__ 180 #ifdef CONFIG_TRACE_IRQFLAGS 181 # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; 182 # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; 183 #else 184 # define TRACE_IRQS_ON 185 # define TRACE_IRQS_OFF 186 #endif 187 #ifdef CONFIG_DEBUG_LOCK_ALLOC 188 # ifdef CONFIG_X86_64 189 # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk 190 # define LOCKDEP_SYS_EXIT_IRQ \ 191 TRACE_IRQS_ON; \ 192 sti; \ 193 call lockdep_sys_exit_thunk; \ 194 cli; \ 195 TRACE_IRQS_OFF; 196 # else 197 # define LOCKDEP_SYS_EXIT \ 198 pushl %eax; \ 199 pushl %ecx; \ 200 pushl %edx; \ 201 call lockdep_sys_exit; \ 202 popl %edx; \ 203 popl %ecx; \ 204 popl %eax; 205 # define LOCKDEP_SYS_EXIT_IRQ 206 # endif 207 #else 208 # define LOCKDEP_SYS_EXIT 209 # define LOCKDEP_SYS_EXIT_IRQ 210 #endif 211 #endif /* __ASSEMBLY__ */ 212 213 #endif 214