1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
5
6 #include <linux/static_key.h>
7 #include <linux/objtool.h>
8 #include <linux/linkage.h>
9
10 #include <asm/alternative.h>
11 #include <asm/cpufeatures.h>
12 #include <asm/msr-index.h>
13 #include <asm/unwind_hints.h>
14 #include <asm/percpu.h>
15
16 #define RETPOLINE_THUNK_SIZE 32
17
18 /*
19 * Fill the CPU return stack buffer.
20 *
21 * Each entry in the RSB, if used for a speculative 'ret', contains an
22 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
23 *
24 * This is required in various cases for retpoline and IBRS-based
25 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
26 * eliminate potentially bogus entries from the RSB, and sometimes
27 * purely to ensure that it doesn't get empty, which on some CPUs would
28 * allow predictions from other (unwanted!) sources to be used.
29 *
30 * We define a CPP macro such that it can be used from both .S files and
31 * inline assembly. It's possible to do a .macro and then include that
32 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
33 */
34
35 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
36
37 /*
38 * Google experimented with loop-unrolling and this turned out to be
39 * the optimal version — two calls, each with their own speculation
40 * trap should their return address end up getting used, in a loop.
41 */
42 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
43 mov $(nr/2), reg; \
44 771: \
45 ANNOTATE_INTRA_FUNCTION_CALL; \
46 call 772f; \
47 773: /* speculation trap */ \
48 UNWIND_HINT_EMPTY; \
49 pause; \
50 lfence; \
51 jmp 773b; \
52 772: \
53 ANNOTATE_INTRA_FUNCTION_CALL; \
54 call 774f; \
55 775: /* speculation trap */ \
56 UNWIND_HINT_EMPTY; \
57 pause; \
58 lfence; \
59 jmp 775b; \
60 774: \
61 add $(BITS_PER_LONG/8) * 2, sp; \
62 dec reg; \
63 jnz 771b; \
64 /* barrier for jnz misprediction */ \
65 lfence;
66
67 #ifdef __ASSEMBLY__
68
69 /*
70 * This should be used immediately before an indirect jump/call. It tells
71 * objtool the subsequent indirect jump/call is vouched safe for retpoline
72 * builds.
73 */
74 .macro ANNOTATE_RETPOLINE_SAFE
75 .Lannotate_\@:
76 .pushsection .discard.retpoline_safe
77 _ASM_PTR .Lannotate_\@
78 .popsection
79 .endm
80
81 /*
82 * (ab)use RETPOLINE_SAFE on RET to annotate away 'bare' RET instructions
83 * vs RETBleed validation.
84 */
85 #define ANNOTATE_UNRET_SAFE ANNOTATE_RETPOLINE_SAFE
86
87 /*
88 * Abuse ANNOTATE_RETPOLINE_SAFE on a NOP to indicate UNRET_END, should
89 * eventually turn into it's own annotation.
90 */
91 .macro ANNOTATE_UNRET_END
92 #ifdef CONFIG_DEBUG_ENTRY
93 ANNOTATE_RETPOLINE_SAFE
94 nop
95 #endif
96 .endm
97
98 /*
99 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
100 * indirect jmp/call which may be susceptible to the Spectre variant 2
101 * attack.
102 */
103 .macro JMP_NOSPEC reg:req
104 #ifdef CONFIG_RETPOLINE
105 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), \
106 __stringify(jmp __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
107 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), X86_FEATURE_RETPOLINE_LFENCE
108 #else
109 jmp *%\reg
110 #endif
111 .endm
112
113 .macro CALL_NOSPEC reg:req
114 #ifdef CONFIG_RETPOLINE
115 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *%\reg), \
116 __stringify(call __x86_indirect_thunk_\reg), X86_FEATURE_RETPOLINE, \
117 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *%\reg), X86_FEATURE_RETPOLINE_LFENCE
118 #else
119 call *%\reg
120 #endif
121 .endm
122
123 .macro ISSUE_UNBALANCED_RET_GUARD
124 ANNOTATE_INTRA_FUNCTION_CALL
125 call .Lunbalanced_ret_guard_\@
126 int3
127 .Lunbalanced_ret_guard_\@:
128 add $(BITS_PER_LONG/8), %_ASM_SP
129 lfence
130 .endm
131
132 /*
133 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
134 * monstrosity above, manually.
135 */
136 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2
137 .ifb \ftr2
138 ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr
139 .else
140 ALTERNATIVE_2 "jmp .Lskip_rsb_\@", "", \ftr, "jmp .Lunbalanced_\@", \ftr2
141 .endif
142 __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)
143 .Lunbalanced_\@:
144 ISSUE_UNBALANCED_RET_GUARD
145 .Lskip_rsb_\@:
146 .endm
147
148 #ifdef CONFIG_CPU_UNRET_ENTRY
149 #define CALL_ZEN_UNTRAIN_RET "call zen_untrain_ret"
150 #else
151 #define CALL_ZEN_UNTRAIN_RET ""
152 #endif
153
154 /*
155 * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the
156 * return thunk isn't mapped into the userspace tables (then again, AMD
157 * typically has NO_MELTDOWN).
158 *
159 * While zen_untrain_ret() doesn't clobber anything but requires stack,
160 * entry_ibpb() will clobber AX, CX, DX.
161 *
162 * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point
163 * where we have a stack but before any RET instruction.
164 */
165 .macro UNTRAIN_RET
166 #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_IBPB_ENTRY)
167 ANNOTATE_UNRET_END
168 ALTERNATIVE_2 "", \
169 CALL_ZEN_UNTRAIN_RET, X86_FEATURE_UNRET, \
170 "call entry_ibpb", X86_FEATURE_ENTRY_IBPB
171 #endif
172 .endm
173
174 #else /* __ASSEMBLY__ */
175
176 #define ANNOTATE_RETPOLINE_SAFE \
177 "999:\n\t" \
178 ".pushsection .discard.retpoline_safe\n\t" \
179 _ASM_PTR " 999b\n\t" \
180 ".popsection\n\t"
181
182 extern void __x86_return_thunk(void);
183 extern void zen_untrain_ret(void);
184 extern void entry_ibpb(void);
185
186 #ifdef CONFIG_RETPOLINE
187
188 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
189
190 #define GEN(reg) \
191 extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;
192 #include <asm/GEN-for-each-reg.h>
193 #undef GEN
194
195 extern retpoline_thunk_t __x86_indirect_thunk_array[];
196
197 #ifdef CONFIG_X86_64
198
199 /*
200 * Inline asm uses the %V modifier which is only in newer GCC
201 * which is ensured when CONFIG_RETPOLINE is defined.
202 */
203 # define CALL_NOSPEC \
204 ALTERNATIVE_2( \
205 ANNOTATE_RETPOLINE_SAFE \
206 "call *%[thunk_target]\n", \
207 "call __x86_indirect_thunk_%V[thunk_target]\n", \
208 X86_FEATURE_RETPOLINE, \
209 "lfence;\n" \
210 ANNOTATE_RETPOLINE_SAFE \
211 "call *%[thunk_target]\n", \
212 X86_FEATURE_RETPOLINE_LFENCE)
213
214 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
215
216 #else /* CONFIG_X86_32 */
217 /*
218 * For i386 we use the original ret-equivalent retpoline, because
219 * otherwise we'll run out of registers. We don't care about CET
220 * here, anyway.
221 */
222 # define CALL_NOSPEC \
223 ALTERNATIVE_2( \
224 ANNOTATE_RETPOLINE_SAFE \
225 "call *%[thunk_target]\n", \
226 " jmp 904f;\n" \
227 " .align 16\n" \
228 "901: call 903f;\n" \
229 "902: pause;\n" \
230 " lfence;\n" \
231 " jmp 902b;\n" \
232 " .align 16\n" \
233 "903: lea 4(%%esp), %%esp;\n" \
234 " pushl %[thunk_target];\n" \
235 " ret;\n" \
236 " .align 16\n" \
237 "904: call 901b;\n", \
238 X86_FEATURE_RETPOLINE, \
239 "lfence;\n" \
240 ANNOTATE_RETPOLINE_SAFE \
241 "call *%[thunk_target]\n", \
242 X86_FEATURE_RETPOLINE_LFENCE)
243
244 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
245 #endif
246 #else /* No retpoline for C / inline asm */
247 # define CALL_NOSPEC "call *%[thunk_target]\n"
248 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
249 #endif
250
251 /* The Spectre V2 mitigation variants */
252 enum spectre_v2_mitigation {
253 SPECTRE_V2_NONE,
254 SPECTRE_V2_RETPOLINE,
255 SPECTRE_V2_LFENCE,
256 SPECTRE_V2_EIBRS,
257 SPECTRE_V2_EIBRS_RETPOLINE,
258 SPECTRE_V2_EIBRS_LFENCE,
259 SPECTRE_V2_IBRS,
260 };
261
262 /* The indirect branch speculation control variants */
263 enum spectre_v2_user_mitigation {
264 SPECTRE_V2_USER_NONE,
265 SPECTRE_V2_USER_STRICT,
266 SPECTRE_V2_USER_STRICT_PREFERRED,
267 SPECTRE_V2_USER_PRCTL,
268 SPECTRE_V2_USER_SECCOMP,
269 };
270
271 /* The Speculative Store Bypass disable variants */
272 enum ssb_mitigation {
273 SPEC_STORE_BYPASS_NONE,
274 SPEC_STORE_BYPASS_DISABLE,
275 SPEC_STORE_BYPASS_PRCTL,
276 SPEC_STORE_BYPASS_SECCOMP,
277 };
278
279 extern char __indirect_thunk_start[];
280 extern char __indirect_thunk_end[];
281
282 static __always_inline
alternative_msr_write(unsigned int msr,u64 val,unsigned int feature)283 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
284 {
285 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
286 : : "c" (msr),
287 "a" ((u32)val),
288 "d" ((u32)(val >> 32)),
289 [feature] "i" (feature)
290 : "memory");
291 }
292
indirect_branch_prediction_barrier(void)293 static inline void indirect_branch_prediction_barrier(void)
294 {
295 u64 val = PRED_CMD_IBPB;
296
297 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
298 }
299
300 /* The Intel SPEC CTRL MSR base value cache */
301 extern u64 x86_spec_ctrl_base;
302 DECLARE_PER_CPU(u64, x86_spec_ctrl_current);
303 extern void write_spec_ctrl_current(u64 val, bool force);
304 extern u64 spec_ctrl_current(void);
305
306 /*
307 * With retpoline, we must use IBRS to restrict branch prediction
308 * before calling into firmware.
309 *
310 * (Implemented as CPP macros due to header hell.)
311 */
312 #define firmware_restrict_branch_speculation_start() \
313 do { \
314 preempt_disable(); \
315 alternative_msr_write(MSR_IA32_SPEC_CTRL, \
316 spec_ctrl_current() | SPEC_CTRL_IBRS, \
317 X86_FEATURE_USE_IBRS_FW); \
318 } while (0)
319
320 #define firmware_restrict_branch_speculation_end() \
321 do { \
322 alternative_msr_write(MSR_IA32_SPEC_CTRL, \
323 spec_ctrl_current(), \
324 X86_FEATURE_USE_IBRS_FW); \
325 preempt_enable(); \
326 } while (0)
327
328 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
329 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
330 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
331
332 DECLARE_STATIC_KEY_FALSE(mds_user_clear);
333 DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
334
335 DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
336
337 #include <asm/segment.h>
338
339 /**
340 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
341 *
342 * This uses the otherwise unused and obsolete VERW instruction in
343 * combination with microcode which triggers a CPU buffer flush when the
344 * instruction is executed.
345 */
mds_clear_cpu_buffers(void)346 static __always_inline void mds_clear_cpu_buffers(void)
347 {
348 static const u16 ds = __KERNEL_DS;
349
350 /*
351 * Has to be the memory-operand variant because only that
352 * guarantees the CPU buffer flush functionality according to
353 * documentation. The register-operand variant does not.
354 * Works with any segment selector, but a valid writable
355 * data segment is the fastest variant.
356 *
357 * "cc" clobber is required because VERW modifies ZF.
358 */
359 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
360 }
361
362 /**
363 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
364 *
365 * Clear CPU buffers if the corresponding static key is enabled
366 */
mds_user_clear_cpu_buffers(void)367 static __always_inline void mds_user_clear_cpu_buffers(void)
368 {
369 if (static_branch_likely(&mds_user_clear))
370 mds_clear_cpu_buffers();
371 }
372
373 /**
374 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
375 *
376 * Clear CPU buffers if the corresponding static key is enabled
377 */
mds_idle_clear_cpu_buffers(void)378 static inline void mds_idle_clear_cpu_buffers(void)
379 {
380 if (static_branch_likely(&mds_idle_clear))
381 mds_clear_cpu_buffers();
382 }
383
384 #endif /* __ASSEMBLY__ */
385
386 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
387