1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
5
6 #include <linux/static_key.h>
7
8 #include <asm/alternative.h>
9 #include <asm/alternative-asm.h>
10 #include <asm/cpufeatures.h>
11 #include <asm/msr-index.h>
12
13 /*
14 * Fill the CPU return stack buffer.
15 *
16 * Each entry in the RSB, if used for a speculative 'ret', contains an
17 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
18 *
19 * This is required in various cases for retpoline and IBRS-based
20 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
21 * eliminate potentially bogus entries from the RSB, and sometimes
22 * purely to ensure that it doesn't get empty, which on some CPUs would
23 * allow predictions from other (unwanted!) sources to be used.
24 *
25 * We define a CPP macro such that it can be used from both .S files and
26 * inline assembly. It's possible to do a .macro and then include that
27 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
28 */
29
30 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
31 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
32
33 /*
34 * Google experimented with loop-unrolling and this turned out to be
35 * the optimal version — two calls, each with their own speculation
36 * trap should their return address end up getting used, in a loop.
37 */
38 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
39 mov $(nr/2), reg; \
40 771: \
41 call 772f; \
42 773: /* speculation trap */ \
43 pause; \
44 lfence; \
45 jmp 773b; \
46 772: \
47 call 774f; \
48 775: /* speculation trap */ \
49 pause; \
50 lfence; \
51 jmp 775b; \
52 774: \
53 dec reg; \
54 jnz 771b; \
55 add $(BITS_PER_LONG/8) * nr, sp;
56
57 #ifdef __ASSEMBLY__
58
59 /*
60 * These are the bare retpoline primitives for indirect jmp and call.
61 * Do not use these directly; they only exist to make the ALTERNATIVE
62 * invocation below less ugly.
63 */
64 .macro RETPOLINE_JMP reg:req
65 call .Ldo_rop_\@
66 .Lspec_trap_\@:
67 pause
68 lfence
69 jmp .Lspec_trap_\@
70 .Ldo_rop_\@:
71 mov \reg, (%_ASM_SP)
72 ret
73 .endm
74
75 /*
76 * This is a wrapper around RETPOLINE_JMP so the called function in reg
77 * returns to the instruction after the macro.
78 */
79 .macro RETPOLINE_CALL reg:req
80 jmp .Ldo_call_\@
81 .Ldo_retpoline_jmp_\@:
82 RETPOLINE_JMP \reg
83 .Ldo_call_\@:
84 call .Ldo_retpoline_jmp_\@
85 .endm
86
87 /*
88 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
89 * indirect jmp/call which may be susceptible to the Spectre variant 2
90 * attack.
91 */
92 .macro JMP_NOSPEC reg:req
93 #ifdef CONFIG_RETPOLINE
94 ALTERNATIVE_2 __stringify(jmp *\reg), \
95 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
96 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
97 #else
98 jmp *\reg
99 #endif
100 .endm
101
102 .macro CALL_NOSPEC reg:req
103 #ifdef CONFIG_RETPOLINE
104 ALTERNATIVE_2 __stringify(call *\reg), \
105 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
106 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
107 #else
108 call *\reg
109 #endif
110 .endm
111
112 /*
113 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
114 * monstrosity above, manually.
115 */
116 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
117 #ifdef CONFIG_RETPOLINE
118 ALTERNATIVE "jmp .Lskip_rsb_\@", \
119 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
120 \ftr
121 .Lskip_rsb_\@:
122 #endif
123 .endm
124
125 #else /* __ASSEMBLY__ */
126
127 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
128
129 /*
130 * Since the inline asm uses the %V modifier which is only in newer GCC,
131 * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
132 */
133 # define CALL_NOSPEC \
134 ALTERNATIVE( \
135 "call *%[thunk_target]\n", \
136 "call __x86_indirect_thunk_%V[thunk_target]\n", \
137 X86_FEATURE_RETPOLINE)
138 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
139
140 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
141 /*
142 * For i386 we use the original ret-equivalent retpoline, because
143 * otherwise we'll run out of registers. We don't care about CET
144 * here, anyway.
145 */
146 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n", \
147 " jmp 904f;\n" \
148 " .align 16\n" \
149 "901: call 903f;\n" \
150 "902: pause;\n" \
151 " lfence;\n" \
152 " jmp 902b;\n" \
153 " .align 16\n" \
154 "903: lea 4(%%esp), %%esp;\n" \
155 " pushl %[thunk_target];\n" \
156 " ret;\n" \
157 " .align 16\n" \
158 "904: call 901b;\n", \
159 X86_FEATURE_RETPOLINE)
160
161 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
162 #else /* No retpoline for C / inline asm */
163 # define CALL_NOSPEC "call *%[thunk_target]\n"
164 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
165 #endif
166
167 /* The Spectre V2 mitigation variants */
168 enum spectre_v2_mitigation {
169 SPECTRE_V2_NONE,
170 SPECTRE_V2_RETPOLINE_MINIMAL,
171 SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
172 SPECTRE_V2_RETPOLINE_GENERIC,
173 SPECTRE_V2_RETPOLINE_AMD,
174 SPECTRE_V2_IBRS_ENHANCED,
175 };
176
177 /* The indirect branch speculation control variants */
178 enum spectre_v2_user_mitigation {
179 SPECTRE_V2_USER_NONE,
180 SPECTRE_V2_USER_STRICT,
181 SPECTRE_V2_USER_STRICT_PREFERRED,
182 SPECTRE_V2_USER_PRCTL,
183 SPECTRE_V2_USER_SECCOMP,
184 };
185
186 /* The Speculative Store Bypass disable variants */
187 enum ssb_mitigation {
188 SPEC_STORE_BYPASS_NONE,
189 SPEC_STORE_BYPASS_DISABLE,
190 SPEC_STORE_BYPASS_PRCTL,
191 SPEC_STORE_BYPASS_SECCOMP,
192 };
193
194 extern char __indirect_thunk_start[];
195 extern char __indirect_thunk_end[];
196
197 /*
198 * On VMEXIT we must ensure that no RSB predictions learned in the guest
199 * can be followed in the host, by overwriting the RSB completely. Both
200 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
201 * CPUs with IBRS_ALL *might* it be avoided.
202 */
vmexit_fill_RSB(void)203 static inline void vmexit_fill_RSB(void)
204 {
205 #ifdef CONFIG_RETPOLINE
206 unsigned long loops;
207
208 asm volatile (ALTERNATIVE("jmp 910f",
209 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
210 X86_FEATURE_RETPOLINE)
211 "910:"
212 : "=r" (loops), ASM_CALL_CONSTRAINT
213 : : "memory" );
214 #endif
215 }
216
217 static __always_inline
alternative_msr_write(unsigned int msr,u64 val,unsigned int feature)218 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
219 {
220 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
221 : : "c" (msr),
222 "a" ((u32)val),
223 "d" ((u32)(val >> 32)),
224 [feature] "i" (feature)
225 : "memory");
226 }
227
indirect_branch_prediction_barrier(void)228 static inline void indirect_branch_prediction_barrier(void)
229 {
230 u64 val = PRED_CMD_IBPB;
231
232 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
233 }
234
235 /* The Intel SPEC CTRL MSR base value cache */
236 extern u64 x86_spec_ctrl_base;
237
238 /*
239 * With retpoline, we must use IBRS to restrict branch prediction
240 * before calling into firmware.
241 *
242 * (Implemented as CPP macros due to header hell.)
243 */
244 #define firmware_restrict_branch_speculation_start() \
245 do { \
246 u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \
247 \
248 preempt_disable(); \
249 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
250 X86_FEATURE_USE_IBRS_FW); \
251 } while (0)
252
253 #define firmware_restrict_branch_speculation_end() \
254 do { \
255 u64 val = x86_spec_ctrl_base; \
256 \
257 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
258 X86_FEATURE_USE_IBRS_FW); \
259 preempt_enable(); \
260 } while (0)
261
262 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
263 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
264 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
265
266 DECLARE_STATIC_KEY_FALSE(mds_user_clear);
267 DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
268
269 #include <asm/segment.h>
270
271 /**
272 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
273 *
274 * This uses the otherwise unused and obsolete VERW instruction in
275 * combination with microcode which triggers a CPU buffer flush when the
276 * instruction is executed.
277 */
mds_clear_cpu_buffers(void)278 static __always_inline void mds_clear_cpu_buffers(void)
279 {
280 static const u16 ds = __KERNEL_DS;
281
282 /*
283 * Has to be the memory-operand variant because only that
284 * guarantees the CPU buffer flush functionality according to
285 * documentation. The register-operand variant does not.
286 * Works with any segment selector, but a valid writable
287 * data segment is the fastest variant.
288 *
289 * "cc" clobber is required because VERW modifies ZF.
290 */
291 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
292 }
293
294 /**
295 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
296 *
297 * Clear CPU buffers if the corresponding static key is enabled
298 */
mds_user_clear_cpu_buffers(void)299 static __always_inline void mds_user_clear_cpu_buffers(void)
300 {
301 if (static_branch_likely(&mds_user_clear))
302 mds_clear_cpu_buffers();
303 }
304
305 /**
306 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
307 *
308 * Clear CPU buffers if the corresponding static key is enabled
309 */
mds_idle_clear_cpu_buffers(void)310 static inline void mds_idle_clear_cpu_buffers(void)
311 {
312 if (static_branch_likely(&mds_idle_clear))
313 mds_clear_cpu_buffers();
314 }
315
316 #endif /* __ASSEMBLY__ */
317
318 /*
319 * Below is used in the eBPF JIT compiler and emits the byte sequence
320 * for the following assembly:
321 *
322 * With retpolines configured:
323 *
324 * callq do_rop
325 * spec_trap:
326 * pause
327 * lfence
328 * jmp spec_trap
329 * do_rop:
330 * mov %rax,(%rsp)
331 * retq
332 *
333 * Without retpolines configured:
334 *
335 * jmp *%rax
336 */
337 #ifdef CONFIG_RETPOLINE
338 # define RETPOLINE_RAX_BPF_JIT_SIZE 17
339 # define RETPOLINE_RAX_BPF_JIT() \
340 EMIT1_off32(0xE8, 7); /* callq do_rop */ \
341 /* spec_trap: */ \
342 EMIT2(0xF3, 0x90); /* pause */ \
343 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
344 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
345 /* do_rop: */ \
346 EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
347 EMIT1(0xC3); /* retq */
348 #else
349 # define RETPOLINE_RAX_BPF_JIT_SIZE 2
350 # define RETPOLINE_RAX_BPF_JIT() \
351 EMIT2(0xFF, 0xE0); /* jmp *%rax */
352 #endif
353
354 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
355