1 /*
2 * Copyright (C) 1994 Linus Torvalds
3 *
4 * Cyrix stuff, June 1998 by:
5 * - Rafael R. Reilova (moved everything from head.S),
6 * <rreilova@ececs.uc.edu>
7 * - Channing Corn (tests & fixes),
8 * - Andrew D. Balsa (code cleanup).
9 */
10 #include <linux/init.h>
11 #include <linux/utsname.h>
12 #include <linux/cpu.h>
13 #include <linux/module.h>
14
15 #include <asm/nospec-branch.h>
16 #include <asm/cmdline.h>
17 #include <asm/bugs.h>
18 #include <asm/processor.h>
19 #include <asm/processor-flags.h>
20 #include <asm/fpu/internal.h>
21 #include <asm/msr.h>
22 #include <asm/paravirt.h>
23 #include <asm/alternative.h>
24 #include <asm/pgtable.h>
25 #include <asm/cacheflush.h>
26 #include <asm/intel-family.h>
27
28 static void __init spectre_v2_select_mitigation(void);
29
check_bugs(void)30 void __init check_bugs(void)
31 {
32 identify_boot_cpu();
33
34 if (!IS_ENABLED(CONFIG_SMP)) {
35 pr_info("CPU: ");
36 print_cpu_info(&boot_cpu_data);
37 }
38
39 /* Select the proper spectre mitigation before patching alternatives */
40 spectre_v2_select_mitigation();
41
42 #ifdef CONFIG_X86_32
43 /*
44 * Check whether we are able to run this kernel safely on SMP.
45 *
46 * - i386 is no longer supported.
47 * - In order to run on anything without a TSC, we need to be
48 * compiled for a i486.
49 */
50 if (boot_cpu_data.x86 < 4)
51 panic("Kernel requires i486+ for 'invlpg' and other features");
52
53 init_utsname()->machine[1] =
54 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
55 alternative_instructions();
56
57 fpu__init_check_bugs();
58 #else /* CONFIG_X86_64 */
59 alternative_instructions();
60
61 /*
62 * Make sure the first 2MB area is not mapped by huge pages
63 * There are typically fixed size MTRRs in there and overlapping
64 * MTRRs into large pages causes slow downs.
65 *
66 * Right now we don't do that with gbpages because there seems
67 * very little benefit for that case.
68 */
69 if (!direct_gbpages)
70 set_memory_4k((unsigned long)__va(0), 1);
71 #endif
72 }
73
74 /* The kernel command line selection */
75 enum spectre_v2_mitigation_cmd {
76 SPECTRE_V2_CMD_NONE,
77 SPECTRE_V2_CMD_AUTO,
78 SPECTRE_V2_CMD_FORCE,
79 SPECTRE_V2_CMD_RETPOLINE,
80 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
81 SPECTRE_V2_CMD_RETPOLINE_AMD,
82 };
83
84 static const char *spectre_v2_strings[] = {
85 [SPECTRE_V2_NONE] = "Vulnerable",
86 [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline",
87 [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline",
88 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
89 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
90 };
91
92 #undef pr_fmt
93 #define pr_fmt(fmt) "Spectre V2 : " fmt
94
95 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
96
97 #ifdef RETPOLINE
98 static bool spectre_v2_bad_module;
99
retpoline_module_ok(bool has_retpoline)100 bool retpoline_module_ok(bool has_retpoline)
101 {
102 if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
103 return true;
104
105 pr_err("System may be vulnerable to spectre v2\n");
106 spectre_v2_bad_module = true;
107 return false;
108 }
109
spectre_v2_module_string(void)110 static inline const char *spectre_v2_module_string(void)
111 {
112 return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
113 }
114 #else
spectre_v2_module_string(void)115 static inline const char *spectre_v2_module_string(void) { return ""; }
116 #endif
117
spec2_print_if_insecure(const char * reason)118 static void __init spec2_print_if_insecure(const char *reason)
119 {
120 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
121 pr_info("%s selected on command line.\n", reason);
122 }
123
spec2_print_if_secure(const char * reason)124 static void __init spec2_print_if_secure(const char *reason)
125 {
126 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
127 pr_info("%s selected on command line.\n", reason);
128 }
129
retp_compiler(void)130 static inline bool retp_compiler(void)
131 {
132 return __is_defined(RETPOLINE);
133 }
134
match_option(const char * arg,int arglen,const char * opt)135 static inline bool match_option(const char *arg, int arglen, const char *opt)
136 {
137 int len = strlen(opt);
138
139 return len == arglen && !strncmp(arg, opt, len);
140 }
141
142 static const struct {
143 const char *option;
144 enum spectre_v2_mitigation_cmd cmd;
145 bool secure;
146 } mitigation_options[] = {
147 { "off", SPECTRE_V2_CMD_NONE, false },
148 { "on", SPECTRE_V2_CMD_FORCE, true },
149 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
150 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false },
151 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
152 { "auto", SPECTRE_V2_CMD_AUTO, false },
153 };
154
spectre_v2_parse_cmdline(void)155 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
156 {
157 char arg[20];
158 int ret, i;
159 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
160
161 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
162 return SPECTRE_V2_CMD_NONE;
163 else {
164 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
165 if (ret < 0)
166 return SPECTRE_V2_CMD_AUTO;
167
168 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
169 if (!match_option(arg, ret, mitigation_options[i].option))
170 continue;
171 cmd = mitigation_options[i].cmd;
172 break;
173 }
174
175 if (i >= ARRAY_SIZE(mitigation_options)) {
176 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
177 return SPECTRE_V2_CMD_AUTO;
178 }
179 }
180
181 if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
182 cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
183 cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
184 !IS_ENABLED(CONFIG_RETPOLINE)) {
185 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
186 return SPECTRE_V2_CMD_AUTO;
187 }
188
189 if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
190 boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
191 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
192 return SPECTRE_V2_CMD_AUTO;
193 }
194
195 if (mitigation_options[i].secure)
196 spec2_print_if_secure(mitigation_options[i].option);
197 else
198 spec2_print_if_insecure(mitigation_options[i].option);
199
200 return cmd;
201 }
202
203 /* Check for Skylake-like CPUs (for RSB handling) */
is_skylake_era(void)204 static bool __init is_skylake_era(void)
205 {
206 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
207 boot_cpu_data.x86 == 6) {
208 switch (boot_cpu_data.x86_model) {
209 case INTEL_FAM6_SKYLAKE_MOBILE:
210 case INTEL_FAM6_SKYLAKE_DESKTOP:
211 case INTEL_FAM6_SKYLAKE_X:
212 case INTEL_FAM6_KABYLAKE_MOBILE:
213 case INTEL_FAM6_KABYLAKE_DESKTOP:
214 return true;
215 }
216 }
217 return false;
218 }
219
spectre_v2_select_mitigation(void)220 static void __init spectre_v2_select_mitigation(void)
221 {
222 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
223 enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
224
225 /*
226 * If the CPU is not affected and the command line mode is NONE or AUTO
227 * then nothing to do.
228 */
229 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
230 (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
231 return;
232
233 switch (cmd) {
234 case SPECTRE_V2_CMD_NONE:
235 return;
236
237 case SPECTRE_V2_CMD_FORCE:
238 case SPECTRE_V2_CMD_AUTO:
239 if (IS_ENABLED(CONFIG_RETPOLINE))
240 goto retpoline_auto;
241 break;
242 case SPECTRE_V2_CMD_RETPOLINE_AMD:
243 if (IS_ENABLED(CONFIG_RETPOLINE))
244 goto retpoline_amd;
245 break;
246 case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
247 if (IS_ENABLED(CONFIG_RETPOLINE))
248 goto retpoline_generic;
249 break;
250 case SPECTRE_V2_CMD_RETPOLINE:
251 if (IS_ENABLED(CONFIG_RETPOLINE))
252 goto retpoline_auto;
253 break;
254 }
255 pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
256 return;
257
258 retpoline_auto:
259 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
260 retpoline_amd:
261 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
262 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
263 goto retpoline_generic;
264 }
265 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
266 SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
267 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
268 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
269 } else {
270 retpoline_generic:
271 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
272 SPECTRE_V2_RETPOLINE_MINIMAL;
273 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
274 }
275
276 spectre_v2_enabled = mode;
277 pr_info("%s\n", spectre_v2_strings[mode]);
278
279 /*
280 * If neither SMEP nor PTI are available, there is a risk of
281 * hitting userspace addresses in the RSB after a context switch
282 * from a shallow call stack to a deeper one. To prevent this fill
283 * the entire RSB, even when using IBRS.
284 *
285 * Skylake era CPUs have a separate issue with *underflow* of the
286 * RSB, when they will predict 'ret' targets from the generic BTB.
287 * The proper mitigation for this is IBRS. If IBRS is not supported
288 * or deactivated in favour of retpolines the RSB fill on context
289 * switch is required.
290 */
291 if ((!boot_cpu_has(X86_FEATURE_KAISER) &&
292 !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
293 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
294 pr_info("Spectre v2 mitigation: Filling RSB on context switch\n");
295 }
296
297 /* Initialize Indirect Branch Prediction Barrier if supported */
298 if (boot_cpu_has(X86_FEATURE_IBPB)) {
299 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
300 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
301 }
302
303 /*
304 * Retpoline means the kernel is safe because it has no indirect
305 * branches. But firmware isn't, so use IBRS to protect that.
306 */
307 if (boot_cpu_has(X86_FEATURE_IBRS)) {
308 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
309 pr_info("Enabling Restricted Speculation for firmware calls\n");
310 }
311 }
312
313 #undef pr_fmt
314
315 #ifdef CONFIG_SYSFS
cpu_show_meltdown(struct device * dev,struct device_attribute * attr,char * buf)316 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
317 {
318 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
319 return sprintf(buf, "Not affected\n");
320 if (boot_cpu_has(X86_FEATURE_KAISER))
321 return sprintf(buf, "Mitigation: PTI\n");
322 return sprintf(buf, "Vulnerable\n");
323 }
324
cpu_show_spectre_v1(struct device * dev,struct device_attribute * attr,char * buf)325 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
326 {
327 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
328 return sprintf(buf, "Not affected\n");
329 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
330 }
331
cpu_show_spectre_v2(struct device * dev,struct device_attribute * attr,char * buf)332 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
333 {
334 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
335 return sprintf(buf, "Not affected\n");
336
337 return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
338 boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
339 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
340 spectre_v2_module_string());
341 }
342 #endif
343