1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12 #include <asm/setup_data.h>
13
14 #ifndef __BOOT_COMPRESSED
15 #define error(v) pr_err(v)
16 #define has_cpuflag(f) boot_cpu_has(f)
17 #define sev_printk(fmt, ...) printk(fmt, ##__VA_ARGS__)
18 #define sev_printk_rtl(fmt, ...) printk_ratelimited(fmt, ##__VA_ARGS__)
19 #else
20 #undef WARN
21 #define WARN(condition, format...) (!!(condition))
22 #define sev_printk(fmt, ...)
23 #define sev_printk_rtl(fmt, ...)
24 #undef vc_forward_exception
25 #define vc_forward_exception(c) panic("SNP: Hypervisor requested exception\n")
26 #endif
27
28 /*
29 * SVSM related information:
30 * When running under an SVSM, the VMPL that Linux is executing at must be
31 * non-zero. The VMPL is therefore used to indicate the presence of an SVSM.
32 *
33 * During boot, the page tables are set up as identity mapped and later
34 * changed to use kernel virtual addresses. Maintain separate virtual and
35 * physical addresses for the CAA to allow SVSM functions to be used during
36 * early boot, both with identity mapped virtual addresses and proper kernel
37 * virtual addresses.
38 */
39 u8 snp_vmpl __ro_after_init;
40 EXPORT_SYMBOL_GPL(snp_vmpl);
41 static struct svsm_ca *boot_svsm_caa __ro_after_init;
42 static u64 boot_svsm_caa_pa __ro_after_init;
43
44 static struct svsm_ca *svsm_get_caa(void);
45 static u64 svsm_get_caa_pa(void);
46 static int svsm_perform_call_protocol(struct svsm_call *call);
47
48 /* I/O parameters for CPUID-related helpers */
49 struct cpuid_leaf {
50 u32 fn;
51 u32 subfn;
52 u32 eax;
53 u32 ebx;
54 u32 ecx;
55 u32 edx;
56 };
57
58 /*
59 * Individual entries of the SNP CPUID table, as defined by the SNP
60 * Firmware ABI, Revision 0.9, Section 7.1, Table 14.
61 */
62 struct snp_cpuid_fn {
63 u32 eax_in;
64 u32 ecx_in;
65 u64 xcr0_in;
66 u64 xss_in;
67 u32 eax;
68 u32 ebx;
69 u32 ecx;
70 u32 edx;
71 u64 __reserved;
72 } __packed;
73
74 /*
75 * SNP CPUID table, as defined by the SNP Firmware ABI, Revision 0.9,
76 * Section 8.14.2.6. Also noted there is the SNP firmware-enforced limit
77 * of 64 entries per CPUID table.
78 */
79 #define SNP_CPUID_COUNT_MAX 64
80
81 struct snp_cpuid_table {
82 u32 count;
83 u32 __reserved1;
84 u64 __reserved2;
85 struct snp_cpuid_fn fn[SNP_CPUID_COUNT_MAX];
86 } __packed;
87
88 /*
89 * Since feature negotiation related variables are set early in the boot
90 * process they must reside in the .data section so as not to be zeroed
91 * out when the .bss section is later cleared.
92 *
93 * GHCB protocol version negotiated with the hypervisor.
94 */
95 static u16 ghcb_version __ro_after_init;
96
97 /* Copy of the SNP firmware's CPUID page. */
98 static struct snp_cpuid_table cpuid_table_copy __ro_after_init;
99
100 /*
101 * These will be initialized based on CPUID table so that non-present
102 * all-zero leaves (for sparse tables) can be differentiated from
103 * invalid/out-of-range leaves. This is needed since all-zero leaves
104 * still need to be post-processed.
105 */
106 static u32 cpuid_std_range_max __ro_after_init;
107 static u32 cpuid_hyp_range_max __ro_after_init;
108 static u32 cpuid_ext_range_max __ro_after_init;
109
sev_es_check_cpu_features(void)110 static bool __init sev_es_check_cpu_features(void)
111 {
112 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
113 error("RDRAND instruction not supported - no trusted source of randomness available\n");
114 return false;
115 }
116
117 return true;
118 }
119
120 static void __head __noreturn
sev_es_terminate(unsigned int set,unsigned int reason)121 sev_es_terminate(unsigned int set, unsigned int reason)
122 {
123 u64 val = GHCB_MSR_TERM_REQ;
124
125 /* Tell the hypervisor what went wrong. */
126 val |= GHCB_SEV_TERM_REASON(set, reason);
127
128 /* Request Guest Termination from Hypervisor */
129 sev_es_wr_ghcb_msr(val);
130 VMGEXIT();
131
132 while (true)
133 asm volatile("hlt\n" : : : "memory");
134 }
135
136 /*
137 * The hypervisor features are available from GHCB version 2 onward.
138 */
get_hv_features(void)139 static u64 get_hv_features(void)
140 {
141 u64 val;
142
143 if (ghcb_version < 2)
144 return 0;
145
146 sev_es_wr_ghcb_msr(GHCB_MSR_HV_FT_REQ);
147 VMGEXIT();
148
149 val = sev_es_rd_ghcb_msr();
150 if (GHCB_RESP_CODE(val) != GHCB_MSR_HV_FT_RESP)
151 return 0;
152
153 return GHCB_MSR_HV_FT_RESP_VAL(val);
154 }
155
snp_register_ghcb_early(unsigned long paddr)156 static void snp_register_ghcb_early(unsigned long paddr)
157 {
158 unsigned long pfn = paddr >> PAGE_SHIFT;
159 u64 val;
160
161 sev_es_wr_ghcb_msr(GHCB_MSR_REG_GPA_REQ_VAL(pfn));
162 VMGEXIT();
163
164 val = sev_es_rd_ghcb_msr();
165
166 /* If the response GPA is not ours then abort the guest */
167 if ((GHCB_RESP_CODE(val) != GHCB_MSR_REG_GPA_RESP) ||
168 (GHCB_MSR_REG_GPA_RESP_VAL(val) != pfn))
169 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_REGISTER);
170 }
171
sev_es_negotiate_protocol(void)172 static bool sev_es_negotiate_protocol(void)
173 {
174 u64 val;
175
176 /* Do the GHCB protocol version negotiation */
177 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
178 VMGEXIT();
179 val = sev_es_rd_ghcb_msr();
180
181 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
182 return false;
183
184 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTOCOL_MIN ||
185 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTOCOL_MAX)
186 return false;
187
188 ghcb_version = min_t(size_t, GHCB_MSR_PROTO_MAX(val), GHCB_PROTOCOL_MAX);
189
190 return true;
191 }
192
vc_ghcb_invalidate(struct ghcb * ghcb)193 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
194 {
195 ghcb->save.sw_exit_code = 0;
196 __builtin_memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
197 }
198
vc_decoding_needed(unsigned long exit_code)199 static bool vc_decoding_needed(unsigned long exit_code)
200 {
201 /* Exceptions don't require to decode the instruction */
202 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
203 exit_code <= SVM_EXIT_LAST_EXCP);
204 }
205
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)206 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
207 struct pt_regs *regs,
208 unsigned long exit_code)
209 {
210 enum es_result ret = ES_OK;
211
212 memset(ctxt, 0, sizeof(*ctxt));
213 ctxt->regs = regs;
214
215 if (vc_decoding_needed(exit_code))
216 ret = vc_decode_insn(ctxt);
217
218 return ret;
219 }
220
vc_finish_insn(struct es_em_ctxt * ctxt)221 static void vc_finish_insn(struct es_em_ctxt *ctxt)
222 {
223 ctxt->regs->ip += ctxt->insn.length;
224 }
225
verify_exception_info(struct ghcb * ghcb,struct es_em_ctxt * ctxt)226 static enum es_result verify_exception_info(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
227 {
228 u32 ret;
229
230 ret = ghcb->save.sw_exit_info_1 & GENMASK_ULL(31, 0);
231 if (!ret)
232 return ES_OK;
233
234 if (ret == 1) {
235 u64 info = ghcb->save.sw_exit_info_2;
236 unsigned long v = info & SVM_EVTINJ_VEC_MASK;
237
238 /* Check if exception information from hypervisor is sane. */
239 if ((info & SVM_EVTINJ_VALID) &&
240 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
241 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
242 ctxt->fi.vector = v;
243
244 if (info & SVM_EVTINJ_VALID_ERR)
245 ctxt->fi.error_code = info >> 32;
246
247 return ES_EXCEPTION;
248 }
249 }
250
251 return ES_VMM_ERROR;
252 }
253
svsm_process_result_codes(struct svsm_call * call)254 static inline int svsm_process_result_codes(struct svsm_call *call)
255 {
256 switch (call->rax_out) {
257 case SVSM_SUCCESS:
258 return 0;
259 case SVSM_ERR_INCOMPLETE:
260 case SVSM_ERR_BUSY:
261 return -EAGAIN;
262 default:
263 return -EINVAL;
264 }
265 }
266
267 /*
268 * Issue a VMGEXIT to call the SVSM:
269 * - Load the SVSM register state (RAX, RCX, RDX, R8 and R9)
270 * - Set the CA call pending field to 1
271 * - Issue VMGEXIT
272 * - Save the SVSM return register state (RAX, RCX, RDX, R8 and R9)
273 * - Perform atomic exchange of the CA call pending field
274 *
275 * - See the "Secure VM Service Module for SEV-SNP Guests" specification for
276 * details on the calling convention.
277 * - The calling convention loosely follows the Microsoft X64 calling
278 * convention by putting arguments in RCX, RDX, R8 and R9.
279 * - RAX specifies the SVSM protocol/callid as input and the return code
280 * as output.
281 */
svsm_issue_call(struct svsm_call * call,u8 * pending)282 static __always_inline void svsm_issue_call(struct svsm_call *call, u8 *pending)
283 {
284 register unsigned long rax asm("rax") = call->rax;
285 register unsigned long rcx asm("rcx") = call->rcx;
286 register unsigned long rdx asm("rdx") = call->rdx;
287 register unsigned long r8 asm("r8") = call->r8;
288 register unsigned long r9 asm("r9") = call->r9;
289
290 call->caa->call_pending = 1;
291
292 asm volatile("rep; vmmcall\n\t"
293 : "+r" (rax), "+r" (rcx), "+r" (rdx), "+r" (r8), "+r" (r9)
294 : : "memory");
295
296 *pending = xchg(&call->caa->call_pending, *pending);
297
298 call->rax_out = rax;
299 call->rcx_out = rcx;
300 call->rdx_out = rdx;
301 call->r8_out = r8;
302 call->r9_out = r9;
303 }
304
svsm_perform_msr_protocol(struct svsm_call * call)305 static int svsm_perform_msr_protocol(struct svsm_call *call)
306 {
307 u8 pending = 0;
308 u64 val, resp;
309
310 /*
311 * When using the MSR protocol, be sure to save and restore
312 * the current MSR value.
313 */
314 val = sev_es_rd_ghcb_msr();
315
316 sev_es_wr_ghcb_msr(GHCB_MSR_VMPL_REQ_LEVEL(0));
317
318 svsm_issue_call(call, &pending);
319
320 resp = sev_es_rd_ghcb_msr();
321
322 sev_es_wr_ghcb_msr(val);
323
324 if (pending)
325 return -EINVAL;
326
327 if (GHCB_RESP_CODE(resp) != GHCB_MSR_VMPL_RESP)
328 return -EINVAL;
329
330 if (GHCB_MSR_VMPL_RESP_VAL(resp))
331 return -EINVAL;
332
333 return svsm_process_result_codes(call);
334 }
335
svsm_perform_ghcb_protocol(struct ghcb * ghcb,struct svsm_call * call)336 static int svsm_perform_ghcb_protocol(struct ghcb *ghcb, struct svsm_call *call)
337 {
338 struct es_em_ctxt ctxt;
339 u8 pending = 0;
340
341 vc_ghcb_invalidate(ghcb);
342
343 /*
344 * Fill in protocol and format specifiers. This can be called very early
345 * in the boot, so use rip-relative references as needed.
346 */
347 ghcb->protocol_version = RIP_REL_REF(ghcb_version);
348 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
349
350 ghcb_set_sw_exit_code(ghcb, SVM_VMGEXIT_SNP_RUN_VMPL);
351 ghcb_set_sw_exit_info_1(ghcb, 0);
352 ghcb_set_sw_exit_info_2(ghcb, 0);
353
354 sev_es_wr_ghcb_msr(__pa(ghcb));
355
356 svsm_issue_call(call, &pending);
357
358 if (pending)
359 return -EINVAL;
360
361 switch (verify_exception_info(ghcb, &ctxt)) {
362 case ES_OK:
363 break;
364 case ES_EXCEPTION:
365 vc_forward_exception(&ctxt);
366 fallthrough;
367 default:
368 return -EINVAL;
369 }
370
371 return svsm_process_result_codes(call);
372 }
373
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)374 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
375 struct es_em_ctxt *ctxt,
376 u64 exit_code, u64 exit_info_1,
377 u64 exit_info_2)
378 {
379 /* Fill in protocol and format specifiers */
380 ghcb->protocol_version = ghcb_version;
381 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
382
383 ghcb_set_sw_exit_code(ghcb, exit_code);
384 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
385 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
386
387 sev_es_wr_ghcb_msr(__pa(ghcb));
388 VMGEXIT();
389
390 return verify_exception_info(ghcb, ctxt);
391 }
392
__sev_cpuid_hv(u32 fn,int reg_idx,u32 * reg)393 static int __sev_cpuid_hv(u32 fn, int reg_idx, u32 *reg)
394 {
395 u64 val;
396
397 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, reg_idx));
398 VMGEXIT();
399 val = sev_es_rd_ghcb_msr();
400 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
401 return -EIO;
402
403 *reg = (val >> 32);
404
405 return 0;
406 }
407
__sev_cpuid_hv_msr(struct cpuid_leaf * leaf)408 static int __sev_cpuid_hv_msr(struct cpuid_leaf *leaf)
409 {
410 int ret;
411
412 /*
413 * MSR protocol does not support fetching non-zero subfunctions, but is
414 * sufficient to handle current early-boot cases. Should that change,
415 * make sure to report an error rather than ignoring the index and
416 * grabbing random values. If this issue arises in the future, handling
417 * can be added here to use GHCB-page protocol for cases that occur late
418 * enough in boot that GHCB page is available.
419 */
420 if (cpuid_function_is_indexed(leaf->fn) && leaf->subfn)
421 return -EINVAL;
422
423 ret = __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EAX, &leaf->eax);
424 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EBX, &leaf->ebx);
425 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_ECX, &leaf->ecx);
426 ret = ret ? : __sev_cpuid_hv(leaf->fn, GHCB_CPUID_REQ_EDX, &leaf->edx);
427
428 return ret;
429 }
430
__sev_cpuid_hv_ghcb(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)431 static int __sev_cpuid_hv_ghcb(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
432 {
433 u32 cr4 = native_read_cr4();
434 int ret;
435
436 ghcb_set_rax(ghcb, leaf->fn);
437 ghcb_set_rcx(ghcb, leaf->subfn);
438
439 if (cr4 & X86_CR4_OSXSAVE)
440 /* Safe to read xcr0 */
441 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
442 else
443 /* xgetbv will cause #UD - use reset value for xcr0 */
444 ghcb_set_xcr0(ghcb, 1);
445
446 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
447 if (ret != ES_OK)
448 return ret;
449
450 if (!(ghcb_rax_is_valid(ghcb) &&
451 ghcb_rbx_is_valid(ghcb) &&
452 ghcb_rcx_is_valid(ghcb) &&
453 ghcb_rdx_is_valid(ghcb)))
454 return ES_VMM_ERROR;
455
456 leaf->eax = ghcb->save.rax;
457 leaf->ebx = ghcb->save.rbx;
458 leaf->ecx = ghcb->save.rcx;
459 leaf->edx = ghcb->save.rdx;
460
461 return ES_OK;
462 }
463
sev_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)464 static int sev_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
465 {
466 return ghcb ? __sev_cpuid_hv_ghcb(ghcb, ctxt, leaf)
467 : __sev_cpuid_hv_msr(leaf);
468 }
469
470 /*
471 * This may be called early while still running on the initial identity
472 * mapping. Use RIP-relative addressing to obtain the correct address
473 * while running with the initial identity mapping as well as the
474 * switch-over to kernel virtual addresses later.
475 */
snp_cpuid_get_table(void)476 static const struct snp_cpuid_table *snp_cpuid_get_table(void)
477 {
478 return &RIP_REL_REF(cpuid_table_copy);
479 }
480
481 /*
482 * The SNP Firmware ABI, Revision 0.9, Section 7.1, details the use of
483 * XCR0_IN and XSS_IN to encode multiple versions of 0xD subfunctions 0
484 * and 1 based on the corresponding features enabled by a particular
485 * combination of XCR0 and XSS registers so that a guest can look up the
486 * version corresponding to the features currently enabled in its XCR0/XSS
487 * registers. The only values that differ between these versions/table
488 * entries is the enabled XSAVE area size advertised via EBX.
489 *
490 * While hypervisors may choose to make use of this support, it is more
491 * robust/secure for a guest to simply find the entry corresponding to the
492 * base/legacy XSAVE area size (XCR0=1 or XCR0=3), and then calculate the
493 * XSAVE area size using subfunctions 2 through 64, as documented in APM
494 * Volume 3, Rev 3.31, Appendix E.3.8, which is what is done here.
495 *
496 * Since base/legacy XSAVE area size is documented as 0x240, use that value
497 * directly rather than relying on the base size in the CPUID table.
498 *
499 * Return: XSAVE area size on success, 0 otherwise.
500 */
snp_cpuid_calc_xsave_size(u64 xfeatures_en,bool compacted)501 static u32 snp_cpuid_calc_xsave_size(u64 xfeatures_en, bool compacted)
502 {
503 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
504 u64 xfeatures_found = 0;
505 u32 xsave_size = 0x240;
506 int i;
507
508 for (i = 0; i < cpuid_table->count; i++) {
509 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
510
511 if (!(e->eax_in == 0xD && e->ecx_in > 1 && e->ecx_in < 64))
512 continue;
513 if (!(xfeatures_en & (BIT_ULL(e->ecx_in))))
514 continue;
515 if (xfeatures_found & (BIT_ULL(e->ecx_in)))
516 continue;
517
518 xfeatures_found |= (BIT_ULL(e->ecx_in));
519
520 if (compacted)
521 xsave_size += e->eax;
522 else
523 xsave_size = max(xsave_size, e->eax + e->ebx);
524 }
525
526 /*
527 * Either the guest set unsupported XCR0/XSS bits, or the corresponding
528 * entries in the CPUID table were not present. This is not a valid
529 * state to be in.
530 */
531 if (xfeatures_found != (xfeatures_en & GENMASK_ULL(63, 2)))
532 return 0;
533
534 return xsave_size;
535 }
536
537 static bool __head
snp_cpuid_get_validated_func(struct cpuid_leaf * leaf)538 snp_cpuid_get_validated_func(struct cpuid_leaf *leaf)
539 {
540 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
541 int i;
542
543 for (i = 0; i < cpuid_table->count; i++) {
544 const struct snp_cpuid_fn *e = &cpuid_table->fn[i];
545
546 if (e->eax_in != leaf->fn)
547 continue;
548
549 if (cpuid_function_is_indexed(leaf->fn) && e->ecx_in != leaf->subfn)
550 continue;
551
552 /*
553 * For 0xD subfunctions 0 and 1, only use the entry corresponding
554 * to the base/legacy XSAVE area size (XCR0=1 or XCR0=3, XSS=0).
555 * See the comments above snp_cpuid_calc_xsave_size() for more
556 * details.
557 */
558 if (e->eax_in == 0xD && (e->ecx_in == 0 || e->ecx_in == 1))
559 if (!(e->xcr0_in == 1 || e->xcr0_in == 3) || e->xss_in)
560 continue;
561
562 leaf->eax = e->eax;
563 leaf->ebx = e->ebx;
564 leaf->ecx = e->ecx;
565 leaf->edx = e->edx;
566
567 return true;
568 }
569
570 return false;
571 }
572
snp_cpuid_hv(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)573 static void snp_cpuid_hv(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
574 {
575 if (sev_cpuid_hv(ghcb, ctxt, leaf))
576 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID_HV);
577 }
578
snp_cpuid_postprocess(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)579 static int snp_cpuid_postprocess(struct ghcb *ghcb, struct es_em_ctxt *ctxt,
580 struct cpuid_leaf *leaf)
581 {
582 struct cpuid_leaf leaf_hv = *leaf;
583
584 switch (leaf->fn) {
585 case 0x1:
586 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
587
588 /* initial APIC ID */
589 leaf->ebx = (leaf_hv.ebx & GENMASK(31, 24)) | (leaf->ebx & GENMASK(23, 0));
590 /* APIC enabled bit */
591 leaf->edx = (leaf_hv.edx & BIT(9)) | (leaf->edx & ~BIT(9));
592
593 /* OSXSAVE enabled bit */
594 if (native_read_cr4() & X86_CR4_OSXSAVE)
595 leaf->ecx |= BIT(27);
596 break;
597 case 0x7:
598 /* OSPKE enabled bit */
599 leaf->ecx &= ~BIT(4);
600 if (native_read_cr4() & X86_CR4_PKE)
601 leaf->ecx |= BIT(4);
602 break;
603 case 0xB:
604 leaf_hv.subfn = 0;
605 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
606
607 /* extended APIC ID */
608 leaf->edx = leaf_hv.edx;
609 break;
610 case 0xD: {
611 bool compacted = false;
612 u64 xcr0 = 1, xss = 0;
613 u32 xsave_size;
614
615 if (leaf->subfn != 0 && leaf->subfn != 1)
616 return 0;
617
618 if (native_read_cr4() & X86_CR4_OSXSAVE)
619 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
620 if (leaf->subfn == 1) {
621 /* Get XSS value if XSAVES is enabled. */
622 if (leaf->eax & BIT(3)) {
623 unsigned long lo, hi;
624
625 asm volatile("rdmsr" : "=a" (lo), "=d" (hi)
626 : "c" (MSR_IA32_XSS));
627 xss = (hi << 32) | lo;
628 }
629
630 /*
631 * The PPR and APM aren't clear on what size should be
632 * encoded in 0xD:0x1:EBX when compaction is not enabled
633 * by either XSAVEC (feature bit 1) or XSAVES (feature
634 * bit 3) since SNP-capable hardware has these feature
635 * bits fixed as 1. KVM sets it to 0 in this case, but
636 * to avoid this becoming an issue it's safer to simply
637 * treat this as unsupported for SNP guests.
638 */
639 if (!(leaf->eax & (BIT(1) | BIT(3))))
640 return -EINVAL;
641
642 compacted = true;
643 }
644
645 xsave_size = snp_cpuid_calc_xsave_size(xcr0 | xss, compacted);
646 if (!xsave_size)
647 return -EINVAL;
648
649 leaf->ebx = xsave_size;
650 }
651 break;
652 case 0x8000001E:
653 snp_cpuid_hv(ghcb, ctxt, &leaf_hv);
654
655 /* extended APIC ID */
656 leaf->eax = leaf_hv.eax;
657 /* compute ID */
658 leaf->ebx = (leaf->ebx & GENMASK(31, 8)) | (leaf_hv.ebx & GENMASK(7, 0));
659 /* node ID */
660 leaf->ecx = (leaf->ecx & GENMASK(31, 8)) | (leaf_hv.ecx & GENMASK(7, 0));
661 break;
662 default:
663 /* No fix-ups needed, use values as-is. */
664 break;
665 }
666
667 return 0;
668 }
669
670 /*
671 * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value
672 * should be treated as fatal by caller.
673 */
674 static int __head
snp_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt,struct cpuid_leaf * leaf)675 snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf)
676 {
677 const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table();
678
679 if (!cpuid_table->count)
680 return -EOPNOTSUPP;
681
682 if (!snp_cpuid_get_validated_func(leaf)) {
683 /*
684 * Some hypervisors will avoid keeping track of CPUID entries
685 * where all values are zero, since they can be handled the
686 * same as out-of-range values (all-zero). This is useful here
687 * as well as it allows virtually all guest configurations to
688 * work using a single SNP CPUID table.
689 *
690 * To allow for this, there is a need to distinguish between
691 * out-of-range entries and in-range zero entries, since the
692 * CPUID table entries are only a template that may need to be
693 * augmented with additional values for things like
694 * CPU-specific information during post-processing. So if it's
695 * not in the table, set the values to zero. Then, if they are
696 * within a valid CPUID range, proceed with post-processing
697 * using zeros as the initial values. Otherwise, skip
698 * post-processing and just return zeros immediately.
699 */
700 leaf->eax = leaf->ebx = leaf->ecx = leaf->edx = 0;
701
702 /* Skip post-processing for out-of-range zero leafs. */
703 if (!(leaf->fn <= RIP_REL_REF(cpuid_std_range_max) ||
704 (leaf->fn >= 0x40000000 && leaf->fn <= RIP_REL_REF(cpuid_hyp_range_max)) ||
705 (leaf->fn >= 0x80000000 && leaf->fn <= RIP_REL_REF(cpuid_ext_range_max))))
706 return 0;
707 }
708
709 return snp_cpuid_postprocess(ghcb, ctxt, leaf);
710 }
711
712 /*
713 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
714 * page yet, so it only supports the MSR based communication with the
715 * hypervisor and only the CPUID exit-code.
716 */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)717 void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
718 {
719 unsigned int subfn = lower_bits(regs->cx, 32);
720 unsigned int fn = lower_bits(regs->ax, 32);
721 u16 opcode = *(unsigned short *)regs->ip;
722 struct cpuid_leaf leaf;
723 int ret;
724
725 /* Only CPUID is supported via MSR protocol */
726 if (exit_code != SVM_EXIT_CPUID)
727 goto fail;
728
729 /* Is it really a CPUID insn? */
730 if (opcode != 0xa20f)
731 goto fail;
732
733 leaf.fn = fn;
734 leaf.subfn = subfn;
735
736 ret = snp_cpuid(NULL, NULL, &leaf);
737 if (!ret)
738 goto cpuid_done;
739
740 if (ret != -EOPNOTSUPP)
741 goto fail;
742
743 if (__sev_cpuid_hv_msr(&leaf))
744 goto fail;
745
746 cpuid_done:
747 regs->ax = leaf.eax;
748 regs->bx = leaf.ebx;
749 regs->cx = leaf.ecx;
750 regs->dx = leaf.edx;
751
752 /*
753 * This is a VC handler and the #VC is only raised when SEV-ES is
754 * active, which means SEV must be active too. Do sanity checks on the
755 * CPUID results to make sure the hypervisor does not trick the kernel
756 * into the no-sev path. This could map sensitive data unencrypted and
757 * make it accessible to the hypervisor.
758 *
759 * In particular, check for:
760 * - Availability of CPUID leaf 0x8000001f
761 * - SEV CPUID bit.
762 *
763 * The hypervisor might still report the wrong C-bit position, but this
764 * can't be checked here.
765 */
766
767 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
768 /* SEV leaf check */
769 goto fail;
770 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
771 /* SEV bit */
772 goto fail;
773
774 /* Skip over the CPUID two-byte opcode */
775 regs->ip += 2;
776
777 return;
778
779 fail:
780 /* Terminate the guest */
781 sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SEV_ES_GEN_REQ);
782 }
783
vc_insn_string_check(struct es_em_ctxt * ctxt,unsigned long address,bool write)784 static enum es_result vc_insn_string_check(struct es_em_ctxt *ctxt,
785 unsigned long address,
786 bool write)
787 {
788 if (user_mode(ctxt->regs) && fault_in_kernel_space(address)) {
789 ctxt->fi.vector = X86_TRAP_PF;
790 ctxt->fi.error_code = X86_PF_USER;
791 ctxt->fi.cr2 = address;
792 if (write)
793 ctxt->fi.error_code |= X86_PF_WRITE;
794
795 return ES_EXCEPTION;
796 }
797
798 return ES_OK;
799 }
800
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)801 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
802 void *src, char *buf,
803 unsigned int data_size,
804 unsigned int count,
805 bool backwards)
806 {
807 int i, b = backwards ? -1 : 1;
808 unsigned long address = (unsigned long)src;
809 enum es_result ret;
810
811 ret = vc_insn_string_check(ctxt, address, false);
812 if (ret != ES_OK)
813 return ret;
814
815 for (i = 0; i < count; i++) {
816 void *s = src + (i * data_size * b);
817 char *d = buf + (i * data_size);
818
819 ret = vc_read_mem(ctxt, s, d, data_size);
820 if (ret != ES_OK)
821 break;
822 }
823
824 return ret;
825 }
826
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)827 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
828 void *dst, char *buf,
829 unsigned int data_size,
830 unsigned int count,
831 bool backwards)
832 {
833 int i, s = backwards ? -1 : 1;
834 unsigned long address = (unsigned long)dst;
835 enum es_result ret;
836
837 ret = vc_insn_string_check(ctxt, address, true);
838 if (ret != ES_OK)
839 return ret;
840
841 for (i = 0; i < count; i++) {
842 void *d = dst + (i * data_size * s);
843 char *b = buf + (i * data_size);
844
845 ret = vc_write_mem(ctxt, d, b, data_size);
846 if (ret != ES_OK)
847 break;
848 }
849
850 return ret;
851 }
852
853 #define IOIO_TYPE_STR BIT(2)
854 #define IOIO_TYPE_IN 1
855 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
856 #define IOIO_TYPE_OUT 0
857 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
858
859 #define IOIO_REP BIT(3)
860
861 #define IOIO_ADDR_64 BIT(9)
862 #define IOIO_ADDR_32 BIT(8)
863 #define IOIO_ADDR_16 BIT(7)
864
865 #define IOIO_DATA_32 BIT(6)
866 #define IOIO_DATA_16 BIT(5)
867 #define IOIO_DATA_8 BIT(4)
868
869 #define IOIO_SEG_ES (0 << 10)
870 #define IOIO_SEG_DS (3 << 10)
871
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)872 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
873 {
874 struct insn *insn = &ctxt->insn;
875 size_t size;
876 u64 port;
877
878 *exitinfo = 0;
879
880 switch (insn->opcode.bytes[0]) {
881 /* INS opcodes */
882 case 0x6c:
883 case 0x6d:
884 *exitinfo |= IOIO_TYPE_INS;
885 *exitinfo |= IOIO_SEG_ES;
886 port = ctxt->regs->dx & 0xffff;
887 break;
888
889 /* OUTS opcodes */
890 case 0x6e:
891 case 0x6f:
892 *exitinfo |= IOIO_TYPE_OUTS;
893 *exitinfo |= IOIO_SEG_DS;
894 port = ctxt->regs->dx & 0xffff;
895 break;
896
897 /* IN immediate opcodes */
898 case 0xe4:
899 case 0xe5:
900 *exitinfo |= IOIO_TYPE_IN;
901 port = (u8)insn->immediate.value & 0xffff;
902 break;
903
904 /* OUT immediate opcodes */
905 case 0xe6:
906 case 0xe7:
907 *exitinfo |= IOIO_TYPE_OUT;
908 port = (u8)insn->immediate.value & 0xffff;
909 break;
910
911 /* IN register opcodes */
912 case 0xec:
913 case 0xed:
914 *exitinfo |= IOIO_TYPE_IN;
915 port = ctxt->regs->dx & 0xffff;
916 break;
917
918 /* OUT register opcodes */
919 case 0xee:
920 case 0xef:
921 *exitinfo |= IOIO_TYPE_OUT;
922 port = ctxt->regs->dx & 0xffff;
923 break;
924
925 default:
926 return ES_DECODE_FAILED;
927 }
928
929 *exitinfo |= port << 16;
930
931 switch (insn->opcode.bytes[0]) {
932 case 0x6c:
933 case 0x6e:
934 case 0xe4:
935 case 0xe6:
936 case 0xec:
937 case 0xee:
938 /* Single byte opcodes */
939 *exitinfo |= IOIO_DATA_8;
940 size = 1;
941 break;
942 default:
943 /* Length determined by instruction parsing */
944 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
945 : IOIO_DATA_32;
946 size = (insn->opnd_bytes == 2) ? 2 : 4;
947 }
948
949 switch (insn->addr_bytes) {
950 case 2:
951 *exitinfo |= IOIO_ADDR_16;
952 break;
953 case 4:
954 *exitinfo |= IOIO_ADDR_32;
955 break;
956 case 8:
957 *exitinfo |= IOIO_ADDR_64;
958 break;
959 }
960
961 if (insn_has_rep_prefix(insn))
962 *exitinfo |= IOIO_REP;
963
964 return vc_ioio_check(ctxt, (u16)port, size);
965 }
966
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)967 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
968 {
969 struct pt_regs *regs = ctxt->regs;
970 u64 exit_info_1, exit_info_2;
971 enum es_result ret;
972
973 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
974 if (ret != ES_OK)
975 return ret;
976
977 if (exit_info_1 & IOIO_TYPE_STR) {
978
979 /* (REP) INS/OUTS */
980
981 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
982 unsigned int io_bytes, exit_bytes;
983 unsigned int ghcb_count, op_count;
984 unsigned long es_base;
985 u64 sw_scratch;
986
987 /*
988 * For the string variants with rep prefix the amount of in/out
989 * operations per #VC exception is limited so that the kernel
990 * has a chance to take interrupts and re-schedule while the
991 * instruction is emulated.
992 */
993 io_bytes = (exit_info_1 >> 4) & 0x7;
994 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
995
996 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
997 exit_info_2 = min(op_count, ghcb_count);
998 exit_bytes = exit_info_2 * io_bytes;
999
1000 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
1001
1002 /* Read bytes of OUTS into the shared buffer */
1003 if (!(exit_info_1 & IOIO_TYPE_IN)) {
1004 ret = vc_insn_string_read(ctxt,
1005 (void *)(es_base + regs->si),
1006 ghcb->shared_buffer, io_bytes,
1007 exit_info_2, df);
1008 if (ret)
1009 return ret;
1010 }
1011
1012 /*
1013 * Issue an VMGEXIT to the HV to consume the bytes from the
1014 * shared buffer or to have it write them into the shared buffer
1015 * depending on the instruction: OUTS or INS.
1016 */
1017 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
1018 ghcb_set_sw_scratch(ghcb, sw_scratch);
1019 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
1020 exit_info_1, exit_info_2);
1021 if (ret != ES_OK)
1022 return ret;
1023
1024 /* Read bytes from shared buffer into the guest's destination. */
1025 if (exit_info_1 & IOIO_TYPE_IN) {
1026 ret = vc_insn_string_write(ctxt,
1027 (void *)(es_base + regs->di),
1028 ghcb->shared_buffer, io_bytes,
1029 exit_info_2, df);
1030 if (ret)
1031 return ret;
1032
1033 if (df)
1034 regs->di -= exit_bytes;
1035 else
1036 regs->di += exit_bytes;
1037 } else {
1038 if (df)
1039 regs->si -= exit_bytes;
1040 else
1041 regs->si += exit_bytes;
1042 }
1043
1044 if (exit_info_1 & IOIO_REP)
1045 regs->cx -= exit_info_2;
1046
1047 ret = regs->cx ? ES_RETRY : ES_OK;
1048
1049 } else {
1050
1051 /* IN/OUT into/from rAX */
1052
1053 int bits = (exit_info_1 & 0x70) >> 1;
1054 u64 rax = 0;
1055
1056 if (!(exit_info_1 & IOIO_TYPE_IN))
1057 rax = lower_bits(regs->ax, bits);
1058
1059 ghcb_set_rax(ghcb, rax);
1060
1061 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
1062 if (ret != ES_OK)
1063 return ret;
1064
1065 if (exit_info_1 & IOIO_TYPE_IN) {
1066 if (!ghcb_rax_is_valid(ghcb))
1067 return ES_VMM_ERROR;
1068 regs->ax = lower_bits(ghcb->save.rax, bits);
1069 }
1070 }
1071
1072 return ret;
1073 }
1074
vc_handle_cpuid_snp(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1075 static int vc_handle_cpuid_snp(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
1076 {
1077 struct pt_regs *regs = ctxt->regs;
1078 struct cpuid_leaf leaf;
1079 int ret;
1080
1081 leaf.fn = regs->ax;
1082 leaf.subfn = regs->cx;
1083 ret = snp_cpuid(ghcb, ctxt, &leaf);
1084 if (!ret) {
1085 regs->ax = leaf.eax;
1086 regs->bx = leaf.ebx;
1087 regs->cx = leaf.ecx;
1088 regs->dx = leaf.edx;
1089 }
1090
1091 return ret;
1092 }
1093
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)1094 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
1095 struct es_em_ctxt *ctxt)
1096 {
1097 struct pt_regs *regs = ctxt->regs;
1098 u32 cr4 = native_read_cr4();
1099 enum es_result ret;
1100 int snp_cpuid_ret;
1101
1102 snp_cpuid_ret = vc_handle_cpuid_snp(ghcb, ctxt);
1103 if (!snp_cpuid_ret)
1104 return ES_OK;
1105 if (snp_cpuid_ret != -EOPNOTSUPP)
1106 return ES_VMM_ERROR;
1107
1108 ghcb_set_rax(ghcb, regs->ax);
1109 ghcb_set_rcx(ghcb, regs->cx);
1110
1111 if (cr4 & X86_CR4_OSXSAVE)
1112 /* Safe to read xcr0 */
1113 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
1114 else
1115 /* xgetbv will cause #GP - use reset value for xcr0 */
1116 ghcb_set_xcr0(ghcb, 1);
1117
1118 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
1119 if (ret != ES_OK)
1120 return ret;
1121
1122 if (!(ghcb_rax_is_valid(ghcb) &&
1123 ghcb_rbx_is_valid(ghcb) &&
1124 ghcb_rcx_is_valid(ghcb) &&
1125 ghcb_rdx_is_valid(ghcb)))
1126 return ES_VMM_ERROR;
1127
1128 regs->ax = ghcb->save.rax;
1129 regs->bx = ghcb->save.rbx;
1130 regs->cx = ghcb->save.rcx;
1131 regs->dx = ghcb->save.rdx;
1132
1133 return ES_OK;
1134 }
1135
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)1136 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
1137 struct es_em_ctxt *ctxt,
1138 unsigned long exit_code)
1139 {
1140 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
1141 enum es_result ret;
1142
1143 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
1144 if (ret != ES_OK)
1145 return ret;
1146
1147 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
1148 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
1149 return ES_VMM_ERROR;
1150
1151 ctxt->regs->ax = ghcb->save.rax;
1152 ctxt->regs->dx = ghcb->save.rdx;
1153 if (rdtscp)
1154 ctxt->regs->cx = ghcb->save.rcx;
1155
1156 return ES_OK;
1157 }
1158
1159 struct cc_setup_data {
1160 struct setup_data header;
1161 u32 cc_blob_address;
1162 };
1163
1164 /*
1165 * Search for a Confidential Computing blob passed in as a setup_data entry
1166 * via the Linux Boot Protocol.
1167 */
1168 static __head
find_cc_blob_setup_data(struct boot_params * bp)1169 struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp)
1170 {
1171 struct cc_setup_data *sd = NULL;
1172 struct setup_data *hdr;
1173
1174 hdr = (struct setup_data *)bp->hdr.setup_data;
1175
1176 while (hdr) {
1177 if (hdr->type == SETUP_CC_BLOB) {
1178 sd = (struct cc_setup_data *)hdr;
1179 return (struct cc_blob_sev_info *)(unsigned long)sd->cc_blob_address;
1180 }
1181 hdr = (struct setup_data *)hdr->next;
1182 }
1183
1184 return NULL;
1185 }
1186
1187 /*
1188 * Initialize the kernel's copy of the SNP CPUID table, and set up the
1189 * pointer that will be used to access it.
1190 *
1191 * Maintaining a direct mapping of the SNP CPUID table used by firmware would
1192 * be possible as an alternative, but the approach is brittle since the
1193 * mapping needs to be updated in sync with all the changes to virtual memory
1194 * layout and related mapping facilities throughout the boot process.
1195 */
setup_cpuid_table(const struct cc_blob_sev_info * cc_info)1196 static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info)
1197 {
1198 const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table;
1199 int i;
1200
1201 if (!cc_info || !cc_info->cpuid_phys || cc_info->cpuid_len < PAGE_SIZE)
1202 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1203
1204 cpuid_table_fw = (const struct snp_cpuid_table *)cc_info->cpuid_phys;
1205 if (!cpuid_table_fw->count || cpuid_table_fw->count > SNP_CPUID_COUNT_MAX)
1206 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_CPUID);
1207
1208 cpuid_table = snp_cpuid_get_table();
1209 memcpy((void *)cpuid_table, cpuid_table_fw, sizeof(*cpuid_table));
1210
1211 /* Initialize CPUID ranges for range-checking. */
1212 for (i = 0; i < cpuid_table->count; i++) {
1213 const struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1214
1215 if (fn->eax_in == 0x0)
1216 RIP_REL_REF(cpuid_std_range_max) = fn->eax;
1217 else if (fn->eax_in == 0x40000000)
1218 RIP_REL_REF(cpuid_hyp_range_max) = fn->eax;
1219 else if (fn->eax_in == 0x80000000)
1220 RIP_REL_REF(cpuid_ext_range_max) = fn->eax;
1221 }
1222 }
1223
__pval_terminate(u64 pfn,bool action,unsigned int page_size,int ret,u64 svsm_ret)1224 static inline void __pval_terminate(u64 pfn, bool action, unsigned int page_size,
1225 int ret, u64 svsm_ret)
1226 {
1227 WARN(1, "PVALIDATE failure: pfn: 0x%llx, action: %u, size: %u, ret: %d, svsm_ret: 0x%llx\n",
1228 pfn, action, page_size, ret, svsm_ret);
1229
1230 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
1231 }
1232
svsm_pval_terminate(struct svsm_pvalidate_call * pc,int ret,u64 svsm_ret)1233 static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svsm_ret)
1234 {
1235 unsigned int page_size;
1236 bool action;
1237 u64 pfn;
1238
1239 pfn = pc->entry[pc->cur_index].pfn;
1240 action = pc->entry[pc->cur_index].action;
1241 page_size = pc->entry[pc->cur_index].page_size;
1242
1243 __pval_terminate(pfn, action, page_size, ret, svsm_ret);
1244 }
1245
sev_evict_cache(void * va,int npages)1246 static inline void sev_evict_cache(void *va, int npages)
1247 {
1248 volatile u8 val __always_unused;
1249 u8 *bytes = va;
1250 int page_idx;
1251
1252 /*
1253 * For SEV guests, a read from the first/last cache-lines of a 4K page
1254 * using the guest key is sufficient to cause a flush of all cache-lines
1255 * associated with that 4K page without incurring all the overhead of a
1256 * full CLFLUSH sequence.
1257 */
1258 for (page_idx = 0; page_idx < npages; page_idx++) {
1259 val = bytes[page_idx * PAGE_SIZE];
1260 val = bytes[page_idx * PAGE_SIZE + PAGE_SIZE - 1];
1261 }
1262 }
1263
svsm_pval_4k_page(unsigned long paddr,bool validate)1264 static void svsm_pval_4k_page(unsigned long paddr, bool validate)
1265 {
1266 struct svsm_pvalidate_call *pc;
1267 struct svsm_call call = {};
1268 unsigned long flags;
1269 u64 pc_pa;
1270 int ret;
1271
1272 /*
1273 * This can be called very early in the boot, use native functions in
1274 * order to avoid paravirt issues.
1275 */
1276 flags = native_local_irq_save();
1277
1278 call.caa = svsm_get_caa();
1279
1280 pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
1281 pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
1282
1283 pc->num_entries = 1;
1284 pc->cur_index = 0;
1285 pc->entry[0].page_size = RMP_PG_SIZE_4K;
1286 pc->entry[0].action = validate;
1287 pc->entry[0].ignore_cf = 0;
1288 pc->entry[0].rsvd = 0;
1289 pc->entry[0].pfn = paddr >> PAGE_SHIFT;
1290
1291 /* Protocol 0, Call ID 1 */
1292 call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
1293 call.rcx = pc_pa;
1294
1295 ret = svsm_perform_call_protocol(&call);
1296 if (ret)
1297 svsm_pval_terminate(pc, ret, call.rax_out);
1298
1299 native_local_irq_restore(flags);
1300 }
1301
pvalidate_4k_page(unsigned long vaddr,unsigned long paddr,bool validate)1302 static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, bool validate)
1303 {
1304 int ret;
1305
1306 /*
1307 * This can be called very early during boot, so use rIP-relative
1308 * references as needed.
1309 */
1310 if (RIP_REL_REF(snp_vmpl)) {
1311 svsm_pval_4k_page(paddr, validate);
1312 } else {
1313 ret = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
1314 if (ret)
1315 __pval_terminate(PHYS_PFN(paddr), validate, RMP_PG_SIZE_4K, ret, 0);
1316 }
1317
1318 /*
1319 * If validating memory (making it private) and affected by the
1320 * cache-coherency vulnerability, perform the cache eviction mitigation.
1321 */
1322 if (validate && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO))
1323 sev_evict_cache((void *)vaddr, 1);
1324 }
1325
pval_pages(struct snp_psc_desc * desc)1326 static void pval_pages(struct snp_psc_desc *desc)
1327 {
1328 struct psc_entry *e;
1329 unsigned long vaddr;
1330 unsigned int size;
1331 unsigned int i;
1332 bool validate;
1333 u64 pfn;
1334 int rc;
1335
1336 for (i = 0; i <= desc->hdr.end_entry; i++) {
1337 e = &desc->entries[i];
1338
1339 pfn = e->gfn;
1340 vaddr = (unsigned long)pfn_to_kaddr(pfn);
1341 size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
1342 validate = e->operation == SNP_PAGE_STATE_PRIVATE;
1343
1344 rc = pvalidate(vaddr, size, validate);
1345 if (!rc)
1346 continue;
1347
1348 if (rc == PVALIDATE_FAIL_SIZEMISMATCH && size == RMP_PG_SIZE_2M) {
1349 unsigned long vaddr_end = vaddr + PMD_SIZE;
1350
1351 for (; vaddr < vaddr_end; vaddr += PAGE_SIZE, pfn++) {
1352 rc = pvalidate(vaddr, RMP_PG_SIZE_4K, validate);
1353 if (rc)
1354 __pval_terminate(pfn, validate, RMP_PG_SIZE_4K, rc, 0);
1355 }
1356 } else {
1357 __pval_terminate(pfn, validate, size, rc, 0);
1358 }
1359 }
1360 }
1361
svsm_build_ca_from_pfn_range(u64 pfn,u64 pfn_end,bool action,struct svsm_pvalidate_call * pc)1362 static u64 svsm_build_ca_from_pfn_range(u64 pfn, u64 pfn_end, bool action,
1363 struct svsm_pvalidate_call *pc)
1364 {
1365 struct svsm_pvalidate_entry *pe;
1366
1367 /* Nothing in the CA yet */
1368 pc->num_entries = 0;
1369 pc->cur_index = 0;
1370
1371 pe = &pc->entry[0];
1372
1373 while (pfn < pfn_end) {
1374 pe->page_size = RMP_PG_SIZE_4K;
1375 pe->action = action;
1376 pe->ignore_cf = 0;
1377 pe->rsvd = 0;
1378 pe->pfn = pfn;
1379
1380 pe++;
1381 pfn++;
1382
1383 pc->num_entries++;
1384 if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
1385 break;
1386 }
1387
1388 return pfn;
1389 }
1390
svsm_build_ca_from_psc_desc(struct snp_psc_desc * desc,unsigned int desc_entry,struct svsm_pvalidate_call * pc)1391 static int svsm_build_ca_from_psc_desc(struct snp_psc_desc *desc, unsigned int desc_entry,
1392 struct svsm_pvalidate_call *pc)
1393 {
1394 struct svsm_pvalidate_entry *pe;
1395 struct psc_entry *e;
1396
1397 /* Nothing in the CA yet */
1398 pc->num_entries = 0;
1399 pc->cur_index = 0;
1400
1401 pe = &pc->entry[0];
1402 e = &desc->entries[desc_entry];
1403
1404 while (desc_entry <= desc->hdr.end_entry) {
1405 pe->page_size = e->pagesize ? RMP_PG_SIZE_2M : RMP_PG_SIZE_4K;
1406 pe->action = e->operation == SNP_PAGE_STATE_PRIVATE;
1407 pe->ignore_cf = 0;
1408 pe->rsvd = 0;
1409 pe->pfn = e->gfn;
1410
1411 pe++;
1412 e++;
1413
1414 desc_entry++;
1415 pc->num_entries++;
1416 if (pc->num_entries == SVSM_PVALIDATE_MAX_COUNT)
1417 break;
1418 }
1419
1420 return desc_entry;
1421 }
1422
svsm_pval_pages(struct snp_psc_desc * desc)1423 static void svsm_pval_pages(struct snp_psc_desc *desc)
1424 {
1425 struct svsm_pvalidate_entry pv_4k[VMGEXIT_PSC_MAX_ENTRY];
1426 unsigned int i, pv_4k_count = 0;
1427 struct svsm_pvalidate_call *pc;
1428 struct svsm_call call = {};
1429 unsigned long flags;
1430 bool action;
1431 u64 pc_pa;
1432 int ret;
1433
1434 /*
1435 * This can be called very early in the boot, use native functions in
1436 * order to avoid paravirt issues.
1437 */
1438 flags = native_local_irq_save();
1439
1440 /*
1441 * The SVSM calling area (CA) can support processing 510 entries at a
1442 * time. Loop through the Page State Change descriptor until the CA is
1443 * full or the last entry in the descriptor is reached, at which time
1444 * the SVSM is invoked. This repeats until all entries in the descriptor
1445 * are processed.
1446 */
1447 call.caa = svsm_get_caa();
1448
1449 pc = (struct svsm_pvalidate_call *)call.caa->svsm_buffer;
1450 pc_pa = svsm_get_caa_pa() + offsetof(struct svsm_ca, svsm_buffer);
1451
1452 /* Protocol 0, Call ID 1 */
1453 call.rax = SVSM_CORE_CALL(SVSM_CORE_PVALIDATE);
1454 call.rcx = pc_pa;
1455
1456 for (i = 0; i <= desc->hdr.end_entry;) {
1457 i = svsm_build_ca_from_psc_desc(desc, i, pc);
1458
1459 do {
1460 ret = svsm_perform_call_protocol(&call);
1461 if (!ret)
1462 continue;
1463
1464 /*
1465 * Check if the entry failed because of an RMP mismatch (a
1466 * PVALIDATE at 2M was requested, but the page is mapped in
1467 * the RMP as 4K).
1468 */
1469
1470 if (call.rax_out == SVSM_PVALIDATE_FAIL_SIZEMISMATCH &&
1471 pc->entry[pc->cur_index].page_size == RMP_PG_SIZE_2M) {
1472 /* Save this entry for post-processing at 4K */
1473 pv_4k[pv_4k_count++] = pc->entry[pc->cur_index];
1474
1475 /* Skip to the next one unless at the end of the list */
1476 pc->cur_index++;
1477 if (pc->cur_index < pc->num_entries)
1478 ret = -EAGAIN;
1479 else
1480 ret = 0;
1481 }
1482 } while (ret == -EAGAIN);
1483
1484 if (ret)
1485 svsm_pval_terminate(pc, ret, call.rax_out);
1486 }
1487
1488 /* Process any entries that failed to be validated at 2M and validate them at 4K */
1489 for (i = 0; i < pv_4k_count; i++) {
1490 u64 pfn, pfn_end;
1491
1492 action = pv_4k[i].action;
1493 pfn = pv_4k[i].pfn;
1494 pfn_end = pfn + 512;
1495
1496 while (pfn < pfn_end) {
1497 pfn = svsm_build_ca_from_pfn_range(pfn, pfn_end, action, pc);
1498
1499 ret = svsm_perform_call_protocol(&call);
1500 if (ret)
1501 svsm_pval_terminate(pc, ret, call.rax_out);
1502 }
1503 }
1504
1505 native_local_irq_restore(flags);
1506 }
1507
pvalidate_pages(struct snp_psc_desc * desc)1508 static void pvalidate_pages(struct snp_psc_desc *desc)
1509 {
1510 struct psc_entry *e;
1511 unsigned int i;
1512
1513 if (snp_vmpl)
1514 svsm_pval_pages(desc);
1515 else
1516 pval_pages(desc);
1517
1518 /*
1519 * If not affected by the cache-coherency vulnerability there is no need
1520 * to perform the cache eviction mitigation.
1521 */
1522 if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO))
1523 return;
1524
1525 for (i = 0; i <= desc->hdr.end_entry; i++) {
1526 e = &desc->entries[i];
1527
1528 /*
1529 * If validating memory (making it private) perform the cache
1530 * eviction mitigation.
1531 */
1532 if (e->operation == SNP_PAGE_STATE_PRIVATE)
1533 sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1);
1534 }
1535 }
1536
vmgexit_psc(struct ghcb * ghcb,struct snp_psc_desc * desc)1537 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)
1538 {
1539 int cur_entry, end_entry, ret = 0;
1540 struct snp_psc_desc *data;
1541 struct es_em_ctxt ctxt;
1542
1543 vc_ghcb_invalidate(ghcb);
1544
1545 /* Copy the input desc into GHCB shared buffer */
1546 data = (struct snp_psc_desc *)ghcb->shared_buffer;
1547 memcpy(ghcb->shared_buffer, desc, min_t(int, GHCB_SHARED_BUF_SIZE, sizeof(*desc)));
1548
1549 /*
1550 * As per the GHCB specification, the hypervisor can resume the guest
1551 * before processing all the entries. Check whether all the entries
1552 * are processed. If not, then keep retrying. Note, the hypervisor
1553 * will update the data memory directly to indicate the status, so
1554 * reference the data->hdr everywhere.
1555 *
1556 * The strategy here is to wait for the hypervisor to change the page
1557 * state in the RMP table before guest accesses the memory pages. If the
1558 * page state change was not successful, then later memory access will
1559 * result in a crash.
1560 */
1561 cur_entry = data->hdr.cur_entry;
1562 end_entry = data->hdr.end_entry;
1563
1564 while (data->hdr.cur_entry <= data->hdr.end_entry) {
1565 ghcb_set_sw_scratch(ghcb, (u64)__pa(data));
1566
1567 /* This will advance the shared buffer data points to. */
1568 ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_PSC, 0, 0);
1569
1570 /*
1571 * Page State Change VMGEXIT can pass error code through
1572 * exit_info_2.
1573 */
1574 if (WARN(ret || ghcb->save.sw_exit_info_2,
1575 "SNP: PSC failed ret=%d exit_info_2=%llx\n",
1576 ret, ghcb->save.sw_exit_info_2)) {
1577 ret = 1;
1578 goto out;
1579 }
1580
1581 /* Verify that reserved bit is not set */
1582 if (WARN(data->hdr.reserved, "Reserved bit is set in the PSC header\n")) {
1583 ret = 1;
1584 goto out;
1585 }
1586
1587 /*
1588 * Sanity check that entry processing is not going backwards.
1589 * This will happen only if hypervisor is tricking us.
1590 */
1591 if (WARN(data->hdr.end_entry > end_entry || cur_entry > data->hdr.cur_entry,
1592 "SNP: PSC processing going backward, end_entry %d (got %d) cur_entry %d (got %d)\n",
1593 end_entry, data->hdr.end_entry, cur_entry, data->hdr.cur_entry)) {
1594 ret = 1;
1595 goto out;
1596 }
1597 }
1598
1599 out:
1600 return ret;
1601 }
1602
vc_check_opcode_bytes(struct es_em_ctxt * ctxt,unsigned long exit_code)1603 static enum es_result vc_check_opcode_bytes(struct es_em_ctxt *ctxt,
1604 unsigned long exit_code)
1605 {
1606 unsigned int opcode = (unsigned int)ctxt->insn.opcode.value;
1607 u8 modrm = ctxt->insn.modrm.value;
1608
1609 switch (exit_code) {
1610
1611 case SVM_EXIT_IOIO:
1612 case SVM_EXIT_NPF:
1613 /* handled separately */
1614 return ES_OK;
1615
1616 case SVM_EXIT_CPUID:
1617 if (opcode == 0xa20f)
1618 return ES_OK;
1619 break;
1620
1621 case SVM_EXIT_INVD:
1622 if (opcode == 0x080f)
1623 return ES_OK;
1624 break;
1625
1626 case SVM_EXIT_MONITOR:
1627 /* MONITOR and MONITORX instructions generate the same error code */
1628 if (opcode == 0x010f && (modrm == 0xc8 || modrm == 0xfa))
1629 return ES_OK;
1630 break;
1631
1632 case SVM_EXIT_MWAIT:
1633 /* MWAIT and MWAITX instructions generate the same error code */
1634 if (opcode == 0x010f && (modrm == 0xc9 || modrm == 0xfb))
1635 return ES_OK;
1636 break;
1637
1638 case SVM_EXIT_MSR:
1639 /* RDMSR */
1640 if (opcode == 0x320f ||
1641 /* WRMSR */
1642 opcode == 0x300f)
1643 return ES_OK;
1644 break;
1645
1646 case SVM_EXIT_RDPMC:
1647 if (opcode == 0x330f)
1648 return ES_OK;
1649 break;
1650
1651 case SVM_EXIT_RDTSC:
1652 if (opcode == 0x310f)
1653 return ES_OK;
1654 break;
1655
1656 case SVM_EXIT_RDTSCP:
1657 if (opcode == 0x010f && modrm == 0xf9)
1658 return ES_OK;
1659 break;
1660
1661 case SVM_EXIT_READ_DR7:
1662 if (opcode == 0x210f &&
1663 X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
1664 return ES_OK;
1665 break;
1666
1667 case SVM_EXIT_VMMCALL:
1668 if (opcode == 0x010f && modrm == 0xd9)
1669 return ES_OK;
1670
1671 break;
1672
1673 case SVM_EXIT_WRITE_DR7:
1674 if (opcode == 0x230f &&
1675 X86_MODRM_REG(ctxt->insn.modrm.value) == 7)
1676 return ES_OK;
1677 break;
1678
1679 case SVM_EXIT_WBINVD:
1680 if (opcode == 0x90f)
1681 return ES_OK;
1682 break;
1683
1684 default:
1685 break;
1686 }
1687
1688 sev_printk(KERN_ERR "Wrong/unhandled opcode bytes: 0x%x, exit_code: 0x%lx, rIP: 0x%lx\n",
1689 opcode, exit_code, ctxt->regs->ip);
1690
1691 return ES_UNSUPPORTED;
1692 }
1693
1694 /*
1695 * Maintain the GPA of the SVSM Calling Area (CA) in order to utilize the SVSM
1696 * services needed when not running in VMPL0.
1697 */
svsm_setup_ca(const struct cc_blob_sev_info * cc_info)1698 static bool __head svsm_setup_ca(const struct cc_blob_sev_info *cc_info)
1699 {
1700 struct snp_secrets_page *secrets_page;
1701 struct snp_cpuid_table *cpuid_table;
1702 unsigned int i;
1703 u64 caa;
1704
1705 BUILD_BUG_ON(sizeof(*secrets_page) != PAGE_SIZE);
1706
1707 /*
1708 * Check if running at VMPL0.
1709 *
1710 * Use RMPADJUST (see the rmpadjust() function for a description of what
1711 * the instruction does) to update the VMPL1 permissions of a page. If
1712 * the guest is running at VMPL0, this will succeed and implies there is
1713 * no SVSM. If the guest is running at any other VMPL, this will fail.
1714 * Linux SNP guests only ever run at a single VMPL level so permission mask
1715 * changes of a lesser-privileged VMPL are a don't-care.
1716 *
1717 * Use a rip-relative reference to obtain the proper address, since this
1718 * routine is running identity mapped when called, both by the decompressor
1719 * code and the early kernel code.
1720 */
1721 if (!rmpadjust((unsigned long)&RIP_REL_REF(boot_ghcb_page), RMP_PG_SIZE_4K, 1))
1722 return false;
1723
1724 /*
1725 * Not running at VMPL0, ensure everything has been properly supplied
1726 * for running under an SVSM.
1727 */
1728 if (!cc_info || !cc_info->secrets_phys || cc_info->secrets_len != PAGE_SIZE)
1729 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SECRETS_PAGE);
1730
1731 secrets_page = (struct snp_secrets_page *)cc_info->secrets_phys;
1732 if (!secrets_page->svsm_size)
1733 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_NO_SVSM);
1734
1735 if (!secrets_page->svsm_guest_vmpl)
1736 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_VMPL0);
1737
1738 RIP_REL_REF(snp_vmpl) = secrets_page->svsm_guest_vmpl;
1739
1740 caa = secrets_page->svsm_caa;
1741
1742 /*
1743 * An open-coded PAGE_ALIGNED() in order to avoid including
1744 * kernel-proper headers into the decompressor.
1745 */
1746 if (caa & (PAGE_SIZE - 1))
1747 sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_SVSM_CAA);
1748
1749 /*
1750 * The CA is identity mapped when this routine is called, both by the
1751 * decompressor code and the early kernel code.
1752 */
1753 RIP_REL_REF(boot_svsm_caa) = (struct svsm_ca *)caa;
1754 RIP_REL_REF(boot_svsm_caa_pa) = caa;
1755
1756 /* Advertise the SVSM presence via CPUID. */
1757 cpuid_table = (struct snp_cpuid_table *)snp_cpuid_get_table();
1758 for (i = 0; i < cpuid_table->count; i++) {
1759 struct snp_cpuid_fn *fn = &cpuid_table->fn[i];
1760
1761 if (fn->eax_in == 0x8000001f)
1762 fn->eax |= BIT(28);
1763 }
1764
1765 return true;
1766 }
1767