1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12 #ifndef __BOOT_COMPRESSED
13 #define error(v) pr_err(v)
14 #define has_cpuflag(f) boot_cpu_has(f)
15 #endif
16
sev_es_check_cpu_features(void)17 static bool __init sev_es_check_cpu_features(void)
18 {
19 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
20 error("RDRAND instruction not supported - no trusted source of randomness available\n");
21 return false;
22 }
23
24 return true;
25 }
26
sev_es_terminate(unsigned int reason)27 static void sev_es_terminate(unsigned int reason)
28 {
29 u64 val = GHCB_SEV_TERMINATE;
30
31 /*
32 * Tell the hypervisor what went wrong - only reason-set 0 is
33 * currently supported.
34 */
35 val |= GHCB_SEV_TERMINATE_REASON(0, reason);
36
37 /* Request Guest Termination from Hypvervisor */
38 sev_es_wr_ghcb_msr(val);
39 VMGEXIT();
40
41 while (true)
42 asm volatile("hlt\n" : : : "memory");
43 }
44
sev_es_negotiate_protocol(void)45 static bool sev_es_negotiate_protocol(void)
46 {
47 u64 val;
48
49 /* Do the GHCB protocol version negotiation */
50 sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ);
51 VMGEXIT();
52 val = sev_es_rd_ghcb_msr();
53
54 if (GHCB_INFO(val) != GHCB_SEV_INFO)
55 return false;
56
57 if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR ||
58 GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR)
59 return false;
60
61 return true;
62 }
63
vc_ghcb_invalidate(struct ghcb * ghcb)64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
65 {
66 ghcb->save.sw_exit_code = 0;
67 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
68 }
69
vc_decoding_needed(unsigned long exit_code)70 static bool vc_decoding_needed(unsigned long exit_code)
71 {
72 /* Exceptions don't require to decode the instruction */
73 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
74 exit_code <= SVM_EXIT_LAST_EXCP);
75 }
76
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)77 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
78 struct pt_regs *regs,
79 unsigned long exit_code)
80 {
81 enum es_result ret = ES_OK;
82
83 memset(ctxt, 0, sizeof(*ctxt));
84 ctxt->regs = regs;
85
86 if (vc_decoding_needed(exit_code))
87 ret = vc_decode_insn(ctxt);
88
89 return ret;
90 }
91
vc_finish_insn(struct es_em_ctxt * ctxt)92 static void vc_finish_insn(struct es_em_ctxt *ctxt)
93 {
94 ctxt->regs->ip += ctxt->insn.length;
95 }
96
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)97 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
98 struct es_em_ctxt *ctxt,
99 u64 exit_code, u64 exit_info_1,
100 u64 exit_info_2)
101 {
102 enum es_result ret;
103
104 /* Fill in protocol and format specifiers */
105 ghcb->protocol_version = GHCB_PROTOCOL_MAX;
106 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
107
108 ghcb_set_sw_exit_code(ghcb, exit_code);
109 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
110 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
111
112 sev_es_wr_ghcb_msr(__pa(ghcb));
113 VMGEXIT();
114
115 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
116 u64 info = ghcb->save.sw_exit_info_2;
117 unsigned long v;
118
119 info = ghcb->save.sw_exit_info_2;
120 v = info & SVM_EVTINJ_VEC_MASK;
121
122 /* Check if exception information from hypervisor is sane. */
123 if ((info & SVM_EVTINJ_VALID) &&
124 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
125 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
126 ctxt->fi.vector = v;
127 if (info & SVM_EVTINJ_VALID_ERR)
128 ctxt->fi.error_code = info >> 32;
129 ret = ES_EXCEPTION;
130 } else {
131 ret = ES_VMM_ERROR;
132 }
133 } else if (ghcb->save.sw_exit_info_1 & 0xffffffff) {
134 ret = ES_VMM_ERROR;
135 } else {
136 ret = ES_OK;
137 }
138
139 return ret;
140 }
141
142 /*
143 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
144 * page yet, so it only supports the MSR based communication with the
145 * hypervisor and only the CPUID exit-code.
146 */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)147 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
148 {
149 unsigned int fn = lower_bits(regs->ax, 32);
150 unsigned long val;
151
152 /* Only CPUID is supported via MSR protocol */
153 if (exit_code != SVM_EXIT_CPUID)
154 goto fail;
155
156 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
157 VMGEXIT();
158 val = sev_es_rd_ghcb_msr();
159 if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
160 goto fail;
161 regs->ax = val >> 32;
162
163 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
164 VMGEXIT();
165 val = sev_es_rd_ghcb_msr();
166 if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
167 goto fail;
168 regs->bx = val >> 32;
169
170 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
171 VMGEXIT();
172 val = sev_es_rd_ghcb_msr();
173 if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
174 goto fail;
175 regs->cx = val >> 32;
176
177 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
178 VMGEXIT();
179 val = sev_es_rd_ghcb_msr();
180 if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
181 goto fail;
182 regs->dx = val >> 32;
183
184 /*
185 * This is a VC handler and the #VC is only raised when SEV-ES is
186 * active, which means SEV must be active too. Do sanity checks on the
187 * CPUID results to make sure the hypervisor does not trick the kernel
188 * into the no-sev path. This could map sensitive data unencrypted and
189 * make it accessible to the hypervisor.
190 *
191 * In particular, check for:
192 * - Availability of CPUID leaf 0x8000001f
193 * - SEV CPUID bit.
194 *
195 * The hypervisor might still report the wrong C-bit position, but this
196 * can't be checked here.
197 */
198
199 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
200 /* SEV leaf check */
201 goto fail;
202 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
203 /* SEV bit */
204 goto fail;
205
206 /* Skip over the CPUID two-byte opcode */
207 regs->ip += 2;
208
209 return;
210
211 fail:
212 sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE);
213 VMGEXIT();
214
215 /* Shouldn't get here - if we do halt the machine */
216 while (true)
217 asm volatile("hlt\n");
218 }
219
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)220 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
221 void *src, char *buf,
222 unsigned int data_size,
223 unsigned int count,
224 bool backwards)
225 {
226 int i, b = backwards ? -1 : 1;
227 enum es_result ret = ES_OK;
228
229 for (i = 0; i < count; i++) {
230 void *s = src + (i * data_size * b);
231 char *d = buf + (i * data_size);
232
233 ret = vc_read_mem(ctxt, s, d, data_size);
234 if (ret != ES_OK)
235 break;
236 }
237
238 return ret;
239 }
240
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)241 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
242 void *dst, char *buf,
243 unsigned int data_size,
244 unsigned int count,
245 bool backwards)
246 {
247 int i, s = backwards ? -1 : 1;
248 enum es_result ret = ES_OK;
249
250 for (i = 0; i < count; i++) {
251 void *d = dst + (i * data_size * s);
252 char *b = buf + (i * data_size);
253
254 ret = vc_write_mem(ctxt, d, b, data_size);
255 if (ret != ES_OK)
256 break;
257 }
258
259 return ret;
260 }
261
262 #define IOIO_TYPE_STR BIT(2)
263 #define IOIO_TYPE_IN 1
264 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
265 #define IOIO_TYPE_OUT 0
266 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
267
268 #define IOIO_REP BIT(3)
269
270 #define IOIO_ADDR_64 BIT(9)
271 #define IOIO_ADDR_32 BIT(8)
272 #define IOIO_ADDR_16 BIT(7)
273
274 #define IOIO_DATA_32 BIT(6)
275 #define IOIO_DATA_16 BIT(5)
276 #define IOIO_DATA_8 BIT(4)
277
278 #define IOIO_SEG_ES (0 << 10)
279 #define IOIO_SEG_DS (3 << 10)
280
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)281 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
282 {
283 struct insn *insn = &ctxt->insn;
284 *exitinfo = 0;
285
286 switch (insn->opcode.bytes[0]) {
287 /* INS opcodes */
288 case 0x6c:
289 case 0x6d:
290 *exitinfo |= IOIO_TYPE_INS;
291 *exitinfo |= IOIO_SEG_ES;
292 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
293 break;
294
295 /* OUTS opcodes */
296 case 0x6e:
297 case 0x6f:
298 *exitinfo |= IOIO_TYPE_OUTS;
299 *exitinfo |= IOIO_SEG_DS;
300 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
301 break;
302
303 /* IN immediate opcodes */
304 case 0xe4:
305 case 0xe5:
306 *exitinfo |= IOIO_TYPE_IN;
307 *exitinfo |= (u8)insn->immediate.value << 16;
308 break;
309
310 /* OUT immediate opcodes */
311 case 0xe6:
312 case 0xe7:
313 *exitinfo |= IOIO_TYPE_OUT;
314 *exitinfo |= (u8)insn->immediate.value << 16;
315 break;
316
317 /* IN register opcodes */
318 case 0xec:
319 case 0xed:
320 *exitinfo |= IOIO_TYPE_IN;
321 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
322 break;
323
324 /* OUT register opcodes */
325 case 0xee:
326 case 0xef:
327 *exitinfo |= IOIO_TYPE_OUT;
328 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
329 break;
330
331 default:
332 return ES_DECODE_FAILED;
333 }
334
335 switch (insn->opcode.bytes[0]) {
336 case 0x6c:
337 case 0x6e:
338 case 0xe4:
339 case 0xe6:
340 case 0xec:
341 case 0xee:
342 /* Single byte opcodes */
343 *exitinfo |= IOIO_DATA_8;
344 break;
345 default:
346 /* Length determined by instruction parsing */
347 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
348 : IOIO_DATA_32;
349 }
350 switch (insn->addr_bytes) {
351 case 2:
352 *exitinfo |= IOIO_ADDR_16;
353 break;
354 case 4:
355 *exitinfo |= IOIO_ADDR_32;
356 break;
357 case 8:
358 *exitinfo |= IOIO_ADDR_64;
359 break;
360 }
361
362 if (insn_has_rep_prefix(insn))
363 *exitinfo |= IOIO_REP;
364
365 return ES_OK;
366 }
367
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)368 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
369 {
370 struct pt_regs *regs = ctxt->regs;
371 u64 exit_info_1, exit_info_2;
372 enum es_result ret;
373
374 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
375 if (ret != ES_OK)
376 return ret;
377
378 if (exit_info_1 & IOIO_TYPE_STR) {
379
380 /* (REP) INS/OUTS */
381
382 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
383 unsigned int io_bytes, exit_bytes;
384 unsigned int ghcb_count, op_count;
385 unsigned long es_base;
386 u64 sw_scratch;
387
388 /*
389 * For the string variants with rep prefix the amount of in/out
390 * operations per #VC exception is limited so that the kernel
391 * has a chance to take interrupts and re-schedule while the
392 * instruction is emulated.
393 */
394 io_bytes = (exit_info_1 >> 4) & 0x7;
395 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
396
397 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
398 exit_info_2 = min(op_count, ghcb_count);
399 exit_bytes = exit_info_2 * io_bytes;
400
401 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
402
403 /* Read bytes of OUTS into the shared buffer */
404 if (!(exit_info_1 & IOIO_TYPE_IN)) {
405 ret = vc_insn_string_read(ctxt,
406 (void *)(es_base + regs->si),
407 ghcb->shared_buffer, io_bytes,
408 exit_info_2, df);
409 if (ret)
410 return ret;
411 }
412
413 /*
414 * Issue an VMGEXIT to the HV to consume the bytes from the
415 * shared buffer or to have it write them into the shared buffer
416 * depending on the instruction: OUTS or INS.
417 */
418 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
419 ghcb_set_sw_scratch(ghcb, sw_scratch);
420 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
421 exit_info_1, exit_info_2);
422 if (ret != ES_OK)
423 return ret;
424
425 /* Read bytes from shared buffer into the guest's destination. */
426 if (exit_info_1 & IOIO_TYPE_IN) {
427 ret = vc_insn_string_write(ctxt,
428 (void *)(es_base + regs->di),
429 ghcb->shared_buffer, io_bytes,
430 exit_info_2, df);
431 if (ret)
432 return ret;
433
434 if (df)
435 regs->di -= exit_bytes;
436 else
437 regs->di += exit_bytes;
438 } else {
439 if (df)
440 regs->si -= exit_bytes;
441 else
442 regs->si += exit_bytes;
443 }
444
445 if (exit_info_1 & IOIO_REP)
446 regs->cx -= exit_info_2;
447
448 ret = regs->cx ? ES_RETRY : ES_OK;
449
450 } else {
451
452 /* IN/OUT into/from rAX */
453
454 int bits = (exit_info_1 & 0x70) >> 1;
455 u64 rax = 0;
456
457 if (!(exit_info_1 & IOIO_TYPE_IN))
458 rax = lower_bits(regs->ax, bits);
459
460 ghcb_set_rax(ghcb, rax);
461
462 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
463 if (ret != ES_OK)
464 return ret;
465
466 if (exit_info_1 & IOIO_TYPE_IN) {
467 if (!ghcb_rax_is_valid(ghcb))
468 return ES_VMM_ERROR;
469 regs->ax = lower_bits(ghcb->save.rax, bits);
470 }
471 }
472
473 return ret;
474 }
475
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)476 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
477 struct es_em_ctxt *ctxt)
478 {
479 struct pt_regs *regs = ctxt->regs;
480 u32 cr4 = native_read_cr4();
481 enum es_result ret;
482
483 ghcb_set_rax(ghcb, regs->ax);
484 ghcb_set_rcx(ghcb, regs->cx);
485
486 if (cr4 & X86_CR4_OSXSAVE)
487 /* Safe to read xcr0 */
488 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
489 else
490 /* xgetbv will cause #GP - use reset value for xcr0 */
491 ghcb_set_xcr0(ghcb, 1);
492
493 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
494 if (ret != ES_OK)
495 return ret;
496
497 if (!(ghcb_rax_is_valid(ghcb) &&
498 ghcb_rbx_is_valid(ghcb) &&
499 ghcb_rcx_is_valid(ghcb) &&
500 ghcb_rdx_is_valid(ghcb)))
501 return ES_VMM_ERROR;
502
503 regs->ax = ghcb->save.rax;
504 regs->bx = ghcb->save.rbx;
505 regs->cx = ghcb->save.rcx;
506 regs->dx = ghcb->save.rdx;
507
508 return ES_OK;
509 }
510
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)511 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
512 struct es_em_ctxt *ctxt,
513 unsigned long exit_code)
514 {
515 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
516 enum es_result ret;
517
518 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
519 if (ret != ES_OK)
520 return ret;
521
522 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
523 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
524 return ES_VMM_ERROR;
525
526 ctxt->regs->ax = ghcb->save.rax;
527 ctxt->regs->dx = ghcb->save.rdx;
528 if (rdtscp)
529 ctxt->regs->cx = ghcb->save.rcx;
530
531 return ES_OK;
532 }
533