1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * tools/testing/selftests/kvm/lib/x86_64/processor.c
4 *
5 * Copyright (C) 2018, Google LLC.
6 */
7
8 #include "test_util.h"
9 #include "kvm_util.h"
10 #include "../kvm_util_internal.h"
11 #include "processor.h"
12
13 #ifndef NUM_INTERRUPTS
14 #define NUM_INTERRUPTS 256
15 #endif
16
17 #define DEFAULT_CODE_SELECTOR 0x8
18 #define DEFAULT_DATA_SELECTOR 0x10
19
20 vm_vaddr_t exception_handlers;
21
regs_dump(FILE * stream,struct kvm_regs * regs,uint8_t indent)22 void regs_dump(FILE *stream, struct kvm_regs *regs,
23 uint8_t indent)
24 {
25 fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
26 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
27 indent, "",
28 regs->rax, regs->rbx, regs->rcx, regs->rdx);
29 fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
30 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
31 indent, "",
32 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
33 fprintf(stream, "%*sr8: 0x%.16llx r9: 0x%.16llx "
34 "r10: 0x%.16llx r11: 0x%.16llx\n",
35 indent, "",
36 regs->r8, regs->r9, regs->r10, regs->r11);
37 fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
38 "r14: 0x%.16llx r15: 0x%.16llx\n",
39 indent, "",
40 regs->r12, regs->r13, regs->r14, regs->r15);
41 fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
42 indent, "",
43 regs->rip, regs->rflags);
44 }
45
46 /*
47 * Segment Dump
48 *
49 * Input Args:
50 * stream - Output FILE stream
51 * segment - KVM segment
52 * indent - Left margin indent amount
53 *
54 * Output Args: None
55 *
56 * Return: None
57 *
58 * Dumps the state of the KVM segment given by @segment, to the FILE stream
59 * given by @stream.
60 */
segment_dump(FILE * stream,struct kvm_segment * segment,uint8_t indent)61 static void segment_dump(FILE *stream, struct kvm_segment *segment,
62 uint8_t indent)
63 {
64 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
65 "selector: 0x%.4x type: 0x%.2x\n",
66 indent, "", segment->base, segment->limit,
67 segment->selector, segment->type);
68 fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
69 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
70 indent, "", segment->present, segment->dpl,
71 segment->db, segment->s, segment->l);
72 fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
73 "unusable: 0x%.2x padding: 0x%.2x\n",
74 indent, "", segment->g, segment->avl,
75 segment->unusable, segment->padding);
76 }
77
78 /*
79 * dtable Dump
80 *
81 * Input Args:
82 * stream - Output FILE stream
83 * dtable - KVM dtable
84 * indent - Left margin indent amount
85 *
86 * Output Args: None
87 *
88 * Return: None
89 *
90 * Dumps the state of the KVM dtable given by @dtable, to the FILE stream
91 * given by @stream.
92 */
dtable_dump(FILE * stream,struct kvm_dtable * dtable,uint8_t indent)93 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
94 uint8_t indent)
95 {
96 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
97 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
98 indent, "", dtable->base, dtable->limit,
99 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
100 }
101
sregs_dump(FILE * stream,struct kvm_sregs * sregs,uint8_t indent)102 void sregs_dump(FILE *stream, struct kvm_sregs *sregs,
103 uint8_t indent)
104 {
105 unsigned int i;
106
107 fprintf(stream, "%*scs:\n", indent, "");
108 segment_dump(stream, &sregs->cs, indent + 2);
109 fprintf(stream, "%*sds:\n", indent, "");
110 segment_dump(stream, &sregs->ds, indent + 2);
111 fprintf(stream, "%*ses:\n", indent, "");
112 segment_dump(stream, &sregs->es, indent + 2);
113 fprintf(stream, "%*sfs:\n", indent, "");
114 segment_dump(stream, &sregs->fs, indent + 2);
115 fprintf(stream, "%*sgs:\n", indent, "");
116 segment_dump(stream, &sregs->gs, indent + 2);
117 fprintf(stream, "%*sss:\n", indent, "");
118 segment_dump(stream, &sregs->ss, indent + 2);
119 fprintf(stream, "%*str:\n", indent, "");
120 segment_dump(stream, &sregs->tr, indent + 2);
121 fprintf(stream, "%*sldt:\n", indent, "");
122 segment_dump(stream, &sregs->ldt, indent + 2);
123
124 fprintf(stream, "%*sgdt:\n", indent, "");
125 dtable_dump(stream, &sregs->gdt, indent + 2);
126 fprintf(stream, "%*sidt:\n", indent, "");
127 dtable_dump(stream, &sregs->idt, indent + 2);
128
129 fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
130 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
131 indent, "",
132 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
133 fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
134 "apic_base: 0x%.16llx\n",
135 indent, "",
136 sregs->cr8, sregs->efer, sregs->apic_base);
137
138 fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
139 for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
140 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
141 sregs->interrupt_bitmap[i]);
142 }
143 }
144
virt_pgd_alloc(struct kvm_vm * vm)145 void virt_pgd_alloc(struct kvm_vm *vm)
146 {
147 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
148 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
149
150 /* If needed, create page map l4 table. */
151 if (!vm->pgd_created) {
152 vm->pgd = vm_alloc_page_table(vm);
153 vm->pgd_created = true;
154 }
155 }
156
virt_get_pte(struct kvm_vm * vm,uint64_t pt_pfn,uint64_t vaddr,int level)157 static void *virt_get_pte(struct kvm_vm *vm, uint64_t pt_pfn, uint64_t vaddr,
158 int level)
159 {
160 uint64_t *page_table = addr_gpa2hva(vm, pt_pfn << vm->page_shift);
161 int index = vaddr >> (vm->page_shift + level * 9) & 0x1ffu;
162
163 return &page_table[index];
164 }
165
virt_create_upper_pte(struct kvm_vm * vm,uint64_t pt_pfn,uint64_t vaddr,uint64_t paddr,int level,enum x86_page_size page_size)166 static uint64_t *virt_create_upper_pte(struct kvm_vm *vm,
167 uint64_t pt_pfn,
168 uint64_t vaddr,
169 uint64_t paddr,
170 int level,
171 enum x86_page_size page_size)
172 {
173 uint64_t *pte = virt_get_pte(vm, pt_pfn, vaddr, level);
174
175 if (!(*pte & PTE_PRESENT_MASK)) {
176 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK;
177 if (level == page_size)
178 *pte |= PTE_LARGE_MASK | (paddr & PHYSICAL_PAGE_MASK);
179 else
180 *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
181 } else {
182 /*
183 * Entry already present. Assert that the caller doesn't want
184 * a hugepage at this level, and that there isn't a hugepage at
185 * this level.
186 */
187 TEST_ASSERT(level != page_size,
188 "Cannot create hugepage at level: %u, vaddr: 0x%lx\n",
189 page_size, vaddr);
190 TEST_ASSERT(!(*pte & PTE_LARGE_MASK),
191 "Cannot create page table at level: %u, vaddr: 0x%lx\n",
192 level, vaddr);
193 }
194 return pte;
195 }
196
__virt_pg_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr,enum x86_page_size page_size)197 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
198 enum x86_page_size page_size)
199 {
200 const uint64_t pg_size = 1ull << ((page_size * 9) + 12);
201 uint64_t *pml4e, *pdpe, *pde;
202 uint64_t *pte;
203
204 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K,
205 "Unknown or unsupported guest mode, mode: 0x%x", vm->mode);
206
207 TEST_ASSERT((vaddr % pg_size) == 0,
208 "Virtual address not aligned,\n"
209 "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size);
210 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)),
211 "Invalid virtual address, vaddr: 0x%lx", vaddr);
212 TEST_ASSERT((paddr % pg_size) == 0,
213 "Physical address not aligned,\n"
214 " paddr: 0x%lx page size: 0x%lx", paddr, pg_size);
215 TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
216 "Physical address beyond maximum supported,\n"
217 " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
218 paddr, vm->max_gfn, vm->page_size);
219
220 /*
221 * Allocate upper level page tables, if not already present. Return
222 * early if a hugepage was created.
223 */
224 pml4e = virt_create_upper_pte(vm, vm->pgd >> vm->page_shift,
225 vaddr, paddr, 3, page_size);
226 if (*pml4e & PTE_LARGE_MASK)
227 return;
228
229 pdpe = virt_create_upper_pte(vm, PTE_GET_PFN(*pml4e), vaddr, paddr, 2, page_size);
230 if (*pdpe & PTE_LARGE_MASK)
231 return;
232
233 pde = virt_create_upper_pte(vm, PTE_GET_PFN(*pdpe), vaddr, paddr, 1, page_size);
234 if (*pde & PTE_LARGE_MASK)
235 return;
236
237 /* Fill in page table entry. */
238 pte = virt_get_pte(vm, PTE_GET_PFN(*pde), vaddr, 0);
239 TEST_ASSERT(!(*pte & PTE_PRESENT_MASK),
240 "PTE already present for 4k page at vaddr: 0x%lx\n", vaddr);
241 *pte = PTE_PRESENT_MASK | PTE_WRITABLE_MASK | (paddr & PHYSICAL_PAGE_MASK);
242 }
243
virt_pg_map(struct kvm_vm * vm,uint64_t vaddr,uint64_t paddr)244 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
245 {
246 __virt_pg_map(vm, vaddr, paddr, X86_PAGE_SIZE_4K);
247 }
248
_vm_get_page_table_entry(struct kvm_vm * vm,int vcpuid,uint64_t vaddr)249 static uint64_t *_vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid,
250 uint64_t vaddr)
251 {
252 uint16_t index[4];
253 uint64_t *pml4e, *pdpe, *pde;
254 uint64_t *pte;
255 struct kvm_cpuid_entry2 *entry;
256 struct kvm_sregs sregs;
257 int max_phy_addr;
258 /* Set the bottom 52 bits. */
259 uint64_t rsvd_mask = 0x000fffffffffffff;
260
261 entry = kvm_get_supported_cpuid_index(0x80000008, 0);
262 max_phy_addr = entry->eax & 0x000000ff;
263 /* Clear the bottom bits of the reserved mask. */
264 rsvd_mask = (rsvd_mask >> max_phy_addr) << max_phy_addr;
265
266 /*
267 * SDM vol 3, fig 4-11 "Formats of CR3 and Paging-Structure Entries
268 * with 4-Level Paging and 5-Level Paging".
269 * If IA32_EFER.NXE = 0 and the P flag of a paging-structure entry is 1,
270 * the XD flag (bit 63) is reserved.
271 */
272 vcpu_sregs_get(vm, vcpuid, &sregs);
273 if ((sregs.efer & EFER_NX) == 0) {
274 rsvd_mask |= (1ull << 63);
275 }
276
277 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
278 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
279 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
280 (vaddr >> vm->page_shift)),
281 "Invalid virtual address, vaddr: 0x%lx",
282 vaddr);
283 /*
284 * Based on the mode check above there are 48 bits in the vaddr, so
285 * shift 16 to sign extend the last bit (bit-47),
286 */
287 TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16),
288 "Canonical check failed. The virtual address is invalid.");
289
290 index[0] = (vaddr >> 12) & 0x1ffu;
291 index[1] = (vaddr >> 21) & 0x1ffu;
292 index[2] = (vaddr >> 30) & 0x1ffu;
293 index[3] = (vaddr >> 39) & 0x1ffu;
294
295 pml4e = addr_gpa2hva(vm, vm->pgd);
296 TEST_ASSERT(pml4e[index[3]] & PTE_PRESENT_MASK,
297 "Expected pml4e to be present for gva: 0x%08lx", vaddr);
298 TEST_ASSERT((pml4e[index[3]] & (rsvd_mask | PTE_LARGE_MASK)) == 0,
299 "Unexpected reserved bits set.");
300
301 pdpe = addr_gpa2hva(vm, PTE_GET_PFN(pml4e[index[3]]) * vm->page_size);
302 TEST_ASSERT(pdpe[index[2]] & PTE_PRESENT_MASK,
303 "Expected pdpe to be present for gva: 0x%08lx", vaddr);
304 TEST_ASSERT(!(pdpe[index[2]] & PTE_LARGE_MASK),
305 "Expected pdpe to map a pde not a 1-GByte page.");
306 TEST_ASSERT((pdpe[index[2]] & rsvd_mask) == 0,
307 "Unexpected reserved bits set.");
308
309 pde = addr_gpa2hva(vm, PTE_GET_PFN(pdpe[index[2]]) * vm->page_size);
310 TEST_ASSERT(pde[index[1]] & PTE_PRESENT_MASK,
311 "Expected pde to be present for gva: 0x%08lx", vaddr);
312 TEST_ASSERT(!(pde[index[1]] & PTE_LARGE_MASK),
313 "Expected pde to map a pte not a 2-MByte page.");
314 TEST_ASSERT((pde[index[1]] & rsvd_mask) == 0,
315 "Unexpected reserved bits set.");
316
317 pte = addr_gpa2hva(vm, PTE_GET_PFN(pde[index[1]]) * vm->page_size);
318 TEST_ASSERT(pte[index[0]] & PTE_PRESENT_MASK,
319 "Expected pte to be present for gva: 0x%08lx", vaddr);
320
321 return &pte[index[0]];
322 }
323
vm_get_page_table_entry(struct kvm_vm * vm,int vcpuid,uint64_t vaddr)324 uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr)
325 {
326 uint64_t *pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
327
328 return *(uint64_t *)pte;
329 }
330
vm_set_page_table_entry(struct kvm_vm * vm,int vcpuid,uint64_t vaddr,uint64_t pte)331 void vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr,
332 uint64_t pte)
333 {
334 uint64_t *new_pte = _vm_get_page_table_entry(vm, vcpuid, vaddr);
335
336 *(uint64_t *)new_pte = pte;
337 }
338
virt_dump(FILE * stream,struct kvm_vm * vm,uint8_t indent)339 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
340 {
341 uint64_t *pml4e, *pml4e_start;
342 uint64_t *pdpe, *pdpe_start;
343 uint64_t *pde, *pde_start;
344 uint64_t *pte, *pte_start;
345
346 if (!vm->pgd_created)
347 return;
348
349 fprintf(stream, "%*s "
350 " no\n", indent, "");
351 fprintf(stream, "%*s index hvaddr gpaddr "
352 "addr w exec dirty\n",
353 indent, "");
354 pml4e_start = (uint64_t *) addr_gpa2hva(vm, vm->pgd);
355 for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
356 pml4e = &pml4e_start[n1];
357 if (!(*pml4e & PTE_PRESENT_MASK))
358 continue;
359 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "
360 " %u\n",
361 indent, "",
362 pml4e - pml4e_start, pml4e,
363 addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),
364 !!(*pml4e & PTE_WRITABLE_MASK), !!(*pml4e & PTE_NX_MASK));
365
366 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
367 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
368 pdpe = &pdpe_start[n2];
369 if (!(*pdpe & PTE_PRESENT_MASK))
370 continue;
371 fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10llx "
372 "%u %u\n",
373 indent, "",
374 pdpe - pdpe_start, pdpe,
375 addr_hva2gpa(vm, pdpe),
376 PTE_GET_PFN(*pdpe), !!(*pdpe & PTE_WRITABLE_MASK),
377 !!(*pdpe & PTE_NX_MASK));
378
379 pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
380 for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
381 pde = &pde_start[n3];
382 if (!(*pde & PTE_PRESENT_MASK))
383 continue;
384 fprintf(stream, "%*spde 0x%-3zx %p "
385 "0x%-12lx 0x%-10llx %u %u\n",
386 indent, "", pde - pde_start, pde,
387 addr_hva2gpa(vm, pde),
388 PTE_GET_PFN(*pde), !!(*pde & PTE_WRITABLE_MASK),
389 !!(*pde & PTE_NX_MASK));
390
391 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
392 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
393 pte = &pte_start[n4];
394 if (!(*pte & PTE_PRESENT_MASK))
395 continue;
396 fprintf(stream, "%*spte 0x%-3zx %p "
397 "0x%-12lx 0x%-10llx %u %u "
398 " %u 0x%-10lx\n",
399 indent, "",
400 pte - pte_start, pte,
401 addr_hva2gpa(vm, pte),
402 PTE_GET_PFN(*pte),
403 !!(*pte & PTE_WRITABLE_MASK),
404 !!(*pte & PTE_NX_MASK),
405 !!(*pte & PTE_DIRTY_MASK),
406 ((uint64_t) n1 << 27)
407 | ((uint64_t) n2 << 18)
408 | ((uint64_t) n3 << 9)
409 | ((uint64_t) n4));
410 }
411 }
412 }
413 }
414 }
415
416 /*
417 * Set Unusable Segment
418 *
419 * Input Args: None
420 *
421 * Output Args:
422 * segp - Pointer to segment register
423 *
424 * Return: None
425 *
426 * Sets the segment register pointed to by @segp to an unusable state.
427 */
kvm_seg_set_unusable(struct kvm_segment * segp)428 static void kvm_seg_set_unusable(struct kvm_segment *segp)
429 {
430 memset(segp, 0, sizeof(*segp));
431 segp->unusable = true;
432 }
433
kvm_seg_fill_gdt_64bit(struct kvm_vm * vm,struct kvm_segment * segp)434 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
435 {
436 void *gdt = addr_gva2hva(vm, vm->gdt);
437 struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
438
439 desc->limit0 = segp->limit & 0xFFFF;
440 desc->base0 = segp->base & 0xFFFF;
441 desc->base1 = segp->base >> 16;
442 desc->type = segp->type;
443 desc->s = segp->s;
444 desc->dpl = segp->dpl;
445 desc->p = segp->present;
446 desc->limit1 = segp->limit >> 16;
447 desc->avl = segp->avl;
448 desc->l = segp->l;
449 desc->db = segp->db;
450 desc->g = segp->g;
451 desc->base2 = segp->base >> 24;
452 if (!segp->s)
453 desc->base3 = segp->base >> 32;
454 }
455
456
457 /*
458 * Set Long Mode Flat Kernel Code Segment
459 *
460 * Input Args:
461 * vm - VM whose GDT is being filled, or NULL to only write segp
462 * selector - selector value
463 *
464 * Output Args:
465 * segp - Pointer to KVM segment
466 *
467 * Return: None
468 *
469 * Sets up the KVM segment pointed to by @segp, to be a code segment
470 * with the selector value given by @selector.
471 */
kvm_seg_set_kernel_code_64bit(struct kvm_vm * vm,uint16_t selector,struct kvm_segment * segp)472 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
473 struct kvm_segment *segp)
474 {
475 memset(segp, 0, sizeof(*segp));
476 segp->selector = selector;
477 segp->limit = 0xFFFFFFFFu;
478 segp->s = 0x1; /* kTypeCodeData */
479 segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
480 * | kFlagCodeReadable
481 */
482 segp->g = true;
483 segp->l = true;
484 segp->present = 1;
485 if (vm)
486 kvm_seg_fill_gdt_64bit(vm, segp);
487 }
488
489 /*
490 * Set Long Mode Flat Kernel Data Segment
491 *
492 * Input Args:
493 * vm - VM whose GDT is being filled, or NULL to only write segp
494 * selector - selector value
495 *
496 * Output Args:
497 * segp - Pointer to KVM segment
498 *
499 * Return: None
500 *
501 * Sets up the KVM segment pointed to by @segp, to be a data segment
502 * with the selector value given by @selector.
503 */
kvm_seg_set_kernel_data_64bit(struct kvm_vm * vm,uint16_t selector,struct kvm_segment * segp)504 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
505 struct kvm_segment *segp)
506 {
507 memset(segp, 0, sizeof(*segp));
508 segp->selector = selector;
509 segp->limit = 0xFFFFFFFFu;
510 segp->s = 0x1; /* kTypeCodeData */
511 segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
512 * | kFlagDataWritable
513 */
514 segp->g = true;
515 segp->present = true;
516 if (vm)
517 kvm_seg_fill_gdt_64bit(vm, segp);
518 }
519
addr_gva2gpa(struct kvm_vm * vm,vm_vaddr_t gva)520 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
521 {
522 uint16_t index[4];
523 uint64_t *pml4e, *pdpe, *pde;
524 uint64_t *pte;
525
526 TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use "
527 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
528
529 index[0] = (gva >> 12) & 0x1ffu;
530 index[1] = (gva >> 21) & 0x1ffu;
531 index[2] = (gva >> 30) & 0x1ffu;
532 index[3] = (gva >> 39) & 0x1ffu;
533
534 if (!vm->pgd_created)
535 goto unmapped_gva;
536 pml4e = addr_gpa2hva(vm, vm->pgd);
537 if (!(pml4e[index[3]] & PTE_PRESENT_MASK))
538 goto unmapped_gva;
539
540 pdpe = addr_gpa2hva(vm, PTE_GET_PFN(pml4e[index[3]]) * vm->page_size);
541 if (!(pdpe[index[2]] & PTE_PRESENT_MASK))
542 goto unmapped_gva;
543
544 pde = addr_gpa2hva(vm, PTE_GET_PFN(pdpe[index[2]]) * vm->page_size);
545 if (!(pde[index[1]] & PTE_PRESENT_MASK))
546 goto unmapped_gva;
547
548 pte = addr_gpa2hva(vm, PTE_GET_PFN(pde[index[1]]) * vm->page_size);
549 if (!(pte[index[0]] & PTE_PRESENT_MASK))
550 goto unmapped_gva;
551
552 return (PTE_GET_PFN(pte[index[0]]) * vm->page_size) + (gva & 0xfffu);
553
554 unmapped_gva:
555 TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
556 exit(EXIT_FAILURE);
557 }
558
kvm_setup_gdt(struct kvm_vm * vm,struct kvm_dtable * dt)559 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt)
560 {
561 if (!vm->gdt)
562 vm->gdt = vm_vaddr_alloc_page(vm);
563
564 dt->base = vm->gdt;
565 dt->limit = getpagesize();
566 }
567
kvm_setup_tss_64bit(struct kvm_vm * vm,struct kvm_segment * segp,int selector)568 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
569 int selector)
570 {
571 if (!vm->tss)
572 vm->tss = vm_vaddr_alloc_page(vm);
573
574 memset(segp, 0, sizeof(*segp));
575 segp->base = vm->tss;
576 segp->limit = 0x67;
577 segp->selector = selector;
578 segp->type = 0xb;
579 segp->present = 1;
580 kvm_seg_fill_gdt_64bit(vm, segp);
581 }
582
vcpu_setup(struct kvm_vm * vm,int vcpuid)583 static void vcpu_setup(struct kvm_vm *vm, int vcpuid)
584 {
585 struct kvm_sregs sregs;
586
587 /* Set mode specific system register values. */
588 vcpu_sregs_get(vm, vcpuid, &sregs);
589
590 sregs.idt.limit = 0;
591
592 kvm_setup_gdt(vm, &sregs.gdt);
593
594 switch (vm->mode) {
595 case VM_MODE_PXXV48_4K:
596 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
597 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
598 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
599
600 kvm_seg_set_unusable(&sregs.ldt);
601 kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs);
602 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds);
603 kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es);
604 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18);
605 break;
606
607 default:
608 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
609 }
610
611 sregs.cr3 = vm->pgd;
612 vcpu_sregs_set(vm, vcpuid, &sregs);
613 }
614
vm_vcpu_add_default(struct kvm_vm * vm,uint32_t vcpuid,void * guest_code)615 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
616 {
617 struct kvm_mp_state mp_state;
618 struct kvm_regs regs;
619 vm_vaddr_t stack_vaddr;
620 stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
621 DEFAULT_GUEST_STACK_VADDR_MIN);
622
623 /* Create VCPU */
624 vm_vcpu_add(vm, vcpuid);
625 vcpu_setup(vm, vcpuid);
626
627 /* Setup guest general purpose registers */
628 vcpu_regs_get(vm, vcpuid, ®s);
629 regs.rflags = regs.rflags | 0x2;
630 regs.rsp = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize());
631 regs.rip = (unsigned long) guest_code;
632 vcpu_regs_set(vm, vcpuid, ®s);
633
634 /* Setup the MP state */
635 mp_state.mp_state = 0;
636 vcpu_set_mp_state(vm, vcpuid, &mp_state);
637
638 /* Setup supported CPUIDs */
639 vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid());
640 }
641
642 /*
643 * Allocate an instance of struct kvm_cpuid2
644 *
645 * Input Args: None
646 *
647 * Output Args: None
648 *
649 * Return: A pointer to the allocated struct. The caller is responsible
650 * for freeing this struct.
651 *
652 * Since kvm_cpuid2 uses a 0-length array to allow a the size of the
653 * array to be decided at allocation time, allocation is slightly
654 * complicated. This function uses a reasonable default length for
655 * the array and performs the appropriate allocation.
656 */
allocate_kvm_cpuid2(void)657 static struct kvm_cpuid2 *allocate_kvm_cpuid2(void)
658 {
659 struct kvm_cpuid2 *cpuid;
660 int nent = 100;
661 size_t size;
662
663 size = sizeof(*cpuid);
664 size += nent * sizeof(struct kvm_cpuid_entry2);
665 cpuid = malloc(size);
666 if (!cpuid) {
667 perror("malloc");
668 abort();
669 }
670
671 cpuid->nent = nent;
672
673 return cpuid;
674 }
675
676 /*
677 * KVM Supported CPUID Get
678 *
679 * Input Args: None
680 *
681 * Output Args:
682 *
683 * Return: The supported KVM CPUID
684 *
685 * Get the guest CPUID supported by KVM.
686 */
kvm_get_supported_cpuid(void)687 struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
688 {
689 static struct kvm_cpuid2 *cpuid;
690 int ret;
691 int kvm_fd;
692
693 if (cpuid)
694 return cpuid;
695
696 cpuid = allocate_kvm_cpuid2();
697 kvm_fd = open_kvm_dev_path_or_exit();
698
699 ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, cpuid);
700 TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_CPUID failed %d %d\n",
701 ret, errno);
702
703 close(kvm_fd);
704 return cpuid;
705 }
706
707 /*
708 * KVM Get MSR
709 *
710 * Input Args:
711 * msr_index - Index of MSR
712 *
713 * Output Args: None
714 *
715 * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
716 *
717 * Get value of MSR for VCPU.
718 */
kvm_get_feature_msr(uint64_t msr_index)719 uint64_t kvm_get_feature_msr(uint64_t msr_index)
720 {
721 struct {
722 struct kvm_msrs header;
723 struct kvm_msr_entry entry;
724 } buffer = {};
725 int r, kvm_fd;
726
727 buffer.header.nmsrs = 1;
728 buffer.entry.index = msr_index;
729 kvm_fd = open_kvm_dev_path_or_exit();
730
731 r = ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
732 TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
733 " rc: %i errno: %i", r, errno);
734
735 close(kvm_fd);
736 return buffer.entry.data;
737 }
738
739 /*
740 * VM VCPU CPUID Set
741 *
742 * Input Args:
743 * vm - Virtual Machine
744 * vcpuid - VCPU id
745 *
746 * Output Args: None
747 *
748 * Return: KVM CPUID (KVM_GET_CPUID2)
749 *
750 * Set the VCPU's CPUID.
751 */
vcpu_get_cpuid(struct kvm_vm * vm,uint32_t vcpuid)752 struct kvm_cpuid2 *vcpu_get_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
753 {
754 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
755 struct kvm_cpuid2 *cpuid;
756 int max_ent;
757 int rc = -1;
758
759 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
760
761 cpuid = allocate_kvm_cpuid2();
762 max_ent = cpuid->nent;
763
764 for (cpuid->nent = 1; cpuid->nent <= max_ent; cpuid->nent++) {
765 rc = ioctl(vcpu->fd, KVM_GET_CPUID2, cpuid);
766 if (!rc)
767 break;
768
769 TEST_ASSERT(rc == -1 && errno == E2BIG,
770 "KVM_GET_CPUID2 should either succeed or give E2BIG: %d %d",
771 rc, errno);
772 }
773
774 TEST_ASSERT(rc == 0, "KVM_GET_CPUID2 failed, rc: %i errno: %i",
775 rc, errno);
776
777 return cpuid;
778 }
779
780
781
782 /*
783 * Locate a cpuid entry.
784 *
785 * Input Args:
786 * function: The function of the cpuid entry to find.
787 * index: The index of the cpuid entry.
788 *
789 * Output Args: None
790 *
791 * Return: A pointer to the cpuid entry. Never returns NULL.
792 */
793 struct kvm_cpuid_entry2 *
kvm_get_supported_cpuid_index(uint32_t function,uint32_t index)794 kvm_get_supported_cpuid_index(uint32_t function, uint32_t index)
795 {
796 struct kvm_cpuid2 *cpuid;
797 struct kvm_cpuid_entry2 *entry = NULL;
798 int i;
799
800 cpuid = kvm_get_supported_cpuid();
801 for (i = 0; i < cpuid->nent; i++) {
802 if (cpuid->entries[i].function == function &&
803 cpuid->entries[i].index == index) {
804 entry = &cpuid->entries[i];
805 break;
806 }
807 }
808
809 TEST_ASSERT(entry, "Guest CPUID entry not found: (EAX=%x, ECX=%x).",
810 function, index);
811 return entry;
812 }
813
814 /*
815 * VM VCPU CPUID Set
816 *
817 * Input Args:
818 * vm - Virtual Machine
819 * vcpuid - VCPU id
820 * cpuid - The CPUID values to set.
821 *
822 * Output Args: None
823 *
824 * Return: void
825 *
826 * Set the VCPU's CPUID.
827 */
vcpu_set_cpuid(struct kvm_vm * vm,uint32_t vcpuid,struct kvm_cpuid2 * cpuid)828 void vcpu_set_cpuid(struct kvm_vm *vm,
829 uint32_t vcpuid, struct kvm_cpuid2 *cpuid)
830 {
831 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
832 int rc;
833
834 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
835
836 rc = ioctl(vcpu->fd, KVM_SET_CPUID2, cpuid);
837 TEST_ASSERT(rc == 0, "KVM_SET_CPUID2 failed, rc: %i errno: %i",
838 rc, errno);
839
840 }
841
842 /*
843 * VCPU Get MSR
844 *
845 * Input Args:
846 * vm - Virtual Machine
847 * vcpuid - VCPU ID
848 * msr_index - Index of MSR
849 *
850 * Output Args: None
851 *
852 * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
853 *
854 * Get value of MSR for VCPU.
855 */
vcpu_get_msr(struct kvm_vm * vm,uint32_t vcpuid,uint64_t msr_index)856 uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index)
857 {
858 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
859 struct {
860 struct kvm_msrs header;
861 struct kvm_msr_entry entry;
862 } buffer = {};
863 int r;
864
865 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
866 buffer.header.nmsrs = 1;
867 buffer.entry.index = msr_index;
868 r = ioctl(vcpu->fd, KVM_GET_MSRS, &buffer.header);
869 TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
870 " rc: %i errno: %i", r, errno);
871
872 return buffer.entry.data;
873 }
874
875 /*
876 * _VCPU Set MSR
877 *
878 * Input Args:
879 * vm - Virtual Machine
880 * vcpuid - VCPU ID
881 * msr_index - Index of MSR
882 * msr_value - New value of MSR
883 *
884 * Output Args: None
885 *
886 * Return: The result of KVM_SET_MSRS.
887 *
888 * Sets the value of an MSR for the given VCPU.
889 */
_vcpu_set_msr(struct kvm_vm * vm,uint32_t vcpuid,uint64_t msr_index,uint64_t msr_value)890 int _vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
891 uint64_t msr_value)
892 {
893 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
894 struct {
895 struct kvm_msrs header;
896 struct kvm_msr_entry entry;
897 } buffer = {};
898 int r;
899
900 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
901 memset(&buffer, 0, sizeof(buffer));
902 buffer.header.nmsrs = 1;
903 buffer.entry.index = msr_index;
904 buffer.entry.data = msr_value;
905 r = ioctl(vcpu->fd, KVM_SET_MSRS, &buffer.header);
906 return r;
907 }
908
909 /*
910 * VCPU Set MSR
911 *
912 * Input Args:
913 * vm - Virtual Machine
914 * vcpuid - VCPU ID
915 * msr_index - Index of MSR
916 * msr_value - New value of MSR
917 *
918 * Output Args: None
919 *
920 * Return: On success, nothing. On failure a TEST_ASSERT is produced.
921 *
922 * Set value of MSR for VCPU.
923 */
vcpu_set_msr(struct kvm_vm * vm,uint32_t vcpuid,uint64_t msr_index,uint64_t msr_value)924 void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
925 uint64_t msr_value)
926 {
927 int r;
928
929 r = _vcpu_set_msr(vm, vcpuid, msr_index, msr_value);
930 TEST_ASSERT(r == 1, "KVM_SET_MSRS IOCTL failed,\n"
931 " rc: %i errno: %i", r, errno);
932 }
933
vcpu_args_set(struct kvm_vm * vm,uint32_t vcpuid,unsigned int num,...)934 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
935 {
936 va_list ap;
937 struct kvm_regs regs;
938
939 TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
940 " num: %u\n",
941 num);
942
943 va_start(ap, num);
944 vcpu_regs_get(vm, vcpuid, ®s);
945
946 if (num >= 1)
947 regs.rdi = va_arg(ap, uint64_t);
948
949 if (num >= 2)
950 regs.rsi = va_arg(ap, uint64_t);
951
952 if (num >= 3)
953 regs.rdx = va_arg(ap, uint64_t);
954
955 if (num >= 4)
956 regs.rcx = va_arg(ap, uint64_t);
957
958 if (num >= 5)
959 regs.r8 = va_arg(ap, uint64_t);
960
961 if (num >= 6)
962 regs.r9 = va_arg(ap, uint64_t);
963
964 vcpu_regs_set(vm, vcpuid, ®s);
965 va_end(ap);
966 }
967
vcpu_dump(FILE * stream,struct kvm_vm * vm,uint32_t vcpuid,uint8_t indent)968 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
969 {
970 struct kvm_regs regs;
971 struct kvm_sregs sregs;
972
973 fprintf(stream, "%*scpuid: %u\n", indent, "", vcpuid);
974
975 fprintf(stream, "%*sregs:\n", indent + 2, "");
976 vcpu_regs_get(vm, vcpuid, ®s);
977 regs_dump(stream, ®s, indent + 4);
978
979 fprintf(stream, "%*ssregs:\n", indent + 2, "");
980 vcpu_sregs_get(vm, vcpuid, &sregs);
981 sregs_dump(stream, &sregs, indent + 4);
982 }
983
984 struct kvm_x86_state {
985 struct kvm_vcpu_events events;
986 struct kvm_mp_state mp_state;
987 struct kvm_regs regs;
988 struct kvm_xsave xsave;
989 struct kvm_xcrs xcrs;
990 struct kvm_sregs sregs;
991 struct kvm_debugregs debugregs;
992 union {
993 struct kvm_nested_state nested;
994 char nested_[16384];
995 };
996 struct kvm_msrs msrs;
997 };
998
kvm_get_num_msrs_fd(int kvm_fd)999 static int kvm_get_num_msrs_fd(int kvm_fd)
1000 {
1001 struct kvm_msr_list nmsrs;
1002 int r;
1003
1004 nmsrs.nmsrs = 0;
1005 r = ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
1006 TEST_ASSERT(r == -1 && errno == E2BIG, "Unexpected result from KVM_GET_MSR_INDEX_LIST probe, r: %i",
1007 r);
1008
1009 return nmsrs.nmsrs;
1010 }
1011
kvm_get_num_msrs(struct kvm_vm * vm)1012 static int kvm_get_num_msrs(struct kvm_vm *vm)
1013 {
1014 return kvm_get_num_msrs_fd(vm->kvm_fd);
1015 }
1016
kvm_get_msr_index_list(void)1017 struct kvm_msr_list *kvm_get_msr_index_list(void)
1018 {
1019 struct kvm_msr_list *list;
1020 int nmsrs, r, kvm_fd;
1021
1022 kvm_fd = open_kvm_dev_path_or_exit();
1023
1024 nmsrs = kvm_get_num_msrs_fd(kvm_fd);
1025 list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
1026 list->nmsrs = nmsrs;
1027 r = ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1028 close(kvm_fd);
1029
1030 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1031 r);
1032
1033 return list;
1034 }
1035
vcpu_save_state(struct kvm_vm * vm,uint32_t vcpuid)1036 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid)
1037 {
1038 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1039 struct kvm_msr_list *list;
1040 struct kvm_x86_state *state;
1041 int nmsrs, r, i;
1042 static int nested_size = -1;
1043
1044 if (nested_size == -1) {
1045 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
1046 TEST_ASSERT(nested_size <= sizeof(state->nested_),
1047 "Nested state size too big, %i > %zi",
1048 nested_size, sizeof(state->nested_));
1049 }
1050
1051 /*
1052 * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
1053 * guest state is consistent only after userspace re-enters the
1054 * kernel with KVM_RUN. Complete IO prior to migrating state
1055 * to a new VM.
1056 */
1057 vcpu_run_complete_io(vm, vcpuid);
1058
1059 nmsrs = kvm_get_num_msrs(vm);
1060 list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
1061 list->nmsrs = nmsrs;
1062 r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1063 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1064 r);
1065
1066 state = malloc(sizeof(*state) + nmsrs * sizeof(state->msrs.entries[0]));
1067 r = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, &state->events);
1068 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_VCPU_EVENTS, r: %i",
1069 r);
1070
1071 r = ioctl(vcpu->fd, KVM_GET_MP_STATE, &state->mp_state);
1072 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MP_STATE, r: %i",
1073 r);
1074
1075 r = ioctl(vcpu->fd, KVM_GET_REGS, &state->regs);
1076 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_REGS, r: %i",
1077 r);
1078
1079 r = ioctl(vcpu->fd, KVM_GET_XSAVE, &state->xsave);
1080 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XSAVE, r: %i",
1081 r);
1082
1083 if (kvm_check_cap(KVM_CAP_XCRS)) {
1084 r = ioctl(vcpu->fd, KVM_GET_XCRS, &state->xcrs);
1085 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XCRS, r: %i",
1086 r);
1087 }
1088
1089 r = ioctl(vcpu->fd, KVM_GET_SREGS, &state->sregs);
1090 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_SREGS, r: %i",
1091 r);
1092
1093 if (nested_size) {
1094 state->nested.size = sizeof(state->nested_);
1095 r = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, &state->nested);
1096 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_NESTED_STATE, r: %i",
1097 r);
1098 TEST_ASSERT(state->nested.size <= nested_size,
1099 "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1100 state->nested.size, nested_size);
1101 } else
1102 state->nested.size = 0;
1103
1104 state->msrs.nmsrs = nmsrs;
1105 for (i = 0; i < nmsrs; i++)
1106 state->msrs.entries[i].index = list->indices[i];
1107 r = ioctl(vcpu->fd, KVM_GET_MSRS, &state->msrs);
1108 TEST_ASSERT(r == nmsrs, "Unexpected result from KVM_GET_MSRS, r: %i (failed MSR was 0x%x)",
1109 r, r == nmsrs ? -1 : list->indices[r]);
1110
1111 r = ioctl(vcpu->fd, KVM_GET_DEBUGREGS, &state->debugregs);
1112 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_DEBUGREGS, r: %i",
1113 r);
1114
1115 free(list);
1116 return state;
1117 }
1118
vcpu_load_state(struct kvm_vm * vm,uint32_t vcpuid,struct kvm_x86_state * state)1119 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *state)
1120 {
1121 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1122 int r;
1123
1124 r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
1125 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
1126 r);
1127
1128 if (kvm_check_cap(KVM_CAP_XCRS)) {
1129 r = ioctl(vcpu->fd, KVM_SET_XCRS, &state->xcrs);
1130 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XCRS, r: %i",
1131 r);
1132 }
1133
1134 r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
1135 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
1136 r);
1137
1138 r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
1139 TEST_ASSERT(r == state->msrs.nmsrs, "Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)",
1140 r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
1141
1142 r = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, &state->events);
1143 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
1144 r);
1145
1146 r = ioctl(vcpu->fd, KVM_SET_MP_STATE, &state->mp_state);
1147 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
1148 r);
1149
1150 r = ioctl(vcpu->fd, KVM_SET_DEBUGREGS, &state->debugregs);
1151 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_DEBUGREGS, r: %i",
1152 r);
1153
1154 r = ioctl(vcpu->fd, KVM_SET_REGS, &state->regs);
1155 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
1156 r);
1157
1158 if (state->nested.size) {
1159 r = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, &state->nested);
1160 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_NESTED_STATE, r: %i",
1161 r);
1162 }
1163 }
1164
is_intel_cpu(void)1165 bool is_intel_cpu(void)
1166 {
1167 int eax, ebx, ecx, edx;
1168 const uint32_t *chunk;
1169 const int leaf = 0;
1170
1171 __asm__ __volatile__(
1172 "cpuid"
1173 : /* output */ "=a"(eax), "=b"(ebx),
1174 "=c"(ecx), "=d"(edx)
1175 : /* input */ "0"(leaf), "2"(0));
1176
1177 chunk = (const uint32_t *)("GenuineIntel");
1178 return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);
1179 }
1180
kvm_get_cpuid_max_basic(void)1181 uint32_t kvm_get_cpuid_max_basic(void)
1182 {
1183 return kvm_get_supported_cpuid_entry(0)->eax;
1184 }
1185
kvm_get_cpuid_max_extended(void)1186 uint32_t kvm_get_cpuid_max_extended(void)
1187 {
1188 return kvm_get_supported_cpuid_entry(0x80000000)->eax;
1189 }
1190
kvm_get_cpu_address_width(unsigned int * pa_bits,unsigned int * va_bits)1191 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits)
1192 {
1193 struct kvm_cpuid_entry2 *entry;
1194 bool pae;
1195
1196 /* SDM 4.1.4 */
1197 if (kvm_get_cpuid_max_extended() < 0x80000008) {
1198 pae = kvm_get_supported_cpuid_entry(1)->edx & (1 << 6);
1199 *pa_bits = pae ? 36 : 32;
1200 *va_bits = 32;
1201 } else {
1202 entry = kvm_get_supported_cpuid_entry(0x80000008);
1203 *pa_bits = entry->eax & 0xff;
1204 *va_bits = (entry->eax >> 8) & 0xff;
1205 }
1206 }
1207
1208 struct idt_entry {
1209 uint16_t offset0;
1210 uint16_t selector;
1211 uint16_t ist : 3;
1212 uint16_t : 5;
1213 uint16_t type : 4;
1214 uint16_t : 1;
1215 uint16_t dpl : 2;
1216 uint16_t p : 1;
1217 uint16_t offset1;
1218 uint32_t offset2; uint32_t reserved;
1219 };
1220
set_idt_entry(struct kvm_vm * vm,int vector,unsigned long addr,int dpl,unsigned short selector)1221 static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr,
1222 int dpl, unsigned short selector)
1223 {
1224 struct idt_entry *base =
1225 (struct idt_entry *)addr_gva2hva(vm, vm->idt);
1226 struct idt_entry *e = &base[vector];
1227
1228 memset(e, 0, sizeof(*e));
1229 e->offset0 = addr;
1230 e->selector = selector;
1231 e->ist = 0;
1232 e->type = 14;
1233 e->dpl = dpl;
1234 e->p = 1;
1235 e->offset1 = addr >> 16;
1236 e->offset2 = addr >> 32;
1237 }
1238
kvm_exit_unexpected_vector(uint32_t value)1239 void kvm_exit_unexpected_vector(uint32_t value)
1240 {
1241 ucall(UCALL_UNHANDLED, 1, value);
1242 }
1243
route_exception(struct ex_regs * regs)1244 void route_exception(struct ex_regs *regs)
1245 {
1246 typedef void(*handler)(struct ex_regs *);
1247 handler *handlers = (handler *)exception_handlers;
1248
1249 if (handlers && handlers[regs->vector]) {
1250 handlers[regs->vector](regs);
1251 return;
1252 }
1253
1254 kvm_exit_unexpected_vector(regs->vector);
1255 }
1256
vm_init_descriptor_tables(struct kvm_vm * vm)1257 void vm_init_descriptor_tables(struct kvm_vm *vm)
1258 {
1259 extern void *idt_handlers;
1260 int i;
1261
1262 vm->idt = vm_vaddr_alloc_page(vm);
1263 vm->handlers = vm_vaddr_alloc_page(vm);
1264 /* Handlers have the same address in both address spaces.*/
1265 for (i = 0; i < NUM_INTERRUPTS; i++)
1266 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0,
1267 DEFAULT_CODE_SELECTOR);
1268 }
1269
vcpu_init_descriptor_tables(struct kvm_vm * vm,uint32_t vcpuid)1270 void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid)
1271 {
1272 struct kvm_sregs sregs;
1273
1274 vcpu_sregs_get(vm, vcpuid, &sregs);
1275 sregs.idt.base = vm->idt;
1276 sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1;
1277 sregs.gdt.base = vm->gdt;
1278 sregs.gdt.limit = getpagesize() - 1;
1279 kvm_seg_set_kernel_data_64bit(NULL, DEFAULT_DATA_SELECTOR, &sregs.gs);
1280 vcpu_sregs_set(vm, vcpuid, &sregs);
1281 *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
1282 }
1283
vm_install_exception_handler(struct kvm_vm * vm,int vector,void (* handler)(struct ex_regs *))1284 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
1285 void (*handler)(struct ex_regs *))
1286 {
1287 vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers);
1288
1289 handlers[vector] = (vm_vaddr_t)handler;
1290 }
1291
assert_on_unhandled_exception(struct kvm_vm * vm,uint32_t vcpuid)1292 void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)
1293 {
1294 struct ucall uc;
1295
1296 if (get_ucall(vm, vcpuid, &uc) == UCALL_UNHANDLED) {
1297 uint64_t vector = uc.args[0];
1298
1299 TEST_FAIL("Unexpected vectored event in guest (vector:0x%lx)",
1300 vector);
1301 }
1302 }
1303
set_cpuid(struct kvm_cpuid2 * cpuid,struct kvm_cpuid_entry2 * ent)1304 bool set_cpuid(struct kvm_cpuid2 *cpuid,
1305 struct kvm_cpuid_entry2 *ent)
1306 {
1307 int i;
1308
1309 for (i = 0; i < cpuid->nent; i++) {
1310 struct kvm_cpuid_entry2 *cur = &cpuid->entries[i];
1311
1312 if (cur->function != ent->function || cur->index != ent->index)
1313 continue;
1314
1315 memcpy(cur, ent, sizeof(struct kvm_cpuid_entry2));
1316 return true;
1317 }
1318
1319 return false;
1320 }
1321
kvm_hypercall(uint64_t nr,uint64_t a0,uint64_t a1,uint64_t a2,uint64_t a3)1322 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
1323 uint64_t a3)
1324 {
1325 uint64_t r;
1326
1327 asm volatile("vmcall"
1328 : "=a"(r)
1329 : "a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1330 return r;
1331 }
1332
kvm_get_supported_hv_cpuid(void)1333 struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void)
1334 {
1335 static struct kvm_cpuid2 *cpuid;
1336 int ret;
1337 int kvm_fd;
1338
1339 if (cpuid)
1340 return cpuid;
1341
1342 cpuid = allocate_kvm_cpuid2();
1343 kvm_fd = open_kvm_dev_path_or_exit();
1344
1345 ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1346 TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_HV_CPUID failed %d %d\n",
1347 ret, errno);
1348
1349 close(kvm_fd);
1350 return cpuid;
1351 }
1352
vcpu_set_hv_cpuid(struct kvm_vm * vm,uint32_t vcpuid)1353 void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
1354 {
1355 static struct kvm_cpuid2 *cpuid_full;
1356 struct kvm_cpuid2 *cpuid_sys, *cpuid_hv;
1357 int i, nent = 0;
1358
1359 if (!cpuid_full) {
1360 cpuid_sys = kvm_get_supported_cpuid();
1361 cpuid_hv = kvm_get_supported_hv_cpuid();
1362
1363 cpuid_full = malloc(sizeof(*cpuid_full) +
1364 (cpuid_sys->nent + cpuid_hv->nent) *
1365 sizeof(struct kvm_cpuid_entry2));
1366 if (!cpuid_full) {
1367 perror("malloc");
1368 abort();
1369 }
1370
1371 /* Need to skip KVM CPUID leaves 0x400000xx */
1372 for (i = 0; i < cpuid_sys->nent; i++) {
1373 if (cpuid_sys->entries[i].function >= 0x40000000 &&
1374 cpuid_sys->entries[i].function < 0x40000100)
1375 continue;
1376 cpuid_full->entries[nent] = cpuid_sys->entries[i];
1377 nent++;
1378 }
1379
1380 memcpy(&cpuid_full->entries[nent], cpuid_hv->entries,
1381 cpuid_hv->nent * sizeof(struct kvm_cpuid_entry2));
1382 cpuid_full->nent = nent + cpuid_hv->nent;
1383 }
1384
1385 vcpu_set_cpuid(vm, vcpuid, cpuid_full);
1386 }
1387
vcpu_get_supported_hv_cpuid(struct kvm_vm * vm,uint32_t vcpuid)1388 struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid)
1389 {
1390 static struct kvm_cpuid2 *cpuid;
1391
1392 cpuid = allocate_kvm_cpuid2();
1393
1394 vcpu_ioctl(vm, vcpuid, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1395
1396 return cpuid;
1397 }
1398
1399 #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541
1400 #define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163
1401 #define X86EMUL_CPUID_VENDOR_AuthenticAMD_edx 0x69746e65
1402
x86_family(unsigned int eax)1403 static inline unsigned x86_family(unsigned int eax)
1404 {
1405 unsigned int x86;
1406
1407 x86 = (eax >> 8) & 0xf;
1408
1409 if (x86 == 0xf)
1410 x86 += (eax >> 20) & 0xff;
1411
1412 return x86;
1413 }
1414
vm_compute_max_gfn(struct kvm_vm * vm)1415 unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
1416 {
1417 const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
1418 unsigned long ht_gfn, max_gfn, max_pfn;
1419 uint32_t eax, ebx, ecx, edx, max_ext_leaf;
1420
1421 max_gfn = (1ULL << (vm->pa_bits - vm->page_shift)) - 1;
1422
1423 /* Avoid reserved HyperTransport region on AMD processors. */
1424 eax = ecx = 0;
1425 cpuid(&eax, &ebx, &ecx, &edx);
1426 if (ebx != X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx ||
1427 ecx != X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx ||
1428 edx != X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
1429 return max_gfn;
1430
1431 /* On parts with <40 physical address bits, the area is fully hidden */
1432 if (vm->pa_bits < 40)
1433 return max_gfn;
1434
1435 /* Before family 17h, the HyperTransport area is just below 1T. */
1436 ht_gfn = (1 << 28) - num_ht_pages;
1437 eax = 1;
1438 cpuid(&eax, &ebx, &ecx, &edx);
1439 if (x86_family(eax) < 0x17)
1440 goto done;
1441
1442 /*
1443 * Otherwise it's at the top of the physical address space, possibly
1444 * reduced due to SME by bits 11:6 of CPUID[0x8000001f].EBX. Use
1445 * the old conservative value if MAXPHYADDR is not enumerated.
1446 */
1447 eax = 0x80000000;
1448 cpuid(&eax, &ebx, &ecx, &edx);
1449 max_ext_leaf = eax;
1450 if (max_ext_leaf < 0x80000008)
1451 goto done;
1452
1453 eax = 0x80000008;
1454 cpuid(&eax, &ebx, &ecx, &edx);
1455 max_pfn = (1ULL << ((eax & 0xff) - vm->page_shift)) - 1;
1456 if (max_ext_leaf >= 0x8000001f) {
1457 eax = 0x8000001f;
1458 cpuid(&eax, &ebx, &ecx, &edx);
1459 max_pfn >>= (ebx >> 6) & 0x3f;
1460 }
1461
1462 ht_gfn = max_pfn - num_ht_pages;
1463 done:
1464 return min(max_gfn, ht_gfn - 1);
1465 }
1466