1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2021 SUSE LLC <mdoucha@suse.cz>
4 *
5 * x86-specific KVM helper functions
6 */
7
8 #include "kvm_x86_svm.h"
9
10 void kvm_svm_guest_entry(void);
11
12 struct kvm_interrupt_frame {
13 uintptr_t eip, cs, eflags, esp, ss;
14 };
15
16 const char *tst_interrupt_names[INTERRUPT_COUNT] = {
17 "Division by zero",
18 "Debug interrupt",
19 "Non-maskable interrupt",
20 "Breakpoint",
21 "Arithmetic overflow",
22 "Bound range exception",
23 "Illegal instruction error",
24 "Device not available error",
25 "Double fault",
26 NULL,
27 "Invalid TSS error",
28 "Segment not present error",
29 "Stack segment fault",
30 "General protection fault",
31 "Page fault",
32 NULL,
33 "Floating point exception",
34 "Alignment error",
35 "Machine check exception",
36 "SIMD floating point exception",
37 "Virtualization exception",
38 "Control protection exception",
39 NULL,
40 NULL,
41 NULL,
42 NULL,
43 NULL,
44 NULL,
45 "Hypervisor injection exception",
46 "VMM communication exception",
47 "Security exception",
48 NULL
49 };
50
51 static uintptr_t intr_handlers[] = {
52 (uintptr_t)kvm_handle_zerodiv,
53 (uintptr_t)kvm_handle_debug,
54 (uintptr_t)kvm_handle_nmi,
55 (uintptr_t)kvm_handle_breakpoint,
56 (uintptr_t)kvm_handle_overflow,
57 (uintptr_t)kvm_handle_bound_range_exc,
58 (uintptr_t)kvm_handle_bad_opcode,
59 (uintptr_t)kvm_handle_device_error,
60 (uintptr_t)kvm_handle_double_fault,
61 (uintptr_t)kvm_handle_bad_exception,
62 (uintptr_t)kvm_handle_invalid_tss,
63 (uintptr_t)kvm_handle_segfault,
64 (uintptr_t)kvm_handle_stack_fault,
65 (uintptr_t)kvm_handle_gpf,
66 (uintptr_t)kvm_handle_page_fault,
67 (uintptr_t)kvm_handle_bad_exception,
68 (uintptr_t)kvm_handle_fpu_error,
69 (uintptr_t)kvm_handle_alignment_error,
70 (uintptr_t)kvm_handle_machine_check,
71 (uintptr_t)kvm_handle_simd_error,
72 (uintptr_t)kvm_handle_virt_error,
73 (uintptr_t)kvm_handle_cpe,
74 (uintptr_t)kvm_handle_bad_exception,
75 (uintptr_t)kvm_handle_bad_exception,
76 (uintptr_t)kvm_handle_bad_exception,
77 (uintptr_t)kvm_handle_bad_exception,
78 (uintptr_t)kvm_handle_bad_exception,
79 (uintptr_t)kvm_handle_bad_exception,
80 (uintptr_t)kvm_handle_hv_injection,
81 (uintptr_t)kvm_handle_vmm_comm,
82 (uintptr_t)kvm_handle_security_error,
83 (uintptr_t)kvm_handle_bad_exception,
84 0
85 };
86
kvm_set_intr_handler(unsigned int id,uintptr_t func)87 static void kvm_set_intr_handler(unsigned int id, uintptr_t func)
88 {
89 memset(kvm_idt + id, 0, sizeof(kvm_idt[0]));
90 kvm_idt[id].offset_lo = func & 0xffff;
91 kvm_idt[id].offset_hi = func >> 16;
92 kvm_idt[id].selector = 8;
93 kvm_idt[id].flags = 0x8f; /* type = 0xf, P = 1 */
94 }
95
kvm_init_interrupts(void)96 void kvm_init_interrupts(void)
97 {
98 int i;
99
100 for (i = 0; intr_handlers[i]; i++)
101 kvm_set_intr_handler(i, intr_handlers[i]);
102
103 for (; i < X86_INTR_COUNT; i++)
104 kvm_set_intr_handler(i, (uintptr_t)kvm_handle_bad_exception);
105 }
106
kvm_get_page_address_pae(const struct page_table_entry_pae * entry)107 uintptr_t kvm_get_page_address_pae(const struct page_table_entry_pae *entry)
108 {
109 if (!entry->present)
110 return 0;
111
112 return entry->address << 12;
113 }
114
115 #ifdef __x86_64__
kvm_set_segment_descriptor64(struct segment_descriptor64 * dst,uint64_t baseaddr,uint32_t limit,unsigned int flags)116 static void kvm_set_segment_descriptor64(struct segment_descriptor64 *dst,
117 uint64_t baseaddr, uint32_t limit, unsigned int flags)
118 {
119
120 dst->baseaddr_lo = baseaddr & 0xffffff;
121 dst->baseaddr_hi = baseaddr >> 24;
122 dst->limit_lo = limit & 0xffff;
123 dst->limit_hi = limit >> 16;
124 dst->flags_lo = flags & 0xff;
125 dst->flags_hi = (flags >> 8) & 0xf;
126 dst->reserved = 0;
127 }
128 #endif
129
kvm_set_segment_descriptor(struct segment_descriptor * dst,uint64_t baseaddr,uint32_t limit,unsigned int flags)130 void kvm_set_segment_descriptor(struct segment_descriptor *dst,
131 uint64_t baseaddr, uint32_t limit, unsigned int flags)
132 {
133 if (limit >> 20)
134 tst_brk(TBROK, "Segment limit out of range");
135
136 #ifdef __x86_64__
137 /* System descriptors have double size in 64bit mode */
138 if (!(flags & SEGFLAG_NSYSTEM)) {
139 kvm_set_segment_descriptor64((struct segment_descriptor64 *)dst,
140 baseaddr, limit, flags);
141 return;
142 }
143 #endif
144
145 if (baseaddr >> 32)
146 tst_brk(TBROK, "Segment base address out of range");
147
148 dst->baseaddr_lo = baseaddr & 0xffffff;
149 dst->baseaddr_hi = baseaddr >> 24;
150 dst->limit_lo = limit & 0xffff;
151 dst->limit_hi = limit >> 16;
152 dst->flags_lo = flags & 0xff;
153 dst->flags_hi = (flags >> 8) & 0xf;
154 }
155
kvm_parse_segment_descriptor(struct segment_descriptor * src,uint64_t * baseaddr,uint32_t * limit,unsigned int * flags)156 void kvm_parse_segment_descriptor(struct segment_descriptor *src,
157 uint64_t *baseaddr, uint32_t *limit, unsigned int *flags)
158 {
159 if (baseaddr) {
160 *baseaddr = (((uint64_t)src->baseaddr_hi) << 24) |
161 src->baseaddr_lo;
162 }
163
164 if (limit)
165 *limit = (((uint32_t)src->limit_hi) << 16) | src->limit_lo;
166
167 if (flags)
168 *flags = (((uint32_t)src->flags_hi) << 8) | src->flags_lo;
169 }
170
kvm_find_free_descriptor(const struct segment_descriptor * table,size_t size)171 int kvm_find_free_descriptor(const struct segment_descriptor *table,
172 size_t size)
173 {
174 const struct segment_descriptor *ptr;
175 size_t i;
176
177 for (i = 0, ptr = table; i < size; i++, ptr++) {
178 if (!(ptr->flags_lo & SEGFLAG_PRESENT))
179 return i;
180
181 #ifdef __x86_64__
182 /* System descriptors have double size in 64bit mode */
183 if (!(ptr->flags_lo & SEGFLAG_NSYSTEM)) {
184 ptr++;
185 i++;
186 }
187 #endif
188 }
189
190 return -1;
191 }
192
kvm_create_stack_descriptor(struct segment_descriptor * table,size_t tabsize,void * stack_base)193 unsigned int kvm_create_stack_descriptor(struct segment_descriptor *table,
194 size_t tabsize, void *stack_base)
195 {
196 int ret = kvm_find_free_descriptor(table, tabsize);
197
198 if (ret < 0)
199 tst_brk(TBROK, "Descriptor table is full");
200
201 kvm_set_segment_descriptor(table + ret, 0,
202 (((uintptr_t)stack_base) - 1) >> 12, SEGTYPE_STACK |
203 SEGFLAG_PRESENT | SEGFLAG_32BIT | SEGFLAG_PAGE_LIMIT);
204 return ret;
205 }
206
kvm_get_cpuid(unsigned int eax,unsigned int ecx,struct kvm_cpuid * buf)207 void kvm_get_cpuid(unsigned int eax, unsigned int ecx, struct kvm_cpuid *buf)
208 {
209 asm (
210 "cpuid\n"
211 : "=a" (buf->eax), "=b" (buf->ebx), "=c" (buf->ecx),
212 "=d" (buf->edx)
213 : "0" (eax), "2" (ecx)
214 );
215 }
216
kvm_rdmsr(unsigned int msr)217 uint64_t kvm_rdmsr(unsigned int msr)
218 {
219 unsigned int ret_lo, ret_hi;
220
221 asm (
222 "rdmsr\n"
223 : "=a" (ret_lo), "=d" (ret_hi)
224 : "c" (msr)
225 );
226
227 return (((uint64_t)ret_hi) << 32) | ret_lo;
228 }
229
kvm_wrmsr(unsigned int msr,uint64_t value)230 void kvm_wrmsr(unsigned int msr, uint64_t value)
231 {
232 uint32_t val_lo = value & 0xffffffff, val_hi = value >> 32;
233
234 asm (
235 "wrmsr\n"
236 :
237 : "a" (val_lo), "d" (val_hi), "c" (msr)
238 );
239 }
240
kvm_get_interrupt_ip(const struct kvm_interrupt_frame * ifrm)241 uintptr_t kvm_get_interrupt_ip(const struct kvm_interrupt_frame *ifrm)
242 {
243 return ifrm->eip;
244 }
245
kvm_is_svm_supported(void)246 int kvm_is_svm_supported(void)
247 {
248 struct kvm_cpuid buf;
249
250 kvm_get_cpuid(CPUID_GET_INPUT_RANGE, 0, &buf);
251
252 if (buf.eax < CPUID_GET_EXT_FEATURES)
253 return 0;
254
255 kvm_get_cpuid(CPUID_GET_EXT_FEATURES, 0, &buf);
256 return buf.ecx & 0x4;
257 }
258
kvm_get_svm_state(void)259 int kvm_get_svm_state(void)
260 {
261 return kvm_rdmsr(MSR_EFER) & EFER_SVME;
262 }
263
kvm_set_svm_state(int enabled)264 void kvm_set_svm_state(int enabled)
265 {
266 uint64_t value;
267
268 if (!kvm_is_svm_supported())
269 tst_brk(TCONF, "CPU does not support SVM");
270
271 if (kvm_rdmsr(MSR_VM_CR) & VM_CR_SVMDIS)
272 tst_brk(TCONF, "SVM is supported but disabled");
273
274 value = kvm_rdmsr(MSR_EFER);
275
276 if (enabled)
277 value |= EFER_SVME;
278 else
279 value &= ~EFER_SVME;
280
281 kvm_wrmsr(MSR_EFER, value);
282 }
283
kvm_alloc_vmcb(void)284 struct kvm_vmcb *kvm_alloc_vmcb(void)
285 {
286 struct kvm_vmcb *ret;
287
288 ret = tst_heap_alloc_aligned(sizeof(struct kvm_vmcb), PAGESIZE);
289 memset(ret, 0, sizeof(struct kvm_vmcb));
290 return ret;
291 }
292
kvm_init_svm(void)293 void kvm_init_svm(void)
294 {
295 kvm_set_svm_state(1);
296 kvm_wrmsr(MSR_VM_HSAVE_PA, (uintptr_t)kvm_alloc_vmcb());
297 }
298
kvm_vmcb_copy_gdt_descriptor(struct kvm_vmcb_descriptor * dst,unsigned int gdt_id)299 void kvm_vmcb_copy_gdt_descriptor(struct kvm_vmcb_descriptor *dst,
300 unsigned int gdt_id)
301 {
302 uint64_t baseaddr;
303 uint32_t limit;
304 unsigned int flags;
305
306 if (gdt_id >= KVM_GDT_SIZE)
307 tst_brk(TBROK, "GDT descriptor ID out of range");
308
309 kvm_parse_segment_descriptor(kvm_gdt + gdt_id, &baseaddr, &limit,
310 &flags);
311
312 if (!(flags & SEGFLAG_PRESENT)) {
313 memset(dst, 0, sizeof(struct kvm_vmcb_descriptor));
314 return;
315 }
316
317 if (flags & SEGFLAG_PAGE_LIMIT)
318 limit = (limit << 12) | 0xfff;
319
320 dst->selector = gdt_id << 3;
321 dst->attrib = flags;
322 dst->limit = limit;
323 dst->base = baseaddr;
324 }
325
kvm_vmcb_set_intercept(struct kvm_vmcb * vmcb,unsigned int id,unsigned int state)326 void kvm_vmcb_set_intercept(struct kvm_vmcb *vmcb, unsigned int id,
327 unsigned int state)
328 {
329 unsigned int addr = id / 8, bit = 1 << (id % 8);
330
331 if (id >= SVM_INTERCEPT_MAX)
332 tst_brk(TBROK, "Invalid SVM intercept ID");
333
334 if (state)
335 vmcb->intercepts[addr] |= bit;
336 else
337 vmcb->intercepts[addr] &= ~bit;
338 }
339
kvm_init_guest_vmcb(struct kvm_vmcb * vmcb,uint32_t asid,uint16_t ss,void * rsp,int (* guest_main)(void))340 void kvm_init_guest_vmcb(struct kvm_vmcb *vmcb, uint32_t asid, uint16_t ss,
341 void *rsp, int (*guest_main)(void))
342 {
343 struct kvm_cregs cregs;
344 struct kvm_sregs sregs;
345
346 kvm_read_cregs(&cregs);
347 kvm_read_sregs(&sregs);
348
349 kvm_vmcb_set_intercept(vmcb, SVM_INTERCEPT_VMRUN, 1);
350 kvm_vmcb_set_intercept(vmcb, SVM_INTERCEPT_HLT, 1);
351
352 kvm_vmcb_copy_gdt_descriptor(&vmcb->es, sregs.es >> 3);
353 kvm_vmcb_copy_gdt_descriptor(&vmcb->cs, sregs.cs >> 3);
354 kvm_vmcb_copy_gdt_descriptor(&vmcb->ss, ss);
355 kvm_vmcb_copy_gdt_descriptor(&vmcb->ds, sregs.ds >> 3);
356 kvm_vmcb_copy_gdt_descriptor(&vmcb->fs, sregs.fs >> 3);
357 kvm_vmcb_copy_gdt_descriptor(&vmcb->gs, sregs.gs >> 3);
358 vmcb->gdtr.base = (uintptr_t)kvm_gdt;
359 vmcb->gdtr.limit = (KVM_GDT_SIZE*sizeof(struct segment_descriptor)) - 1;
360 vmcb->idtr.base = (uintptr_t)kvm_idt;
361 vmcb->idtr.limit = (X86_INTR_COUNT*sizeof(struct intr_descriptor)) - 1;
362
363 vmcb->guest_asid = asid;
364 vmcb->efer = kvm_rdmsr(MSR_EFER);
365 vmcb->cr0 = cregs.cr0;
366 vmcb->cr3 = cregs.cr3;
367 vmcb->cr4 = cregs.cr4;
368 vmcb->rip = (uintptr_t)kvm_svm_guest_entry;
369 vmcb->rax = (uintptr_t)guest_main;
370 vmcb->rsp = (uintptr_t)rsp;
371 vmcb->rflags = 0x200; /* Interrupts enabled */
372 }
373
kvm_create_svm_vcpu(int (* guest_main)(void),int alloc_stack)374 struct kvm_svm_vcpu *kvm_create_svm_vcpu(int (*guest_main)(void),
375 int alloc_stack)
376 {
377 uint16_t ss = 0;
378 char *stack = NULL;
379 struct kvm_vmcb *vmcb;
380 struct kvm_svm_vcpu *ret;
381
382 vmcb = kvm_alloc_vmcb();
383
384 if (alloc_stack) {
385 stack = tst_heap_alloc_aligned(2 * PAGESIZE, PAGESIZE);
386 ss = kvm_create_stack_descriptor(kvm_gdt, KVM_GDT_SIZE, stack);
387 stack += 2 * PAGESIZE;
388 }
389
390 kvm_init_guest_vmcb(vmcb, 1, ss, stack, guest_main);
391 ret = tst_heap_alloc(sizeof(struct kvm_svm_vcpu));
392 memset(ret, 0, sizeof(struct kvm_svm_vcpu));
393 ret->vmcb = vmcb;
394 return ret;
395 }
396