• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2021 SUSE LLC <mdoucha@suse.cz>
4  *
5  * x86-specific KVM helper functions
6  */
7 
8 #include "kvm_x86_svm.h"
9 
10 void kvm_svm_guest_entry(void);
11 
12 struct kvm_interrupt_frame {
13 	uintptr_t eip, cs, eflags, esp, ss;
14 };
15 
16 const char *tst_interrupt_names[INTERRUPT_COUNT] = {
17 	"Division by zero",
18 	"Debug interrupt",
19 	"Non-maskable interrupt",
20 	"Breakpoint",
21 	"Arithmetic overflow",
22 	"Bound range exception",
23 	"Illegal instruction error",
24 	"Device not available error",
25 	"Double fault",
26 	NULL,
27 	"Invalid TSS error",
28 	"Segment not present error",
29 	"Stack segment fault",
30 	"General protection fault",
31 	"Page fault",
32 	NULL,
33 	"Floating point exception",
34 	"Alignment error",
35 	"Machine check exception",
36 	"SIMD floating point exception",
37 	"Virtualization exception",
38 	"Control protection exception",
39 	NULL,
40 	NULL,
41 	NULL,
42 	NULL,
43 	NULL,
44 	NULL,
45 	"Hypervisor injection exception",
46 	"VMM communication exception",
47 	"Security exception",
48 	NULL
49 };
50 
51 static uintptr_t intr_handlers[] = {
52 	(uintptr_t)kvm_handle_zerodiv,
53 	(uintptr_t)kvm_handle_debug,
54 	(uintptr_t)kvm_handle_nmi,
55 	(uintptr_t)kvm_handle_breakpoint,
56 	(uintptr_t)kvm_handle_overflow,
57 	(uintptr_t)kvm_handle_bound_range_exc,
58 	(uintptr_t)kvm_handle_bad_opcode,
59 	(uintptr_t)kvm_handle_device_error,
60 	(uintptr_t)kvm_handle_double_fault,
61 	(uintptr_t)kvm_handle_bad_exception,
62 	(uintptr_t)kvm_handle_invalid_tss,
63 	(uintptr_t)kvm_handle_segfault,
64 	(uintptr_t)kvm_handle_stack_fault,
65 	(uintptr_t)kvm_handle_gpf,
66 	(uintptr_t)kvm_handle_page_fault,
67 	(uintptr_t)kvm_handle_bad_exception,
68 	(uintptr_t)kvm_handle_fpu_error,
69 	(uintptr_t)kvm_handle_alignment_error,
70 	(uintptr_t)kvm_handle_machine_check,
71 	(uintptr_t)kvm_handle_simd_error,
72 	(uintptr_t)kvm_handle_virt_error,
73 	(uintptr_t)kvm_handle_cpe,
74 	(uintptr_t)kvm_handle_bad_exception,
75 	(uintptr_t)kvm_handle_bad_exception,
76 	(uintptr_t)kvm_handle_bad_exception,
77 	(uintptr_t)kvm_handle_bad_exception,
78 	(uintptr_t)kvm_handle_bad_exception,
79 	(uintptr_t)kvm_handle_bad_exception,
80 	(uintptr_t)kvm_handle_hv_injection,
81 	(uintptr_t)kvm_handle_vmm_comm,
82 	(uintptr_t)kvm_handle_security_error,
83 	(uintptr_t)kvm_handle_bad_exception,
84 	0
85 };
86 
kvm_set_intr_handler(unsigned int id,uintptr_t func)87 static void kvm_set_intr_handler(unsigned int id, uintptr_t func)
88 {
89 	memset(kvm_idt + id, 0, sizeof(kvm_idt[0]));
90 	kvm_idt[id].offset_lo = func & 0xffff;
91 	kvm_idt[id].offset_hi = func >> 16;
92 	kvm_idt[id].selector = 8;
93 	kvm_idt[id].flags = 0x8f;	/* type = 0xf, P = 1 */
94 }
95 
kvm_init_interrupts(void)96 void kvm_init_interrupts(void)
97 {
98 	int i;
99 
100 	for (i = 0; intr_handlers[i]; i++)
101 		kvm_set_intr_handler(i, intr_handlers[i]);
102 
103 	for (; i < X86_INTR_COUNT; i++)
104 		kvm_set_intr_handler(i, (uintptr_t)kvm_handle_bad_exception);
105 }
106 
kvm_get_page_address_pae(const struct page_table_entry_pae * entry)107 uintptr_t kvm_get_page_address_pae(const struct page_table_entry_pae *entry)
108 {
109 	if (!entry->present)
110 		return 0;
111 
112 	return entry->address << 12;
113 }
114 
115 #ifdef __x86_64__
kvm_set_segment_descriptor64(struct segment_descriptor64 * dst,uint64_t baseaddr,uint32_t limit,unsigned int flags)116 static void kvm_set_segment_descriptor64(struct segment_descriptor64 *dst,
117 	uint64_t baseaddr, uint32_t limit, unsigned int flags)
118 {
119 
120 	dst->baseaddr_lo = baseaddr & 0xffffff;
121 	dst->baseaddr_hi = baseaddr >> 24;
122 	dst->limit_lo = limit & 0xffff;
123 	dst->limit_hi = limit >> 16;
124 	dst->flags_lo = flags & 0xff;
125 	dst->flags_hi = (flags >> 8) & 0xf;
126 	dst->reserved = 0;
127 }
128 #endif
129 
kvm_set_segment_descriptor(struct segment_descriptor * dst,uint64_t baseaddr,uint32_t limit,unsigned int flags)130 void kvm_set_segment_descriptor(struct segment_descriptor *dst,
131 	uint64_t baseaddr, uint32_t limit, unsigned int flags)
132 {
133 	if (limit >> 20)
134 		tst_brk(TBROK, "Segment limit out of range");
135 
136 #ifdef __x86_64__
137 	/* System descriptors have double size in 64bit mode */
138 	if (!(flags & SEGFLAG_NSYSTEM)) {
139 		kvm_set_segment_descriptor64((struct segment_descriptor64 *)dst,
140 			baseaddr, limit, flags);
141 		return;
142 	}
143 #endif
144 
145 	if (baseaddr >> 32)
146 		tst_brk(TBROK, "Segment base address out of range");
147 
148 	dst->baseaddr_lo = baseaddr & 0xffffff;
149 	dst->baseaddr_hi = baseaddr >> 24;
150 	dst->limit_lo = limit & 0xffff;
151 	dst->limit_hi = limit >> 16;
152 	dst->flags_lo = flags & 0xff;
153 	dst->flags_hi = (flags >> 8) & 0xf;
154 }
155 
kvm_parse_segment_descriptor(struct segment_descriptor * src,uint64_t * baseaddr,uint32_t * limit,unsigned int * flags)156 void kvm_parse_segment_descriptor(struct segment_descriptor *src,
157 	uint64_t *baseaddr, uint32_t *limit, unsigned int *flags)
158 {
159 	if (baseaddr) {
160 		*baseaddr = (((uint64_t)src->baseaddr_hi) << 24) |
161 			src->baseaddr_lo;
162 	}
163 
164 	if (limit)
165 		*limit = (((uint32_t)src->limit_hi) << 16) | src->limit_lo;
166 
167 	if (flags)
168 		*flags = (((uint32_t)src->flags_hi) << 8) | src->flags_lo;
169 }
170 
kvm_find_free_descriptor(const struct segment_descriptor * table,size_t size)171 int kvm_find_free_descriptor(const struct segment_descriptor *table,
172 	size_t size)
173 {
174 	const struct segment_descriptor *ptr;
175 	size_t i;
176 
177 	for (i = 0, ptr = table; i < size; i++, ptr++) {
178 		if (!(ptr->flags_lo & SEGFLAG_PRESENT))
179 			return i;
180 
181 #ifdef __x86_64__
182 		/* System descriptors have double size in 64bit mode */
183 		if (!(ptr->flags_lo & SEGFLAG_NSYSTEM)) {
184 			ptr++;
185 			i++;
186 		}
187 #endif
188 	}
189 
190 	return -1;
191 }
192 
kvm_create_stack_descriptor(struct segment_descriptor * table,size_t tabsize,void * stack_base)193 unsigned int kvm_create_stack_descriptor(struct segment_descriptor *table,
194 	size_t tabsize, void *stack_base)
195 {
196 	int ret = kvm_find_free_descriptor(table, tabsize);
197 
198 	if (ret < 0)
199 		tst_brk(TBROK, "Descriptor table is full");
200 
201 	kvm_set_segment_descriptor(table + ret, 0,
202 		(((uintptr_t)stack_base) - 1) >> 12, SEGTYPE_STACK |
203 		SEGFLAG_PRESENT | SEGFLAG_32BIT | SEGFLAG_PAGE_LIMIT);
204 	return ret;
205 }
206 
kvm_get_cpuid(unsigned int eax,unsigned int ecx,struct kvm_cpuid * buf)207 void kvm_get_cpuid(unsigned int eax, unsigned int ecx, struct kvm_cpuid *buf)
208 {
209 	asm (
210 		"cpuid\n"
211 		: "=a" (buf->eax), "=b" (buf->ebx), "=c" (buf->ecx),
212 			"=d" (buf->edx)
213 		: "0" (eax), "2" (ecx)
214 	);
215 }
216 
kvm_rdmsr(unsigned int msr)217 uint64_t kvm_rdmsr(unsigned int msr)
218 {
219 	unsigned int ret_lo, ret_hi;
220 
221 	asm (
222 		"rdmsr\n"
223 		: "=a" (ret_lo), "=d" (ret_hi)
224 		: "c" (msr)
225 	);
226 
227 	return (((uint64_t)ret_hi) << 32) | ret_lo;
228 }
229 
kvm_wrmsr(unsigned int msr,uint64_t value)230 void kvm_wrmsr(unsigned int msr, uint64_t value)
231 {
232 	uint32_t val_lo = value & 0xffffffff, val_hi = value >> 32;
233 
234 	asm (
235 		"wrmsr\n"
236 		:
237 		: "a" (val_lo), "d" (val_hi), "c" (msr)
238 	);
239 }
240 
kvm_get_interrupt_ip(const struct kvm_interrupt_frame * ifrm)241 uintptr_t kvm_get_interrupt_ip(const struct kvm_interrupt_frame *ifrm)
242 {
243 	return ifrm->eip;
244 }
245 
kvm_is_svm_supported(void)246 int kvm_is_svm_supported(void)
247 {
248 	struct kvm_cpuid buf;
249 
250 	kvm_get_cpuid(CPUID_GET_INPUT_RANGE, 0, &buf);
251 
252 	if (buf.eax < CPUID_GET_EXT_FEATURES)
253 		return 0;
254 
255 	kvm_get_cpuid(CPUID_GET_EXT_FEATURES, 0, &buf);
256 	return buf.ecx & 0x4;
257 }
258 
kvm_get_svm_state(void)259 int kvm_get_svm_state(void)
260 {
261 	return kvm_rdmsr(MSR_EFER) & EFER_SVME;
262 }
263 
kvm_set_svm_state(int enabled)264 void kvm_set_svm_state(int enabled)
265 {
266 	uint64_t value;
267 
268 	if (!kvm_is_svm_supported())
269 		tst_brk(TCONF, "CPU does not support SVM");
270 
271 	if (kvm_rdmsr(MSR_VM_CR) & VM_CR_SVMDIS)
272 		tst_brk(TCONF, "SVM is supported but disabled");
273 
274 	value = kvm_rdmsr(MSR_EFER);
275 
276 	if (enabled)
277 		value |= EFER_SVME;
278 	else
279 		value &= ~EFER_SVME;
280 
281 	kvm_wrmsr(MSR_EFER, value);
282 }
283 
kvm_alloc_vmcb(void)284 struct kvm_vmcb *kvm_alloc_vmcb(void)
285 {
286 	struct kvm_vmcb *ret;
287 
288 	ret = tst_heap_alloc_aligned(sizeof(struct kvm_vmcb), PAGESIZE);
289 	memset(ret, 0, sizeof(struct kvm_vmcb));
290 	return ret;
291 }
292 
kvm_init_svm(void)293 void kvm_init_svm(void)
294 {
295 	kvm_set_svm_state(1);
296 	kvm_wrmsr(MSR_VM_HSAVE_PA, (uintptr_t)kvm_alloc_vmcb());
297 }
298 
kvm_vmcb_copy_gdt_descriptor(struct kvm_vmcb_descriptor * dst,unsigned int gdt_id)299 void kvm_vmcb_copy_gdt_descriptor(struct kvm_vmcb_descriptor *dst,
300 	unsigned int gdt_id)
301 {
302 	uint64_t baseaddr;
303 	uint32_t limit;
304 	unsigned int flags;
305 
306 	if (gdt_id >= KVM_GDT_SIZE)
307 		tst_brk(TBROK, "GDT descriptor ID out of range");
308 
309 	kvm_parse_segment_descriptor(kvm_gdt + gdt_id, &baseaddr, &limit,
310 		&flags);
311 
312 	if (!(flags & SEGFLAG_PRESENT)) {
313 		memset(dst, 0, sizeof(struct kvm_vmcb_descriptor));
314 		return;
315 	}
316 
317 	if (flags & SEGFLAG_PAGE_LIMIT)
318 		limit = (limit << 12) | 0xfff;
319 
320 	dst->selector = gdt_id << 3;
321 	dst->attrib = flags;
322 	dst->limit = limit;
323 	dst->base = baseaddr;
324 }
325 
kvm_vmcb_set_intercept(struct kvm_vmcb * vmcb,unsigned int id,unsigned int state)326 void kvm_vmcb_set_intercept(struct kvm_vmcb *vmcb, unsigned int id,
327 	unsigned int state)
328 {
329 	unsigned int addr = id / 8, bit = 1 << (id % 8);
330 
331 	if (id >= SVM_INTERCEPT_MAX)
332 		tst_brk(TBROK, "Invalid SVM intercept ID");
333 
334 	if (state)
335 		vmcb->intercepts[addr] |= bit;
336 	else
337 		vmcb->intercepts[addr] &= ~bit;
338 }
339 
kvm_init_guest_vmcb(struct kvm_vmcb * vmcb,uint32_t asid,uint16_t ss,void * rsp,int (* guest_main)(void))340 void kvm_init_guest_vmcb(struct kvm_vmcb *vmcb, uint32_t asid, uint16_t ss,
341 	void *rsp, int (*guest_main)(void))
342 {
343 	struct kvm_cregs cregs;
344 	struct kvm_sregs sregs;
345 
346 	kvm_read_cregs(&cregs);
347 	kvm_read_sregs(&sregs);
348 
349 	kvm_vmcb_set_intercept(vmcb, SVM_INTERCEPT_VMRUN, 1);
350 	kvm_vmcb_set_intercept(vmcb, SVM_INTERCEPT_HLT, 1);
351 
352 	kvm_vmcb_copy_gdt_descriptor(&vmcb->es, sregs.es >> 3);
353 	kvm_vmcb_copy_gdt_descriptor(&vmcb->cs, sregs.cs >> 3);
354 	kvm_vmcb_copy_gdt_descriptor(&vmcb->ss, ss);
355 	kvm_vmcb_copy_gdt_descriptor(&vmcb->ds, sregs.ds >> 3);
356 	kvm_vmcb_copy_gdt_descriptor(&vmcb->fs, sregs.fs >> 3);
357 	kvm_vmcb_copy_gdt_descriptor(&vmcb->gs, sregs.gs >> 3);
358 	vmcb->gdtr.base = (uintptr_t)kvm_gdt;
359 	vmcb->gdtr.limit = (KVM_GDT_SIZE*sizeof(struct segment_descriptor)) - 1;
360 	vmcb->idtr.base = (uintptr_t)kvm_idt;
361 	vmcb->idtr.limit = (X86_INTR_COUNT*sizeof(struct intr_descriptor)) - 1;
362 
363 	vmcb->guest_asid = asid;
364 	vmcb->efer = kvm_rdmsr(MSR_EFER);
365 	vmcb->cr0 = cregs.cr0;
366 	vmcb->cr3 = cregs.cr3;
367 	vmcb->cr4 = cregs.cr4;
368 	vmcb->rip = (uintptr_t)kvm_svm_guest_entry;
369 	vmcb->rax = (uintptr_t)guest_main;
370 	vmcb->rsp = (uintptr_t)rsp;
371 	vmcb->rflags = 0x200;	/* Interrupts enabled */
372 }
373 
kvm_create_svm_vcpu(int (* guest_main)(void),int alloc_stack)374 struct kvm_svm_vcpu *kvm_create_svm_vcpu(int (*guest_main)(void),
375 	int alloc_stack)
376 {
377 	uint16_t ss = 0;
378 	char *stack = NULL;
379 	struct kvm_vmcb *vmcb;
380 	struct kvm_svm_vcpu *ret;
381 
382 	vmcb = kvm_alloc_vmcb();
383 
384 	if (alloc_stack) {
385 		stack = tst_heap_alloc_aligned(2 * PAGESIZE, PAGESIZE);
386 		ss = kvm_create_stack_descriptor(kvm_gdt, KVM_GDT_SIZE, stack);
387 		stack += 2 * PAGESIZE;
388 	}
389 
390 	kvm_init_guest_vmcb(vmcb, 1, ss, stack, guest_main);
391 	ret = tst_heap_alloc(sizeof(struct kvm_svm_vcpu));
392 	memset(ret, 0, sizeof(struct kvm_svm_vcpu));
393 	ret->vmcb = vmcb;
394 	return ret;
395 }
396