1 // SPDX-License-Identifier: GPL-2.0
2 #define _GNU_SOURCE /* for program_invocation_short_name */
3 #include <fcntl.h>
4 #include <pthread.h>
5 #include <sched.h>
6 #include <semaphore.h>
7 #include <signal.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <sys/ioctl.h>
12 #include <sys/mman.h>
13
14 #include <linux/compiler.h>
15
16 #include <test_util.h>
17 #include <kvm_util.h>
18 #include <processor.h>
19
20 #define VCPU_ID 0
21
22 /*
23 * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a
24 * 2MB sized and aligned region so that the initial region corresponds to
25 * exactly one large page.
26 */
27 #define MEM_REGION_SIZE 0x200000
28
29 #ifdef __x86_64__
30 /*
31 * Somewhat arbitrary location and slot, intended to not overlap anything.
32 */
33 #define MEM_REGION_GPA 0xc0000000
34 #define MEM_REGION_SLOT 10
35
36 static const uint64_t MMIO_VAL = 0xbeefull;
37
38 extern const uint64_t final_rip_start;
39 extern const uint64_t final_rip_end;
40
41 static sem_t vcpu_ready;
42
guest_spin_on_val(uint64_t spin_val)43 static inline uint64_t guest_spin_on_val(uint64_t spin_val)
44 {
45 uint64_t val;
46
47 do {
48 val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA));
49 } while (val == spin_val);
50
51 GUEST_SYNC(0);
52 return val;
53 }
54
vcpu_worker(void * data)55 static void *vcpu_worker(void *data)
56 {
57 struct kvm_vm *vm = data;
58 struct kvm_run *run;
59 struct ucall uc;
60 uint64_t cmd;
61
62 /*
63 * Loop until the guest is done. Re-enter the guest on all MMIO exits,
64 * which will occur if the guest attempts to access a memslot after it
65 * has been deleted or while it is being moved .
66 */
67 run = vcpu_state(vm, VCPU_ID);
68
69 while (1) {
70 vcpu_run(vm, VCPU_ID);
71
72 if (run->exit_reason == KVM_EXIT_IO) {
73 cmd = get_ucall(vm, VCPU_ID, &uc);
74 if (cmd != UCALL_SYNC)
75 break;
76
77 sem_post(&vcpu_ready);
78 continue;
79 }
80
81 if (run->exit_reason != KVM_EXIT_MMIO)
82 break;
83
84 TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write");
85 TEST_ASSERT(run->mmio.len == 8,
86 "Unexpected exit mmio size = %u", run->mmio.len);
87
88 TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA,
89 "Unexpected exit mmio address = 0x%llx",
90 run->mmio.phys_addr);
91 memcpy(run->mmio.data, &MMIO_VAL, 8);
92 }
93
94 if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT)
95 TEST_FAIL("%s at %s:%ld, val = %lu", (const char *)uc.args[0],
96 __FILE__, uc.args[1], uc.args[2]);
97
98 return NULL;
99 }
100
wait_for_vcpu(void)101 static void wait_for_vcpu(void)
102 {
103 struct timespec ts;
104
105 TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),
106 "clock_gettime() failed: %d\n", errno);
107
108 ts.tv_sec += 2;
109 TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),
110 "sem_timedwait() failed: %d\n", errno);
111
112 /* Wait for the vCPU thread to reenter the guest. */
113 usleep(100000);
114 }
115
spawn_vm(pthread_t * vcpu_thread,void * guest_code)116 static struct kvm_vm *spawn_vm(pthread_t *vcpu_thread, void *guest_code)
117 {
118 struct kvm_vm *vm;
119 uint64_t *hva;
120 uint64_t gpa;
121
122 vm = vm_create_default(VCPU_ID, 0, guest_code);
123
124 vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid());
125
126 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
127 MEM_REGION_GPA, MEM_REGION_SLOT,
128 MEM_REGION_SIZE / getpagesize(), 0);
129
130 /*
131 * Allocate and map two pages so that the GPA accessed by guest_code()
132 * stays valid across the memslot move.
133 */
134 gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT);
135 TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n");
136
137 virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2, 0);
138
139 /* Ditto for the host mapping so that both pages can be zeroed. */
140 hva = addr_gpa2hva(vm, MEM_REGION_GPA);
141 memset(hva, 0, 2 * 4096);
142
143 pthread_create(vcpu_thread, NULL, vcpu_worker, vm);
144
145 /* Ensure the guest thread is spun up. */
146 wait_for_vcpu();
147
148 return vm;
149 }
150
151
guest_code_move_memory_region(void)152 static void guest_code_move_memory_region(void)
153 {
154 uint64_t val;
155
156 GUEST_SYNC(0);
157
158 /*
159 * Spin until the memory region starts getting moved to a
160 * misaligned address.
161 * Every region move may or may not trigger MMIO, as the
162 * window where the memslot is invalid is usually quite small.
163 */
164 val = guest_spin_on_val(0);
165 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val);
166
167 /* Spin until the misaligning memory region move completes. */
168 val = guest_spin_on_val(MMIO_VAL);
169 GUEST_ASSERT_1(val == 1 || val == 0, val);
170
171 /* Spin until the memory region starts to get re-aligned. */
172 val = guest_spin_on_val(0);
173 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val);
174
175 /* Spin until the re-aligning memory region move completes. */
176 val = guest_spin_on_val(MMIO_VAL);
177 GUEST_ASSERT_1(val == 1, val);
178
179 GUEST_DONE();
180 }
181
test_move_memory_region(void)182 static void test_move_memory_region(void)
183 {
184 pthread_t vcpu_thread;
185 struct kvm_vm *vm;
186 uint64_t *hva;
187
188 vm = spawn_vm(&vcpu_thread, guest_code_move_memory_region);
189
190 hva = addr_gpa2hva(vm, MEM_REGION_GPA);
191
192 /*
193 * Shift the region's base GPA. The guest should not see "2" as the
194 * hva->gpa translation is misaligned, i.e. the guest is accessing a
195 * different host pfn.
196 */
197 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096);
198 WRITE_ONCE(*hva, 2);
199
200 /*
201 * The guest _might_ see an invalid memslot and trigger MMIO, but it's
202 * a tiny window. Spin and defer the sync until the memslot is
203 * restored and guest behavior is once again deterministic.
204 */
205 usleep(100000);
206
207 /*
208 * Note, value in memory needs to be changed *before* restoring the
209 * memslot, else the guest could race the update and see "2".
210 */
211 WRITE_ONCE(*hva, 1);
212
213 /* Restore the original base, the guest should see "1". */
214 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA);
215 wait_for_vcpu();
216 /* Defered sync from when the memslot was misaligned (above). */
217 wait_for_vcpu();
218
219 pthread_join(vcpu_thread, NULL);
220
221 kvm_vm_free(vm);
222 }
223
guest_code_delete_memory_region(void)224 static void guest_code_delete_memory_region(void)
225 {
226 uint64_t val;
227
228 GUEST_SYNC(0);
229
230 /* Spin until the memory region is deleted. */
231 val = guest_spin_on_val(0);
232 GUEST_ASSERT_1(val == MMIO_VAL, val);
233
234 /* Spin until the memory region is recreated. */
235 val = guest_spin_on_val(MMIO_VAL);
236 GUEST_ASSERT_1(val == 0, val);
237
238 /* Spin until the memory region is deleted. */
239 val = guest_spin_on_val(0);
240 GUEST_ASSERT_1(val == MMIO_VAL, val);
241
242 asm("1:\n\t"
243 ".pushsection .rodata\n\t"
244 ".global final_rip_start\n\t"
245 "final_rip_start: .quad 1b\n\t"
246 ".popsection");
247
248 /* Spin indefinitely (until the code memslot is deleted). */
249 guest_spin_on_val(MMIO_VAL);
250
251 asm("1:\n\t"
252 ".pushsection .rodata\n\t"
253 ".global final_rip_end\n\t"
254 "final_rip_end: .quad 1b\n\t"
255 ".popsection");
256
257 GUEST_ASSERT_1(0, 0);
258 }
259
test_delete_memory_region(void)260 static void test_delete_memory_region(void)
261 {
262 pthread_t vcpu_thread;
263 struct kvm_regs regs;
264 struct kvm_run *run;
265 struct kvm_vm *vm;
266
267 vm = spawn_vm(&vcpu_thread, guest_code_delete_memory_region);
268
269 /* Delete the memory region, the guest should not die. */
270 vm_mem_region_delete(vm, MEM_REGION_SLOT);
271 wait_for_vcpu();
272
273 /* Recreate the memory region. The guest should see "0". */
274 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
275 MEM_REGION_GPA, MEM_REGION_SLOT,
276 MEM_REGION_SIZE / getpagesize(), 0);
277 wait_for_vcpu();
278
279 /* Delete the region again so that there's only one memslot left. */
280 vm_mem_region_delete(vm, MEM_REGION_SLOT);
281 wait_for_vcpu();
282
283 /*
284 * Delete the primary memslot. This should cause an emulation error or
285 * shutdown due to the page tables getting nuked.
286 */
287 vm_mem_region_delete(vm, 0);
288
289 pthread_join(vcpu_thread, NULL);
290
291 run = vcpu_state(vm, VCPU_ID);
292
293 TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN ||
294 run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
295 "Unexpected exit reason = %d", run->exit_reason);
296
297 vcpu_regs_get(vm, VCPU_ID, ®s);
298
299 /*
300 * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already,
301 * so the instruction pointer would point to the reset vector.
302 */
303 if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR)
304 TEST_ASSERT(regs.rip >= final_rip_start &&
305 regs.rip < final_rip_end,
306 "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n",
307 final_rip_start, final_rip_end, regs.rip);
308
309 kvm_vm_free(vm);
310 }
311
test_zero_memory_regions(void)312 static void test_zero_memory_regions(void)
313 {
314 struct kvm_run *run;
315 struct kvm_vm *vm;
316
317 pr_info("Testing KVM_RUN with zero added memory regions\n");
318
319 vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
320 vm_vcpu_add(vm, VCPU_ID);
321
322 TEST_ASSERT(!ioctl(vm_get_fd(vm), KVM_SET_NR_MMU_PAGES, 64),
323 "KVM_SET_NR_MMU_PAGES failed, errno = %d\n", errno);
324 vcpu_run(vm, VCPU_ID);
325
326 run = vcpu_state(vm, VCPU_ID);
327 TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
328 "Unexpected exit_reason = %u\n", run->exit_reason);
329
330 kvm_vm_free(vm);
331 }
332 #endif /* __x86_64__ */
333
334 /*
335 * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any
336 * tentative to add further slots should fail.
337 */
test_add_max_memory_regions(void)338 static void test_add_max_memory_regions(void)
339 {
340 int ret;
341 struct kvm_vm *vm;
342 uint32_t max_mem_slots;
343 uint32_t slot;
344 uint64_t guest_addr = 0x0;
345 uint64_t mem_reg_npages;
346 void *mem;
347
348 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
349 TEST_ASSERT(max_mem_slots > 0,
350 "KVM_CAP_NR_MEMSLOTS should be greater than 0");
351 pr_info("Allowed number of memory slots: %i\n", max_mem_slots);
352
353 vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
354
355 mem_reg_npages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, MEM_REGION_SIZE);
356
357 /* Check it can be added memory slots up to the maximum allowed */
358 pr_info("Adding slots 0..%i, each memory region with %dK size\n",
359 (max_mem_slots - 1), MEM_REGION_SIZE >> 10);
360 for (slot = 0; slot < max_mem_slots; slot++) {
361 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
362 guest_addr, slot, mem_reg_npages,
363 0);
364 guest_addr += MEM_REGION_SIZE;
365 }
366
367 /* Check it cannot be added memory slots beyond the limit */
368 mem = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE,
369 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
370 TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host");
371
372 ret = ioctl(vm_get_fd(vm), KVM_SET_USER_MEMORY_REGION,
373 &(struct kvm_userspace_memory_region) {slot, 0, guest_addr,
374 MEM_REGION_SIZE, (uint64_t) mem});
375 TEST_ASSERT(ret == -1 && errno == EINVAL,
376 "Adding one more memory slot should fail with EINVAL");
377
378 munmap(mem, MEM_REGION_SIZE);
379 kvm_vm_free(vm);
380 }
381
main(int argc,char * argv[])382 int main(int argc, char *argv[])
383 {
384 #ifdef __x86_64__
385 int i, loops;
386 #endif
387
388 /* Tell stdout not to buffer its content */
389 setbuf(stdout, NULL);
390
391 #ifdef __x86_64__
392 /*
393 * FIXME: the zero-memslot test fails on aarch64 and s390x because
394 * KVM_RUN fails with ENOEXEC or EFAULT.
395 */
396 test_zero_memory_regions();
397 #endif
398
399 test_add_max_memory_regions();
400
401 #ifdef __x86_64__
402 if (argc > 1)
403 loops = atoi(argv[1]);
404 else
405 loops = 10;
406
407 pr_info("Testing MOVE of in-use region, %d loops\n", loops);
408 for (i = 0; i < loops; i++)
409 test_move_memory_region();
410
411 pr_info("Testing DELETE of in-use region, %d loops\n", loops);
412 for (i = 0; i < loops; i++)
413 test_delete_memory_region();
414 #endif
415
416 return 0;
417 }
418