1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * AMD SVM-SEV support
6 *
7 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
8 */
9
10 #include <linux/kvm_types.h>
11 #include <linux/kvm_host.h>
12 #include <linux/kernel.h>
13 #include <linux/highmem.h>
14 #include <linux/psp-sev.h>
15 #include <linux/pagemap.h>
16 #include <linux/swap.h>
17
18 #include "x86.h"
19 #include "svm.h"
20
21 static int sev_flush_asids(void);
22 static DECLARE_RWSEM(sev_deactivate_lock);
23 static DEFINE_MUTEX(sev_bitmap_lock);
24 unsigned int max_sev_asid;
25 static unsigned int min_sev_asid;
26 static unsigned long *sev_asid_bitmap;
27 static unsigned long *sev_reclaim_asid_bitmap;
28 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
29
30 struct enc_region {
31 struct list_head list;
32 unsigned long npages;
33 struct page **pages;
34 unsigned long uaddr;
35 unsigned long size;
36 };
37
sev_flush_asids(void)38 static int sev_flush_asids(void)
39 {
40 int ret, error = 0;
41
42 /*
43 * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
44 * so it must be guarded.
45 */
46 down_write(&sev_deactivate_lock);
47
48 wbinvd_on_all_cpus();
49 ret = sev_guest_df_flush(&error);
50
51 up_write(&sev_deactivate_lock);
52
53 if (ret)
54 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error);
55
56 return ret;
57 }
58
59 /* Must be called with the sev_bitmap_lock held */
__sev_recycle_asids(void)60 static bool __sev_recycle_asids(void)
61 {
62 int pos;
63
64 /* Check if there are any ASIDs to reclaim before performing a flush */
65 pos = find_next_bit(sev_reclaim_asid_bitmap,
66 max_sev_asid, min_sev_asid - 1);
67 if (pos >= max_sev_asid)
68 return false;
69
70 if (sev_flush_asids())
71 return false;
72
73 bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
74 max_sev_asid);
75 bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
76
77 return true;
78 }
79
sev_asid_new(void)80 static int sev_asid_new(void)
81 {
82 bool retry = true;
83 int pos;
84
85 mutex_lock(&sev_bitmap_lock);
86
87 /*
88 * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
89 */
90 again:
91 pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
92 if (pos >= max_sev_asid) {
93 if (retry && __sev_recycle_asids()) {
94 retry = false;
95 goto again;
96 }
97 mutex_unlock(&sev_bitmap_lock);
98 return -EBUSY;
99 }
100
101 __set_bit(pos, sev_asid_bitmap);
102
103 mutex_unlock(&sev_bitmap_lock);
104
105 return pos + 1;
106 }
107
sev_get_asid(struct kvm * kvm)108 static int sev_get_asid(struct kvm *kvm)
109 {
110 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
111
112 return sev->asid;
113 }
114
sev_asid_free(int asid)115 static void sev_asid_free(int asid)
116 {
117 struct svm_cpu_data *sd;
118 int cpu, pos;
119
120 mutex_lock(&sev_bitmap_lock);
121
122 pos = asid - 1;
123 __set_bit(pos, sev_reclaim_asid_bitmap);
124
125 for_each_possible_cpu(cpu) {
126 sd = per_cpu(svm_data, cpu);
127 sd->sev_vmcbs[asid] = NULL;
128 }
129
130 mutex_unlock(&sev_bitmap_lock);
131 }
132
sev_decommission(unsigned int handle)133 static void sev_decommission(unsigned int handle)
134 {
135 struct sev_data_decommission *decommission;
136
137 if (!handle)
138 return;
139
140 decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
141 if (!decommission)
142 return;
143
144 decommission->handle = handle;
145 sev_guest_decommission(decommission, NULL);
146
147 kfree(decommission);
148 }
149
sev_unbind_asid(struct kvm * kvm,unsigned int handle)150 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
151 {
152 struct sev_data_deactivate *data;
153
154 if (!handle)
155 return;
156
157 data = kzalloc(sizeof(*data), GFP_KERNEL);
158 if (!data)
159 return;
160
161 /* deactivate handle */
162 data->handle = handle;
163
164 /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
165 down_read(&sev_deactivate_lock);
166 sev_guest_deactivate(data, NULL);
167 up_read(&sev_deactivate_lock);
168
169 kfree(data);
170
171 sev_decommission(handle);
172 }
173
sev_guest_init(struct kvm * kvm,struct kvm_sev_cmd * argp)174 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
175 {
176 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
177 int asid, ret;
178
179 if (kvm->created_vcpus)
180 return -EINVAL;
181
182 ret = -EBUSY;
183 if (unlikely(sev->active))
184 return ret;
185
186 asid = sev_asid_new();
187 if (asid < 0)
188 return ret;
189
190 ret = sev_platform_init(&argp->error);
191 if (ret)
192 goto e_free;
193
194 sev->active = true;
195 sev->asid = asid;
196 INIT_LIST_HEAD(&sev->regions_list);
197
198 return 0;
199
200 e_free:
201 sev_asid_free(asid);
202 return ret;
203 }
204
sev_bind_asid(struct kvm * kvm,unsigned int handle,int * error)205 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
206 {
207 struct sev_data_activate *data;
208 int asid = sev_get_asid(kvm);
209 int ret;
210
211 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
212 if (!data)
213 return -ENOMEM;
214
215 /* activate ASID on the given handle */
216 data->handle = handle;
217 data->asid = asid;
218 ret = sev_guest_activate(data, error);
219 kfree(data);
220
221 return ret;
222 }
223
__sev_issue_cmd(int fd,int id,void * data,int * error)224 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
225 {
226 struct fd f;
227 int ret;
228
229 f = fdget(fd);
230 if (!f.file)
231 return -EBADF;
232
233 ret = sev_issue_cmd_external_user(f.file, id, data, error);
234
235 fdput(f);
236 return ret;
237 }
238
sev_issue_cmd(struct kvm * kvm,int id,void * data,int * error)239 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
240 {
241 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
242
243 return __sev_issue_cmd(sev->fd, id, data, error);
244 }
245
sev_launch_start(struct kvm * kvm,struct kvm_sev_cmd * argp)246 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
247 {
248 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
249 struct sev_data_launch_start *start;
250 struct kvm_sev_launch_start params;
251 void *dh_blob, *session_blob;
252 int *error = &argp->error;
253 int ret;
254
255 if (!sev_guest(kvm))
256 return -ENOTTY;
257
258 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params)))
259 return -EFAULT;
260
261 start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
262 if (!start)
263 return -ENOMEM;
264
265 dh_blob = NULL;
266 if (params.dh_uaddr) {
267 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
268 if (IS_ERR(dh_blob)) {
269 ret = PTR_ERR(dh_blob);
270 goto e_free;
271 }
272
273 start->dh_cert_address = __sme_set(__pa(dh_blob));
274 start->dh_cert_len = params.dh_len;
275 }
276
277 session_blob = NULL;
278 if (params.session_uaddr) {
279 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
280 if (IS_ERR(session_blob)) {
281 ret = PTR_ERR(session_blob);
282 goto e_free_dh;
283 }
284
285 start->session_address = __sme_set(__pa(session_blob));
286 start->session_len = params.session_len;
287 }
288
289 start->handle = params.handle;
290 start->policy = params.policy;
291
292 /* create memory encryption context */
293 ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
294 if (ret)
295 goto e_free_session;
296
297 /* Bind ASID to this guest */
298 ret = sev_bind_asid(kvm, start->handle, error);
299 if (ret) {
300 sev_decommission(start->handle);
301 goto e_free_session;
302 }
303
304 /* return handle to userspace */
305 params.handle = start->handle;
306 if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params))) {
307 sev_unbind_asid(kvm, start->handle);
308 ret = -EFAULT;
309 goto e_free_session;
310 }
311
312 sev->handle = start->handle;
313 sev->fd = argp->sev_fd;
314
315 e_free_session:
316 kfree(session_blob);
317 e_free_dh:
318 kfree(dh_blob);
319 e_free:
320 kfree(start);
321 return ret;
322 }
323
sev_pin_memory(struct kvm * kvm,unsigned long uaddr,unsigned long ulen,unsigned long * n,int write)324 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
325 unsigned long ulen, unsigned long *n,
326 int write)
327 {
328 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
329 unsigned long npages, size;
330 int npinned;
331 unsigned long locked, lock_limit;
332 struct page **pages;
333 unsigned long first, last;
334 int ret;
335
336 lockdep_assert_held(&kvm->lock);
337
338 if (ulen == 0 || uaddr + ulen < uaddr)
339 return ERR_PTR(-EINVAL);
340
341 /* Calculate number of pages. */
342 first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
343 last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
344 npages = (last - first + 1);
345
346 locked = sev->pages_locked + npages;
347 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
348 if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
349 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
350 return ERR_PTR(-ENOMEM);
351 }
352
353 if (WARN_ON_ONCE(npages > INT_MAX))
354 return ERR_PTR(-EINVAL);
355
356 /* Avoid using vmalloc for smaller buffers. */
357 size = npages * sizeof(struct page *);
358 if (size > PAGE_SIZE)
359 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
360 else
361 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
362
363 if (!pages)
364 return ERR_PTR(-ENOMEM);
365
366 /* Pin the user virtual address. */
367 npinned = pin_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
368 if (npinned != npages) {
369 pr_err("SEV: Failure locking %lu pages.\n", npages);
370 ret = -ENOMEM;
371 goto err;
372 }
373
374 *n = npages;
375 sev->pages_locked = locked;
376
377 return pages;
378
379 err:
380 if (npinned > 0)
381 unpin_user_pages(pages, npinned);
382
383 kvfree(pages);
384 return ERR_PTR(ret);
385 }
386
sev_unpin_memory(struct kvm * kvm,struct page ** pages,unsigned long npages)387 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
388 unsigned long npages)
389 {
390 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
391
392 unpin_user_pages(pages, npages);
393 kvfree(pages);
394 sev->pages_locked -= npages;
395 }
396
sev_clflush_pages(struct page * pages[],unsigned long npages)397 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
398 {
399 uint8_t *page_virtual;
400 unsigned long i;
401
402 if (this_cpu_has(X86_FEATURE_SME_COHERENT) || npages == 0 ||
403 pages == NULL)
404 return;
405
406 for (i = 0; i < npages; i++) {
407 page_virtual = kmap_atomic(pages[i]);
408 clflush_cache_range(page_virtual, PAGE_SIZE);
409 kunmap_atomic(page_virtual);
410 }
411 }
412
get_num_contig_pages(unsigned long idx,struct page ** inpages,unsigned long npages)413 static unsigned long get_num_contig_pages(unsigned long idx,
414 struct page **inpages, unsigned long npages)
415 {
416 unsigned long paddr, next_paddr;
417 unsigned long i = idx + 1, pages = 1;
418
419 /* find the number of contiguous pages starting from idx */
420 paddr = __sme_page_pa(inpages[idx]);
421 while (i < npages) {
422 next_paddr = __sme_page_pa(inpages[i++]);
423 if ((paddr + PAGE_SIZE) == next_paddr) {
424 pages++;
425 paddr = next_paddr;
426 continue;
427 }
428 break;
429 }
430
431 return pages;
432 }
433
sev_launch_update_data(struct kvm * kvm,struct kvm_sev_cmd * argp)434 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
435 {
436 unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
437 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
438 struct kvm_sev_launch_update_data params;
439 struct sev_data_launch_update_data *data;
440 struct page **inpages;
441 int ret;
442
443 if (!sev_guest(kvm))
444 return -ENOTTY;
445
446 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params)))
447 return -EFAULT;
448
449 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
450 if (!data)
451 return -ENOMEM;
452
453 vaddr = params.uaddr;
454 size = params.len;
455 vaddr_end = vaddr + size;
456
457 /* Lock the user memory. */
458 inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
459 if (IS_ERR(inpages)) {
460 ret = PTR_ERR(inpages);
461 goto e_free;
462 }
463
464 /*
465 * Flush (on non-coherent CPUs) before LAUNCH_UPDATE encrypts pages in
466 * place; the cache may contain the data that was written unencrypted.
467 */
468 sev_clflush_pages(inpages, npages);
469
470 for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
471 int offset, len;
472
473 /*
474 * If the user buffer is not page-aligned, calculate the offset
475 * within the page.
476 */
477 offset = vaddr & (PAGE_SIZE - 1);
478
479 /* Calculate the number of pages that can be encrypted in one go. */
480 pages = get_num_contig_pages(i, inpages, npages);
481
482 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
483
484 data->handle = sev->handle;
485 data->len = len;
486 data->address = __sme_page_pa(inpages[i]) + offset;
487 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
488 if (ret)
489 goto e_unpin;
490
491 size -= len;
492 next_vaddr = vaddr + len;
493 }
494
495 e_unpin:
496 /* content of memory is updated, mark pages dirty */
497 for (i = 0; i < npages; i++) {
498 set_page_dirty_lock(inpages[i]);
499 mark_page_accessed(inpages[i]);
500 }
501 /* unlock the user pages */
502 sev_unpin_memory(kvm, inpages, npages);
503 e_free:
504 kfree(data);
505 return ret;
506 }
507
sev_launch_measure(struct kvm * kvm,struct kvm_sev_cmd * argp)508 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
509 {
510 void __user *measure = (void __user *)(uintptr_t)argp->data;
511 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
512 struct sev_data_launch_measure *data;
513 struct kvm_sev_launch_measure params;
514 void __user *p = NULL;
515 void *blob = NULL;
516 int ret;
517
518 if (!sev_guest(kvm))
519 return -ENOTTY;
520
521 if (copy_from_user(¶ms, measure, sizeof(params)))
522 return -EFAULT;
523
524 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
525 if (!data)
526 return -ENOMEM;
527
528 /* User wants to query the blob length */
529 if (!params.len)
530 goto cmd;
531
532 p = (void __user *)(uintptr_t)params.uaddr;
533 if (p) {
534 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
535 ret = -EINVAL;
536 goto e_free;
537 }
538
539 ret = -ENOMEM;
540 blob = kmalloc(params.len, GFP_KERNEL);
541 if (!blob)
542 goto e_free;
543
544 data->address = __psp_pa(blob);
545 data->len = params.len;
546 }
547
548 cmd:
549 data->handle = sev->handle;
550 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
551
552 /*
553 * If we query the session length, FW responded with expected data.
554 */
555 if (!params.len)
556 goto done;
557
558 if (ret)
559 goto e_free_blob;
560
561 if (blob) {
562 if (copy_to_user(p, blob, params.len))
563 ret = -EFAULT;
564 }
565
566 done:
567 params.len = data->len;
568 if (copy_to_user(measure, ¶ms, sizeof(params)))
569 ret = -EFAULT;
570 e_free_blob:
571 kfree(blob);
572 e_free:
573 kfree(data);
574 return ret;
575 }
576
sev_launch_finish(struct kvm * kvm,struct kvm_sev_cmd * argp)577 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
578 {
579 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
580 struct sev_data_launch_finish *data;
581 int ret;
582
583 if (!sev_guest(kvm))
584 return -ENOTTY;
585
586 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
587 if (!data)
588 return -ENOMEM;
589
590 data->handle = sev->handle;
591 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
592
593 kfree(data);
594 return ret;
595 }
596
sev_guest_status(struct kvm * kvm,struct kvm_sev_cmd * argp)597 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
598 {
599 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
600 struct kvm_sev_guest_status params;
601 struct sev_data_guest_status *data;
602 int ret;
603
604 if (!sev_guest(kvm))
605 return -ENOTTY;
606
607 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
608 if (!data)
609 return -ENOMEM;
610
611 data->handle = sev->handle;
612 ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
613 if (ret)
614 goto e_free;
615
616 params.policy = data->policy;
617 params.state = data->state;
618 params.handle = data->handle;
619
620 if (copy_to_user((void __user *)(uintptr_t)argp->data, ¶ms, sizeof(params)))
621 ret = -EFAULT;
622 e_free:
623 kfree(data);
624 return ret;
625 }
626
__sev_issue_dbg_cmd(struct kvm * kvm,unsigned long src,unsigned long dst,int size,int * error,bool enc)627 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
628 unsigned long dst, int size,
629 int *error, bool enc)
630 {
631 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
632 struct sev_data_dbg *data;
633 int ret;
634
635 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
636 if (!data)
637 return -ENOMEM;
638
639 data->handle = sev->handle;
640 data->dst_addr = dst;
641 data->src_addr = src;
642 data->len = size;
643
644 ret = sev_issue_cmd(kvm,
645 enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
646 data, error);
647 kfree(data);
648 return ret;
649 }
650
__sev_dbg_decrypt(struct kvm * kvm,unsigned long src_paddr,unsigned long dst_paddr,int sz,int * err)651 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
652 unsigned long dst_paddr, int sz, int *err)
653 {
654 int offset;
655
656 /*
657 * Its safe to read more than we are asked, caller should ensure that
658 * destination has enough space.
659 */
660 offset = src_paddr & 15;
661 src_paddr = round_down(src_paddr, 16);
662 sz = round_up(sz + offset, 16);
663
664 return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
665 }
666
__sev_dbg_decrypt_user(struct kvm * kvm,unsigned long paddr,unsigned long __user dst_uaddr,unsigned long dst_paddr,int size,int * err)667 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
668 unsigned long __user dst_uaddr,
669 unsigned long dst_paddr,
670 int size, int *err)
671 {
672 struct page *tpage = NULL;
673 int ret, offset;
674
675 /* if inputs are not 16-byte then use intermediate buffer */
676 if (!IS_ALIGNED(dst_paddr, 16) ||
677 !IS_ALIGNED(paddr, 16) ||
678 !IS_ALIGNED(size, 16)) {
679 tpage = (void *)alloc_page(GFP_KERNEL);
680 if (!tpage)
681 return -ENOMEM;
682
683 dst_paddr = __sme_page_pa(tpage);
684 }
685
686 ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
687 if (ret)
688 goto e_free;
689
690 if (tpage) {
691 offset = paddr & 15;
692 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
693 page_address(tpage) + offset, size))
694 ret = -EFAULT;
695 }
696
697 e_free:
698 if (tpage)
699 __free_page(tpage);
700
701 return ret;
702 }
703
__sev_dbg_encrypt_user(struct kvm * kvm,unsigned long paddr,unsigned long __user vaddr,unsigned long dst_paddr,unsigned long __user dst_vaddr,int size,int * error)704 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
705 unsigned long __user vaddr,
706 unsigned long dst_paddr,
707 unsigned long __user dst_vaddr,
708 int size, int *error)
709 {
710 struct page *src_tpage = NULL;
711 struct page *dst_tpage = NULL;
712 int ret, len = size;
713
714 /* If source buffer is not aligned then use an intermediate buffer */
715 if (!IS_ALIGNED(vaddr, 16)) {
716 src_tpage = alloc_page(GFP_KERNEL);
717 if (!src_tpage)
718 return -ENOMEM;
719
720 if (copy_from_user(page_address(src_tpage),
721 (void __user *)(uintptr_t)vaddr, size)) {
722 __free_page(src_tpage);
723 return -EFAULT;
724 }
725
726 paddr = __sme_page_pa(src_tpage);
727 }
728
729 /*
730 * If destination buffer or length is not aligned then do read-modify-write:
731 * - decrypt destination in an intermediate buffer
732 * - copy the source buffer in an intermediate buffer
733 * - use the intermediate buffer as source buffer
734 */
735 if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
736 int dst_offset;
737
738 dst_tpage = alloc_page(GFP_KERNEL);
739 if (!dst_tpage) {
740 ret = -ENOMEM;
741 goto e_free;
742 }
743
744 ret = __sev_dbg_decrypt(kvm, dst_paddr,
745 __sme_page_pa(dst_tpage), size, error);
746 if (ret)
747 goto e_free;
748
749 /*
750 * If source is kernel buffer then use memcpy() otherwise
751 * copy_from_user().
752 */
753 dst_offset = dst_paddr & 15;
754
755 if (src_tpage)
756 memcpy(page_address(dst_tpage) + dst_offset,
757 page_address(src_tpage), size);
758 else {
759 if (copy_from_user(page_address(dst_tpage) + dst_offset,
760 (void __user *)(uintptr_t)vaddr, size)) {
761 ret = -EFAULT;
762 goto e_free;
763 }
764 }
765
766 paddr = __sme_page_pa(dst_tpage);
767 dst_paddr = round_down(dst_paddr, 16);
768 len = round_up(size, 16);
769 }
770
771 ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
772
773 e_free:
774 if (src_tpage)
775 __free_page(src_tpage);
776 if (dst_tpage)
777 __free_page(dst_tpage);
778 return ret;
779 }
780
sev_dbg_crypt(struct kvm * kvm,struct kvm_sev_cmd * argp,bool dec)781 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
782 {
783 unsigned long vaddr, vaddr_end, next_vaddr;
784 unsigned long dst_vaddr;
785 struct page **src_p, **dst_p;
786 struct kvm_sev_dbg debug;
787 unsigned long n;
788 unsigned int size;
789 int ret;
790
791 if (!sev_guest(kvm))
792 return -ENOTTY;
793
794 if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
795 return -EFAULT;
796
797 if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
798 return -EINVAL;
799 if (!debug.dst_uaddr)
800 return -EINVAL;
801
802 vaddr = debug.src_uaddr;
803 size = debug.len;
804 vaddr_end = vaddr + size;
805 dst_vaddr = debug.dst_uaddr;
806
807 for (; vaddr < vaddr_end; vaddr = next_vaddr) {
808 int len, s_off, d_off;
809
810 /* lock userspace source and destination page */
811 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
812 if (IS_ERR(src_p))
813 return PTR_ERR(src_p);
814
815 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
816 if (IS_ERR(dst_p)) {
817 sev_unpin_memory(kvm, src_p, n);
818 return PTR_ERR(dst_p);
819 }
820
821 /*
822 * Flush (on non-coherent CPUs) before DBG_{DE,EN}CRYPT read or modify
823 * the pages; flush the destination too so that future accesses do not
824 * see stale data.
825 */
826 sev_clflush_pages(src_p, 1);
827 sev_clflush_pages(dst_p, 1);
828
829 /*
830 * Since user buffer may not be page aligned, calculate the
831 * offset within the page.
832 */
833 s_off = vaddr & ~PAGE_MASK;
834 d_off = dst_vaddr & ~PAGE_MASK;
835 len = min_t(size_t, (PAGE_SIZE - s_off), size);
836
837 if (dec)
838 ret = __sev_dbg_decrypt_user(kvm,
839 __sme_page_pa(src_p[0]) + s_off,
840 dst_vaddr,
841 __sme_page_pa(dst_p[0]) + d_off,
842 len, &argp->error);
843 else
844 ret = __sev_dbg_encrypt_user(kvm,
845 __sme_page_pa(src_p[0]) + s_off,
846 vaddr,
847 __sme_page_pa(dst_p[0]) + d_off,
848 dst_vaddr,
849 len, &argp->error);
850
851 sev_unpin_memory(kvm, src_p, n);
852 sev_unpin_memory(kvm, dst_p, n);
853
854 if (ret)
855 goto err;
856
857 next_vaddr = vaddr + len;
858 dst_vaddr = dst_vaddr + len;
859 size -= len;
860 }
861 err:
862 return ret;
863 }
864
sev_launch_secret(struct kvm * kvm,struct kvm_sev_cmd * argp)865 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
866 {
867 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
868 struct sev_data_launch_secret *data;
869 struct kvm_sev_launch_secret params;
870 struct page **pages;
871 void *blob, *hdr;
872 unsigned long n, i;
873 int ret, offset;
874
875 if (!sev_guest(kvm))
876 return -ENOTTY;
877
878 if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params)))
879 return -EFAULT;
880
881 pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
882 if (IS_ERR(pages))
883 return PTR_ERR(pages);
884
885 /*
886 * Flush (on non-coherent CPUs) before LAUNCH_SECRET encrypts pages in
887 * place; the cache may contain the data that was written unencrypted.
888 */
889 sev_clflush_pages(pages, n);
890
891 /*
892 * The secret must be copied into contiguous memory region, lets verify
893 * that userspace memory pages are contiguous before we issue command.
894 */
895 if (get_num_contig_pages(0, pages, n) != n) {
896 ret = -EINVAL;
897 goto e_unpin_memory;
898 }
899
900 ret = -ENOMEM;
901 data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
902 if (!data)
903 goto e_unpin_memory;
904
905 offset = params.guest_uaddr & (PAGE_SIZE - 1);
906 data->guest_address = __sme_page_pa(pages[0]) + offset;
907 data->guest_len = params.guest_len;
908
909 blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
910 if (IS_ERR(blob)) {
911 ret = PTR_ERR(blob);
912 goto e_free;
913 }
914
915 data->trans_address = __psp_pa(blob);
916 data->trans_len = params.trans_len;
917
918 hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
919 if (IS_ERR(hdr)) {
920 ret = PTR_ERR(hdr);
921 goto e_free_blob;
922 }
923 data->hdr_address = __psp_pa(hdr);
924 data->hdr_len = params.hdr_len;
925
926 data->handle = sev->handle;
927 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
928
929 kfree(hdr);
930
931 e_free_blob:
932 kfree(blob);
933 e_free:
934 kfree(data);
935 e_unpin_memory:
936 /* content of memory is updated, mark pages dirty */
937 for (i = 0; i < n; i++) {
938 set_page_dirty_lock(pages[i]);
939 mark_page_accessed(pages[i]);
940 }
941 sev_unpin_memory(kvm, pages, n);
942 return ret;
943 }
944
svm_mem_enc_op(struct kvm * kvm,void __user * argp)945 int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
946 {
947 struct kvm_sev_cmd sev_cmd;
948 int r;
949
950 if (!svm_sev_enabled())
951 return -ENOTTY;
952
953 if (!argp)
954 return 0;
955
956 if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
957 return -EFAULT;
958
959 mutex_lock(&kvm->lock);
960
961 switch (sev_cmd.id) {
962 case KVM_SEV_INIT:
963 r = sev_guest_init(kvm, &sev_cmd);
964 break;
965 case KVM_SEV_LAUNCH_START:
966 r = sev_launch_start(kvm, &sev_cmd);
967 break;
968 case KVM_SEV_LAUNCH_UPDATE_DATA:
969 r = sev_launch_update_data(kvm, &sev_cmd);
970 break;
971 case KVM_SEV_LAUNCH_MEASURE:
972 r = sev_launch_measure(kvm, &sev_cmd);
973 break;
974 case KVM_SEV_LAUNCH_FINISH:
975 r = sev_launch_finish(kvm, &sev_cmd);
976 break;
977 case KVM_SEV_GUEST_STATUS:
978 r = sev_guest_status(kvm, &sev_cmd);
979 break;
980 case KVM_SEV_DBG_DECRYPT:
981 r = sev_dbg_crypt(kvm, &sev_cmd, true);
982 break;
983 case KVM_SEV_DBG_ENCRYPT:
984 r = sev_dbg_crypt(kvm, &sev_cmd, false);
985 break;
986 case KVM_SEV_LAUNCH_SECRET:
987 r = sev_launch_secret(kvm, &sev_cmd);
988 break;
989 default:
990 r = -EINVAL;
991 goto out;
992 }
993
994 if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
995 r = -EFAULT;
996
997 out:
998 mutex_unlock(&kvm->lock);
999 return r;
1000 }
1001
svm_register_enc_region(struct kvm * kvm,struct kvm_enc_region * range)1002 int svm_register_enc_region(struct kvm *kvm,
1003 struct kvm_enc_region *range)
1004 {
1005 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1006 struct enc_region *region;
1007 int ret = 0;
1008
1009 if (!sev_guest(kvm))
1010 return -ENOTTY;
1011
1012 if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
1013 return -EINVAL;
1014
1015 region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
1016 if (!region)
1017 return -ENOMEM;
1018
1019 mutex_lock(&kvm->lock);
1020 region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, 1);
1021 if (IS_ERR(region->pages)) {
1022 ret = PTR_ERR(region->pages);
1023 mutex_unlock(&kvm->lock);
1024 goto e_free;
1025 }
1026
1027 region->uaddr = range->addr;
1028 region->size = range->size;
1029
1030 list_add_tail(®ion->list, &sev->regions_list);
1031 mutex_unlock(&kvm->lock);
1032
1033 /*
1034 * The guest may change the memory encryption attribute from C=0 -> C=1
1035 * or vice versa for this memory range. Lets make sure caches are
1036 * flushed to ensure that guest data gets written into memory with
1037 * correct C-bit.
1038 */
1039 sev_clflush_pages(region->pages, region->npages);
1040
1041 return ret;
1042
1043 e_free:
1044 kfree(region);
1045 return ret;
1046 }
1047
1048 static struct enc_region *
find_enc_region(struct kvm * kvm,struct kvm_enc_region * range)1049 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
1050 {
1051 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1052 struct list_head *head = &sev->regions_list;
1053 struct enc_region *i;
1054
1055 list_for_each_entry(i, head, list) {
1056 if (i->uaddr == range->addr &&
1057 i->size == range->size)
1058 return i;
1059 }
1060
1061 return NULL;
1062 }
1063
__unregister_enc_region_locked(struct kvm * kvm,struct enc_region * region)1064 static void __unregister_enc_region_locked(struct kvm *kvm,
1065 struct enc_region *region)
1066 {
1067 sev_unpin_memory(kvm, region->pages, region->npages);
1068 list_del(®ion->list);
1069 kfree(region);
1070 }
1071
svm_unregister_enc_region(struct kvm * kvm,struct kvm_enc_region * range)1072 int svm_unregister_enc_region(struct kvm *kvm,
1073 struct kvm_enc_region *range)
1074 {
1075 struct enc_region *region;
1076 int ret;
1077
1078 mutex_lock(&kvm->lock);
1079
1080 if (!sev_guest(kvm)) {
1081 ret = -ENOTTY;
1082 goto failed;
1083 }
1084
1085 region = find_enc_region(kvm, range);
1086 if (!region) {
1087 ret = -EINVAL;
1088 goto failed;
1089 }
1090
1091 /*
1092 * Ensure that all guest tagged cache entries are flushed before
1093 * releasing the pages back to the system for use. CLFLUSH will
1094 * not do this, so issue a WBINVD.
1095 */
1096 wbinvd_on_all_cpus();
1097
1098 __unregister_enc_region_locked(kvm, region);
1099
1100 mutex_unlock(&kvm->lock);
1101 return 0;
1102
1103 failed:
1104 mutex_unlock(&kvm->lock);
1105 return ret;
1106 }
1107
sev_vm_destroy(struct kvm * kvm)1108 void sev_vm_destroy(struct kvm *kvm)
1109 {
1110 struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1111 struct list_head *head = &sev->regions_list;
1112 struct list_head *pos, *q;
1113
1114 if (!sev_guest(kvm))
1115 return;
1116
1117 mutex_lock(&kvm->lock);
1118
1119 /*
1120 * Ensure that all guest tagged cache entries are flushed before
1121 * releasing the pages back to the system for use. CLFLUSH will
1122 * not do this, so issue a WBINVD.
1123 */
1124 wbinvd_on_all_cpus();
1125
1126 /*
1127 * if userspace was terminated before unregistering the memory regions
1128 * then lets unpin all the registered memory.
1129 */
1130 if (!list_empty(head)) {
1131 list_for_each_safe(pos, q, head) {
1132 __unregister_enc_region_locked(kvm,
1133 list_entry(pos, struct enc_region, list));
1134 cond_resched();
1135 }
1136 }
1137
1138 mutex_unlock(&kvm->lock);
1139
1140 sev_unbind_asid(kvm, sev->handle);
1141 sev_asid_free(sev->asid);
1142 }
1143
sev_hardware_setup(void)1144 int __init sev_hardware_setup(void)
1145 {
1146 /* Maximum number of encrypted guests supported simultaneously */
1147 max_sev_asid = cpuid_ecx(0x8000001F);
1148
1149 if (!svm_sev_enabled())
1150 return 1;
1151
1152 /* Minimum ASID value that should be used for SEV guest */
1153 min_sev_asid = cpuid_edx(0x8000001F);
1154
1155 /* Initialize SEV ASID bitmaps */
1156 sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1157 if (!sev_asid_bitmap)
1158 return 1;
1159
1160 sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1161 if (!sev_reclaim_asid_bitmap)
1162 return 1;
1163
1164 pr_info("SEV supported\n");
1165
1166 return 0;
1167 }
1168
sev_hardware_teardown(void)1169 void sev_hardware_teardown(void)
1170 {
1171 if (!svm_sev_enabled())
1172 return;
1173
1174 bitmap_free(sev_asid_bitmap);
1175 bitmap_free(sev_reclaim_asid_bitmap);
1176
1177 sev_flush_asids();
1178 }
1179
pre_sev_run(struct vcpu_svm * svm,int cpu)1180 void pre_sev_run(struct vcpu_svm *svm, int cpu)
1181 {
1182 struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
1183 int asid = sev_get_asid(svm->vcpu.kvm);
1184
1185 /* Assign the asid allocated with this SEV guest */
1186 svm->vmcb->control.asid = asid;
1187
1188 /*
1189 * Flush guest TLB:
1190 *
1191 * 1) when different VMCB for the same ASID is to be run on the same host CPU.
1192 * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
1193 */
1194 if (sd->sev_vmcbs[asid] == svm->vmcb &&
1195 svm->vcpu.arch.last_vmentry_cpu == cpu)
1196 return;
1197
1198 sd->sev_vmcbs[asid] = svm->vmcb;
1199 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
1200 vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1201 }
1202