1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2023 Google LLC
4 * Author: Mostafa Saleh <smostafa@google.com>
5 */
6
7 #include <asm/kvm_mmu.h>
8 #include <linux/kvm_host.h>
9 #include <kvm/iommu.h>
10
11 struct kvm_iommu_driver *iommu_driver;
12 extern struct kvm_iommu_ops *kvm_nvhe_sym(kvm_iommu_ops);
13
kvm_iommu_register_driver(struct kvm_iommu_driver * kern_ops)14 int kvm_iommu_register_driver(struct kvm_iommu_driver *kern_ops)
15 {
16 BUG_ON(!kern_ops);
17
18 /*
19 * Paired with smp_load_acquire(&iommu_driver)
20 * Ensure memory stores happening during a driver
21 * init are observed before executing kvm iommu callbacks.
22 */
23 return cmpxchg_release(&iommu_driver, NULL, kern_ops) ? -EBUSY : 0;
24 }
25 EXPORT_SYMBOL(kvm_iommu_register_driver);
26
kvm_iommu_init_hyp(struct kvm_iommu_ops * hyp_ops,struct kvm_hyp_memcache * atomic_mc,unsigned long init_arg)27 int kvm_iommu_init_hyp(struct kvm_iommu_ops *hyp_ops,
28 struct kvm_hyp_memcache *atomic_mc,
29 unsigned long init_arg)
30 {
31 BUG_ON(!hyp_ops);
32
33 return kvm_call_hyp_nvhe(__pkvm_iommu_init, hyp_ops,
34 atomic_mc->head, atomic_mc->nr_pages, init_arg);
35 }
36 EXPORT_SYMBOL(kvm_iommu_init_hyp);
37
kvm_iommu_init_driver(void)38 int kvm_iommu_init_driver(void)
39 {
40 if (!smp_load_acquire(&iommu_driver) || !iommu_driver->get_iommu_id) {
41 kvm_err("pKVM enabled with no IOMMU driver, do not run confidential" \
42 " workloads in virtual machines\n");
43 return -ENODEV;
44 }
45 /*
46 * init_driver is optional as the driver already registered it self.
47 * This call mainly notify the driver we are about to drop privilege.
48 */
49 if (!iommu_driver->init_driver)
50 return 0;
51 kvm_hyp_iommu_domains = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
52 get_order(KVM_IOMMU_DOMAINS_ROOT_SIZE));
53 kvm_hyp_iommu_domains = kern_hyp_va(kvm_hyp_iommu_domains);
54 if (!kvm_hyp_iommu_domains) {
55 kvm_err("No enough mem for IOMMU domains");
56 return -ENOMEM;
57 }
58
59 return iommu_driver->init_driver();
60 }
61
kvm_iommu_remove_driver(void)62 void kvm_iommu_remove_driver(void)
63 {
64 if (smp_load_acquire(&iommu_driver))
65 iommu_driver->remove_driver();
66 }
67
kvm_get_iommu_id(struct device * dev)68 pkvm_handle_t kvm_get_iommu_id(struct device *dev)
69 {
70 return iommu_driver->get_iommu_id(dev);
71 }
72
pkvm_iommu_suspend(struct device * dev)73 int pkvm_iommu_suspend(struct device *dev)
74 {
75 int device_id = kvm_get_iommu_id(dev);
76
77 return kvm_call_hyp_nvhe(__pkvm_host_hvc_pd, device_id, 0);
78 }
79 EXPORT_SYMBOL(pkvm_iommu_suspend);
80
pkvm_iommu_resume(struct device * dev)81 int pkvm_iommu_resume(struct device *dev)
82 {
83 int device_id = kvm_get_iommu_id(dev);
84
85 return kvm_call_hyp_nvhe(__pkvm_host_hvc_pd, device_id, 1);
86 }
87 EXPORT_SYMBOL(pkvm_iommu_resume);
88