1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2024 Google LLC
4 * Author: Mostafa Saleh <smostafa@google.com>
5 */
6
7 #include <kvm/arm_hypercalls.h>
8
9 #include <nvhe/alloc.h>
10 #include <nvhe/iommu.h>
11 #include <nvhe/mem_protect.h>
12 #include <nvhe/pkvm.h>
13 #include <nvhe/pviommu.h>
14 #include <nvhe/pviommu-host.h>
15
16 struct pviommu_guest_domain {
17 pkvm_handle_t id;
18 struct list_head list;
19 };
20
21 static DEFINE_HYP_SPINLOCK(pviommu_guest_domain_lock);
22
23 #define KVM_IOMMU_MAX_GUEST_DOMAINS (KVM_IOMMU_MAX_DOMAINS >> 1)
24 static unsigned long guest_domains[KVM_IOMMU_MAX_GUEST_DOMAINS / BITS_PER_LONG];
25
26 /*
27 * Guests doens't have separate domain space as the host, but they share the upper half
28 * of the domain ids, so they would ask for a domain and get a domain id as a return.
29 * This is a rare operation for guests, so bruteforcing the domain space should be fine
30 * for now, however we can improve this by having a hint for last allocated domain_id or
31 * use a pseudo-random number.
32 */
pkvm_guest_iommu_alloc_id(void)33 static int pkvm_guest_iommu_alloc_id(void)
34 {
35 int i;
36
37 for (i = 0 ; i < ARRAY_SIZE(guest_domains) ; ++i) {
38 if (guest_domains[i] != ~0UL) {
39 int domain_off = ffz(guest_domains[i]);
40
41 guest_domains[i] |= (1UL << domain_off);
42 return domain_off + i * BITS_PER_LONG +
43 (KVM_IOMMU_MAX_DOMAINS >> 1);
44 }
45 }
46
47 return -EBUSY;
48 }
49
pkvm_guest_iommu_free_id(int domain_id)50 static void pkvm_guest_iommu_free_id(int domain_id)
51 {
52 domain_id -= (KVM_IOMMU_MAX_DOMAINS >> 1);
53 if (WARN_ON(domain_id < 0) || (domain_id >= KVM_IOMMU_MAX_GUEST_DOMAINS))
54 return;
55
56 guest_domains[domain_id / BITS_PER_LONG] &= ~(1UL << (domain_id % BITS_PER_LONG));
57 }
58
59 /*
60 * check if vcpu has requested memory before
61 */
__need_req(struct kvm_vcpu * vcpu)62 static bool __need_req(struct kvm_vcpu *vcpu)
63 {
64 struct kvm_hyp_req *hyp_req = vcpu->arch.hyp_reqs;
65
66 return hyp_req->type != KVM_HYP_LAST_REQ;
67 }
68
pkvm_pviommu_hyp_req(u64 * exit_code)69 static void pkvm_pviommu_hyp_req(u64 *exit_code)
70 {
71 write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR);
72 *exit_code = ARM_EXCEPTION_HYP_REQ;
73 }
74
pkvm_guest_iommu_attach_dev(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)75 static bool pkvm_guest_iommu_attach_dev(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
76 {
77 int ret;
78 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
79 u64 iommu_id = smccc_get_arg2(vcpu);
80 u64 sid = smccc_get_arg3(vcpu);
81 u64 pasid = smccc_get_arg4(vcpu);
82 u64 domain_id = smccc_get_arg5(vcpu);
83 u64 pasid_bits = smccc_get_arg6(vcpu);
84 struct pviommu_route route;
85 struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
86
87 ret = pkvm_pviommu_get_route(vm, iommu_id, sid, &route);
88 if (ret)
89 goto out_ret;
90 iommu_id = route.iommu;
91 sid = route.sid;
92
93 ret = kvm_iommu_attach_dev(iommu_id, domain_id, sid, pasid, pasid_bits, 0);
94 if (ret == -ENOMEM) {
95 /*
96 * The driver will request memory when returning -ENOMEM, so go back to host to
97 * fulfill the request and repeat the HVC.
98 */
99 pkvm_pviommu_hyp_req(exit_code);
100 return false;
101 }
102
103 out_ret:
104 smccc_set_retval(vcpu, ret ? SMCCC_RET_INVALID_PARAMETER : SMCCC_RET_SUCCESS,
105 0, 0, 0);
106 return true;
107 }
108
pkvm_guest_iommu_detach_dev(struct pkvm_hyp_vcpu * hyp_vcpu)109 static bool pkvm_guest_iommu_detach_dev(struct pkvm_hyp_vcpu *hyp_vcpu)
110 {
111 int ret;
112 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
113 u64 iommu_id = smccc_get_arg2(vcpu);
114 u64 sid = smccc_get_arg3(vcpu);
115 u64 pasid = smccc_get_arg4(vcpu);
116 u64 domain_id = smccc_get_arg5(vcpu);
117 struct pviommu_route route;
118 struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
119
120 /* MBZ */
121 if (smccc_get_arg6(vcpu)) {
122 ret = -EINVAL;
123 goto out_ret;
124 }
125
126 ret = pkvm_pviommu_get_route(vm, iommu_id, sid, &route);
127 if (ret)
128 goto out_ret;
129 iommu_id = route.iommu;
130 sid = route.sid;
131
132 ret = kvm_iommu_detach_dev(iommu_id, domain_id, sid, pasid);
133
134 out_ret:
135 smccc_set_retval(vcpu, ret ? SMCCC_RET_INVALID_PARAMETER : SMCCC_RET_SUCCESS,
136 0, 0, 0);
137 return true;
138 }
139
pkvm_guest_iommu_alloc_domain(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)140 static bool pkvm_guest_iommu_alloc_domain(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
141 {
142 int ret;
143 int domain_id = 0;
144 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
145 struct pviommu_guest_domain *guest_domain;
146 struct kvm_hyp_req *req;
147 struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
148
149 guest_domain = hyp_alloc(sizeof(*guest_domain));
150 if (!guest_domain) {
151 BUG_ON(hyp_alloc_errno() != -ENOMEM);
152 req = pkvm_hyp_req_reserve(hyp_vcpu, REQ_MEM_DEST_HYP_ALLOC);
153 req->mem.nr_pages = hyp_alloc_missing_donations();
154 req->mem.sz_alloc = PAGE_SIZE;
155 pkvm_pviommu_hyp_req(exit_code);
156 return false;
157 }
158
159 /* MBZ */
160 if (smccc_get_arg2(vcpu) || smccc_get_arg3(vcpu) || smccc_get_arg4(vcpu) ||
161 smccc_get_arg5(vcpu) || smccc_get_arg6(vcpu))
162 goto out_inval;
163
164 hyp_spin_lock(&pviommu_guest_domain_lock);
165 domain_id = pkvm_guest_iommu_alloc_id();
166 if (domain_id < 0)
167 goto out_inval;
168
169 ret = kvm_iommu_alloc_domain(domain_id, KVM_IOMMU_DOMAIN_ANY_TYPE);
170 if (ret == -ENOMEM) {
171 pkvm_guest_iommu_free_id(domain_id);
172 hyp_spin_unlock(&pviommu_guest_domain_lock);
173 hyp_free(guest_domain);
174 pkvm_pviommu_hyp_req(exit_code);
175 return false;
176 } else if (ret) {
177 pkvm_guest_iommu_free_id(domain_id);
178 goto out_inval;
179 }
180
181 guest_domain->id = domain_id;
182 list_add_tail(&guest_domain->list, &vm->domains);
183 hyp_spin_unlock(&pviommu_guest_domain_lock);
184 smccc_set_retval(vcpu, SMCCC_RET_SUCCESS, domain_id, 0, 0);
185 return true;
186
187 out_inval:
188 hyp_spin_unlock(&pviommu_guest_domain_lock);
189 hyp_free(guest_domain);
190 smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
191 return true;
192 }
193
pkvm_guest_iommu_free_domain(struct pkvm_hyp_vcpu * hyp_vcpu)194 static bool pkvm_guest_iommu_free_domain(struct pkvm_hyp_vcpu *hyp_vcpu)
195 {
196 int ret;
197 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
198 u64 domain_id = smccc_get_arg2(vcpu);
199 struct pviommu_guest_domain *guest_domain, *temp;
200 struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
201
202 if (smccc_get_arg3(vcpu) || smccc_get_arg4(vcpu) || smccc_get_arg5(vcpu) ||
203 smccc_get_arg6(vcpu)) {
204 ret = -EINVAL;
205 goto out_ret;
206 }
207
208 hyp_spin_lock(&pviommu_guest_domain_lock);
209 ret = kvm_iommu_free_domain(domain_id);
210 if (ret)
211 goto out_unlock;
212 list_for_each_entry_safe(guest_domain, temp, &vm->domains, list) {
213 if (guest_domain->id == domain_id) {
214 pkvm_guest_iommu_free_id(domain_id);
215 list_del(&guest_domain->list);
216 hyp_free(guest_domain);
217 break;
218 }
219 }
220
221 out_unlock:
222 hyp_spin_unlock(&pviommu_guest_domain_lock);
223
224 out_ret:
225 smccc_set_retval(vcpu, ret ? SMCCC_RET_INVALID_PARAMETER : SMCCC_RET_SUCCESS,
226 0, 0, 0);
227 return true;
228 }
229
__smccc_prot_linux(u64 prot)230 static int __smccc_prot_linux(u64 prot)
231 {
232 int iommu_prot = 0;
233
234 if (prot & ARM_SMCCC_KVM_PVIOMMU_READ)
235 iommu_prot |= IOMMU_READ;
236 if (prot & ARM_SMCCC_KVM_PVIOMMU_WRITE)
237 iommu_prot |= IOMMU_WRITE;
238 if (prot & ARM_SMCCC_KVM_PVIOMMU_CACHE)
239 iommu_prot |= IOMMU_CACHE;
240 if (prot & ARM_SMCCC_KVM_PVIOMMU_NOEXEC)
241 iommu_prot |= IOMMU_NOEXEC;
242 if (prot & ARM_SMCCC_KVM_PVIOMMU_MMIO)
243 iommu_prot |= IOMMU_MMIO;
244 if (prot & ARM_SMCCC_KVM_PVIOMMU_PRIV)
245 iommu_prot |= IOMMU_PRIV;
246
247 return iommu_prot;
248 }
249
pkvm_guest_iommu_map(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)250 static bool pkvm_guest_iommu_map(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
251 {
252 size_t mapped, total_mapped = 0;
253 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
254 u64 domain = smccc_get_arg2(vcpu);
255 u64 iova = smccc_get_arg3(vcpu);
256 u64 ipa = smccc_get_arg4(vcpu);
257 u64 size = smccc_get_arg5(vcpu);
258 u64 prot = smccc_get_arg6(vcpu);
259 u64 paddr;
260 int ret;
261 s8 level;
262 u64 smccc_ret = SMCCC_RET_SUCCESS;
263
264 if (!IS_ALIGNED(size, PAGE_SIZE) ||
265 !IS_ALIGNED(ipa, PAGE_SIZE) ||
266 !IS_ALIGNED(iova, PAGE_SIZE)) {
267 smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
268 return true;
269 }
270
271 while (size) {
272 /*
273 * We need to get the PA and atomically use the page temporarily to avoid
274 * racing with relinquish.
275 */
276 ret = pkvm_get_guest_pa_request_use_dma(hyp_vcpu, ipa, size,
277 &paddr, &level);
278 if (ret == -ENOENT) {
279 /*
280 * Pages are not mapped and a request was created, updated the guest
281 * state and go back to host
282 */
283 goto out_host_request;
284 } else if (ret) {
285 smccc_ret = SMCCC_RET_INVALID_PARAMETER;
286 break;
287 }
288
289 kvm_iommu_map_pages(domain, iova, paddr,
290 PAGE_SIZE, min(size, kvm_granule_size(level)) / PAGE_SIZE,
291 __smccc_prot_linux(prot), &mapped);
292 WARN_ON(__pkvm_unuse_dma(paddr, kvm_granule_size(level), hyp_vcpu));
293 if (!mapped) {
294 if (!__need_req(vcpu)) {
295 smccc_ret = SMCCC_RET_INVALID_PARAMETER;
296 break;
297 }
298 /*
299 * Return back to the host with a request to fill the memcache,
300 * and also update the guest state with what was mapped, so the
301 * next time the vcpu runs it can check that not all requested
302 * memory was mapped, and it would repeat the HVC with the rest
303 * of the range.
304 */
305 goto out_host_request;
306 }
307
308 ipa += mapped;
309 iova += mapped;
310 total_mapped += mapped;
311 size -= mapped;
312 }
313
314 smccc_set_retval(vcpu, smccc_ret, total_mapped, 0, 0);
315 return true;
316 out_host_request:
317 *exit_code = ARM_EXCEPTION_HYP_REQ;
318 smccc_set_retval(vcpu, SMCCC_RET_SUCCESS, total_mapped, 0, 0);
319 return false;
320 }
321
pkvm_guest_iommu_unmap(struct pkvm_hyp_vcpu * hyp_vcpu,u64 * exit_code)322 static bool pkvm_guest_iommu_unmap(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
323 {
324 struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
325 u64 domain = smccc_get_arg2(vcpu);
326 u64 iova = smccc_get_arg3(vcpu);
327 u64 size = smccc_get_arg4(vcpu);
328 size_t unmapped;
329 unsigned long ret = SMCCC_RET_SUCCESS;
330
331 if (!IS_ALIGNED(size, PAGE_SIZE) ||
332 !IS_ALIGNED(iova, PAGE_SIZE) ||
333 smccc_get_arg5(vcpu) ||
334 smccc_get_arg6(vcpu)) {
335 smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0);
336 return true;
337 }
338
339 unmapped = kvm_iommu_unmap_pages(domain, iova, PAGE_SIZE, size / PAGE_SIZE);
340 if (unmapped < size) {
341 if (!__need_req(vcpu)) {
342 ret = SMCCC_RET_INVALID_PARAMETER;
343 } else {
344 /* See comment in pkvm_guest_iommu_map(). */
345 *exit_code = ARM_EXCEPTION_HYP_REQ;
346 smccc_set_retval(vcpu, SMCCC_RET_SUCCESS, unmapped, 0, 0);
347 return false;
348 }
349 }
350
351 smccc_set_retval(vcpu, ret, unmapped, 0, 0);
352 return true;
353 }
354
kvm_iommu_teardown_guest_domains(struct pkvm_hyp_vm * hyp_vm)355 void kvm_iommu_teardown_guest_domains(struct pkvm_hyp_vm *hyp_vm)
356 {
357 struct pviommu_guest_domain *guest_domain, *temp;
358
359 hyp_spin_lock(&pviommu_guest_domain_lock);
360 list_for_each_entry_safe(guest_domain, temp, &hyp_vm->domains, list) {
361 kvm_iommu_force_free_domain(guest_domain->id, hyp_vm);
362 pkvm_guest_iommu_free_id(guest_domain->id);
363 list_del(&guest_domain->list);
364 hyp_free(guest_domain);
365 }
366 hyp_spin_unlock(&pviommu_guest_domain_lock);
367 }
368
kvm_handle_pviommu_hvc(struct kvm_vcpu * vcpu,u64 * exit_code)369 bool kvm_handle_pviommu_hvc(struct kvm_vcpu *vcpu, u64 *exit_code)
370 {
371 u64 iommu_op = smccc_get_arg1(vcpu);
372 struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu);
373 struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
374
375 /*
376 * Eagerly fill the vm iommu pool to avoid deadlocks from donation path while
377 * doing IOMMU operations.
378 */
379 refill_hyp_pool(&vm->iommu_pool, &hyp_vcpu->host_vcpu->arch.iommu_mc);
380 switch (iommu_op) {
381 case KVM_PVIOMMU_OP_ALLOC_DOMAIN:
382 return pkvm_guest_iommu_alloc_domain(hyp_vcpu, exit_code);
383 case KVM_PVIOMMU_OP_FREE_DOMAIN:
384 return pkvm_guest_iommu_free_domain(hyp_vcpu);
385 case KVM_PVIOMMU_OP_ATTACH_DEV:
386 return pkvm_guest_iommu_attach_dev(hyp_vcpu, exit_code);
387 case KVM_PVIOMMU_OP_DETACH_DEV:
388 return pkvm_guest_iommu_detach_dev(hyp_vcpu);
389 case KVM_PVIOMMU_OP_MAP_PAGES:
390 return pkvm_guest_iommu_map(hyp_vcpu, exit_code);
391 case KVM_PVIOMMU_OP_UNMAP_PAGES:
392 return pkvm_guest_iommu_unmap(hyp_vcpu, exit_code);
393 }
394
395 smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
396 return true;
397 }
398