1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
4 */
5
6 #define pr_fmt(fmt) "gunyah_vm_mgr: " fmt
7
8 #include <asm/gunyah.h>
9 #include <linux/mm.h>
10 #include <linux/pagemap.h>
11
12 #include "rsc_mgr.h"
13 #include "vm_mgr.h"
14
15 #define WRITE_TAG (1 << 0)
16 #define SHARE_TAG (1 << 1)
17
18 static inline struct gunyah_resource *
__first_resource(struct gunyah_vm_resource_ticket * ticket)19 __first_resource(struct gunyah_vm_resource_ticket *ticket)
20 {
21 return list_first_entry_or_null(&ticket->resources,
22 struct gunyah_resource, list);
23 }
24
25 /*
26 * Once the parcel is converted to paged, vm_mgr only tracks
27 * the pages. The parcel needs to be reclaimed by the caller.
28 */
gunyah_vm_parcel_to_paged(struct gunyah_vm * ghvm,struct gunyah_rm_mem_parcel * parcel,u64 gfn,u64 nr)29 int gunyah_vm_parcel_to_paged(struct gunyah_vm *ghvm,
30 struct gunyah_rm_mem_parcel *parcel, u64 gfn,
31 u64 nr)
32 {
33 struct gunyah_vm_parcel *vm_parcel;
34 struct gunyah_rm_mem_entry *entry;
35 struct gunyah_vm_binding *b;
36 unsigned long i, tag = 0;
37 struct folio *folio;
38 pgoff_t off = 0;
39 u64 size;
40 int ret = 0;
41
42 down_write(&ghvm->bindings_lock);
43 b = mtree_load(&ghvm->bindings, gfn);
44 if (!b) {
45 WARN_ON("No backing binding for the parcel being paged");
46 ret = -ENOENT;
47 goto unlock;
48 }
49
50 vm_parcel = b->vm_parcel;
51 if (!vm_parcel) {
52 WARN_ON("No parcel found");
53 ret = -ENOENT;
54 goto unlock;
55 }
56
57 if (parcel->n_acl_entries > 1)
58 tag |= SHARE_TAG;
59 if (parcel->acl_entries[0].perms & GUNYAH_RM_ACL_W)
60 tag |= WRITE_TAG;
61
62 for (i = 0; i < parcel->n_mem_entries; i++) {
63 entry = &parcel->mem_entries[i];
64 size = entry->size;
65 folio = pfn_folio(PHYS_PFN(le64_to_cpu(entry->phys_addr)));
66 while (size > 0) {
67 ret = mtree_insert_range(&ghvm->mm, gfn + off,
68 gfn + off + folio_nr_pages(folio) - 1,
69 xa_tag_pointer(folio, tag),
70 GFP_KERNEL);
71 if (ret) {
72 WARN_ON(ret != -ENOMEM);
73 gunyah_vm_mm_erase_range(ghvm, gfn, off - 1);
74 goto unlock;
75 }
76 off += folio_nr_pages(folio);
77 size -= folio_size(folio);
78 folio = folio_next(folio);
79 }
80 }
81 BUG_ON(off != nr);
82 vm_parcel->start = 0;
83 b->vm_parcel = NULL;
84
85 unlock:
86 up_write(&ghvm->bindings_lock);
87 return ret;
88 }
89
90 /**
91 * gunyah_vm_mm_erase_range() - Erases a range of folios from ghvm's mm
92 * @ghvm: gunyah vm
93 * @gfn: start guest frame number
94 * @nr: number of pages to erase
95 *
96 * Do not use this function unless rolling back gunyah_vm_parcel_to_paged.
97 */
gunyah_vm_mm_erase_range(struct gunyah_vm * ghvm,u64 gfn,u64 nr)98 void gunyah_vm_mm_erase_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr)
99 {
100 struct folio *folio;
101 u64 off = gfn;
102 void *entry;
103
104 while (off < gfn + nr) {
105 entry = mtree_erase(&ghvm->mm, off);
106 if (!entry)
107 return;
108 folio = xa_untag_pointer(entry);
109 if (!folio)
110 return;
111 off += folio_nr_pages(folio);
112 }
113 }
114
donate_flags(bool share)115 static inline u32 donate_flags(bool share)
116 {
117 if (share)
118 return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
119 GUNYAH_MEMEXTENT_DONATE_TO_SIBLING);
120 else
121 return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
122 GUNYAH_MEMEXTENT_DONATE_TO_PROTECTED);
123 }
124
reclaim_flags(bool share,bool sync)125 static inline u32 reclaim_flags(bool share, bool sync)
126 {
127 u32 flags = 0;
128
129 if (share)
130 flags |= FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
131 GUNYAH_MEMEXTENT_DONATE_TO_SIBLING);
132 else
133 flags |= FIELD_PREP_CONST(
134 GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
135 GUNYAH_MEMEXTENT_DONATE_FROM_PROTECTED);
136
137 if (!sync)
138 flags |= GUNYAH_MEMEXTENT_OPTION_NOSYNC;
139
140 return flags;
141 }
142
gunyah_vm_provide_folio(struct gunyah_vm * ghvm,struct folio * folio,u64 gfn,bool share,bool write)143 int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
144 u64 gfn, bool share, bool write)
145 {
146 struct gunyah_resource *guest_extent, *host_extent, *addrspace;
147 u32 map_flags = BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PARTIAL);
148 u64 extent_attrs, gpa = gunyah_gfn_to_gpa(gfn);
149 phys_addr_t pa = PFN_PHYS(folio_pfn(folio));
150 enum gunyah_pagetable_access access;
151 size_t size = folio_size(folio);
152 enum gunyah_error gunyah_error;
153 unsigned long tag = 0;
154 int ret, tmp;
155
156 /* clang-format off */
157 if (share) {
158 guest_extent = __first_resource(&ghvm->guest_shared_extent_ticket);
159 host_extent = __first_resource(&ghvm->host_shared_extent_ticket);
160 } else {
161 guest_extent = __first_resource(&ghvm->guest_private_extent_ticket);
162 host_extent = __first_resource(&ghvm->host_private_extent_ticket);
163 }
164 /* clang-format on */
165 addrspace = __first_resource(&ghvm->addrspace_ticket);
166
167 if (!addrspace || !guest_extent || !host_extent)
168 return -ENODEV;
169
170 if (share) {
171 map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_VMMIO);
172 tag |= SHARE_TAG;
173 } else {
174 map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PRIVATE);
175 }
176
177 if (write)
178 tag |= WRITE_TAG;
179
180 ret = mtree_insert_range(&ghvm->mm, gfn,
181 gfn + folio_nr_pages(folio) - 1,
182 xa_tag_pointer(folio, tag), GFP_KERNEL);
183 if (ret == -EEXIST)
184 ret = -EAGAIN;
185 if (ret)
186 return ret;
187
188 if (share && write)
189 access = GUNYAH_PAGETABLE_ACCESS_RW;
190 else if (share && !write)
191 access = GUNYAH_PAGETABLE_ACCESS_R;
192 else if (!share && write)
193 access = GUNYAH_PAGETABLE_ACCESS_RWX;
194 else /* !share && !write */
195 access = GUNYAH_PAGETABLE_ACCESS_RX;
196
197 ret = gunyah_rm_platform_pre_demand_page(ghvm->rm, ghvm->vmid, access,
198 folio);
199 if (ret)
200 goto reclaim_host;
201
202 gunyah_error = gunyah_hypercall_memextent_donate(donate_flags(share),
203 host_extent->capid,
204 guest_extent->capid,
205 pa, size);
206 if (gunyah_error != GUNYAH_ERROR_OK) {
207 pr_err("Failed to donate memory for guest address 0x%016llx: %d\n",
208 gpa, gunyah_error);
209 ret = gunyah_error_remap(gunyah_error);
210 goto platform_release;
211 }
212
213 extent_attrs =
214 FIELD_PREP_CONST(GUNYAH_MEMEXTENT_MAPPING_TYPE,
215 ARCH_GUNYAH_DEFAULT_MEMTYPE) |
216 FIELD_PREP(GUNYAH_MEMEXTENT_MAPPING_USER_ACCESS, access) |
217 FIELD_PREP(GUNYAH_MEMEXTENT_MAPPING_KERNEL_ACCESS, access);
218 gunyah_error = gunyah_hypercall_addrspace_map(addrspace->capid,
219 guest_extent->capid, gpa,
220 extent_attrs, map_flags,
221 pa, size);
222 if (gunyah_error != GUNYAH_ERROR_OK) {
223 pr_err("Failed to map guest address 0x%016llx: %d\n", gpa,
224 gunyah_error);
225 ret = gunyah_error_remap(gunyah_error);
226 goto memextent_reclaim;
227 }
228
229 return 0;
230 memextent_reclaim:
231 gunyah_error = gunyah_hypercall_memextent_donate(
232 reclaim_flags(share, true), guest_extent->capid,
233 host_extent->capid, pa, size);
234 if (gunyah_error != GUNYAH_ERROR_OK)
235 pr_err("Failed to reclaim memory donation for guest address 0x%016llx: %d\n",
236 gpa, gunyah_error);
237 platform_release:
238 tmp = gunyah_rm_platform_reclaim_demand_page(ghvm->rm, ghvm->vmid,
239 access, folio);
240 if (tmp) {
241 pr_err("Platform failed to reclaim memory for guest address 0x%016llx: %d",
242 gpa, tmp);
243 return ret;
244 }
245 reclaim_host:
246 gunyah_folio_host_reclaim(folio);
247 mtree_erase(&ghvm->mm, gfn);
248 return ret;
249 }
250
__gunyah_vm_reclaim_folio_locked(struct gunyah_vm * ghvm,void * entry,u64 gfn,const bool sync)251 static int __gunyah_vm_reclaim_folio_locked(struct gunyah_vm *ghvm, void *entry,
252 u64 gfn, const bool sync)
253 {
254 u32 map_flags = BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PARTIAL);
255 struct gunyah_resource *guest_extent, *host_extent, *addrspace;
256 enum gunyah_pagetable_access access;
257 enum gunyah_error gunyah_error;
258 struct folio *folio;
259 bool write, share;
260 phys_addr_t pa;
261 size_t size;
262 int ret;
263
264 addrspace = __first_resource(&ghvm->addrspace_ticket);
265 if (!addrspace)
266 return -ENODEV;
267
268 share = !!(xa_pointer_tag(entry) & SHARE_TAG);
269 write = !!(xa_pointer_tag(entry) & WRITE_TAG);
270 folio = xa_untag_pointer(entry);
271
272 if (!sync)
273 map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_NOSYNC);
274
275 /* clang-format off */
276 if (share) {
277 guest_extent = __first_resource(&ghvm->guest_shared_extent_ticket);
278 host_extent = __first_resource(&ghvm->host_shared_extent_ticket);
279 map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_VMMIO);
280 } else {
281 guest_extent = __first_resource(&ghvm->guest_private_extent_ticket);
282 host_extent = __first_resource(&ghvm->host_private_extent_ticket);
283 map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PRIVATE);
284 }
285 /* clang-format on */
286
287 pa = PFN_PHYS(folio_pfn(folio));
288 size = folio_size(folio);
289
290 gunyah_error = gunyah_hypercall_addrspace_unmap(addrspace->capid,
291 guest_extent->capid,
292 gunyah_gfn_to_gpa(gfn),
293 map_flags, pa, size);
294 if (gunyah_error != GUNYAH_ERROR_OK) {
295 pr_err_ratelimited(
296 "Failed to unmap guest address 0x%016llx: %d\n",
297 gunyah_gfn_to_gpa(gfn), gunyah_error);
298 ret = gunyah_error_remap(gunyah_error);
299 goto err;
300 }
301
302 gunyah_error = gunyah_hypercall_memextent_donate(
303 reclaim_flags(share, sync), guest_extent->capid,
304 host_extent->capid, pa, size);
305 if (gunyah_error != GUNYAH_ERROR_OK) {
306 pr_err_ratelimited(
307 "Failed to reclaim memory donation for guest address 0x%016llx: %d\n",
308 gunyah_gfn_to_gpa(gfn), gunyah_error);
309 ret = gunyah_error_remap(gunyah_error);
310 goto err;
311 }
312
313 if (share && write)
314 access = GUNYAH_PAGETABLE_ACCESS_RW;
315 else if (share && !write)
316 access = GUNYAH_PAGETABLE_ACCESS_R;
317 else if (!share && write)
318 access = GUNYAH_PAGETABLE_ACCESS_RWX;
319 else /* !share && !write */
320 access = GUNYAH_PAGETABLE_ACCESS_RX;
321
322 ret = gunyah_rm_platform_reclaim_demand_page(ghvm->rm, ghvm->vmid,
323 access, folio);
324 if (ret) {
325 pr_err_ratelimited(
326 "Platform failed to reclaim memory for guest address 0x%016llx: %d",
327 gunyah_gfn_to_gpa(gfn), ret);
328 goto err;
329 }
330
331 BUG_ON(mtree_erase(&ghvm->mm, gfn) != entry);
332
333 unpin_user_page(folio_page(folio, 0));
334 account_locked_vm(current->mm, 1, false);
335 return 0;
336 err:
337 return ret;
338 }
339
gunyah_vm_reclaim_folio(struct gunyah_vm * ghvm,u64 gfn,struct folio * folio)340 int gunyah_vm_reclaim_folio(struct gunyah_vm *ghvm, u64 gfn, struct folio *folio)
341 {
342 void *entry;
343
344 entry = mtree_load(&ghvm->mm, gfn);
345 if (!entry)
346 return 0;
347
348 if (folio != xa_untag_pointer(entry))
349 return -EAGAIN;
350
351 return __gunyah_vm_reclaim_folio_locked(ghvm, entry, gfn, true);
352 }
353
gunyah_vm_reclaim_range(struct gunyah_vm * ghvm,u64 gfn,u64 nr)354 int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr)
355 {
356 unsigned long next = gfn, g;
357 struct folio *folio;
358 int ret, ret2 = 0;
359 void *entry;
360 bool sync;
361
362 mt_for_each(&ghvm->mm, entry, next, gfn + nr) {
363 folio = xa_untag_pointer(entry);
364 g = next;
365 sync = !mt_find_after(&ghvm->mm, &g, gfn + nr);
366
367 g = next - folio_nr_pages(folio);
368 folio_get(folio);
369 folio_lock(folio);
370 if (mtree_load(&ghvm->mm, g) == entry)
371 ret = __gunyah_vm_reclaim_folio_locked(ghvm, entry, g, sync);
372 else
373 ret = -EAGAIN;
374 folio_unlock(folio);
375 folio_put(folio);
376 if (ret && ret2 != -EAGAIN)
377 ret2 = ret;
378 }
379
380 return ret2;
381 }
382
gunyah_vm_binding_alloc(struct gunyah_vm * ghvm,struct gunyah_userspace_memory_region * region,bool lend)383 int gunyah_vm_binding_alloc(struct gunyah_vm *ghvm,
384 struct gunyah_userspace_memory_region *region,
385 bool lend)
386 {
387 struct gunyah_vm_binding *binding;
388 int ret = 0;
389
390 if (!region->memory_size || !PAGE_ALIGNED(region->memory_size) ||
391 !PAGE_ALIGNED(region->userspace_addr) ||
392 !PAGE_ALIGNED(region->guest_phys_addr))
393 return -EINVAL;
394
395 if (overflows_type(region->guest_phys_addr + region->memory_size, u64))
396 return -EOVERFLOW;
397
398 binding = kzalloc(sizeof(*binding), GFP_KERNEL_ACCOUNT);
399 if (!binding) {
400 return -ENOMEM;
401 }
402
403 binding->mem_type = VM_MEM_USER;
404 binding->userspace_addr = region->userspace_addr;
405 binding->vm_parcel = NULL;
406 binding->guest_phys_addr = region->guest_phys_addr;
407 binding->size = region->memory_size;
408 binding->flags = region->flags;
409 binding->label = region->label;
410
411 if (lend) {
412 binding->share_type = VM_MEM_LEND;
413 } else {
414 binding->share_type = VM_MEM_SHARE;
415 }
416 down_write(&ghvm->bindings_lock);
417 ret = mtree_insert_range(&ghvm->bindings,
418 gunyah_gpa_to_gfn(binding->guest_phys_addr),
419 gunyah_gpa_to_gfn(binding->guest_phys_addr + region->memory_size - 1),
420 binding, GFP_KERNEL);
421
422 if(ret != 0)
423 kfree(binding);
424 up_write(&ghvm->bindings_lock);
425
426 return ret;
427 }
428
gunyah_gup_demand_page(struct gunyah_vm * ghvm,struct gunyah_vm_binding * b,u64 gpa,bool write)429 static int gunyah_gup_demand_page(struct gunyah_vm *ghvm, struct gunyah_vm_binding *b,
430 u64 gpa, bool write)
431 {
432 unsigned long gfn = gunyah_gpa_to_gfn(gpa);
433 unsigned int gup_flags;
434 u64 offset;
435 int pinned, ret;
436 struct page *page;
437 struct folio *folio;
438
439 if (write && !(b->flags & GUNYAH_MEM_ALLOW_WRITE))
440 return -EPERM;
441 gup_flags = FOLL_LONGTERM;
442 if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
443 gup_flags |= FOLL_WRITE;
444
445 offset = (gunyah_gfn_to_gpa(gfn) - b->guest_phys_addr);
446
447 ret = account_locked_vm(current->mm, 1, true);
448 if (ret)
449 return ret;
450
451 pinned = pin_user_pages_fast(b->userspace_addr + offset, 1,
452 gup_flags, &page);
453
454 if (pinned != 1) {
455 ret = pinned;
456 goto unlock_page;
457 }
458
459 folio = page_folio(page);
460
461 if (!folio_test_swapbacked(folio)) {
462 ret = -EIO;
463 goto unpin_page;
464 }
465
466 folio_lock(folio);
467 ret = gunyah_vm_provide_folio(ghvm, folio, gfn - folio_page_idx(folio, page),
468 !(b->share_type == VM_MEM_LEND),
469 !!(b->flags & GUNYAH_MEM_ALLOW_WRITE));
470 folio_unlock(folio);
471 if (ret) {
472 if (ret != -EAGAIN)
473 pr_err_ratelimited(
474 "Failed to provide folio for guest addr: %016llx: %d\n",
475 gpa, ret);
476 goto unpin_page;
477 }
478 return ret;
479
480 unpin_page:
481 unpin_user_page(page);
482 unlock_page:
483 account_locked_vm(current->mm, 1, false);
484 return ret;
485 }
486
gunyah_demand_page(struct gunyah_vm * ghvm,u64 gpa,bool write)487 int gunyah_demand_page(struct gunyah_vm *ghvm, u64 gpa, bool write)
488 {
489 unsigned long gfn = gunyah_gpa_to_gfn(gpa);
490 struct gunyah_vm_binding *b;
491 int ret;
492
493 down_read(&ghvm->bindings_lock);
494 b = mtree_load(&ghvm->bindings, gfn);
495 if (!b) {
496 ret = -ENOENT;
497 goto unlock;
498 }
499
500 if (b->mem_type == VM_MEM_CMA) {
501 dev_warn(ghvm->parent, "Demand paging of CMA mem not supported\n");
502 ret = -EOPNOTSUPP;
503 } else {
504 ret = gunyah_gup_demand_page(ghvm, b, gpa, write);
505 }
506
507 unlock:
508 up_read(&ghvm->bindings_lock);
509 return ret;
510 }
511 EXPORT_SYMBOL_GPL(gunyah_demand_page);
512
gunyah_gup_share_parcel(struct gunyah_vm * ghvm,struct gunyah_vm_parcel * vm_parcel,struct gunyah_vm_binding * b,u64 * gfn,u64 * nr)513 static int gunyah_gup_share_parcel(struct gunyah_vm *ghvm,
514 struct gunyah_vm_parcel *vm_parcel,
515 struct gunyah_vm_binding *b, u64 *gfn, u64 *nr)
516 {
517 struct gunyah_rm_mem_parcel *parcel = &vm_parcel->parcel;
518 struct page **pages;
519 int pinned, ret;
520 struct folio *folio;
521 unsigned int gup_flags;
522 unsigned long i, offset, entries, entry_size;
523
524 offset = gunyah_gfn_to_gpa(*gfn) - b->guest_phys_addr;
525 pages = kcalloc(*nr, sizeof(*pages), GFP_KERNEL_ACCOUNT);
526 if (!pages)
527 return -ENOMEM;
528
529 gup_flags = FOLL_LONGTERM;
530 if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
531 gup_flags |= FOLL_WRITE;
532
533 pinned = pin_user_pages_fast(b->userspace_addr + offset, *nr,
534 gup_flags, pages);
535 if (pinned < 0) {
536 ret = pinned;
537 goto free_pages;
538 } else if (pinned != *nr) {
539 ret = -EFAULT;
540 goto unpin_pages;
541 }
542
543 ret = account_locked_vm(current->mm, pinned, true);
544 if (ret)
545 goto unpin_pages;
546
547 /* overallocate & assume no large folios */
548 parcel->mem_entries = kcalloc(pinned, sizeof(parcel->mem_entries[0]),
549 GFP_KERNEL_ACCOUNT);
550 if (!parcel->mem_entries) {
551 ret = -ENOMEM;
552 goto unaccount_pages;
553 }
554 folio = page_folio(pages[0]);
555 *gfn -= folio_page_idx(folio, pages[0]);
556 *nr = folio_nr_pages(folio);
557 parcel->mem_entries[0].phys_addr = cpu_to_le64(PFN_PHYS(folio_pfn(folio)));
558 entry_size = cpu_to_le64(folio_size(folio));
559
560 for (i = 1, entries = 0; i < pinned; i++) {
561 folio = page_folio(pages[i]);
562 if (pages[i] == folio_page(folio, 0)) {
563 if (page_to_pfn(pages[i - 1]) + 1 == page_to_pfn(pages[i])) {
564 entry_size += cpu_to_le64(folio_size(folio));
565 *nr += folio_nr_pages(folio);
566 } else {
567 parcel->mem_entries[entries].size = entry_size;
568 entries++;
569 parcel->mem_entries[entries].phys_addr = cpu_to_le64(PFN_PHYS(folio_pfn(folio)));
570 entry_size = cpu_to_le64(folio_size(folio));
571 *nr += folio_nr_pages(folio);
572 }
573 } else {
574 unpin_user_page(pages[i]);
575 account_locked_vm(current->mm, 1, false);
576 }
577 }
578 parcel->mem_entries[entries].size = entry_size;
579 parcel->n_mem_entries = entries + 1;
580 ret = gunyah_rm_mem_share(ghvm->rm, parcel);
581 if (ret)
582 goto free_mem_entries;
583
584 vm_parcel->start = *gfn;
585 vm_parcel->pages = *nr;
586 b->vm_parcel = vm_parcel;
587 goto free_pages;
588
589 free_mem_entries:
590 kfree(parcel->mem_entries);
591 parcel->mem_entries = NULL;
592 parcel->n_mem_entries = 0;
593 unaccount_pages:
594 account_locked_vm(current->mm, pinned, false);
595 unpin_pages:
596 unpin_user_pages(pages, pinned);
597 free_pages:
598 kfree(pages);
599 return ret;
600 }
601
gunyah_share_parcel(struct gunyah_vm * ghvm,struct gunyah_vm_parcel * vm_parcel,u64 * gfn,u64 * nr)602 int gunyah_share_parcel(struct gunyah_vm *ghvm, struct gunyah_vm_parcel *vm_parcel,
603 u64 *gfn, u64 *nr)
604 {
605 struct gunyah_rm_mem_parcel *parcel = &vm_parcel->parcel;
606 struct gunyah_vm_binding *b;
607 bool lend;
608 u16 vmid;
609 int ret;
610
611 if (!*nr)
612 return -EINVAL;
613
614 down_write(&ghvm->bindings_lock);
615 b = mtree_load(&ghvm->bindings, *gfn);
616 if (!b) {
617 ret = -ENOENT;
618 goto unlock;
619 }
620
621 parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
622 if (b->share_type == VM_MEM_LEND) {
623 parcel->n_acl_entries = 1;
624 lend = true;
625 } else {
626 lend = false;
627 parcel->n_acl_entries = 2;
628 parcel->label = b->label;
629 }
630 parcel->acl_entries = kcalloc(parcel->n_acl_entries,
631 sizeof(*parcel->acl_entries), GFP_KERNEL);
632 if (!parcel->acl_entries) {
633 ret = -ENOMEM;
634 goto unlock;
635 }
636
637 /* acl_entries[0].vmid will be this VM's vmid. We'll fill it when the
638 * VM is starting and we know the VM's vmid.
639 */
640 parcel->acl_entries[0].vmid = cpu_to_le16(ghvm->vmid);
641 if (b->flags & GUNYAH_MEM_ALLOW_READ)
642 parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_R;
643 if (b->flags & GUNYAH_MEM_ALLOW_WRITE)
644 parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_W;
645 if (b->flags & GUNYAH_MEM_ALLOW_EXEC)
646 parcel->acl_entries[0].perms |= GUNYAH_RM_ACL_X;
647
648 if (!lend) {
649 ret = gunyah_rm_get_vmid(ghvm->rm, &vmid);
650 if (ret)
651 goto free_acl;
652
653 parcel->acl_entries[1].vmid = cpu_to_le16(vmid);
654 /* Host assumed to have all these permissions. Gunyah will not
655 * grant new permissions if host actually had less than RWX
656 */
657 parcel->acl_entries[1].perms = GUNYAH_RM_ACL_R | GUNYAH_RM_ACL_W | GUNYAH_RM_ACL_X;
658 }
659
660 if (b->mem_type == VM_MEM_CMA) {
661 ret = gunyah_cma_share_parcel(ghvm, vm_parcel, b, gfn, nr);
662 if (ret) {
663 dev_warn(ghvm->parent, "Failed to share CMA memory: %d\n", ret);
664 goto free_acl;
665 }
666 } else {
667 ret = gunyah_gup_share_parcel(ghvm, vm_parcel, b, gfn, nr);
668 if (ret) {
669 dev_warn(ghvm->parent, "Failed to share GUP memory: %d\n", ret);
670 goto free_acl;
671 }
672 }
673 goto unlock;
674
675 free_acl:
676 kfree(parcel->acl_entries);
677 parcel->acl_entries = NULL;
678 unlock:
679 up_write(&ghvm->bindings_lock);
680 return ret;
681 }
682
683 /*
684 * This function will provide the number of bindings from
685 * start_addr to end_addr.
686 * Use ULONG_MAX as the end_addr to get all the bindings of the VM.
687 */
gunyah_count_bindings(struct gunyah_vm * ghvm,u64 start_addr,u64 end_addr)688 static u32 gunyah_count_bindings(struct gunyah_vm *ghvm, u64 start_addr,
689 u64 end_addr)
690 {
691 struct gunyah_vm_binding *b;
692 unsigned long addr = start_addr;
693 u32 count = 0;
694
695 down_read(&ghvm->bindings_lock);
696 mt_for_each(&ghvm->bindings, b, addr, end_addr)
697 count++;
698 up_read(&ghvm->bindings_lock);
699
700 return count;
701 }
702
gunyah_gup_reclaim_parcel(struct gunyah_vm * ghvm,struct gunyah_vm_parcel * vm_parcel,struct gunyah_vm_binding * b)703 static int gunyah_gup_reclaim_parcel(struct gunyah_vm *ghvm,
704 struct gunyah_vm_parcel *vm_parcel, struct gunyah_vm_binding *b)
705 {
706 struct gunyah_rm_mem_parcel *parcel = &vm_parcel->parcel;
707 struct gunyah_rm_mem_entry *entry;
708 struct folio *folio;
709 pgoff_t i;
710 int ret;
711
712 if (parcel->mem_handle == GUNYAH_MEM_HANDLE_INVAL)
713 return 0;
714
715 ret = gunyah_rm_mem_reclaim(ghvm->rm, parcel);
716 if (ret) {
717 dev_err(ghvm->parent, "Failed to reclaim parcel: %d\n",
718 ret);
719 /* We can't reclaim the pages -- hold onto the pages
720 * forever because we don't know what state the memory
721 * is in
722 */
723 return ret;
724 }
725
726 for (i = 0; i < parcel->n_mem_entries; i++) {
727 entry = &parcel->mem_entries[i];
728
729 folio = pfn_folio(PHYS_PFN(le64_to_cpu(entry->phys_addr)));
730
731 if (folio_test_private(folio))
732 gunyah_folio_host_reclaim(folio);
733
734 unpin_user_page(folio_page(folio, 0));
735 account_locked_vm(ghvm->mm_s, 1, false);
736 }
737
738 parcel->mem_handle = GUNYAH_MEM_HANDLE_INVAL;
739 kfree(parcel->mem_entries);
740 kfree(parcel->acl_entries);
741 vm_parcel->start = 0;
742 vm_parcel->pages = 0;
743 b->vm_parcel = NULL;
744 return ret;
745 }
746
gunyah_reclaim_parcel(struct gunyah_vm * ghvm,struct gunyah_vm_parcel * vm_parcel)747 static int gunyah_reclaim_parcel(struct gunyah_vm *ghvm,
748 struct gunyah_vm_parcel *vm_parcel)
749 {
750 struct gunyah_vm_binding *b;
751 int ret;
752
753 down_write(&ghvm->bindings_lock);
754 b = mtree_load(&ghvm->bindings, vm_parcel->start);
755 if (!b) {
756 ret = -ENOENT;
757 goto unlock;
758 }
759
760 if (b->mem_type == VM_MEM_CMA)
761 ret = gunyah_cma_reclaim_parcel(ghvm, vm_parcel, b);
762 else
763 ret = gunyah_gup_reclaim_parcel(ghvm, vm_parcel, b);
764
765 unlock:
766 up_write(&ghvm->bindings_lock);
767 return ret;
768 }
769
gunyah_reclaim_parcels(struct gunyah_vm * ghvm,u64 start_gfn,u64 end_gfn)770 int gunyah_reclaim_parcels(struct gunyah_vm *ghvm, u64 start_gfn,
771 u64 end_gfn)
772 {
773 unsigned long gfn = start_gfn;
774 struct gunyah_vm_binding *b;
775 int ret, ret2 = 0;
776
777 mt_for_each(&ghvm->bindings, b, gfn, end_gfn) {
778 if (b->vm_parcel)
779 ret = gunyah_reclaim_parcel(ghvm, b->vm_parcel);
780 if (ret)
781 ret2 = ret;
782 }
783
784 return ret2;
785 }
786
787 /*
788 * gunyah_share_range_as_parcels() - Share all bindings as parcels from start_gfn to end_gfn
789 * @ghvm - The gunyah vm
790 * @start_gfn: Start guest page number
791 * @end_gfn: Last guest page number
792 * @parcels: Array of parcels allocated.
793 *
794 * Use ULONG_MAX as the end_gfn to share all the bindings of the VM
795 * provided enough space for parcels is present.
796 * Caller is responsible to free the parcels when parcels are done
797 * being used.
798 */
gunyah_share_range_as_parcels(struct gunyah_vm * ghvm,u64 start_gfn,u64 end_gfn,struct gunyah_vm_parcel ** parcels)799 int gunyah_share_range_as_parcels(struct gunyah_vm *ghvm, u64 start_gfn,
800 u64 end_gfn, struct gunyah_vm_parcel **parcels)
801 {
802 struct gunyah_vm_binding *b;
803 unsigned long gfn = start_gfn;
804 u32 count = 0, n;
805 int ret, ret_err;
806
807 /* Find the number of parcels needed to be created within the requested range*/
808 n = gunyah_count_bindings(ghvm, start_gfn, end_gfn);
809
810 *parcels = kzalloc(sizeof(struct gunyah_vm_parcel) * n, GFP_KERNEL);
811 if (!*parcels)
812 return -ENOMEM;
813
814 mt_for_each(&ghvm->bindings, b, gfn, end_gfn) {
815 u64 parcel_start = b->guest_phys_addr >> PAGE_SHIFT;
816 u64 parcel_pages = b->size >> PAGE_SHIFT;
817
818 ret = gunyah_share_parcel(ghvm, &(*parcels)[count++], &parcel_start, &parcel_pages);
819 if (ret) {
820 dev_err(ghvm->parent, "Failed to share parcel of %llx: %d\n",
821 parcel_start, ret);
822 /* Let's roll back.*/
823 while (count--) {
824 if ((*parcels)[count].parcel.mem_handle !=
825 GUNYAH_MEM_HANDLE_INVAL) {
826 ret_err = gunyah_reclaim_parcel(ghvm, &(*parcels)[count]);
827 if (ret_err)
828 dev_err(ghvm->parent, "Failed to reclaim parcel: %d, memory will leak\n",
829 ret_err);
830 }
831 }
832 goto err;
833 }
834 }
835 return ret;
836
837 err:
838 kfree(*parcels);
839 *parcels = NULL;
840 return ret;
841 }
842 EXPORT_SYMBOL_GPL(gunyah_share_range_as_parcels);
843
gunyah_setup_demand_paging(struct gunyah_vm * ghvm,u64 start_gfn,u64 end_gfn)844 int gunyah_setup_demand_paging(struct gunyah_vm *ghvm, u64 start_gfn,
845 u64 end_gfn)
846 {
847 struct gunyah_rm_mem_entry *entries;
848 unsigned long gfn = start_gfn;
849 struct gunyah_vm_binding *b;
850 u32 count = 0, i;
851 int ret = 0;
852
853 down_read(&ghvm->bindings_lock);
854 mt_for_each(&ghvm->bindings, b, gfn, end_gfn)
855 if (b->share_type == VM_MEM_LEND &&
856 (b->guest_phys_addr != ghvm->fw.config.guest_phys_addr))
857 count++;
858
859 if (!count)
860 goto out;
861
862 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
863 if (!entries) {
864 ret = -ENOMEM;
865 goto out;
866 }
867
868 gfn = start_gfn;
869 i = 0;
870 mt_for_each(&ghvm->bindings, b, gfn, end_gfn) {
871 if (b->share_type != VM_MEM_LEND ||
872 (b->guest_phys_addr == ghvm->fw.config.guest_phys_addr))
873 continue;
874 entries[i].phys_addr = cpu_to_le64(b->guest_phys_addr);
875 entries[i].size = cpu_to_le64(b->size);
876 if (++i == count)
877 break;
878 }
879
880 ret = gunyah_rm_vm_set_demand_paging(ghvm->rm, ghvm->vmid, i, entries);
881 kfree(entries);
882 out:
883 up_read(&ghvm->bindings_lock);
884 return ret;
885 }
886