1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2023 MediaTek Inc.
4 */
5
6 #include <linux/soc/mediatek/gzvm_drv.h>
7 #include <trace/hooks/gzvm.h>
8
cmp_ppages(struct rb_node * node,const struct rb_node * parent)9 static int cmp_ppages(struct rb_node *node, const struct rb_node *parent)
10 {
11 struct gzvm_pinned_page *a = container_of(node,
12 struct gzvm_pinned_page,
13 node);
14 struct gzvm_pinned_page *b = container_of(parent,
15 struct gzvm_pinned_page,
16 node);
17
18 if (a->ipa < b->ipa)
19 return -1;
20 if (a->ipa > b->ipa)
21 return 1;
22 return 0;
23 }
24
25 /* Invoker of this function is responsible for locking */
gzvm_insert_ppage(struct gzvm * vm,struct gzvm_pinned_page * ppage)26 static int gzvm_insert_ppage(struct gzvm *vm, struct gzvm_pinned_page *ppage)
27 {
28 if (rb_find_add(&ppage->node, &vm->pinned_pages, cmp_ppages))
29 return -EEXIST;
30 return 0;
31 }
32
rb_ppage_cmp(const void * key,const struct rb_node * node)33 static int rb_ppage_cmp(const void *key, const struct rb_node *node)
34 {
35 struct gzvm_pinned_page *p = container_of(node,
36 struct gzvm_pinned_page,
37 node);
38 phys_addr_t ipa = (phys_addr_t)key;
39
40 return (ipa < p->ipa) ? -1 : (ipa > p->ipa);
41 }
42
43 /* Invoker of this function is responsible for locking */
gzvm_remove_ppage(struct gzvm * vm,phys_addr_t ipa)44 static int gzvm_remove_ppage(struct gzvm *vm, phys_addr_t ipa)
45 {
46 struct gzvm_pinned_page *ppage;
47 struct rb_node *node;
48
49 node = rb_find((void *)ipa, &vm->pinned_pages, rb_ppage_cmp);
50
51 if (node)
52 rb_erase(node, &vm->pinned_pages);
53 else
54 return 0;
55
56 ppage = container_of(node, struct gzvm_pinned_page, node);
57 unpin_user_pages_dirty_lock(&ppage->page, 1, true);
58 kfree(ppage);
59
60 return 0;
61 }
62
pin_one_page(struct gzvm * vm,unsigned long hva,u64 gpa,struct page ** out_page)63 static int pin_one_page(struct gzvm *vm, unsigned long hva, u64 gpa,
64 struct page **out_page)
65 {
66 unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE;
67 struct gzvm_pinned_page *ppage = NULL;
68 struct mm_struct *mm = current->mm;
69 struct page *page = NULL;
70 int ret;
71
72 ppage = kmalloc(sizeof(*ppage), GFP_KERNEL_ACCOUNT);
73 if (!ppage)
74 return -ENOMEM;
75
76 mmap_read_lock(mm);
77 ret = pin_user_pages(hva, 1, flags, &page);
78 mmap_read_unlock(mm);
79
80 if (ret != 1 || !page) {
81 kfree(ppage);
82 return -EFAULT;
83 }
84
85 ppage->page = page;
86 ppage->ipa = gpa;
87
88 mutex_lock(&vm->mem_lock);
89 ret = gzvm_insert_ppage(vm, ppage);
90
91 /**
92 * The return of -EEXIST from gzvm_insert_ppage is considered an
93 * expected behavior in this context.
94 * This situation arises when two or more VCPUs are concurrently
95 * engaged in demand paging handling. The initial VCPU has already
96 * allocated and pinned a page, while the subsequent VCPU attempts
97 * to pin the same page again. As a result, we prompt the unpinning
98 * and release of the allocated structure, followed by a return 0.
99 */
100 if (ret == -EEXIST) {
101 kfree(ppage);
102 unpin_user_pages(&page, 1);
103 ret = 0;
104 }
105 mutex_unlock(&vm->mem_lock);
106 *out_page = page;
107
108 return ret;
109 }
110
111 /**
112 * gzvm_handle_relinquish() - Handle memory relinquish request from hypervisor
113 *
114 * @vcpu: Pointer to struct gzvm_vcpu_run in userspace
115 * @ipa: Start address(gpa) of a reclaimed page
116 *
117 * Return: Always return 0 because there are no cases of failure
118 */
gzvm_handle_relinquish(struct gzvm_vcpu * vcpu,phys_addr_t ipa)119 int gzvm_handle_relinquish(struct gzvm_vcpu *vcpu, phys_addr_t ipa)
120 {
121 struct gzvm *vm = vcpu->gzvm;
122
123 mutex_lock(&vm->mem_lock);
124 gzvm_remove_ppage(vm, ipa);
125 mutex_unlock(&vm->mem_lock);
126
127 return 0;
128 }
129
gzvm_vm_allocate_guest_page(struct gzvm * vm,struct gzvm_memslot * slot,u64 gfn,u64 * pfn)130 int gzvm_vm_allocate_guest_page(struct gzvm *vm, struct gzvm_memslot *slot,
131 u64 gfn, u64 *pfn)
132 {
133 struct page *page = NULL;
134 unsigned long hva;
135 int ret;
136
137 if (gzvm_gfn_to_hva_memslot(slot, gfn, (u64 *)&hva) != 0)
138 return -EINVAL;
139
140 ret = pin_one_page(vm, hva, PFN_PHYS(gfn), &page);
141 if (ret != 0)
142 return ret;
143
144 if (page == NULL)
145 return -EFAULT;
146 /**
147 * As `pin_user_pages` already gets the page struct, we don't need to
148 * call other APIs to reduce function call overhead.
149 */
150 *pfn = page_to_pfn(page);
151
152 return 0;
153 }
154
handle_single_demand_page(struct gzvm * vm,int memslot_id,u64 gfn)155 static int handle_single_demand_page(struct gzvm *vm, int memslot_id, u64 gfn)
156 {
157 int ret;
158 u64 pfn;
159
160 ret = gzvm_vm_allocate_guest_page(vm, &vm->memslot[memslot_id], gfn, &pfn);
161 if (unlikely(ret))
162 return -EFAULT;
163
164 trace_android_vh_gzvm_handle_demand_page_pre(vm, memslot_id, pfn, gfn, 1);
165
166 ret = gzvm_arch_map_guest(vm->vm_id, memslot_id, pfn, gfn, 1);
167 if (unlikely(ret))
168 return -EFAULT;
169
170 trace_android_vh_gzvm_handle_demand_page_post(vm, memslot_id, pfn, gfn, 1);
171
172 return ret;
173 }
174
handle_block_demand_page(struct gzvm * vm,int memslot_id,u64 gfn)175 static int handle_block_demand_page(struct gzvm *vm, int memslot_id, u64 gfn)
176 {
177 u32 nr_entries_all = GZVM_BLOCK_BASED_DEMAND_PAGE_SIZE / PAGE_SIZE;
178 u32 nr_entries = vm->gzvm_drv->demand_paging_batch_pages;
179 struct gzvm_memslot *memslot = &vm->memslot[memslot_id];
180 u64 start_gfn = ALIGN_DOWN(gfn, nr_entries_all);
181 u32 total_pages = memslot->npages;
182 u64 base_gfn = memslot->base_gfn;
183 u64 pfn, __gfn;
184 int ret, i;
185
186 if (start_gfn < base_gfn)
187 start_gfn = base_gfn;
188
189 u64 end_gfn = start_gfn + nr_entries_all;
190
191 if (start_gfn + nr_entries_all > base_gfn + total_pages)
192 end_gfn = base_gfn + total_pages;
193
194 mutex_lock(&vm->demand_paging_lock);
195 for (; start_gfn < end_gfn; start_gfn += nr_entries) {
196 /*
197 * If the start/end gfn of this demand paging block is outside the
198 * memory region of memslot, adjust the start_gfn/nr_entries.
199 */
200 if (start_gfn + nr_entries > base_gfn + total_pages)
201 nr_entries = base_gfn + total_pages - start_gfn;
202
203 for (i = 0, __gfn = start_gfn; i < nr_entries; i++, __gfn++) {
204 ret = gzvm_vm_allocate_guest_page(vm, memslot, __gfn,
205 &pfn);
206 if (unlikely(ret)) {
207 pr_notice("VM-%u failed to allocate page for GFN 0x%llx (%d)\n",
208 vm->vm_id, __gfn, ret);
209 ret = -ERR_FAULT;
210 goto err_unlock;
211 }
212 vm->demand_page_buffer[i] = pfn;
213 }
214
215 trace_android_vh_gzvm_handle_demand_page_pre(vm, memslot_id, 0, gfn, nr_entries);
216
217 ret = gzvm_arch_map_guest_block(vm->vm_id, memslot_id,
218 start_gfn, nr_entries);
219 if (unlikely(ret)) {
220 ret = -EFAULT;
221 goto err_unlock;
222 }
223
224 trace_android_vh_gzvm_handle_demand_page_post(vm, memslot_id, 0, gfn, nr_entries);
225 }
226 err_unlock:
227 mutex_unlock(&vm->demand_paging_lock);
228 return ret;
229 }
230
231 /**
232 * gzvm_handle_page_fault() - Handle guest page fault, find corresponding page
233 * for the faulting gpa
234 * @vcpu: Pointer to struct gzvm_vcpu_run of the faulting vcpu
235 *
236 * Return:
237 * * 0 - Success to handle guest page fault
238 * * -EFAULT - Failed to map phys addr to guest's GPA
239 */
gzvm_handle_page_fault(struct gzvm_vcpu * vcpu)240 int gzvm_handle_page_fault(struct gzvm_vcpu *vcpu)
241 {
242 struct gzvm *vm = vcpu->gzvm;
243 int memslot_id;
244 u64 gfn;
245
246 gfn = PHYS_PFN(vcpu->run->exception.fault_gpa);
247 memslot_id = gzvm_find_memslot(vm, gfn);
248 if (unlikely(memslot_id < 0))
249 return -EFAULT;
250
251 if (unlikely(vm->mem_alloc_mode == GZVM_FULLY_POPULATED))
252 return -EFAULT;
253
254 if (vm->demand_page_gran == PAGE_SIZE)
255 return handle_single_demand_page(vm, memslot_id, gfn);
256 else
257 return handle_block_demand_page(vm, memslot_id, gfn);
258 }
259