1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 */
6
7 #include <linux/kvm_host.h>
8 #include <asm/kvm_hyp.h>
9 #include <asm/kvm_mmu.h>
10 #include <asm/kvm_pgtable.h>
11 #include <asm/kvm_pkvm.h>
12
13 #include <nvhe/early_alloc.h>
14 #include <nvhe/ffa.h>
15 #include <nvhe/gfp.h>
16 #include <nvhe/iommu.h>
17 #include <nvhe/memory.h>
18 #include <nvhe/mem_protect.h>
19 #include <nvhe/mm.h>
20 #include <nvhe/pkvm.h>
21 #include <nvhe/serial.h>
22 #include <nvhe/trap_handler.h>
23
24 unsigned long hyp_nr_cpus;
25
26 phys_addr_t pvmfw_base;
27 phys_addr_t pvmfw_size;
28
29 #define hyp_percpu_size ((unsigned long)__per_cpu_end - \
30 (unsigned long)__per_cpu_start)
31
32 static void *vmemmap_base;
33 static void *vm_table_base;
34 static void *hyp_pgt_base;
35 static void *host_s2_pgt_base;
36 static void *ffa_proxy_pages;
37 static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
38 static struct hyp_pool hpool;
39
divide_memory_pool(void * virt,unsigned long size)40 static int divide_memory_pool(void *virt, unsigned long size)
41 {
42 unsigned long nr_pages;
43
44 hyp_early_alloc_init(virt, size);
45
46 nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page));
47 vmemmap_base = hyp_early_alloc_contig(nr_pages);
48 if (!vmemmap_base)
49 return -ENOMEM;
50
51 nr_pages = hyp_vm_table_pages();
52 vm_table_base = hyp_early_alloc_contig(nr_pages);
53 if (!vm_table_base)
54 return -ENOMEM;
55
56 nr_pages = hyp_s1_pgtable_pages();
57 hyp_pgt_base = hyp_early_alloc_contig(nr_pages);
58 if (!hyp_pgt_base)
59 return -ENOMEM;
60
61 nr_pages = host_s2_pgtable_pages();
62 host_s2_pgt_base = hyp_early_alloc_contig(nr_pages);
63 if (!host_s2_pgt_base)
64 return -ENOMEM;
65
66 nr_pages = hyp_ffa_proxy_pages();
67 ffa_proxy_pages = hyp_early_alloc_contig(nr_pages);
68 if (!ffa_proxy_pages)
69 return -ENOMEM;
70
71 return 0;
72 }
73
create_hyp_host_fp_mappings(void)74 static int create_hyp_host_fp_mappings(void)
75 {
76 void *start, *end;
77 int ret, i;
78
79 for (i = 0; i < hyp_nr_cpus; i++) {
80 start = (void *)kern_hyp_va(kvm_arm_hyp_host_fp_state[i]);
81 end = start + PAGE_ALIGN(pkvm_host_fp_state_size());
82 ret = pkvm_create_mappings(start, end, PAGE_HYP);
83 if (ret)
84 return ret;
85 }
86
87 return 0;
88 }
89
recreate_hyp_mappings(phys_addr_t phys,unsigned long size,unsigned long * per_cpu_base,u32 hyp_va_bits)90 static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
91 unsigned long *per_cpu_base,
92 u32 hyp_va_bits)
93 {
94 void *start, *end, *virt = hyp_phys_to_virt(phys);
95 unsigned long pgt_size = hyp_s1_pgtable_pages() << PAGE_SHIFT;
96 enum kvm_pgtable_prot prot;
97 int ret, i;
98
99 /* Recreate the hyp page-table using the early page allocator */
100 hyp_early_alloc_init(hyp_pgt_base, pgt_size);
101 ret = kvm_pgtable_hyp_init(&pkvm_pgtable, hyp_va_bits,
102 &hyp_early_alloc_mm_ops);
103 if (ret)
104 return ret;
105
106 ret = hyp_create_idmap(hyp_va_bits);
107 if (ret)
108 return ret;
109
110 ret = hyp_map_vectors();
111 if (ret)
112 return ret;
113
114 ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base));
115 if (ret)
116 return ret;
117
118 ret = pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_EXEC);
119 if (ret)
120 return ret;
121
122 ret = pkvm_create_mappings(__hyp_data_start, __hyp_data_end, PAGE_HYP);
123 if (ret)
124 return ret;
125
126 ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO);
127 if (ret)
128 return ret;
129
130 ret = pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP);
131 if (ret)
132 return ret;
133
134 ret = pkvm_create_mappings(virt, virt + size, PAGE_HYP);
135 if (ret)
136 return ret;
137
138 for (i = 0; i < hyp_nr_cpus; i++) {
139 struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i);
140 unsigned long hyp_addr;
141
142 start = (void *)kern_hyp_va(per_cpu_base[i]);
143 end = start + PAGE_ALIGN(hyp_percpu_size);
144 ret = pkvm_create_mappings(start, end, PAGE_HYP);
145 if (ret)
146 return ret;
147
148 /*
149 * Allocate a contiguous HYP private VA range for the stack
150 * and guard page. The allocation is also aligned based on
151 * the order of its size.
152 */
153 ret = pkvm_alloc_private_va_range(NVHE_STACK_SIZE * 2, &hyp_addr);
154 if (ret)
155 return ret;
156
157 /*
158 * Since the stack grows downwards, map the stack to the page
159 * at the higher address and leave the lower guard page
160 * unbacked.
161 *
162 * Any valid stack address now has the NVHE_STACK_SHIFT bit as 1
163 * and addresses corresponding to the guard page have the
164 * NVHE_STACK_SHIFT bit as 0 - this is used for overflow detection.
165 */
166 hyp_spin_lock(&pkvm_pgd_lock);
167 ret = kvm_pgtable_hyp_map(&pkvm_pgtable, hyp_addr + NVHE_STACK_SIZE,
168 NVHE_STACK_SIZE, params->stack_pa, PAGE_HYP);
169 hyp_spin_unlock(&pkvm_pgd_lock);
170 if (ret)
171 return ret;
172
173 /* Update stack_hyp_va to end of the stack's private VA range */
174 params->stack_hyp_va = hyp_addr + (2 * NVHE_STACK_SIZE);
175 }
176
177 create_hyp_host_fp_mappings();
178
179 /*
180 * Map the host sections RO in the hypervisor, but transfer the
181 * ownership from the host to the hypervisor itself to make sure they
182 * can't be donated or shared with another entity.
183 *
184 * The ownership transition requires matching changes in the host
185 * stage-2. This will be done later (see finalize_host_mappings()) once
186 * the hyp_vmemmap is addressable.
187 */
188 prot = pkvm_mkstate(PAGE_HYP_RO, PKVM_PAGE_SHARED_OWNED);
189 ret = pkvm_create_mappings(&kvm_vgic_global_state,
190 &kvm_vgic_global_state + 1, prot);
191 if (ret)
192 return ret;
193
194 start = hyp_phys_to_virt(pvmfw_base);
195 end = start + pvmfw_size;
196 prot = pkvm_mkstate(PAGE_HYP_RO, PKVM_PAGE_OWNED);
197 ret = pkvm_create_mappings(start, end, prot);
198 if (ret)
199 return ret;
200
201 return 0;
202 }
203
update_nvhe_init_params(void)204 static void update_nvhe_init_params(void)
205 {
206 struct kvm_nvhe_init_params *params;
207 unsigned long i;
208
209 for (i = 0; i < hyp_nr_cpus; i++) {
210 params = per_cpu_ptr(&kvm_init_params, i);
211 params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
212 dcache_clean_inval_poc((unsigned long)params,
213 (unsigned long)params + sizeof(*params));
214 }
215 }
216
hyp_zalloc_hyp_page(void * arg)217 static void *hyp_zalloc_hyp_page(void *arg)
218 {
219 return hyp_alloc_pages(&hpool, 0);
220 }
221
hpool_get_page(void * addr)222 static void hpool_get_page(void *addr)
223 {
224 hyp_get_page(&hpool, addr);
225 }
226
hpool_put_page(void * addr)227 static void hpool_put_page(void *addr)
228 {
229 hyp_put_page(&hpool, addr);
230 }
231
fix_host_ownership_walker(u64 addr,u64 end,u32 level,kvm_pte_t * ptep,enum kvm_pgtable_walk_flags flag,void * const arg)232 static int fix_host_ownership_walker(u64 addr, u64 end, u32 level,
233 kvm_pte_t *ptep,
234 enum kvm_pgtable_walk_flags flag,
235 void * const arg)
236 {
237 enum kvm_pgtable_prot prot;
238 enum pkvm_page_state state;
239 kvm_pte_t pte = *ptep;
240 phys_addr_t phys;
241
242 if (!kvm_pte_valid(pte))
243 return 0;
244
245 if (level != (KVM_PGTABLE_MAX_LEVELS - 1))
246 return -EINVAL;
247
248 phys = kvm_pte_to_phys(pte);
249 if (!addr_is_memory(phys))
250 return -EINVAL;
251
252 /*
253 * Adjust the host stage-2 mappings to match the ownership attributes
254 * configured in the hypervisor stage-1.
255 */
256 state = pkvm_getstate(kvm_pgtable_hyp_pte_prot(pte));
257 switch (state) {
258 case PKVM_PAGE_OWNED:
259 return host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_HYP);
260 case PKVM_PAGE_SHARED_OWNED:
261 prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_BORROWED);
262 break;
263 case PKVM_PAGE_SHARED_BORROWED:
264 prot = pkvm_mkstate(PKVM_HOST_MEM_PROT, PKVM_PAGE_SHARED_OWNED);
265 break;
266 default:
267 return -EINVAL;
268 }
269
270 return host_stage2_idmap_locked(phys, PAGE_SIZE, prot, false);
271 }
272
fix_hyp_pgtable_refcnt_walker(u64 addr,u64 end,u32 level,kvm_pte_t * ptep,enum kvm_pgtable_walk_flags flag,void * const arg)273 static int fix_hyp_pgtable_refcnt_walker(u64 addr, u64 end, u32 level,
274 kvm_pte_t *ptep,
275 enum kvm_pgtable_walk_flags flag,
276 void * const arg)
277 {
278 struct kvm_pgtable_mm_ops *mm_ops = arg;
279 kvm_pte_t pte = *ptep;
280
281 /*
282 * Fix-up the refcount for the page-table pages as the early allocator
283 * was unable to access the hyp_vmemmap and so the buddy allocator has
284 * initialised the refcount to '1'.
285 */
286 if (kvm_pte_valid(pte))
287 mm_ops->get_page(ptep);
288
289 return 0;
290 }
291
pin_table_walker(u64 addr,u64 end,u32 level,kvm_pte_t * ptep,enum kvm_pgtable_walk_flags flag,void * const arg)292 static int pin_table_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
293 enum kvm_pgtable_walk_flags flag, void * const arg)
294 {
295 struct kvm_pgtable_mm_ops *mm_ops = arg;
296 kvm_pte_t pte = *ptep;
297
298 if (kvm_pte_valid(pte))
299 mm_ops->get_page(kvm_pte_follow(pte, mm_ops));
300
301 return 0;
302 }
303
pin_host_tables(void)304 static int pin_host_tables(void)
305 {
306 struct kvm_pgtable_walker walker = {
307 .cb = pin_table_walker,
308 .flags = KVM_PGTABLE_WALK_TABLE_POST,
309 .arg = &host_mmu.mm_ops,
310 };
311
312 return kvm_pgtable_walk(&host_mmu.pgt, 0, BIT(host_mmu.pgt.ia_bits), &walker);
313 }
314
fix_host_ownership(void)315 static int fix_host_ownership(void)
316 {
317 struct kvm_pgtable_walker walker = {
318 .cb = fix_host_ownership_walker,
319 .flags = KVM_PGTABLE_WALK_LEAF,
320 };
321 int i, ret;
322
323 for (i = 0; i < hyp_memblock_nr; i++) {
324 struct memblock_region *reg = &hyp_memory[i];
325 u64 start = (u64)hyp_phys_to_virt(reg->base);
326
327 ret = kvm_pgtable_walk(&pkvm_pgtable, start, reg->size, &walker);
328 if (ret)
329 return ret;
330 }
331
332 return 0;
333 }
334
fix_hyp_pgtable_refcnt(void)335 static int fix_hyp_pgtable_refcnt(void)
336 {
337 struct kvm_pgtable_walker walker = {
338 .cb = fix_hyp_pgtable_refcnt_walker,
339 .flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
340 .arg = pkvm_pgtable.mm_ops,
341 };
342
343 return kvm_pgtable_walk(&pkvm_pgtable, 0, BIT(pkvm_pgtable.ia_bits),
344 &walker);
345 }
346
unmap_protected_regions(void)347 static int unmap_protected_regions(void)
348 {
349 struct pkvm_moveable_reg *reg;
350 int i, ret;
351
352 for (i = 0; i < pkvm_moveable_regs_nr; i++) {
353 reg = &pkvm_moveable_regs[i];
354 if (reg->type != PKVM_MREG_PROTECTED_RANGE)
355 continue;
356
357 ret = host_stage2_set_owner_locked(reg->start, reg->size,
358 PKVM_ID_PROTECTED);
359 if (ret)
360 return ret;
361 }
362
363 return 0;
364 }
365
__pkvm_init_finalise(void)366 void __noreturn __pkvm_init_finalise(void)
367 {
368 struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data);
369 struct kvm_cpu_context *host_ctxt = &host_data->host_ctxt;
370 unsigned long nr_pages, reserved_pages, pfn;
371 int ret;
372
373 /* Now that the vmemmap is backed, install the full-fledged allocator */
374 pfn = hyp_virt_to_pfn(hyp_pgt_base);
375 nr_pages = hyp_s1_pgtable_pages();
376 reserved_pages = hyp_early_alloc_nr_used_pages();
377 ret = hyp_pool_init(&hpool, pfn, nr_pages, reserved_pages);
378 if (ret)
379 goto out;
380
381 ret = kvm_host_prepare_stage2(host_s2_pgt_base);
382 if (ret)
383 goto out;
384
385 pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) {
386 .zalloc_page = hyp_zalloc_hyp_page,
387 .phys_to_virt = hyp_phys_to_virt,
388 .virt_to_phys = hyp_virt_to_phys,
389 .get_page = hpool_get_page,
390 .put_page = hpool_put_page,
391 .page_count = hyp_page_count,
392 };
393 pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
394
395 ret = fix_hyp_pgtable_refcnt();
396 if (ret)
397 goto out;
398
399 ret = hyp_create_pcpu_fixmap();
400 if (ret)
401 goto out;
402
403 ret = fix_host_ownership();
404 if (ret)
405 goto out;
406
407 ret = unmap_protected_regions();
408 if (ret)
409 goto out;
410
411 ret = pin_host_tables();
412 if (ret)
413 goto out;
414
415 ret = hyp_ffa_init(ffa_proxy_pages);
416 if (ret)
417 goto out;
418
419 pkvm_hyp_vm_table_init(vm_table_base);
420 out:
421 /*
422 * We tail-called to here from handle___pkvm_init() and will not return,
423 * so make sure to propagate the return value to the host.
424 */
425 cpu_reg(host_ctxt, 1) = ret;
426
427 __host_enter(host_ctxt);
428 }
429
__pkvm_init(phys_addr_t phys,unsigned long size,unsigned long nr_cpus,unsigned long * per_cpu_base,u32 hyp_va_bits)430 int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
431 unsigned long *per_cpu_base, u32 hyp_va_bits)
432 {
433 struct kvm_nvhe_init_params *params;
434 void *virt = hyp_phys_to_virt(phys);
435 void (*fn)(phys_addr_t params_pa, void *finalize_fn_va);
436 int ret;
437
438 BUG_ON(kvm_check_pvm_sysreg_table());
439
440 if (!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(size))
441 return -EINVAL;
442
443 hyp_spin_lock_init(&pkvm_pgd_lock);
444 hyp_nr_cpus = nr_cpus;
445
446 ret = divide_memory_pool(virt, size);
447 if (ret)
448 return ret;
449
450 ret = recreate_hyp_mappings(phys, size, per_cpu_base, hyp_va_bits);
451 if (ret)
452 return ret;
453
454 update_nvhe_init_params();
455
456 /* Jump in the idmap page to switch to the new page-tables */
457 params = this_cpu_ptr(&kvm_init_params);
458 fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd);
459 fn(__hyp_pa(params), __pkvm_init_finalise);
460
461 unreachable();
462 }
463