1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 - Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 */
6
7 #include <linux/io.h>
8 #include <linux/kvm_host.h>
9 #include <linux/memblock.h>
10 #include <linux/mm.h>
11 #include <linux/mutex.h>
12 #include <linux/of_fdt.h>
13 #include <linux/of_reserved_mem.h>
14 #include <linux/sort.h>
15
16 #include <asm/kvm_pkvm.h>
17
18 #include "hyp_constants.h"
19
20 static struct reserved_mem *pkvm_firmware_mem;
21 static phys_addr_t *pvmfw_base = &kvm_nvhe_sym(pvmfw_base);
22 static phys_addr_t *pvmfw_size = &kvm_nvhe_sym(pvmfw_size);
23
24 static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
25 static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
26
27 phys_addr_t hyp_mem_base;
28 phys_addr_t hyp_mem_size;
29
cmp_hyp_memblock(const void * p1,const void * p2)30 static int cmp_hyp_memblock(const void *p1, const void *p2)
31 {
32 const struct memblock_region *r1 = p1;
33 const struct memblock_region *r2 = p2;
34
35 return r1->base < r2->base ? -1 : (r1->base > r2->base);
36 }
37
sort_memblock_regions(void)38 static void __init sort_memblock_regions(void)
39 {
40 sort(hyp_memory,
41 *hyp_memblock_nr_ptr,
42 sizeof(struct memblock_region),
43 cmp_hyp_memblock,
44 NULL);
45 }
46
register_memblock_regions(void)47 static int __init register_memblock_regions(void)
48 {
49 struct memblock_region *reg;
50
51 for_each_mem_region(reg) {
52 if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
53 return -ENOMEM;
54
55 hyp_memory[*hyp_memblock_nr_ptr] = *reg;
56 (*hyp_memblock_nr_ptr)++;
57 }
58 sort_memblock_regions();
59
60 return 0;
61 }
62
kvm_hyp_reserve(void)63 void __init kvm_hyp_reserve(void)
64 {
65 u64 hyp_mem_pages = 0;
66 int ret;
67
68 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
69 return;
70
71 if (kvm_get_mode() != KVM_MODE_PROTECTED)
72 return;
73
74 ret = register_memblock_regions();
75 if (ret) {
76 *hyp_memblock_nr_ptr = 0;
77 kvm_err("Failed to register hyp memblocks: %d\n", ret);
78 return;
79 }
80
81 hyp_mem_pages += hyp_s1_pgtable_pages();
82 hyp_mem_pages += host_s2_pgtable_pages();
83 hyp_mem_pages += hyp_shadow_table_pages(KVM_SHADOW_VM_SIZE);
84 hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
85 hyp_mem_pages += hyp_ffa_proxy_pages();
86
87 /*
88 * Try to allocate a PMD-aligned region to reduce TLB pressure once
89 * this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
90 */
91 hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
92 hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE),
93 PMD_SIZE);
94 if (!hyp_mem_base)
95 hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE);
96 else
97 hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
98
99 if (!hyp_mem_base) {
100 kvm_err("Failed to reserve hyp memory\n");
101 return;
102 }
103
104 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
105 hyp_mem_base);
106 }
107
108 /*
109 * Allocates and donates memory for EL2 shadow structs.
110 *
111 * Allocates space for the shadow state, which includes the shadow vm as well as
112 * the shadow vcpu states.
113 *
114 * Stores an opaque handler in the kvm struct for future reference.
115 *
116 * Return 0 on success, negative error code on failure.
117 */
__create_el2_shadow(struct kvm * kvm)118 static int __create_el2_shadow(struct kvm *kvm)
119 {
120 struct kvm_vcpu *vcpu;
121 size_t pgd_sz, shadow_sz, vcpu_state_sz;
122 void *pgd, *shadow_addr;
123 unsigned long idx;
124 int shadow_handle;
125 int ret;
126
127 if (kvm->created_vcpus < 1)
128 return -EINVAL;
129
130 pgd_sz = kvm_pgtable_stage2_pgd_size(kvm->arch.vtcr);
131 /*
132 * The PGD pages will be reclaimed using a hyp_memcache which implies
133 * page granularity. So, use alloc_pages_exact() to get individual
134 * refcounts.
135 */
136 pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
137 if (!pgd)
138 return -ENOMEM;
139
140 /* Allocate memory to donate to hyp for the kvm and vcpu state pointers. */
141 shadow_sz = PAGE_ALIGN(KVM_SHADOW_VM_SIZE +
142 sizeof(void *) * kvm->created_vcpus);
143 shadow_addr = alloc_pages_exact(shadow_sz, GFP_KERNEL_ACCOUNT);
144 if (!shadow_addr) {
145 ret = -ENOMEM;
146 goto free_pgd;
147 }
148
149 /* Donate the shadow memory to hyp and let hyp initialize it. */
150 ret = kvm_call_hyp_nvhe(__pkvm_init_shadow, kvm, shadow_addr, shadow_sz,
151 pgd);
152 if (ret < 0)
153 goto free_shadow;
154
155 shadow_handle = ret;
156
157 /* Store the shadow handle given by hyp for future call reference. */
158 kvm->arch.pkvm.shadow_handle = shadow_handle;
159
160 /* Donate memory for the vcpu state at hyp and initialize it. */
161 vcpu_state_sz = PAGE_ALIGN(SHADOW_VCPU_STATE_SIZE);
162 kvm_for_each_vcpu (idx, vcpu, kvm) {
163 void *vcpu_state;
164
165 /* Indexing of the vcpus to be sequential starting at 0. */
166 if (WARN_ON(vcpu->vcpu_idx != idx)) {
167 ret = -EINVAL;
168 goto destroy_vm;
169 }
170 vcpu_state = alloc_pages_exact(vcpu_state_sz, GFP_KERNEL_ACCOUNT);
171 if (!vcpu_state) {
172 ret = -ENOMEM;
173 goto destroy_vm;
174 }
175
176 ret = kvm_call_hyp_nvhe(__pkvm_init_shadow_vcpu, shadow_handle,
177 vcpu, vcpu_state);
178 if (ret) {
179 free_pages_exact(vcpu_state, vcpu_state_sz);
180 goto destroy_vm;
181 }
182 }
183
184 return 0;
185
186 destroy_vm:
187 kvm_shadow_destroy(kvm);
188 return ret;
189 free_shadow:
190 free_pages_exact(shadow_addr, shadow_sz);
191 free_pgd:
192 free_pages_exact(pgd, pgd_sz);
193 return ret;
194 }
195
create_el2_shadow(struct kvm * kvm)196 int create_el2_shadow(struct kvm *kvm)
197 {
198 int ret = 0;
199
200 mutex_lock(&kvm->arch.pkvm.shadow_lock);
201 if (!kvm->arch.pkvm.shadow_handle)
202 ret = __create_el2_shadow(kvm);
203 mutex_unlock(&kvm->arch.pkvm.shadow_lock);
204
205 return ret;
206 }
207
kvm_shadow_destroy(struct kvm * kvm)208 void kvm_shadow_destroy(struct kvm *kvm)
209 {
210 struct kvm_pinned_page *ppage, *tmp;
211 struct mm_struct *mm = current->mm;
212 struct list_head *ppages;
213
214 if (kvm->arch.pkvm.shadow_handle)
215 WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_shadow,
216 kvm->arch.pkvm.shadow_handle));
217
218 free_hyp_memcache(&kvm->arch.pkvm.teardown_mc);
219
220 ppages = &kvm->arch.pkvm.pinned_pages;
221 list_for_each_entry_safe(ppage, tmp, ppages, link) {
222 WARN_ON(kvm_call_hyp_nvhe(__pkvm_host_reclaim_page,
223 page_to_pfn(ppage->page)));
224 cond_resched();
225
226 account_locked_vm(mm, 1, false);
227 unpin_user_pages_dirty_lock(&ppage->page, 1, true);
228 list_del(&ppage->link);
229 kfree(ppage);
230 }
231 }
232
pkvm_firmware_rmem_err(struct reserved_mem * rmem,const char * reason)233 static int __init pkvm_firmware_rmem_err(struct reserved_mem *rmem,
234 const char *reason)
235 {
236 phys_addr_t end = rmem->base + rmem->size;
237
238 kvm_err("Ignoring pkvm guest firmware memory reservation [%pa - %pa]: %s\n",
239 &rmem->base, &end, reason);
240 return -EINVAL;
241 }
242
pkvm_firmware_rmem_init(struct reserved_mem * rmem)243 static int __init pkvm_firmware_rmem_init(struct reserved_mem *rmem)
244 {
245 unsigned long node = rmem->fdt_node;
246
247 if (pkvm_firmware_mem)
248 return pkvm_firmware_rmem_err(rmem, "duplicate reservation");
249
250 if (!of_get_flat_dt_prop(node, "no-map", NULL))
251 return pkvm_firmware_rmem_err(rmem, "missing \"no-map\" property");
252
253 if (of_get_flat_dt_prop(node, "reusable", NULL))
254 return pkvm_firmware_rmem_err(rmem, "\"reusable\" property unsupported");
255
256 if (!PAGE_ALIGNED(rmem->base))
257 return pkvm_firmware_rmem_err(rmem, "base is not page-aligned");
258
259 if (!PAGE_ALIGNED(rmem->size))
260 return pkvm_firmware_rmem_err(rmem, "size is not page-aligned");
261
262 *pvmfw_size = rmem->size;
263 *pvmfw_base = rmem->base;
264 pkvm_firmware_mem = rmem;
265 return 0;
266 }
267 RESERVEDMEM_OF_DECLARE(pkvm_firmware, "linux,pkvm-guest-firmware-memory",
268 pkvm_firmware_rmem_init);
269
pkvm_firmware_rmem_clear(void)270 static int __init pkvm_firmware_rmem_clear(void)
271 {
272 void *addr;
273 phys_addr_t size;
274
275 if (likely(!pkvm_firmware_mem) || is_protected_kvm_enabled())
276 return 0;
277
278 kvm_info("Clearing unused pKVM firmware memory\n");
279 size = pkvm_firmware_mem->size;
280 addr = memremap(pkvm_firmware_mem->base, size, MEMREMAP_WB);
281 if (!addr)
282 return -EINVAL;
283
284 memset(addr, 0, size);
285 dcache_clean_poc((unsigned long)addr, (unsigned long)addr + size);
286 memunmap(addr);
287 return 0;
288 }
289 device_initcall_sync(pkvm_firmware_rmem_clear);
290
pkvm_vm_ioctl_set_fw_ipa(struct kvm * kvm,u64 ipa)291 static int pkvm_vm_ioctl_set_fw_ipa(struct kvm *kvm, u64 ipa)
292 {
293 int ret = 0;
294
295 if (!pkvm_firmware_mem)
296 return -EINVAL;
297
298 mutex_lock(&kvm->arch.pkvm.shadow_lock);
299 if (kvm->arch.pkvm.shadow_handle) {
300 ret = -EBUSY;
301 goto out_unlock;
302 }
303
304 kvm->arch.pkvm.pvmfw_load_addr = ipa;
305 out_unlock:
306 mutex_unlock(&kvm->arch.pkvm.shadow_lock);
307 return ret;
308 }
309
pkvm_vm_ioctl_info(struct kvm * kvm,struct kvm_protected_vm_info __user * info)310 static int pkvm_vm_ioctl_info(struct kvm *kvm,
311 struct kvm_protected_vm_info __user *info)
312 {
313 struct kvm_protected_vm_info kinfo = {
314 .firmware_size = pkvm_firmware_mem ?
315 pkvm_firmware_mem->size :
316 0,
317 };
318
319 return copy_to_user(info, &kinfo, sizeof(kinfo)) ? -EFAULT : 0;
320 }
321
kvm_arm_vm_ioctl_pkvm(struct kvm * kvm,struct kvm_enable_cap * cap)322 int kvm_arm_vm_ioctl_pkvm(struct kvm *kvm, struct kvm_enable_cap *cap)
323 {
324 if (cap->args[1] || cap->args[2] || cap->args[3])
325 return -EINVAL;
326
327 switch (cap->flags) {
328 case KVM_CAP_ARM_PROTECTED_VM_FLAGS_SET_FW_IPA:
329 return pkvm_vm_ioctl_set_fw_ipa(kvm, cap->args[0]);
330 case KVM_CAP_ARM_PROTECTED_VM_FLAGS_INFO:
331 return pkvm_vm_ioctl_info(kvm, (void __force __user *)cap->args[0]);
332 default:
333 return -EINVAL;
334 }
335
336 return 0;
337 }
338
kvm_init_pvm(struct kvm * kvm,unsigned long type)339 int kvm_init_pvm(struct kvm *kvm, unsigned long type)
340 {
341 mutex_init(&kvm->arch.pkvm.shadow_lock);
342 kvm->arch.pkvm.pvmfw_load_addr = PVMFW_INVALID_LOAD_ADDR;
343
344 if (!(type & KVM_VM_TYPE_ARM_PROTECTED))
345 return 0;
346
347 if (!is_protected_kvm_enabled())
348 return -EINVAL;
349
350 kvm->arch.pkvm.enabled = true;
351 return 0;
352 }
353