1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 - Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 */
6
7 #include <linux/io.h>
8 #include <linux/kvm_host.h>
9 #include <linux/memblock.h>
10 #include <linux/mm.h>
11 #include <linux/mutex.h>
12 #include <linux/of_fdt.h>
13 #include <linux/of_reserved_mem.h>
14 #include <linux/sort.h>
15
16 #include <asm/kvm_pkvm.h>
17
18 #include "hyp_constants.h"
19
20 static struct reserved_mem *pkvm_firmware_mem;
21 static phys_addr_t *pvmfw_base = &kvm_nvhe_sym(pvmfw_base);
22 static phys_addr_t *pvmfw_size = &kvm_nvhe_sym(pvmfw_size);
23
24 static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
25 static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
26
27 phys_addr_t hyp_mem_base;
28 phys_addr_t hyp_mem_size;
29
cmp_hyp_memblock(const void * p1,const void * p2)30 static int cmp_hyp_memblock(const void *p1, const void *p2)
31 {
32 const struct memblock_region *r1 = p1;
33 const struct memblock_region *r2 = p2;
34
35 return r1->base < r2->base ? -1 : (r1->base > r2->base);
36 }
37
sort_memblock_regions(void)38 static void __init sort_memblock_regions(void)
39 {
40 sort(hyp_memory,
41 *hyp_memblock_nr_ptr,
42 sizeof(struct memblock_region),
43 cmp_hyp_memblock,
44 NULL);
45 }
46
register_memblock_regions(void)47 static int __init register_memblock_regions(void)
48 {
49 struct memblock_region *reg;
50
51 for_each_mem_region(reg) {
52 if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
53 return -ENOMEM;
54
55 hyp_memory[*hyp_memblock_nr_ptr] = *reg;
56 (*hyp_memblock_nr_ptr)++;
57 }
58 sort_memblock_regions();
59
60 return 0;
61 }
62
kvm_hyp_reserve(void)63 void __init kvm_hyp_reserve(void)
64 {
65 u64 hyp_mem_pages = 0;
66 int ret;
67
68 if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
69 return;
70
71 if (kvm_get_mode() != KVM_MODE_PROTECTED)
72 return;
73
74 ret = register_memblock_regions();
75 if (ret) {
76 *hyp_memblock_nr_ptr = 0;
77 kvm_err("Failed to register hyp memblocks: %d\n", ret);
78 return;
79 }
80
81 hyp_mem_pages += hyp_s1_pgtable_pages();
82 hyp_mem_pages += host_s2_pgtable_pages();
83 hyp_mem_pages += hyp_shadow_table_pages(KVM_SHADOW_VM_SIZE);
84 hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
85 hyp_mem_pages += hyp_ffa_proxy_pages();
86
87 /*
88 * Try to allocate a PMD-aligned region to reduce TLB pressure once
89 * this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
90 */
91 hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
92 hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(),
93 ALIGN(hyp_mem_size, PMD_SIZE),
94 PMD_SIZE);
95 if (!hyp_mem_base)
96 hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(),
97 hyp_mem_size, PAGE_SIZE);
98 else
99 hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
100
101 if (!hyp_mem_base) {
102 kvm_err("Failed to reserve hyp memory\n");
103 return;
104 }
105 memblock_reserve(hyp_mem_base, hyp_mem_size);
106
107 kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
108 hyp_mem_base);
109 }
110
111 /*
112 * Allocates and donates memory for EL2 shadow structs.
113 *
114 * Allocates space for the shadow state, which includes the shadow vm as well as
115 * the shadow vcpu states.
116 *
117 * Stores an opaque handler in the kvm struct for future reference.
118 *
119 * Return 0 on success, negative error code on failure.
120 */
__create_el2_shadow(struct kvm * kvm)121 static int __create_el2_shadow(struct kvm *kvm)
122 {
123 struct kvm_vcpu *vcpu;
124 size_t pgd_sz, shadow_sz, vcpu_state_sz;
125 void *pgd, *shadow_addr;
126 unsigned long idx;
127 int shadow_handle;
128 int ret;
129
130 if (kvm->created_vcpus < 1)
131 return -EINVAL;
132
133 pgd_sz = kvm_pgtable_stage2_pgd_size(kvm->arch.vtcr);
134 /*
135 * The PGD pages will be reclaimed using a hyp_memcache which implies
136 * page granularity. So, use alloc_pages_exact() to get individual
137 * refcounts.
138 */
139 pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT);
140 if (!pgd)
141 return -ENOMEM;
142
143 /* Allocate memory to donate to hyp for the kvm and vcpu state pointers. */
144 shadow_sz = PAGE_ALIGN(KVM_SHADOW_VM_SIZE +
145 sizeof(void *) * kvm->created_vcpus);
146 shadow_addr = alloc_pages_exact(shadow_sz, GFP_KERNEL_ACCOUNT);
147 if (!shadow_addr) {
148 ret = -ENOMEM;
149 goto free_pgd;
150 }
151
152 /* Donate the shadow memory to hyp and let hyp initialize it. */
153 ret = kvm_call_hyp_nvhe(__pkvm_init_shadow, kvm, shadow_addr, shadow_sz,
154 pgd);
155 if (ret < 0)
156 goto free_shadow;
157
158 shadow_handle = ret;
159
160 /* Store the shadow handle given by hyp for future call reference. */
161 kvm->arch.pkvm.shadow_handle = shadow_handle;
162
163 /* Donate memory for the vcpu state at hyp and initialize it. */
164 vcpu_state_sz = PAGE_ALIGN(SHADOW_VCPU_STATE_SIZE);
165 kvm_for_each_vcpu (idx, vcpu, kvm) {
166 void *vcpu_state;
167
168 /* Indexing of the vcpus to be sequential starting at 0. */
169 if (WARN_ON(vcpu->vcpu_idx != idx)) {
170 ret = -EINVAL;
171 goto destroy_vm;
172 }
173 vcpu_state = alloc_pages_exact(vcpu_state_sz, GFP_KERNEL_ACCOUNT);
174 if (!vcpu_state) {
175 ret = -ENOMEM;
176 goto destroy_vm;
177 }
178
179 ret = kvm_call_hyp_nvhe(__pkvm_init_shadow_vcpu, shadow_handle,
180 vcpu, vcpu_state);
181 if (ret) {
182 free_pages_exact(vcpu_state, vcpu_state_sz);
183 goto destroy_vm;
184 }
185 }
186
187 return 0;
188
189 destroy_vm:
190 kvm_shadow_destroy(kvm);
191 return ret;
192 free_shadow:
193 free_pages_exact(shadow_addr, shadow_sz);
194 free_pgd:
195 free_pages_exact(pgd, pgd_sz);
196 return ret;
197 }
198
create_el2_shadow(struct kvm * kvm)199 int create_el2_shadow(struct kvm *kvm)
200 {
201 int ret = 0;
202
203 mutex_lock(&kvm->arch.pkvm.shadow_lock);
204 if (!kvm->arch.pkvm.shadow_handle)
205 ret = __create_el2_shadow(kvm);
206 mutex_unlock(&kvm->arch.pkvm.shadow_lock);
207
208 return ret;
209 }
210
kvm_shadow_destroy(struct kvm * kvm)211 void kvm_shadow_destroy(struct kvm *kvm)
212 {
213 struct kvm_pinned_page *ppage, *tmp;
214 struct mm_struct *mm = current->mm;
215 struct list_head *ppages;
216
217 if (kvm->arch.pkvm.shadow_handle)
218 WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_shadow,
219 kvm->arch.pkvm.shadow_handle));
220
221 free_hyp_memcache(&kvm->arch.pkvm.teardown_mc);
222
223 ppages = &kvm->arch.pkvm.pinned_pages;
224 list_for_each_entry_safe(ppage, tmp, ppages, link) {
225 WARN_ON(kvm_call_hyp_nvhe(__pkvm_host_reclaim_page,
226 page_to_pfn(ppage->page)));
227 cond_resched();
228
229 account_locked_vm(mm, 1, false);
230 unpin_user_pages_dirty_lock(&ppage->page, 1, true);
231 list_del(&ppage->link);
232 kfree(ppage);
233 }
234 }
235
pkvm_firmware_rmem_err(struct reserved_mem * rmem,const char * reason)236 static int __init pkvm_firmware_rmem_err(struct reserved_mem *rmem,
237 const char *reason)
238 {
239 phys_addr_t end = rmem->base + rmem->size;
240
241 kvm_err("Ignoring pkvm guest firmware memory reservation [%pa - %pa]: %s\n",
242 &rmem->base, &end, reason);
243 return -EINVAL;
244 }
245
pkvm_firmware_rmem_init(struct reserved_mem * rmem)246 static int __init pkvm_firmware_rmem_init(struct reserved_mem *rmem)
247 {
248 unsigned long node = rmem->fdt_node;
249
250 if (pkvm_firmware_mem)
251 return pkvm_firmware_rmem_err(rmem, "duplicate reservation");
252
253 if (!of_get_flat_dt_prop(node, "no-map", NULL))
254 return pkvm_firmware_rmem_err(rmem, "missing \"no-map\" property");
255
256 if (of_get_flat_dt_prop(node, "reusable", NULL))
257 return pkvm_firmware_rmem_err(rmem, "\"reusable\" property unsupported");
258
259 if (!PAGE_ALIGNED(rmem->base))
260 return pkvm_firmware_rmem_err(rmem, "base is not page-aligned");
261
262 if (!PAGE_ALIGNED(rmem->size))
263 return pkvm_firmware_rmem_err(rmem, "size is not page-aligned");
264
265 *pvmfw_size = rmem->size;
266 *pvmfw_base = rmem->base;
267 pkvm_firmware_mem = rmem;
268 return 0;
269 }
270 RESERVEDMEM_OF_DECLARE(pkvm_firmware, "linux,pkvm-guest-firmware-memory",
271 pkvm_firmware_rmem_init);
272
pkvm_firmware_rmem_clear(void)273 static int __init pkvm_firmware_rmem_clear(void)
274 {
275 void *addr;
276 phys_addr_t size;
277
278 if (likely(!pkvm_firmware_mem) || is_protected_kvm_enabled())
279 return 0;
280
281 kvm_info("Clearing unused pKVM firmware memory\n");
282 size = pkvm_firmware_mem->size;
283 addr = memremap(pkvm_firmware_mem->base, size, MEMREMAP_WB);
284 if (!addr)
285 return -EINVAL;
286
287 memset(addr, 0, size);
288 dcache_clean_poc((unsigned long)addr, (unsigned long)addr + size);
289 memunmap(addr);
290 return 0;
291 }
292 device_initcall_sync(pkvm_firmware_rmem_clear);
293
pkvm_vm_ioctl_set_fw_ipa(struct kvm * kvm,u64 ipa)294 static int pkvm_vm_ioctl_set_fw_ipa(struct kvm *kvm, u64 ipa)
295 {
296 int ret = 0;
297
298 if (!pkvm_firmware_mem)
299 return -EINVAL;
300
301 mutex_lock(&kvm->arch.pkvm.shadow_lock);
302 if (kvm->arch.pkvm.shadow_handle) {
303 ret = -EBUSY;
304 goto out_unlock;
305 }
306
307 kvm->arch.pkvm.pvmfw_load_addr = ipa;
308 out_unlock:
309 mutex_unlock(&kvm->arch.pkvm.shadow_lock);
310 return ret;
311 }
312
pkvm_vm_ioctl_info(struct kvm * kvm,struct kvm_protected_vm_info __user * info)313 static int pkvm_vm_ioctl_info(struct kvm *kvm,
314 struct kvm_protected_vm_info __user *info)
315 {
316 struct kvm_protected_vm_info kinfo = {
317 .firmware_size = pkvm_firmware_mem ?
318 pkvm_firmware_mem->size :
319 0,
320 };
321
322 return copy_to_user(info, &kinfo, sizeof(kinfo)) ? -EFAULT : 0;
323 }
324
kvm_arm_vm_ioctl_pkvm(struct kvm * kvm,struct kvm_enable_cap * cap)325 int kvm_arm_vm_ioctl_pkvm(struct kvm *kvm, struct kvm_enable_cap *cap)
326 {
327 if (cap->args[1] || cap->args[2] || cap->args[3])
328 return -EINVAL;
329
330 switch (cap->flags) {
331 case KVM_CAP_ARM_PROTECTED_VM_FLAGS_SET_FW_IPA:
332 return pkvm_vm_ioctl_set_fw_ipa(kvm, cap->args[0]);
333 case KVM_CAP_ARM_PROTECTED_VM_FLAGS_INFO:
334 return pkvm_vm_ioctl_info(kvm, (void __force __user *)cap->args[0]);
335 default:
336 return -EINVAL;
337 }
338
339 return 0;
340 }
341
kvm_init_pvm(struct kvm * kvm,unsigned long type)342 int kvm_init_pvm(struct kvm *kvm, unsigned long type)
343 {
344 mutex_init(&kvm->arch.pkvm.shadow_lock);
345 kvm->arch.pkvm.pvmfw_load_addr = PVMFW_INVALID_LOAD_ADDR;
346
347 if (!(type & KVM_VM_TYPE_ARM_PROTECTED))
348 return 0;
349
350 if (!is_protected_kvm_enabled())
351 return -EINVAL;
352
353 kvm->arch.pkvm.enabled = true;
354 return 0;
355 }
356