1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 */
6
7 #include <linux/kvm_host.h>
8 #include <asm/kvm_hyp.h>
9 #include <asm/kvm_mmu.h>
10 #include <asm/kvm_pgtable.h>
11 #include <asm/spectre.h>
12
13 #include <nvhe/early_alloc.h>
14 #include <nvhe/gfp.h>
15 #include <nvhe/memory.h>
16 #include <nvhe/mm.h>
17 #include <nvhe/spinlock.h>
18
19 struct kvm_pgtable pkvm_pgtable;
20 hyp_spinlock_t pkvm_pgd_lock;
21 u64 __io_map_base;
22
23 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
24 unsigned int hyp_memblock_nr;
25
__pkvm_create_mappings(unsigned long start,unsigned long size,unsigned long phys,enum kvm_pgtable_prot prot)26 int __pkvm_create_mappings(unsigned long start, unsigned long size,
27 unsigned long phys, enum kvm_pgtable_prot prot)
28 {
29 int err;
30
31 hyp_spin_lock(&pkvm_pgd_lock);
32 err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, size, phys, prot);
33 hyp_spin_unlock(&pkvm_pgd_lock);
34
35 return err;
36 }
37
__pkvm_create_private_mapping(phys_addr_t phys,size_t size,enum kvm_pgtable_prot prot)38 unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
39 enum kvm_pgtable_prot prot)
40 {
41 unsigned long addr;
42 int err;
43
44 hyp_spin_lock(&pkvm_pgd_lock);
45
46 size = PAGE_ALIGN(size + offset_in_page(phys));
47 addr = __io_map_base;
48 __io_map_base += size;
49
50 /* Are we overflowing on the vmemmap ? */
51 if (__io_map_base > __hyp_vmemmap) {
52 __io_map_base -= size;
53 addr = (unsigned long)ERR_PTR(-ENOMEM);
54 goto out;
55 }
56
57 err = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, prot);
58 if (err) {
59 addr = (unsigned long)ERR_PTR(err);
60 goto out;
61 }
62
63 addr = addr + offset_in_page(phys);
64 out:
65 hyp_spin_unlock(&pkvm_pgd_lock);
66
67 return addr;
68 }
69
pkvm_create_mappings(void * from,void * to,enum kvm_pgtable_prot prot)70 int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
71 {
72 unsigned long start = (unsigned long)from;
73 unsigned long end = (unsigned long)to;
74 unsigned long virt_addr;
75 phys_addr_t phys;
76
77 start = start & PAGE_MASK;
78 end = PAGE_ALIGN(end);
79
80 for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
81 int err;
82
83 phys = hyp_virt_to_phys((void *)virt_addr);
84 err = __pkvm_create_mappings(virt_addr, PAGE_SIZE, phys, prot);
85 if (err)
86 return err;
87 }
88
89 return 0;
90 }
91
hyp_back_vmemmap(phys_addr_t phys,unsigned long size,phys_addr_t back)92 int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)
93 {
94 unsigned long start, end;
95
96 hyp_vmemmap_range(phys, size, &start, &end);
97
98 return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);
99 }
100
101 static void *__hyp_bp_vect_base;
pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot)102 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot)
103 {
104 void *vector;
105
106 switch (slot) {
107 case HYP_VECTOR_DIRECT: {
108 vector = __kvm_hyp_vector;
109 break;
110 }
111 case HYP_VECTOR_SPECTRE_DIRECT: {
112 vector = __bp_harden_hyp_vecs;
113 break;
114 }
115 case HYP_VECTOR_INDIRECT:
116 case HYP_VECTOR_SPECTRE_INDIRECT: {
117 vector = (void *)__hyp_bp_vect_base;
118 break;
119 }
120 default:
121 return -EINVAL;
122 }
123
124 vector = __kvm_vector_slot2addr(vector, slot);
125 *this_cpu_ptr(&kvm_hyp_vector) = (unsigned long)vector;
126
127 return 0;
128 }
129
hyp_map_vectors(void)130 int hyp_map_vectors(void)
131 {
132 phys_addr_t phys;
133 void *bp_base;
134
135 if (!kvm_system_needs_idmapped_vectors()) {
136 __hyp_bp_vect_base = __bp_harden_hyp_vecs;
137 return 0;
138 }
139
140 phys = __hyp_pa(__bp_harden_hyp_vecs);
141 bp_base = (void *)__pkvm_create_private_mapping(phys,
142 __BP_HARDEN_HYP_VECS_SZ,
143 PAGE_HYP_EXEC);
144 if (IS_ERR_OR_NULL(bp_base))
145 return PTR_ERR(bp_base);
146
147 __hyp_bp_vect_base = bp_base;
148
149 return 0;
150 }
151
hyp_create_idmap(u32 hyp_va_bits)152 int hyp_create_idmap(u32 hyp_va_bits)
153 {
154 unsigned long start, end;
155
156 start = hyp_virt_to_phys((void *)__hyp_idmap_text_start);
157 start = ALIGN_DOWN(start, PAGE_SIZE);
158
159 end = hyp_virt_to_phys((void *)__hyp_idmap_text_end);
160 end = ALIGN(end, PAGE_SIZE);
161
162 /*
163 * One half of the VA space is reserved to linearly map portions of
164 * memory -- see va_layout.c for more details. The other half of the VA
165 * space contains the trampoline page, and needs some care. Split that
166 * second half in two and find the quarter of VA space not conflicting
167 * with the idmap to place the IOs and the vmemmap. IOs use the lower
168 * half of the quarter and the vmemmap the upper half.
169 */
170 __io_map_base = start & BIT(hyp_va_bits - 2);
171 __io_map_base ^= BIT(hyp_va_bits - 2);
172 __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3);
173
174 return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
175 }
176