1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <linux/kvm_host.h>
8 #include <linux/random.h>
9 #include <linux/memblock.h>
10 #include <asm/alternative.h>
11 #include <asm/debug-monitors.h>
12 #include <asm/insn.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/memory.h>
15 #include <asm/patching.h>
16
17 /*
18 * The LSB of the HYP VA tag
19 */
20 static u8 tag_lsb;
21 /*
22 * The HYP VA tag value with the region bit
23 */
24 static u64 tag_val;
25 static u64 va_mask;
26
27 /*
28 * Compute HYP VA by using the same computation as kern_hyp_va().
29 */
__early_kern_hyp_va(u64 addr)30 static u64 __early_kern_hyp_va(u64 addr)
31 {
32 addr &= va_mask;
33 addr |= tag_val << tag_lsb;
34 return addr;
35 }
36
37 /*
38 * Store a hyp VA <-> PA offset into a EL2-owned variable.
39 */
init_hyp_physvirt_offset(void)40 static void init_hyp_physvirt_offset(void)
41 {
42 u64 kern_va, hyp_va;
43
44 /* Compute the offset from the hyp VA and PA of a random symbol. */
45 kern_va = (u64)lm_alias(__hyp_text_start);
46 hyp_va = __early_kern_hyp_va(kern_va);
47 hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
48 }
49
50 /*
51 * We want to generate a hyp VA with the following format (with V ==
52 * vabits_actual):
53 *
54 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0
55 * ---------------------------------------------------------
56 * | 0000000 | hyp_va_msb | random tag | kern linear VA |
57 * |--------- tag_val -----------|----- va_mask ---|
58 *
59 * which does not conflict with the idmap regions.
60 */
kvm_compute_layout(void)61 __init void kvm_compute_layout(void)
62 {
63 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
64 u64 hyp_va_msb;
65
66 /* Where is my RAM region? */
67 hyp_va_msb = idmap_addr & BIT(vabits_actual - 1);
68 hyp_va_msb ^= BIT(vabits_actual - 1);
69
70 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
71 (u64)(high_memory - 1));
72
73 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
74 tag_val = hyp_va_msb;
75
76 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
77 /* We have some free bits to insert a random tag. */
78 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
79 }
80 tag_val >>= tag_lsb;
81
82 init_hyp_physvirt_offset();
83 }
84
85 /*
86 * The .hyp.reloc ELF section contains a list of kimg positions that
87 * contains kimg VAs but will be accessed only in hyp execution context.
88 * Convert them to hyp VAs. See gen-hyprel.c for more details.
89 */
kvm_apply_hyp_relocations(void)90 __init void kvm_apply_hyp_relocations(void)
91 {
92 int32_t *rel;
93 int32_t *begin = (int32_t *)__hyp_reloc_begin;
94 int32_t *end = (int32_t *)__hyp_reloc_end;
95
96 for (rel = begin; rel < end; ++rel) {
97 uintptr_t *ptr, kimg_va;
98
99 /*
100 * Each entry contains a 32-bit relative offset from itself
101 * to a kimg VA position.
102 */
103 ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
104
105 /* Read the kimg VA value at the relocation address. */
106 kimg_va = *ptr;
107
108 /* Convert to hyp VA and store back to the relocation address. */
109 *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
110 }
111 }
112
kvm_apply_hyp_module_relocations(struct pkvm_el2_module * mod,kvm_nvhe_reloc_t * begin,kvm_nvhe_reloc_t * end)113 void kvm_apply_hyp_module_relocations(struct pkvm_el2_module *mod,
114 kvm_nvhe_reloc_t *begin,
115 kvm_nvhe_reloc_t *end)
116 {
117 kvm_nvhe_reloc_t *rel;
118
119 for (rel = begin; rel < end; ++rel) {
120 u32 **ptr, *va;
121
122 /*
123 * Each entry contains a 32-bit relative offset from itself
124 * to a VA position in the module area.
125 */
126 ptr = (u32 **)((char *)rel + *rel);
127
128 /* Read the module VA value at the relocation address. */
129 va = *ptr;
130
131 /* Convert the module VA of the reloc to a hyp VA */
132 WARN_ON(aarch64_insn_write_literal_u64(ptr,
133 (u64)(((void *)va - mod->sections.start) + mod->hyp_va)));
134 }
135
136 sync_icache_aliases((unsigned long)mod->text.start,
137 (unsigned long)mod->text.end);
138 }
139
compute_instruction(int n,u32 rd,u32 rn)140 static u32 compute_instruction(int n, u32 rd, u32 rn)
141 {
142 u32 insn = AARCH64_BREAK_FAULT;
143
144 switch (n) {
145 case 0:
146 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
147 AARCH64_INSN_VARIANT_64BIT,
148 rn, rd, va_mask);
149 break;
150
151 case 1:
152 /* ROR is a variant of EXTR with Rm = Rn */
153 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
154 rn, rn, rd,
155 tag_lsb);
156 break;
157
158 case 2:
159 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
160 tag_val & GENMASK(11, 0),
161 AARCH64_INSN_VARIANT_64BIT,
162 AARCH64_INSN_ADSB_ADD);
163 break;
164
165 case 3:
166 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
167 tag_val & GENMASK(23, 12),
168 AARCH64_INSN_VARIANT_64BIT,
169 AARCH64_INSN_ADSB_ADD);
170 break;
171
172 case 4:
173 /* ROR is a variant of EXTR with Rm = Rn */
174 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
175 rn, rn, rd, 64 - tag_lsb);
176 break;
177 }
178
179 return insn;
180 }
181
kvm_update_va_mask(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)182 void __init kvm_update_va_mask(struct alt_instr *alt,
183 __le32 *origptr, __le32 *updptr, int nr_inst)
184 {
185 int i;
186
187 BUG_ON(nr_inst != 5);
188
189 for (i = 0; i < nr_inst; i++) {
190 u32 rd, rn, insn, oinsn;
191
192 /*
193 * VHE doesn't need any address translation, let's NOP
194 * everything.
195 *
196 * Alternatively, if the tag is zero (because the layout
197 * dictates it and we don't have any spare bits in the
198 * address), NOP everything after masking the kernel VA.
199 */
200 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
201 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
202 continue;
203 }
204
205 oinsn = le32_to_cpu(origptr[i]);
206 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
207 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
208
209 insn = compute_instruction(i, rd, rn);
210 BUG_ON(insn == AARCH64_BREAK_FAULT);
211
212 updptr[i] = cpu_to_le32(insn);
213 }
214 }
215
kvm_patch_vector_branch(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)216 void kvm_patch_vector_branch(struct alt_instr *alt,
217 __le32 *origptr, __le32 *updptr, int nr_inst)
218 {
219 u64 addr;
220 u32 insn;
221
222 BUG_ON(nr_inst != 4);
223
224 if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
225 WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
226 return;
227
228 /*
229 * Compute HYP VA by using the same computation as kern_hyp_va()
230 */
231 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
232
233 /* Use PC[10:7] to branch to the same vector in KVM */
234 addr |= ((u64)origptr & GENMASK_ULL(10, 7));
235
236 /*
237 * Branch over the preamble in order to avoid the initial store on
238 * the stack (which we already perform in the hardening vectors).
239 */
240 addr += KVM_VECTOR_PREAMBLE;
241
242 /* movz x0, #(addr & 0xffff) */
243 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
244 (u16)addr,
245 0,
246 AARCH64_INSN_VARIANT_64BIT,
247 AARCH64_INSN_MOVEWIDE_ZERO);
248 *updptr++ = cpu_to_le32(insn);
249
250 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
251 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
252 (u16)(addr >> 16),
253 16,
254 AARCH64_INSN_VARIANT_64BIT,
255 AARCH64_INSN_MOVEWIDE_KEEP);
256 *updptr++ = cpu_to_le32(insn);
257
258 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
259 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
260 (u16)(addr >> 32),
261 32,
262 AARCH64_INSN_VARIANT_64BIT,
263 AARCH64_INSN_MOVEWIDE_KEEP);
264 *updptr++ = cpu_to_le32(insn);
265
266 /* br x0 */
267 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
268 AARCH64_INSN_BRANCH_NOLINK);
269 *updptr++ = cpu_to_le32(insn);
270 }
271
generate_mov_q(u64 val,__le32 * origptr,__le32 * updptr,int nr_inst)272 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
273 {
274 u32 insn, oinsn, rd;
275
276 BUG_ON(nr_inst != 4);
277
278 /* Compute target register */
279 oinsn = le32_to_cpu(*origptr);
280 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
281
282 /* movz rd, #(val & 0xffff) */
283 insn = aarch64_insn_gen_movewide(rd,
284 (u16)val,
285 0,
286 AARCH64_INSN_VARIANT_64BIT,
287 AARCH64_INSN_MOVEWIDE_ZERO);
288 *updptr++ = cpu_to_le32(insn);
289
290 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
291 insn = aarch64_insn_gen_movewide(rd,
292 (u16)(val >> 16),
293 16,
294 AARCH64_INSN_VARIANT_64BIT,
295 AARCH64_INSN_MOVEWIDE_KEEP);
296 *updptr++ = cpu_to_le32(insn);
297
298 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
299 insn = aarch64_insn_gen_movewide(rd,
300 (u16)(val >> 32),
301 32,
302 AARCH64_INSN_VARIANT_64BIT,
303 AARCH64_INSN_MOVEWIDE_KEEP);
304 *updptr++ = cpu_to_le32(insn);
305
306 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
307 insn = aarch64_insn_gen_movewide(rd,
308 (u16)(val >> 48),
309 48,
310 AARCH64_INSN_VARIANT_64BIT,
311 AARCH64_INSN_MOVEWIDE_KEEP);
312 *updptr++ = cpu_to_le32(insn);
313 }
314
kvm_get_kimage_voffset(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)315 void kvm_get_kimage_voffset(struct alt_instr *alt,
316 __le32 *origptr, __le32 *updptr, int nr_inst)
317 {
318 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);
319 }
320
kvm_compute_final_ctr_el0(struct alt_instr * alt,__le32 * origptr,__le32 * updptr,int nr_inst)321 void kvm_compute_final_ctr_el0(struct alt_instr *alt,
322 __le32 *origptr, __le32 *updptr, int nr_inst)
323 {
324 generate_mov_q(read_sanitised_ftr_reg(SYS_CTR_EL0),
325 origptr, updptr, nr_inst);
326 }
327