1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <asm/kvm_hyp.h>
8 #include <asm/kvm_mmu.h>
9 #include <asm/tlbflush.h>
10
11 #include <nvhe/mem_protect.h>
12
13 struct tlb_inv_context {
14 struct kvm_s2_mmu *mmu;
15 u64 tcr;
16 u64 sctlr;
17 };
18
enter_vmid_context(struct kvm_s2_mmu * mmu,struct tlb_inv_context * cxt,bool nsh)19 static void enter_vmid_context(struct kvm_s2_mmu *mmu,
20 struct tlb_inv_context *cxt,
21 bool nsh)
22 {
23 struct kvm_s2_mmu *host_s2_mmu = &host_mmu.arch.mmu;
24 struct kvm_cpu_context *host_ctxt;
25 struct kvm_vcpu *vcpu;
26
27 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
28 vcpu = host_ctxt->__hyp_running_vcpu;
29 cxt->mmu = NULL;
30
31 /*
32 * We have two requirements:
33 *
34 * - ensure that the page table updates are visible to all
35 * CPUs, for which a dsb(DOMAIN-st) is what we need, DOMAIN
36 * being either ish or nsh, depending on the invalidation
37 * type.
38 *
39 * - complete any speculative page table walk started before
40 * we trapped to EL2 so that we can mess with the MM
41 * registers out of context, for which dsb(nsh) is enough
42 *
43 * The composition of these two barriers is a dsb(DOMAIN), and
44 * the 'nsh' parameter tracks the distinction between
45 * Inner-Shareable and Non-Shareable, as specified by the
46 * callers.
47 */
48 if (nsh)
49 dsb(nsh);
50 else
51 dsb(ish);
52
53 /*
54 * If we're already in the desired context, then there's nothing
55 * to do.
56 */
57 if (vcpu) {
58 /* We're in guest context */
59 if (mmu == vcpu->arch.hw_mmu || WARN_ON(mmu != host_s2_mmu))
60 return;
61
62 cxt->mmu = vcpu->arch.hw_mmu;
63 } else {
64 /* We're in host context */
65 if (mmu == host_s2_mmu)
66 return;
67
68 cxt->mmu = host_s2_mmu;
69 }
70
71 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
72 u64 val;
73
74 /*
75 * For CPUs that are affected by ARM 1319367, we need to
76 * avoid a Stage-1 walk with the old VMID while we have
77 * the new VMID set in the VTTBR in order to invalidate TLBs.
78 * We're guaranteed that the host S1 MMU is enabled, so
79 * we can simply set the EPD bits to avoid any further
80 * TLB fill. For guests, we ensure that the S1 MMU is
81 * temporarily enabled in the next context.
82 */
83 val = cxt->tcr = read_sysreg_el1(SYS_TCR);
84 val |= TCR_EPD1_MASK | TCR_EPD0_MASK;
85 write_sysreg_el1(val, SYS_TCR);
86 isb();
87
88 if (vcpu) {
89 val = cxt->sctlr = read_sysreg_el1(SYS_SCTLR);
90 if (!(val & SCTLR_ELx_M)) {
91 val |= SCTLR_ELx_M;
92 write_sysreg_el1(val, SYS_SCTLR);
93 isb();
94 }
95 } else {
96 /* The host S1 MMU is always enabled. */
97 cxt->sctlr = SCTLR_ELx_M;
98 }
99 }
100
101 /*
102 * __load_stage2() includes an ISB only when the AT
103 * workaround is applied. Take care of the opposite condition,
104 * ensuring that we always have an ISB, but not two ISBs back
105 * to back.
106 */
107 if (vcpu)
108 __load_host_stage2();
109 else
110 __load_stage2(mmu, kern_hyp_va(mmu->arch));
111
112 asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));
113 }
114
exit_vmid_context(struct tlb_inv_context * cxt)115 static void exit_vmid_context(struct tlb_inv_context *cxt)
116 {
117 struct kvm_s2_mmu *mmu = cxt->mmu;
118 struct kvm_cpu_context *host_ctxt;
119 struct kvm_vcpu *vcpu;
120
121 host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
122 vcpu = host_ctxt->__hyp_running_vcpu;
123
124 if (!mmu)
125 return;
126
127 if (vcpu)
128 __load_stage2(mmu, kern_hyp_va(mmu->arch));
129 else
130 __load_host_stage2();
131
132 /* Ensure write of the old VMID */
133 isb();
134
135 if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
136 if (!(cxt->sctlr & SCTLR_ELx_M)) {
137 write_sysreg_el1(cxt->sctlr, SYS_SCTLR);
138 isb();
139 }
140
141 write_sysreg_el1(cxt->tcr, SYS_TCR);
142 }
143
144 cxt->mmu = NULL;
145 }
146
__kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu * mmu,phys_addr_t ipa,int level)147 void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
148 phys_addr_t ipa, int level)
149 {
150 struct tlb_inv_context cxt;
151
152 /* Switch to requested VMID */
153 enter_vmid_context(mmu, &cxt, false);
154
155 /*
156 * We could do so much better if we had the VA as well.
157 * Instead, we invalidate Stage-2 for this IPA, and the
158 * whole of Stage-1. Weep...
159 */
160 ipa >>= 12;
161 __tlbi_level(ipas2e1is, ipa, level);
162
163 /*
164 * We have to ensure completion of the invalidation at Stage-2,
165 * since a table walk on another CPU could refill a TLB with a
166 * complete (S1 + S2) walk based on the old Stage-2 mapping if
167 * the Stage-1 invalidation happened first.
168 */
169 dsb(ish);
170 __tlbi(vmalle1is);
171 dsb(ish);
172 isb();
173
174 /*
175 * If the host is running at EL1 and we have a VPIPT I-cache,
176 * then we must perform I-cache maintenance at EL2 in order for
177 * it to have an effect on the guest. Since the guest cannot hit
178 * I-cache lines allocated with a different VMID, we don't need
179 * to worry about junk out of guest reset (we nuke the I-cache on
180 * VMID rollover), but we do need to be careful when remapping
181 * executable pages for the same guest. This can happen when KSM
182 * takes a CoW fault on an executable page, copies the page into
183 * a page that was previously mapped in the guest and then needs
184 * to invalidate the guest view of the I-cache for that page
185 * from EL1. To solve this, we invalidate the entire I-cache when
186 * unmapping a page from a guest if we have a VPIPT I-cache but
187 * the host is running at EL1. As above, we could do better if
188 * we had the VA.
189 *
190 * The moral of this story is: if you have a VPIPT I-cache, then
191 * you should be running with VHE enabled.
192 */
193 if (icache_is_vpipt())
194 icache_inval_all_pou();
195
196 exit_vmid_context(&cxt);
197 }
198
__kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu * mmu,phys_addr_t ipa,int level)199 void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,
200 phys_addr_t ipa, int level)
201 {
202 struct tlb_inv_context cxt;
203
204 /* Switch to requested VMID */
205 enter_vmid_context(mmu, &cxt, true);
206
207 /*
208 * We could do so much better if we had the VA as well.
209 * Instead, we invalidate Stage-2 for this IPA, and the
210 * whole of Stage-1. Weep...
211 */
212 ipa >>= 12;
213 __tlbi_level(ipas2e1, ipa, level);
214
215 /*
216 * We have to ensure completion of the invalidation at Stage-2,
217 * since a table walk on another CPU could refill a TLB with a
218 * complete (S1 + S2) walk based on the old Stage-2 mapping if
219 * the Stage-1 invalidation happened first.
220 */
221 dsb(nsh);
222 __tlbi(vmalle1);
223 dsb(nsh);
224 isb();
225
226 /*
227 * If the host is running at EL1 and we have a VPIPT I-cache,
228 * then we must perform I-cache maintenance at EL2 in order for
229 * it to have an effect on the guest. Since the guest cannot hit
230 * I-cache lines allocated with a different VMID, we don't need
231 * to worry about junk out of guest reset (we nuke the I-cache on
232 * VMID rollover), but we do need to be careful when remapping
233 * executable pages for the same guest. This can happen when KSM
234 * takes a CoW fault on an executable page, copies the page into
235 * a page that was previously mapped in the guest and then needs
236 * to invalidate the guest view of the I-cache for that page
237 * from EL1. To solve this, we invalidate the entire I-cache when
238 * unmapping a page from a guest if we have a VPIPT I-cache but
239 * the host is running at EL1. As above, we could do better if
240 * we had the VA.
241 *
242 * The moral of this story is: if you have a VPIPT I-cache, then
243 * you should be running with VHE enabled.
244 */
245 if (icache_is_vpipt())
246 icache_inval_all_pou();
247
248 exit_vmid_context(&cxt);
249 }
250
__kvm_tlb_flush_vmid_range(struct kvm_s2_mmu * mmu,phys_addr_t start,unsigned long pages)251 void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,
252 phys_addr_t start, unsigned long pages)
253 {
254 struct tlb_inv_context cxt;
255 unsigned long stride;
256
257 /*
258 * Since the range of addresses may not be mapped at
259 * the same level, assume the worst case as PAGE_SIZE
260 */
261 stride = PAGE_SIZE;
262 start = round_down(start, stride);
263
264 /* Switch to requested VMID */
265 enter_vmid_context(mmu, &cxt, false);
266
267 __flush_s2_tlb_range_op(ipas2e1is, start, pages, stride, 0);
268
269 dsb(ish);
270 __tlbi(vmalle1is);
271 dsb(ish);
272 isb();
273
274 /* See the comment in __kvm_tlb_flush_vmid_ipa() */
275 if (icache_is_vpipt())
276 icache_inval_all_pou();
277
278 exit_vmid_context(&cxt);
279 }
280
__kvm_tlb_flush_vmid(struct kvm_s2_mmu * mmu)281 void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
282 {
283 struct tlb_inv_context cxt;
284
285 /* Switch to requested VMID */
286 enter_vmid_context(mmu, &cxt, false);
287
288 __tlbi(vmalls12e1is);
289 dsb(ish);
290 isb();
291
292 exit_vmid_context(&cxt);
293 }
294
__kvm_flush_cpu_context(struct kvm_s2_mmu * mmu)295 void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu)
296 {
297 struct tlb_inv_context cxt;
298
299 /* Switch to requested VMID */
300 enter_vmid_context(mmu, &cxt, false);
301
302 __tlbi(vmalle1);
303 asm volatile("ic iallu");
304 dsb(nsh);
305 isb();
306
307 exit_vmid_context(&cxt);
308 }
309
__kvm_flush_vm_context(void)310 void __kvm_flush_vm_context(void)
311 {
312 /* Same remark as in enter_vmid_context() */
313 dsb(ish);
314 __tlbi(alle1is);
315
316 /*
317 * VIPT and PIPT caches are not affected by VMID, so no maintenance
318 * is necessary across a VMID rollover.
319 *
320 * VPIPT caches constrain lookup and maintenance to the active VMID,
321 * so we need to invalidate lines with a stale VMID to avoid an ABA
322 * race after multiple rollovers.
323 *
324 */
325 if (icache_is_vpipt())
326 asm volatile("ic ialluis");
327
328 dsb(ish);
329 }
330