1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 - Google LLC
4 * Author: Quentin Perret <qperret@google.com>
5 * Author: Fuad Tabba <tabba@google.com>
6 */
7 #ifndef __ARM64_KVM_PKVM_H__
8 #define __ARM64_KVM_PKVM_H__
9
10 #include <linux/arm_ffa.h>
11 #include <linux/memblock.h>
12 #include <linux/scatterlist.h>
13 #include <asm/kvm_pgtable.h>
14 #include <asm/sysreg.h>
15
16 /*
17 * Stores the sve state for the host in protected mode.
18 */
19 struct kvm_host_sve_state {
20 u64 zcr_el1;
21
22 /*
23 * Ordering is important since __sve_save_state/__sve_restore_state
24 * relies on it.
25 */
26 u32 fpsr;
27 u32 fpcr;
28
29 /* Must be SVE_VQ_BYTES (128 bit) aligned. */
30 char sve_regs[];
31 };
32
33 /* Maximum number of VMs that can co-exist under pKVM. */
34 #define KVM_MAX_PVMS 255
35
36 #define HYP_MEMBLOCK_REGIONS 128
37 #define PVMFW_INVALID_LOAD_ADDR (-1)
38
39 int pkvm_vm_ioctl_enable_cap(struct kvm *kvm,struct kvm_enable_cap *cap);
40 int pkvm_init_host_vm(struct kvm *kvm, unsigned long type);
41 int pkvm_create_hyp_vm(struct kvm *kvm);
42 void pkvm_destroy_hyp_vm(struct kvm *kvm);
43 void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa);
44
45 /*
46 * Definitions for features to be allowed or restricted for guest virtual
47 * machines, depending on the mode KVM is running in and on the type of guest
48 * that is running.
49 *
50 * Each field in the masks represents the highest supported *unsigned* value for
51 * the feature, if supported by the system.
52 *
53 * If a feature field is not present in either, than it is not supported.
54 *
55 * The approach taken for protected VMs is to allow features that are:
56 * - Needed by common Linux distributions (e.g., floating point)
57 * - Trivial to support, e.g., supporting the feature does not introduce or
58 * require tracking of additional state in KVM
59 * - Cannot be trapped or prevent the guest from using anyway
60 */
61
62 /*
63 * Allow for protected VMs:
64 * - Floating-point and Advanced SIMD
65 * - GICv3(+) system register interface
66 * - Data Independent Timing
67 *
68 * Restrict to the following *unsigned* features for protected VMs:
69 * - AArch64 guests only (no support for AArch32 guests):
70 * AArch32 adds complexity in trap handling, emulation, condition codes,
71 * etc...
72 * - SVE
73 * - RAS (v1)
74 * Supported by KVM
75 */
76 #define PVM_ID_AA64PFR0_ALLOW (\
77 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP) | \
78 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD) | \
79 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC) | \
80 ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_DIT) | \
81 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
82 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
83 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL2), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
84 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL3), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
85 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), ID_AA64PFR0_EL1_SVE_IMP) | \
86 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), ID_AA64PFR0_EL1_RAS_IMP) \
87 )
88
89 /*
90 * Allow for protected VMs:
91 * - Branch Target Identification
92 * - Speculative Store Bypassing
93 */
94 #define PVM_ID_AA64PFR1_ALLOW (\
95 ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_BT) | \
96 ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SSBS) \
97 )
98
99 #define PVM_ID_AA64PFR2_ALLOW (0ULL)
100
101 /*
102 * Allow for protected VMs:
103 * - Mixed-endian
104 * - Distinction between Secure and Non-secure Memory
105 * - Mixed-endian at EL0 only
106 * - Non-context synchronizing exception entry and exit
107 *
108 * Restrict to the following *unsigned* features for protected VMs:
109 * - 40-bit IPA
110 * - 16-bit ASID
111 */
112 #define PVM_ID_AA64MMFR0_ALLOW (\
113 ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGEND) | \
114 ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_SNSMEM) | \
115 ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGENDEL0) | \
116 ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_EXS) | \
117 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_PARANGE), ID_AA64MMFR0_EL1_PARANGE_40) | \
118 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_ASIDBITS), ID_AA64MMFR0_EL1_ASIDBITS_16) \
119 )
120
121 /*
122 * Allow for protected VMs:
123 * - Hardware translation table updates to Access flag and Dirty state
124 * - Number of VMID bits from CPU
125 * - Hierarchical Permission Disables
126 * - Privileged Access Never
127 * - SError interrupt exceptions from speculative reads
128 * - Enhanced Translation Synchronization
129 * - Control for cache maintenance permission
130 */
131 #define PVM_ID_AA64MMFR1_ALLOW (\
132 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS) | \
133 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_VMIDBits) | \
134 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HPDS) | \
135 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_PAN) | \
136 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_SpecSEI) | \
137 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_ETS) | \
138 ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_CMOW) \
139 )
140
141 /*
142 * Allow for protected VMs:
143 * - Common not Private translations
144 * - User Access Override
145 * - IESB bit in the SCTLR_ELx registers
146 * - Unaligned single-copy atomicity and atomic functions
147 * - ESR_ELx.EC value on an exception by read access to feature ID space
148 * - TTL field in address operations.
149 * - Break-before-make sequences when changing translation block size
150 * - E0PDx mechanism
151 */
152 #define PVM_ID_AA64MMFR2_ALLOW (\
153 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_CnP) | \
154 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_UAO) | \
155 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IESB) | \
156 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_AT) | \
157 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IDS) | \
158 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_TTL) | \
159 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_BBM) | \
160 ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_E0PD) \
161 )
162
163 #define PVM_ID_AA64MMFR3_ALLOW (0ULL)
164
165 /*
166 * No restrictions for Scalable Vectors (SVE).
167 */
168 #define PVM_ID_AA64ZFR0_ALLOW (\
169 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_SVEver) | \
170 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_AES) | \
171 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_BitPerm) | \
172 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_BF16) | \
173 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_SHA3) | \
174 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_SM4) | \
175 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_I8MM) | \
176 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_F32MM) | \
177 ARM64_FEATURE_MASK(ID_AA64ZFR0_EL1_F64MM) \
178 )
179
180 /*
181 * No support for debug, including breakpoints, and watchpoints for protected
182 * VMs:
183 * The Arm architecture mandates support for at least the Armv8 debug
184 * architecture, which would include at least 2 hardware breakpoints and
185 * watchpoints. Providing that support to protected guests adds
186 * considerable state and complexity. Therefore, the reserved value of 0 is
187 * used for debug-related fields.
188 */
189 #define PVM_ID_AA64DFR0_ALLOW (0ULL)
190 #define PVM_ID_AA64DFR1_ALLOW (0ULL)
191
192 /*
193 * No support for implementation defined features.
194 */
195 #define PVM_ID_AA64AFR0_ALLOW (0ULL)
196 #define PVM_ID_AA64AFR1_ALLOW (0ULL)
197
198 /*
199 * No restrictions on instructions implemented in AArch64.
200 */
201 #define PVM_ID_AA64ISAR0_ALLOW (\
202 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_AES) | \
203 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_SHA1) | \
204 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_SHA2) | \
205 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_CRC32) | \
206 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_ATOMIC) | \
207 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_RDM) | \
208 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_SHA3) | \
209 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_SM3) | \
210 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_SM4) | \
211 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_DP) | \
212 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_FHM) | \
213 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_TS) | \
214 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_TLB) | \
215 ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_RNDR) \
216 )
217
218 /* Restrict pointer authentication to the basic version. */
219 #define PVM_ID_AA64ISAR1_ALLOW (\
220 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_DPB) | \
221 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_JSCVT) | \
222 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_FCMA) | \
223 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_LRCPC) | \
224 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) | \
225 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI) | \
226 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_FRINTTS) | \
227 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_SB) | \
228 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_SPECRES) | \
229 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_BF16) | \
230 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_DGH) | \
231 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_I8MM) | \
232 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA), ID_AA64ISAR1_EL1_APA_PAuth) | \
233 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API), ID_AA64ISAR1_EL1_API_PAuth) \
234 )
235
236 #define PVM_ID_AA64ISAR2_ALLOW (\
237 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_ATS1A) | \
238 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3) | \
239 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS) | \
240 FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3), ID_AA64ISAR2_EL1_APA3_PAuth) \
241 )
242
243
244 /* All HAFGRTR_EL2 bits are AMU */
245 #define HAFGRTR_AMU __HAFGRTR_EL2_MASK
246
247 #define PVM_HAFGRTR_EL2_SET \
248 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), PVM_ID_AA64PFR0_ALLOW) ? 0ULL : HAFGRTR_AMU)
249
250 #define PVM_HAFGRTR_EL2_CLR (0ULL)
251
252 /* No support for debug, trace, of PMU for protected VMs */
253 #define PVM_HDFGRTR_EL2_SET __HDFGRTR_EL2_MASK
254 #define PVM_HDFGRTR_EL2_CLR __HDFGRTR_EL2_nMASK
255
256 #define PVM_HDFGWTR_EL2_SET __HDFGWTR_EL2_MASK
257 #define PVM_HDFGWTR_EL2_CLR __HDFGWTR_EL2_nMASK
258
259 #define HFGxTR_RAS_IMP (\
260 HFGxTR_EL2_ERXADDR_EL1 | \
261 HFGxTR_EL2_ERXPFGF_EL1 | \
262 HFGxTR_EL2_ERXMISCn_EL1 | \
263 HFGxTR_EL2_ERXSTATUS_EL1 | \
264 HFGxTR_EL2_ERXCTLR_EL1 | \
265 HFGxTR_EL2_ERXFR_EL1 | \
266 HFGxTR_EL2_ERRSELR_EL1 | \
267 HFGxTR_EL2_ERRIDR_EL1 \
268 )
269 #define HFGxTR_RAS_V1P1 (\
270 HFGxTR_EL2_ERXPFGCDN_EL1 | \
271 HFGxTR_EL2_ERXPFGCTL_EL1 \
272 )
273 #define HFGxTR_GIC HFGxTR_EL2_ICC_IGRPENn_EL1
274 #define HFGxTR_CSV2 (\
275 HFGxTR_EL2_SCXTNUM_EL0 | \
276 HFGxTR_EL2_SCXTNUM_EL1 \
277 )
278 #define HFGxTR_LOR (\
279 HFGxTR_EL2_LORSA_EL1 | \
280 HFGxTR_EL2_LORN_EL1 | \
281 HFGxTR_EL2_LORID_EL1 | \
282 HFGxTR_EL2_LOREA_EL1 | \
283 HFGxTR_EL2_LORC_EL1 \
284 )
285 #define HFGxTR_PAUTH (\
286 HFGxTR_EL2_APIBKey | \
287 HFGxTR_EL2_APIAKey | \
288 HFGxTR_EL2_APGAKey | \
289 HFGxTR_EL2_APDBKey | \
290 HFGxTR_EL2_APDAKey \
291 )
292 #define HFGxTR_nAIE (\
293 HFGxTR_EL2_nAMAIR2_EL1 | \
294 HFGxTR_EL2_nMAIR2_EL1 \
295 )
296 #define HFGxTR_nS2POE HFGxTR_EL2_nS2POR_EL1
297 #define HFGxTR_nS1POE (\
298 HFGxTR_EL2_nPOR_EL1 | \
299 HFGxTR_EL2_nPOR_EL0 \
300 )
301 #define HFGxTR_nS1PIE (\
302 HFGxTR_EL2_nPIR_EL1 | \
303 HFGxTR_EL2_nPIRE0_EL1 \
304 )
305 #define HFGxTR_nTHE HFGxTR_EL2_nRCWMASK_EL1
306 #define HFGxTR_nSME (\
307 HFGxTR_EL2_nTPIDR2_EL0 | \
308 HFGxTR_EL2_nSMPRI_EL1 \
309 )
310 #define HFGxTR_nGCS (\
311 HFGxTR_EL2_nGCS_EL1 | \
312 HFGxTR_EL2_nGCS_EL0 \
313 )
314 #define HFGxTR_nLS64 HFGxTR_EL2_nACCDATA_EL1
315
316 #define PVM_HFGXTR_EL2_SET \
317 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), PVM_ID_AA64PFR0_ALLOW) >= ID_AA64PFR0_EL1_RAS_IMP ? 0ULL : HFGxTR_RAS_IMP) | \
318 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), PVM_ID_AA64PFR0_ALLOW) >= ID_AA64PFR0_EL1_RAS_V1P1 ? 0ULL : HFGxTR_RAS_V1P1) | \
319 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC), PVM_ID_AA64PFR0_ALLOW) ? 0ULL : HFGxTR_GIC) | \
320 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), PVM_ID_AA64PFR0_ALLOW) ? 0ULL : HFGxTR_CSV2) | \
321 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), PVM_ID_AA64MMFR1_ALLOW) ? 0ULL : HFGxTR_LOR) | \
322 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HFGxTR_PAUTH) | \
323 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HFGxTR_PAUTH) | \
324 0
325
326 #define PVM_HFGXTR_EL2_CLR \
327 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_AIE), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HFGxTR_nAIE) | \
328 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_S2POE), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HFGxTR_nS2POE) | \
329 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_S1POE), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HFGxTR_nS1POE) | \
330 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_S1PIE), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HFGxTR_nS1PIE) | \
331 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_THE), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HFGxTR_nTHE) | \
332 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HFGxTR_nSME) | \
333 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HFGxTR_nGCS) | \
334 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_LS64), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HFGxTR_nLS64) | \
335 0
336
337 #define PVM_HFGRTR_EL2_SET PVM_HFGXTR_EL2_SET
338 #define PVM_HFGWTR_EL2_SET PVM_HFGXTR_EL2_SET
339 #define PVM_HFGRTR_EL2_CLR PVM_HFGXTR_EL2_CLR
340 #define PVM_HFGWTR_EL2_CLR PVM_HFGXTR_EL2_CLR
341
342 #define HFGITR_SPECRES (\
343 HFGITR_EL2_CPPRCTX | \
344 HFGITR_EL2_DVPRCTX | \
345 HFGITR_EL2_CFPRCTX \
346 )
347 #define HFGITR_TLBIOS (\
348 HFGITR_EL2_TLBIVAALE1OS | \
349 HFGITR_EL2_TLBIVALE1OS | \
350 HFGITR_EL2_TLBIVAAE1OS | \
351 HFGITR_EL2_TLBIASIDE1OS | \
352 HFGITR_EL2_TLBIVAE1OS | \
353 HFGITR_EL2_TLBIVMALLE1OS \
354 )
355 #define HFGITR_TLBIRANGE \
356 (\
357 HFGITR_TLBIOS | \
358 HFGITR_EL2_TLBIRVAALE1 | \
359 HFGITR_EL2_TLBIRVALE1 | \
360 HFGITR_EL2_TLBIRVAAE1 | \
361 HFGITR_EL2_TLBIRVAE1 | \
362 HFGITR_EL2_TLBIRVAE1 | \
363 HFGITR_EL2_TLBIRVAALE1IS | \
364 HFGITR_EL2_TLBIRVALE1IS | \
365 HFGITR_EL2_TLBIRVAAE1IS | \
366 HFGITR_EL2_TLBIRVAE1IS | \
367 HFGITR_EL2_TLBIVAALE1IS | \
368 HFGITR_EL2_TLBIVALE1IS | \
369 HFGITR_EL2_TLBIVAAE1IS | \
370 HFGITR_EL2_TLBIASIDE1IS | \
371 HFGITR_EL2_TLBIVAE1IS | \
372 HFGITR_EL2_TLBIVMALLE1IS | \
373 HFGITR_EL2_TLBIRVAALE1OS | \
374 HFGITR_EL2_TLBIRVALE1OS | \
375 HFGITR_EL2_TLBIRVAAE1OS | \
376 HFGITR_EL2_TLBIRVAE1OS \
377 )
378 #define HFGITR_TLB HFGITR_TLBIRANGE
379 #define HFGITR_PAN2 (\
380 HFGITR_EL2_ATS1E1WP | \
381 HFGITR_EL2_ATS1E1RP | \
382 HFGITR_EL2_ATS1E0W | \
383 HFGITR_EL2_ATS1E0R | \
384 HFGITR_EL2_ATS1E1W | \
385 HFGITR_EL2_ATS1E1R \
386 )
387 #define HFGITR_PAN HFGITR_PAN2
388 #define HFGITR_DPB2 HFGITR_EL2_DCCVADP
389 #define HFGITR_DPB_IMP HFGITR_EL2_DCCVAP
390 #define HFGITR_DPB (HFGITR_DPB_IMP | HFGITR_DPB2)
391 #define HFGITR_nGCS (\
392 HFGITR_EL2_nGCSEPP | \
393 HFGITR_EL2_nGCSSTR_EL1 | \
394 HFGITR_EL2_nGCSPUSHM_EL1 \
395 )
396 #define HFGITR_nBRBE (\
397 HFGITR_EL2_nBRBIALL | \
398 HFGITR_EL2_nBRBINJ \
399 )
400
401 #define PVM_HFGITR_EL2_SET \
402 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_ATS1A), PVM_ID_AA64ISAR2_ALLOW) ? 0ULL : HFGITR_EL2_ATS1E1A) | \
403 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_SPECRES), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HFGITR_SPECRES) | \
404 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR0_EL1_TLB), PVM_ID_AA64ISAR0_ALLOW) ? 0ULL : HFGITR_TLB) | \
405 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_PAN), PVM_ID_AA64MMFR1_ALLOW) ? 0ULL : HFGITR_PAN) | \
406 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_DPB), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HFGITR_DPB) | \
407 0
408
409 #define PVM_HFGITR_EL2_CLR \
410 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HFGITR_nGCS) | \
411 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_BRBE), PVM_ID_AA64DFR0_ALLOW) ? 0ULL : HFGITR_nBRBE) | \
412 0
413
414 #define HCRX_NMI HCRX_EL2_TALLINT
415
416 #define HCRX_nPAuth_LR HCRX_EL2_PACMEn
417 #define HCRX_nFPMR HCRX_EL2_EnFPM
418 #define HCRX_nGCS HCRX_EL2_GCSEn
419 #define HCRX_nSYSREG128 HCRX_EL2_EnIDCP128
420 #define HCRX_nADERR HCRX_EL2_EnSDERR
421 #define HCRX_nDoubleFault2 HCRX_EL2_TMEA
422 #define HCRX_nANERR HCRX_EL2_EnSNERR
423 #define HCRX_nD128 HCRX_EL2_D128En
424 #define HCRX_nTHE HCRX_EL2_PTTWI
425 #define HCRX_nSCTLR2 HCRX_EL2_SCTLR2En
426 #define HCRX_nTCR2 HCRX_EL2_TCR2En
427 #define HCRX_nMOPS (HCRX_EL2_MSCEn | HCRX_EL2_MCE2)
428 #define HCRX_nCMOW HCRX_EL2_CMOW
429 #define HCRX_nNMI (HCRX_EL2_VFNMI | HCRX_EL2_VINMI)
430 #define HCRX_SME HCRX_EL2_SMPME
431 #define HCRX_nXS (HCRX_EL2_FGTnXS | HCRX_EL2_FnXS)
432 #define HCRX_nLS64 (HCRX_EL2_EnASR| HCRX_EL2_EnALS | HCRX_EL2_EnAS0)
433
434 #define PVM_HCRX_EL2_SET \
435 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_NMI), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_NMI) | \
436 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_SME) | \
437 0
438
439 #define PVM_HCRX_EL2_CLR \
440 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA), PVM_ID_AA64ISAR1_ALLOW) < ID_AA64ISAR1_EL1_APA_PAuth_LR ? 0ULL : HCRX_nPAuth_LR) | \
441 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API), PVM_ID_AA64ISAR1_ALLOW) < ID_AA64ISAR1_EL1_APA_PAuth_LR ? 0ULL : HCRX_nPAuth_LR) | \
442 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_nGCS) | \
443 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_SYSREG_128), PVM_ID_AA64ISAR2_ALLOW) ? 0ULL : HCRX_nSYSREG128) | \
444 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_ADERR), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HCRX_nADERR) | \
445 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_DF2), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_nDoubleFault2) | \
446 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_ANERR), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HCRX_nANERR) | \
447 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_PARANGE), PVM_ID_AA64MMFR0_ALLOW) ? 0ULL : HCRX_nD128) | \
448 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_THE), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_nTHE) | \
449 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_SCTLRX), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HCRX_nSCTLR2) | \
450 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR3_EL1_TCRX), PVM_ID_AA64MMFR3_ALLOW) ? 0ULL : HCRX_nTCR2) | \
451 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_MOPS), PVM_ID_AA64ISAR2_ALLOW) ? 0ULL : HCRX_nMOPS) | \
452 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_CMOW), PVM_ID_AA64MMFR1_ALLOW) ? 0ULL : HCRX_nCMOW) | \
453 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_NMI), PVM_ID_AA64PFR1_ALLOW) ? 0ULL : HCRX_nNMI) | \
454 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_XS), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HCRX_nXS) | \
455 (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_LS64), PVM_ID_AA64ISAR1_ALLOW) ? 0ULL : HCRX_nLS64) | \
456 0
457
458 /*
459 * Returns the maximum number of breakpoints supported for protected VMs.
460 */
pkvm_get_max_brps(void)461 static inline int pkvm_get_max_brps(void)
462 {
463 int num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_BRPs),
464 PVM_ID_AA64DFR0_ALLOW);
465
466 /*
467 * If breakpoints are supported, the maximum number is 1 + the field.
468 * Otherwise, return 0, which is not compliant with the architecture,
469 * but is reserved and is used here to indicate no debug support.
470 */
471 return num ? num + 1 : 0;
472 }
473
474 /*
475 * Returns the maximum number of watchpoints supported for protected VMs.
476 */
pkvm_get_max_wrps(void)477 static inline int pkvm_get_max_wrps(void)
478 {
479 int num = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_WRPs),
480 PVM_ID_AA64DFR0_ALLOW);
481
482 return num ? num + 1 : 0;
483 }
484
485 enum pkvm_moveable_reg_type {
486 PKVM_MREG_MEMORY,
487 PKVM_MREG_PROTECTED_RANGE,
488 };
489
490 struct pkvm_moveable_reg {
491 phys_addr_t start;
492 u64 size;
493 enum pkvm_moveable_reg_type type;
494 };
495
496 #define PKVM_NR_MOVEABLE_REGS 512
497 extern struct pkvm_moveable_reg kvm_nvhe_sym(pkvm_moveable_regs)[];
498 extern unsigned int kvm_nvhe_sym(pkvm_moveable_regs_nr);
499
500 extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
501 extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
502
503 extern phys_addr_t kvm_nvhe_sym(pvmfw_base);
504 extern phys_addr_t kvm_nvhe_sym(pvmfw_size);
505
506 static inline unsigned long
hyp_vmemmap_memblock_size(struct memblock_region * reg,size_t vmemmap_entry_size)507 hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size)
508 {
509 unsigned long nr_pages = reg->size >> PAGE_SHIFT;
510 unsigned long start, end;
511
512 start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size;
513 end = start + nr_pages * vmemmap_entry_size;
514 start = ALIGN_DOWN(start, PAGE_SIZE);
515 end = ALIGN(end, PAGE_SIZE);
516
517 return end - start;
518 }
519
hyp_vmemmap_pages(size_t vmemmap_entry_size)520 static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size)
521 {
522 unsigned long res = 0, i;
523
524 for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
525 res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i],
526 vmemmap_entry_size);
527 }
528
529 return res >> PAGE_SHIFT;
530 }
531
hyp_vm_table_pages(void)532 static inline unsigned long hyp_vm_table_pages(void)
533 {
534 return PAGE_ALIGN(KVM_MAX_PVMS * sizeof(void *)) >> PAGE_SHIFT;
535 }
536
__hyp_pgtable_max_pages(unsigned long nr_pages)537 static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
538 {
539 unsigned long total = 0, i;
540
541 /* Provision the worst case scenario */
542 for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
543 nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
544 total += nr_pages;
545 }
546
547 return total;
548 }
549
__hyp_pgtable_moveable_regs_pages(void)550 static inline unsigned long __hyp_pgtable_moveable_regs_pages(void)
551 {
552 unsigned long res = 0, i;
553
554 /* Cover all of moveable regions with page-granularity */
555 for (i = 0; i < kvm_nvhe_sym(pkvm_moveable_regs_nr); i++) {
556 struct pkvm_moveable_reg *reg = &kvm_nvhe_sym(pkvm_moveable_regs)[i];
557 res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
558 }
559
560 return res;
561 }
562
hyp_s1_pgtable_pages(void)563 static inline unsigned long hyp_s1_pgtable_pages(void)
564 {
565 unsigned long res;
566
567 res = __hyp_pgtable_moveable_regs_pages();
568
569 /* Allow 1 GiB for private mappings */
570 res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
571
572 return res;
573 }
574
host_s2_pgtable_pages(void)575 static inline unsigned long host_s2_pgtable_pages(void)
576 {
577 unsigned long res;
578
579 /*
580 * Include an extra 16 pages to safely upper-bound the worst case of
581 * concatenated pgds.
582 */
583 res = __hyp_pgtable_moveable_regs_pages() + 16;
584
585 /* Allow 1 GiB for non-moveable regions */
586 res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
587
588 return res;
589 }
590
591 #define KVM_FFA_MBOX_NR_PAGES 1
592
593 /*
594 * Maximum number of consitutents allowed in a descriptor. This number is
595 * arbitrary, see comment below on SG_MAX_SEGMENTS in hyp_ffa_proxy_pages().
596 */
597 #define KVM_FFA_MAX_NR_CONSTITUENTS 4096
598
hyp_ffa_proxy_pages(void)599 static inline unsigned long hyp_ffa_proxy_pages(void)
600 {
601 size_t desc_max;
602
603 /*
604 * SG_MAX_SEGMENTS is supposed to bound the number of elements in an
605 * sglist, which should match the number of consituents in the
606 * corresponding FFA descriptor. As such, the EL2 buffer needs to be
607 * large enough to hold a descriptor with SG_MAX_SEGMENTS consituents
608 * at least. But the kernel's DMA code doesn't enforce the limit, and
609 * it is sometimes abused, so let's allow larger descriptors and hope
610 * for the best.
611 */
612 BUILD_BUG_ON(KVM_FFA_MAX_NR_CONSTITUENTS < SG_MAX_SEGMENTS);
613
614 /*
615 * The hypervisor FFA proxy needs enough memory to buffer a fragmented
616 * descriptor returned from EL3 in response to a RETRIEVE_REQ call.
617 */
618 desc_max = sizeof(struct ffa_mem_region) +
619 sizeof(struct ffa_mem_region_attributes) +
620 sizeof(struct ffa_composite_mem_region) +
621 KVM_FFA_MAX_NR_CONSTITUENTS * sizeof(struct ffa_mem_region_addr_range);
622
623 /* Plus a page each for the hypervisor's RX and TX mailboxes. */
624 return (2 * KVM_FFA_MBOX_NR_PAGES) + DIV_ROUND_UP(desc_max, PAGE_SIZE);
625 }
626
pkvm_host_fp_state_size(void)627 static inline size_t pkvm_host_fp_state_size(void)
628 {
629 if (system_supports_sve())
630 return size_add(sizeof(struct kvm_host_sve_state),
631 SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)));
632 else
633 return sizeof(struct user_fpsimd_state);
634 }
635
636 int __pkvm_topup_hyp_alloc(unsigned long nr_pages);
637
638 #define kvm_call_refill_hyp_nvhe(f, ...) \
639 ({ \
640 struct arm_smccc_res res; \
641 int __ret; \
642 do { \
643 __ret = -1; \
644 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \
645 ##__VA_ARGS__, &res); \
646 if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \
647 break; \
648 \
649 __ret = res.a1; \
650 if (__ret == -ENOMEM && res.a3) { \
651 __ret = __pkvm_topup_hyp_alloc(res.a3); \
652 } else { \
653 break; \
654 } \
655 } while (!__ret); \
656 __ret; \
657 })
658
659 int pkvm_call_hyp_nvhe_ppage(struct kvm_pinned_page *ppage,
660 int (*call_hyp_nvhe)(u64, u64, u8, void*),
661 void *args, bool unmap);
662 #endif /* __ARM64_KVM_PKVM_H__ */
663