1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2021 - Google LLC
4 * Author: David Brazdil <dbrazdil@google.com>
5 */
6
7 #ifndef __ARM64_KVM_S2MPU_H__
8 #define __ARM64_KVM_S2MPU_H__
9
10 #include <linux/bitfield.h>
11
12 #include <asm/kvm_mmu.h>
13
14 #define S2MPU_MMIO_SIZE SZ_64K
15 #define SYSMMU_SYNC_MMIO_SIZE SZ_64K
16 #define SYSMMU_SYNC_S2_OFFSET SZ_32K
17 #define SYSMMU_SYNC_S2_MMIO_SIZE (SYSMMU_SYNC_MMIO_SIZE - \
18 SYSMMU_SYNC_S2_OFFSET)
19
20 #define NR_VIDS 8
21 #define NR_CTX_IDS 8
22
23 #define ALL_VIDS_BITMAP GENMASK(NR_VIDS - 1, 0)
24
25 #define REG_NS_CTRL0 0x0
26 #define REG_NS_CTRL1 0x4
27 #define REG_NS_CFG 0x10
28 #define REG_NS_INTERRUPT_ENABLE_PER_VID_SET 0x20
29 #define REG_NS_INTERRUPT_CLEAR 0x2c
30 #define REG_NS_VERSION 0x60
31 #define REG_NS_INFO 0x64
32 #define REG_NS_STATUS 0x68
33 #define REG_NS_NUM_CONTEXT 0x100
34 #define REG_NS_CONTEXT_CFG_VALID_VID 0x104
35 #define REG_NS_ALL_INVALIDATION 0x1000
36 #define REG_NS_RANGE_INVALIDATION 0x1020
37 #define REG_NS_RANGE_INVALIDATION_START_PPN 0x1024
38 #define REG_NS_RANGE_INVALIDATION_END_PPN 0x1028
39 #define REG_NS_FAULT_STATUS 0x2000
40 #define REG_NS_FAULT_PA_LOW(vid) (0x2004 + ((vid) * 0x20))
41 #define REG_NS_FAULT_PA_HIGH(vid) (0x2008 + ((vid) * 0x20))
42 #define REG_NS_FAULT_INFO(vid) (0x2010 + ((vid) * 0x20))
43 #define REG_NS_READ_MPTC 0x3000
44 #define REG_NS_READ_MPTC_TAG_PPN 0x3004
45 #define REG_NS_READ_MPTC_TAG_OTHERS 0x3008
46 #define REG_NS_READ_MPTC_DATA 0x3010
47 #define REG_NS_L1ENTRY_L2TABLE_ADDR(vid, gb) (0x4000 + ((vid) * 0x200) + ((gb) * 0x8))
48 #define REG_NS_L1ENTRY_ATTR(vid, gb) (0x4004 + ((vid) * 0x200) + ((gb) * 0x8))
49
50 #define CTRL0_ENABLE BIT(0)
51 #define CTRL0_INTERRUPT_ENABLE BIT(1)
52 #define CTRL0_FAULT_RESP_TYPE_SLVERR BIT(2) /* for v8 */
53 #define CTRL0_FAULT_RESP_TYPE_DECERR BIT(2) /* for v9 */
54 #define CTRL0_MASK (CTRL0_ENABLE | \
55 CTRL0_INTERRUPT_ENABLE | \
56 CTRL0_FAULT_RESP_TYPE_SLVERR | \
57 CTRL0_FAULT_RESP_TYPE_DECERR)
58
59 #define CTRL1_DISABLE_CHK_S1L1PTW BIT(0)
60 #define CTRL1_DISABLE_CHK_S1L2PTW BIT(1)
61 #define CTRL1_ENABLE_PAGE_SIZE_AWARENESS BIT(2)
62 #define CTRL1_DISABLE_CHK_USER_MATCHED_REQ BIT(3)
63 #define CTRL1_MASK (CTRL1_DISABLE_CHK_S1L1PTW | \
64 CTRL1_DISABLE_CHK_S1L2PTW | \
65 CTRL1_ENABLE_PAGE_SIZE_AWARENESS | \
66 CTRL1_DISABLE_CHK_USER_MATCHED_REQ)
67
68 #define CFG_MPTW_CACHE_OVERRIDE BIT(0)
69 #define CFG_MPTW_CACHE_VALUE GENMASK(7, 4)
70 #define CFG_MPTW_QOS_OVERRIDE BIT(8)
71 #define CFG_MPTW_QOS_VALUE GENMASK(15, 12)
72 #define CFG_MPTW_SHAREABLE BIT(16)
73 #define CFG_MASK (CFG_MPTW_CACHE_OVERRIDE | \
74 CFG_MPTW_CACHE_VALUE | \
75 CFG_MPTW_QOS_OVERRIDE | \
76 CFG_MPTW_QOS_VALUE | \
77 CFG_MPTW_SHAREABLE)
78
79 /* For use with hi_lo_readq_relaxed(). */
80 #define REG_NS_FAULT_PA_HIGH_LOW(vid) REG_NS_FAULT_PA_LOW(vid)
81
82 /* Mask used for extracting VID from FAULT_* register offset. */
83 #define REG_NS_FAULT_VID_MASK GENMASK(7, 5)
84
85 #define VERSION_MAJOR_ARCH_VER_MASK GENMASK(31, 28)
86 #define VERSION_MINOR_ARCH_VER_MASK GENMASK(27, 24)
87 #define VERSION_REV_ARCH_VER_MASK GENMASK(23, 16)
88 #define VERSION_RTL_VER_MASK GENMASK(7, 0)
89
90 /* Ignore RTL version in driver version check. */
91 #define VERSION_CHECK_MASK (VERSION_MAJOR_ARCH_VER_MASK | \
92 VERSION_MINOR_ARCH_VER_MASK | \
93 VERSION_REV_ARCH_VER_MASK)
94
95 #define INFO_NUM_SET_MASK GENMASK(15, 0)
96
97 #define STATUS_BUSY BIT(0)
98 #define STATUS_ON_INVALIDATING BIT(1)
99
100 #define NUM_CONTEXT_MASK GENMASK(3, 0)
101
102 #define CONTEXT_CFG_VALID_VID_CTX_VALID(ctx) BIT((4 * (ctx)) + 3)
103 #define CONTEXT_CFG_VALID_VID_CTX_VID(ctx, vid) \
104 FIELD_PREP(GENMASK((4 * (ctx) + 2), 4 * (ctx)), (vid))
105
106 #define INVALIDATION_INVALIDATE BIT(0)
107 #define RANGE_INVALIDATION_PPN_SHIFT 12
108
109 #define NR_FAULT_INFO_REGS 8
110 #define FAULT_INFO_VID_MASK GENMASK(26, 24)
111 #define FAULT_INFO_TYPE_MASK GENMASK(23, 21)
112 #define FAULT_INFO_TYPE_CONTEXT 0x4 /* v9 only */
113 #define FAULT_INFO_TYPE_AP 0x2
114 #define FAULT_INFO_TYPE_MPTW 0x1
115 #define FAULT_INFO_RW_BIT BIT(20)
116 #define FAULT_INFO_LEN_MASK GENMASK(19, 16)
117 #define FAULT_INFO_ID_MASK GENMASK(15, 0)
118
119 #define L1ENTRY_L2TABLE_ADDR_SHIFT 4
120 #define L1ENTRY_L2TABLE_ADDR(pa) ((pa) >> L1ENTRY_L2TABLE_ADDR_SHIFT)
121
122 #define READ_MPTC_WAY_MASK GENMASK(18, 16)
123 #define READ_MPTC_SET_MASK GENMASK(15, 0)
124 #define READ_MPTC_MASK (READ_MPTC_WAY_MASK | READ_MPTC_SET_MASK)
125 #define READ_MPTC_WAY(way) FIELD_PREP(READ_MPTC_WAY_MASK, (way))
126 #define READ_MPTC_SET(set) FIELD_PREP(READ_MPTC_SET_MASK, (set))
127 #define READ_MPTC(set, way) (READ_MPTC_SET(set) | READ_MPTC_WAY(way))
128 #define READ_MPTC_TAG_PPN_MASK GENMASK(23, 0)
129 #define READ_MPTC_TAG_OTHERS_VID_MASK GENMASK(10, 8)
130 #define READ_MPTC_TAG_OTHERS_GRAN_MASK GENMASK(5, 4)
131 #define READ_MPTC_TAG_OTHERS_VALID_BIT BIT(0)
132 #define READ_MPTC_TAG_OTHERS_MASK (READ_MPTC_TAG_OTHERS_VID_MASK | \
133 READ_MPTC_TAG_OTHERS_GRAN_MASK | \
134 READ_MPTC_TAG_OTHERS_VALID_BIT)
135
136 #define L1ENTRY_ATTR_L2TABLE_EN BIT(0)
137 #define L1ENTRY_ATTR_GRAN_4K 0x0
138 #define L1ENTRY_ATTR_GRAN_64K 0x1
139 #define L1ENTRY_ATTR_GRAN_2M 0x2
140 #define L1ENTRY_ATTR_PROT_MASK GENMASK(2, 1)
141 #define L1ENTRY_ATTR_GRAN_MASK GENMASK(5, 4)
142 #define L1ENTRY_ATTR_PROT(prot) FIELD_PREP(L1ENTRY_ATTR_PROT_MASK, prot)
143 #define L1ENTRY_ATTR_GRAN(gran) FIELD_PREP(L1ENTRY_ATTR_GRAN_MASK, gran)
144 #define L1ENTRY_ATTR_1G(prot) L1ENTRY_ATTR_PROT(prot)
145 #define L1ENTRY_ATTR_L2(gran) (L1ENTRY_ATTR_GRAN(gran) | \
146 L1ENTRY_ATTR_L2TABLE_EN)
147
148 #define NR_GIGABYTES 64
149 #define RO_GIGABYTES_FIRST 4
150 #define RO_GIGABYTES_LAST 33
151 #define NR_RO_GIGABYTES (RO_GIGABYTES_LAST - RO_GIGABYTES_FIRST + 1)
152 #define NR_RW_GIGABYTES (NR_GIGABYTES - NR_RO_GIGABYTES)
153
154 #ifdef CONFIG_ARM64_64K_PAGES
155 #define SMPT_GRAN SZ_64K
156 #define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_64K
157 #else
158 #define SMPT_GRAN SZ_4K
159 #define SMPT_GRAN_ATTR L1ENTRY_ATTR_GRAN_4K
160 #endif
161 static_assert(SMPT_GRAN <= PAGE_SIZE);
162
163 #define MPT_PROT_BITS 2
164 #define SMPT_WORD_SIZE sizeof(u32)
165 #define SMPT_ELEMS_PER_BYTE (BITS_PER_BYTE / MPT_PROT_BITS)
166 #define SMPT_ELEMS_PER_WORD (SMPT_WORD_SIZE * SMPT_ELEMS_PER_BYTE)
167 #define SMPT_WORD_BYTE_RANGE (SMPT_GRAN * SMPT_ELEMS_PER_WORD)
168 #define SMPT_NUM_ELEMS (SZ_1G / SMPT_GRAN)
169 #define SMPT_SIZE (SMPT_NUM_ELEMS / SMPT_ELEMS_PER_BYTE)
170 #define SMPT_NUM_WORDS (SMPT_SIZE / SMPT_WORD_SIZE)
171 #define SMPT_NUM_PAGES (SMPT_SIZE / PAGE_SIZE)
172 #define SMPT_ORDER get_order(SMPT_SIZE)
173
174 /* SysMMU_SYNC registers, relative to SYSMMU_SYNC_S2_OFFSET. */
175 #define REG_NS_SYNC_CMD 0x0
176 #define REG_NS_SYNC_COMP 0x4
177
178 #define SYNC_CMD_SYNC BIT(0)
179 #define SYNC_COMP_COMPLETE BIT(0)
180
181 /*
182 * Iterate over S2MPU gigabyte regions. Skip those that cannot be modified
183 * (the MMIO registers are read only, with reset value MPT_PROT_NONE).
184 */
185 #define for_each_gb_in_range(i, first, last) \
186 for ((i) = (first); (i) <= (last) && (i) < NR_GIGABYTES; \
187 (i) = (((i) + 1 == RO_GIGABYTES_FIRST) ? RO_GIGABYTES_LAST : (i)) + 1)
188
189 #define for_each_gb(i) for_each_gb_in_range(i, 0, NR_GIGABYTES - 1)
190 #define for_each_vid(i) for ((i) = 0; (i) < NR_VIDS; (i)++)
191 #define for_each_gb_and_vid(gb, vid) for_each_vid((vid)) for_each_gb((gb))
192
193 enum s2mpu_version {
194 S2MPU_VERSION_8 = 0x11000000,
195 S2MPU_VERSION_9 = 0x20000000,
196 };
197
198 enum mpt_prot {
199 MPT_PROT_NONE = 0,
200 MPT_PROT_R = BIT(0),
201 MPT_PROT_W = BIT(1),
202 MPT_PROT_RW = MPT_PROT_R | MPT_PROT_W,
203 MPT_PROT_MASK = MPT_PROT_RW,
204 };
205
206 static const u64 mpt_prot_doubleword[] = {
207 [MPT_PROT_NONE] = 0x0000000000000000,
208 [MPT_PROT_R] = 0x5555555555555555,
209 [MPT_PROT_W] = 0xaaaaaaaaaaaaaaaa,
210 [MPT_PROT_RW] = 0xffffffffffffffff,
211 };
212
213 enum mpt_update_flags {
214 MPT_UPDATE_L1 = BIT(0),
215 MPT_UPDATE_L2 = BIT(1),
216 };
217
218 struct fmpt {
219 u32 *smpt;
220 bool gran_1g;
221 enum mpt_prot prot;
222 enum mpt_update_flags flags;
223 };
224
225 struct mpt {
226 struct fmpt fmpt[NR_GIGABYTES];
227 };
228
229 /* Set protection bits of SMPT in a given range without using memset. */
__set_smpt_range_slow(u32 * smpt,size_t start_gb_byte,size_t end_gb_byte,enum mpt_prot prot)230 static inline void __set_smpt_range_slow(u32 *smpt, size_t start_gb_byte,
231 size_t end_gb_byte, enum mpt_prot prot)
232 {
233 size_t i, start_word_byte, end_word_byte, word_idx, first_elem, last_elem;
234 u32 val;
235
236 /* Iterate over u32 words. */
237 start_word_byte = start_gb_byte;
238 while (start_word_byte < end_gb_byte) {
239 /* Determine the range of bytes covered by this word. */
240 word_idx = start_word_byte / SMPT_WORD_BYTE_RANGE;
241 end_word_byte = min(
242 ALIGN(start_word_byte + 1, SMPT_WORD_BYTE_RANGE),
243 end_gb_byte);
244
245 /* Identify protection bit offsets within the word. */
246 first_elem = (start_word_byte / SMPT_GRAN) % SMPT_ELEMS_PER_WORD;
247 last_elem = ((end_word_byte - 1) / SMPT_GRAN) % SMPT_ELEMS_PER_WORD;
248
249 /* Modify the corresponding word. */
250 val = READ_ONCE(smpt[word_idx]);
251 for (i = first_elem; i <= last_elem; i++) {
252 val &= ~(MPT_PROT_MASK << (i * MPT_PROT_BITS));
253 val |= prot << (i * MPT_PROT_BITS);
254 }
255 WRITE_ONCE(smpt[word_idx], val);
256
257 start_word_byte = end_word_byte;
258 }
259 }
260
261 /* Set protection bits of SMPT in a given range. */
__set_smpt_range(u32 * smpt,size_t start_gb_byte,size_t end_gb_byte,enum mpt_prot prot)262 static inline void __set_smpt_range(u32 *smpt, size_t start_gb_byte,
263 size_t end_gb_byte, enum mpt_prot prot)
264 {
265 size_t interlude_start, interlude_end, interlude_bytes, word_idx;
266 char prot_byte = (char)mpt_prot_doubleword[prot];
267
268 if (start_gb_byte >= end_gb_byte)
269 return;
270
271 /* Check if range spans at least one full u32 word. */
272 interlude_start = ALIGN(start_gb_byte, SMPT_WORD_BYTE_RANGE);
273 interlude_end = ALIGN_DOWN(end_gb_byte, SMPT_WORD_BYTE_RANGE);
274
275 /* If not, fall back to editing bits in the given range. */
276 if (interlude_start >= interlude_end) {
277 __set_smpt_range_slow(smpt, start_gb_byte, end_gb_byte, prot);
278 return;
279 }
280
281 /* Use bit-editing for prologue/epilogue, memset for interlude. */
282 word_idx = interlude_start / SMPT_WORD_BYTE_RANGE;
283 interlude_bytes = (interlude_end - interlude_start) / SMPT_GRAN / SMPT_ELEMS_PER_BYTE;
284
285 __set_smpt_range_slow(smpt, start_gb_byte, interlude_start, prot);
286 memset(&smpt[word_idx], prot_byte, interlude_bytes);
287 __set_smpt_range_slow(smpt, interlude_end, end_gb_byte, prot);
288 }
289
290 /* Returns true if all SMPT protection bits match 'prot'. */
__is_smpt_uniform(u32 * smpt,enum mpt_prot prot)291 static inline bool __is_smpt_uniform(u32 *smpt, enum mpt_prot prot)
292 {
293 size_t i;
294 u64 *doublewords = (u64 *)smpt;
295
296 for (i = 0; i < SMPT_NUM_WORDS / 2; i++) {
297 if (doublewords[i] != mpt_prot_doubleword[prot])
298 return false;
299 }
300 return true;
301 }
302
303 /*
304 * Set protection bits of FMPT/SMPT in a given range.
305 * Returns flags specifying whether L1/L2 changes need to be made visible
306 * to the device.
307 */
__set_fmpt_range(struct fmpt * fmpt,size_t start_gb_byte,size_t end_gb_byte,enum mpt_prot prot)308 static inline void __set_fmpt_range(struct fmpt *fmpt, size_t start_gb_byte,
309 size_t end_gb_byte, enum mpt_prot prot)
310 {
311 if (start_gb_byte == 0 && end_gb_byte >= SZ_1G) {
312 /* Update covers the entire GB region. */
313 if (fmpt->gran_1g && fmpt->prot == prot) {
314 fmpt->flags = 0;
315 return;
316 }
317
318 fmpt->gran_1g = true;
319 fmpt->prot = prot;
320 fmpt->flags = MPT_UPDATE_L1;
321 return;
322 }
323
324 if (fmpt->gran_1g) {
325 /* GB region currently uses 1G mapping. */
326 if (fmpt->prot == prot) {
327 fmpt->flags = 0;
328 return;
329 }
330
331 /*
332 * Range has different mapping than the rest of the GB.
333 * Convert to PAGE_SIZE mapping.
334 */
335 fmpt->gran_1g = false;
336 __set_smpt_range(fmpt->smpt, 0, start_gb_byte, fmpt->prot);
337 __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
338 __set_smpt_range(fmpt->smpt, end_gb_byte, SZ_1G, fmpt->prot);
339 fmpt->flags = MPT_UPDATE_L1 | MPT_UPDATE_L2;
340 return;
341 }
342
343 /* GB region currently uses PAGE_SIZE mapping. */
344 __set_smpt_range(fmpt->smpt, start_gb_byte, end_gb_byte, prot);
345
346 /* Check if the entire GB region has the same prot bits. */
347 if (!__is_smpt_uniform(fmpt->smpt, prot)) {
348 fmpt->flags = MPT_UPDATE_L2;
349 return;
350 }
351
352 fmpt->gran_1g = true;
353 fmpt->prot = prot;
354 fmpt->flags = MPT_UPDATE_L1;
355 }
356
357 #endif /* __ARM64_KVM_S2MPU_H__ */
358