1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef IO_PGTABLE_H_
3 #define IO_PGTABLE_H_
4
5 #include <linux/io-pgtable.h>
6
7 typedef u64 arm_lpae_iopte;
8
9 struct arm_lpae_io_pgtable {
10 struct io_pgtable iop;
11
12 int pgd_bits;
13 int start_level;
14 int bits_per_level;
15
16 void *pgd;
17
18 bool idmapped; /* Used by hypervisor */
19 };
20
21 struct io_pgtable_walk_data {
22 struct io_pgtable *iop;
23 struct io_pgtable_walk_common *data;
24 int (*visit)(struct io_pgtable_walk_data *walk_data, int lvl,
25 arm_lpae_iopte *ptep, size_t size);
26 unsigned long flags;
27 u64 addr;
28 const u64 end;
29 };
30
31 /* Struct accessors */
32 #define io_pgtable_to_data(x) \
33 container_of((x), struct arm_lpae_io_pgtable, iop)
34
35 #define io_pgtable_ops_to_data(x) \
36 io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))
37
38 /*
39 * Calculate the right shift amount to get to the portion describing level l
40 * in a virtual address mapped by the pagetable in d.
41 */
42 #define ARM_LPAE_LVL_SHIFT(l,d) \
43 (((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
44 ilog2(sizeof(arm_lpae_iopte)))
45
46 #define ARM_LPAE_GRANULE(d) \
47 (sizeof(arm_lpae_iopte) << (d)->bits_per_level)
48 #define ARM_LPAE_PGD_SIZE(d) \
49 (sizeof(arm_lpae_iopte) << (d)->pgd_bits)
50
51 #define ARM_LPAE_PTES_PER_TABLE(d) \
52 (ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))
53
54 /*
55 * Calculate the index at level l used to map virtual address a using the
56 * pagetable in d.
57 */
58 #define ARM_LPAE_PGD_IDX(l,d) \
59 ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)
60
61 #define ARM_LPAE_LVL_IDX(a,l,d) \
62 (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
63 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))
64
65 /* Calculate the block/page mapping size at level l for pagetable in d. */
66 #define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))
67
68 /* Page table bits */
69 #define ARM_LPAE_PTE_TYPE_SHIFT 0
70 #define ARM_LPAE_PTE_TYPE_MASK 0x3
71
72 #define ARM_LPAE_PTE_TYPE_BLOCK 1
73 #define ARM_LPAE_PTE_TYPE_TABLE 3
74 #define ARM_LPAE_PTE_TYPE_PAGE 3
75
76 #define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)
77
78 #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
79 #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
80 #define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51)
81 #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
82 #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
83 #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
84 #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
85 #define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
86 #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)
87
88 #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
89 /* Ignore the contiguous bit for block splitting */
90 #define ARM_LPAE_PTE_ATTR_HI_MASK (ARM_LPAE_PTE_XN | ARM_LPAE_PTE_DBM)
91 #define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \
92 ARM_LPAE_PTE_ATTR_HI_MASK)
93 /* Software bit for solving coherency races */
94 #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)
95
96 /* Stage-1 PTE */
97 #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
98 #define ARM_LPAE_PTE_AP_RDONLY_BIT 7
99 #define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \
100 ARM_LPAE_PTE_AP_RDONLY_BIT)
101 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY | \
102 ARM_LPAE_PTE_DBM)
103 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
104 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)
105
106 /* Stage-2 PTE */
107 #define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
108 #define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
109 #define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
110 #define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
111 #define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
112 #define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)
113
114 /* Register bits */
115 #define ARM_LPAE_VTCR_SL0_MASK 0x3
116
117 #define ARM_LPAE_TCR_T0SZ_SHIFT 0
118
119 #define ARM_LPAE_VTCR_PS_SHIFT 16
120 #define ARM_LPAE_VTCR_PS_MASK 0x7
121
122 #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
123 #define ARM_LPAE_MAIR_ATTR_MASK 0xff
124 #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
125 #define ARM_LPAE_MAIR_ATTR_NC 0x44
126 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
127 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
128 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0
129 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
130 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
131 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3
132
133 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
134 #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
135 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)
136
137 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
138 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
139
140 #define ARM_LPAE_MAX_LEVELS 4
141
142 #define ARM_LPAE_TCR_TG0_4K 0
143 #define ARM_LPAE_TCR_TG0_64K 1
144 #define ARM_LPAE_TCR_TG0_16K 2
145
146 #define ARM_LPAE_TCR_TG1_16K 1
147 #define ARM_LPAE_TCR_TG1_4K 2
148 #define ARM_LPAE_TCR_TG1_64K 3
149
150 #define ARM_LPAE_TCR_SH_NS 0
151 #define ARM_LPAE_TCR_SH_OS 2
152 #define ARM_LPAE_TCR_SH_IS 3
153
154 #define ARM_LPAE_TCR_RGN_NC 0
155 #define ARM_LPAE_TCR_RGN_WBWA 1
156 #define ARM_LPAE_TCR_RGN_WT 2
157 #define ARM_LPAE_TCR_RGN_WB 3
158
159 #define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
160 #define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
161 #define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
162 #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
163 #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
164 #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
165 #define ARM_LPAE_TCR_PS_52_BIT 0x6ULL
166
167 /* IOPTE accessors */
168 #define iopte_type(pte) \
169 (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
170
171 #define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)
172
173 #define iopte_writeable_dirty(pte) \
174 (((pte) & ARM_LPAE_PTE_AP_WR_CLEAN_MASK) == ARM_LPAE_PTE_DBM)
175
176 #define iopte_set_writeable_clean(ptep) \
177 set_bit(ARM_LPAE_PTE_AP_RDONLY_BIT, (unsigned long *)(ptep))
178
179
iopte_leaf(arm_lpae_iopte pte,int lvl,enum io_pgtable_fmt fmt)180 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
181 enum io_pgtable_fmt fmt)
182 {
183 if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
184 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;
185
186 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
187 }
188
iopte_table(arm_lpae_iopte pte,int lvl)189 static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
190 {
191 if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
192 return false;
193 return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
194 }
195
iopte_valid(arm_lpae_iopte pte)196 static inline bool iopte_valid(arm_lpae_iopte pte)
197 {
198 return pte & ARM_LPAE_PTE_VALID;
199 }
200
201 #ifdef __KVM_NVHE_HYPERVISOR__
202 #include <nvhe/memory.h>
203 #define __arm_lpae_virt_to_phys hyp_virt_to_phys
204 #define __arm_lpae_phys_to_virt hyp_phys_to_virt
205
206 struct io_pgtable *kvm_arm_io_pgtable_alloc(struct io_pgtable_cfg *cfg,
207 void *cookie,
208 int *out_ret);
209 int kvm_arm_io_pgtable_free(struct io_pgtable *iop);
210 #else
211 #define __arm_lpae_virt_to_phys __pa
212 #define __arm_lpae_phys_to_virt __va
213 #endif
214
215 /* Generic functions */
216 void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
217 arm_lpae_iopte *ptep);
218
219 int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
220 struct arm_lpae_io_pgtable *data);
221 int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
222 struct arm_lpae_io_pgtable *data);
223 int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
224 struct arm_lpae_io_pgtable *data);
225
226 int __arm_lpae_iopte_walk(struct arm_lpae_io_pgtable *data,
227 struct io_pgtable_walk_data *walk_data,
228 arm_lpae_iopte *ptep,
229 int lvl);
230 /* Host/hyp-specific functions */
231 void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg *cfg, void *cookie);
232 void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg, void *cookie);
233 void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
234 struct io_pgtable_cfg *cfg);
235 int arm_lpae_map_exists(void);
236 void arm_lpae_unmap_empty(void);
237 #endif /* IO_PGTABLE_H_ */
238