1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 */
9
10 #ifndef _ARM_SMMU_H
11 #define _ARM_SMMU_H
12
13 #include <linux/atomic.h>
14 #include <linux/bits.h>
15 #include <linux/clk.h>
16 #include <linux/device.h>
17 #include <linux/io-64-nonatomic-hi-lo.h>
18 #include <linux/io-pgtable.h>
19 #include <linux/iommu.h>
20 #include <linux/mutex.h>
21 #include <linux/spinlock.h>
22 #include <linux/types.h>
23
24 /* Configuration registers */
25 #define ARM_SMMU_GR0_sCR0 0x0
26 #define sCR0_VMID16EN BIT(31)
27 #define sCR0_BSU GENMASK(15, 14)
28 #define sCR0_FB BIT(13)
29 #define sCR0_PTM BIT(12)
30 #define sCR0_VMIDPNE BIT(11)
31 #define sCR0_USFCFG BIT(10)
32 #define sCR0_GCFGFIE BIT(5)
33 #define sCR0_GCFGFRE BIT(4)
34 #define sCR0_EXIDENABLE BIT(3)
35 #define sCR0_GFIE BIT(2)
36 #define sCR0_GFRE BIT(1)
37 #define sCR0_CLIENTPD BIT(0)
38
39 /* Auxiliary Configuration register */
40 #define ARM_SMMU_GR0_sACR 0x10
41
42 /* Identification registers */
43 #define ARM_SMMU_GR0_ID0 0x20
44 #define ID0_S1TS BIT(30)
45 #define ID0_S2TS BIT(29)
46 #define ID0_NTS BIT(28)
47 #define ID0_SMS BIT(27)
48 #define ID0_ATOSNS BIT(26)
49 #define ID0_PTFS_NO_AARCH32 BIT(25)
50 #define ID0_PTFS_NO_AARCH32S BIT(24)
51 #define ID0_NUMIRPT GENMASK(23, 16)
52 #define ID0_CTTW BIT(14)
53 #define ID0_NUMSIDB GENMASK(12, 9)
54 #define ID0_EXIDS BIT(8)
55 #define ID0_NUMSMRG GENMASK(7, 0)
56
57 #define ARM_SMMU_GR0_ID1 0x24
58 #define ID1_PAGESIZE BIT(31)
59 #define ID1_NUMPAGENDXB GENMASK(30, 28)
60 #define ID1_NUMS2CB GENMASK(23, 16)
61 #define ID1_NUMCB GENMASK(7, 0)
62
63 #define ARM_SMMU_GR0_ID2 0x28
64 #define ID2_VMID16 BIT(15)
65 #define ID2_PTFS_64K BIT(14)
66 #define ID2_PTFS_16K BIT(13)
67 #define ID2_PTFS_4K BIT(12)
68 #define ID2_UBS GENMASK(11, 8)
69 #define ID2_OAS GENMASK(7, 4)
70 #define ID2_IAS GENMASK(3, 0)
71
72 #define ARM_SMMU_GR0_ID3 0x2c
73 #define ARM_SMMU_GR0_ID4 0x30
74 #define ARM_SMMU_GR0_ID5 0x34
75 #define ARM_SMMU_GR0_ID6 0x38
76
77 #define ARM_SMMU_GR0_ID7 0x3c
78 #define ID7_MAJOR GENMASK(7, 4)
79 #define ID7_MINOR GENMASK(3, 0)
80
81 #define ARM_SMMU_GR0_sGFSR 0x48
82 #define ARM_SMMU_GR0_sGFSYNR0 0x50
83 #define ARM_SMMU_GR0_sGFSYNR1 0x54
84 #define ARM_SMMU_GR0_sGFSYNR2 0x58
85
86 /* Global TLB invalidation */
87 #define ARM_SMMU_GR0_TLBIVMID 0x64
88 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
89 #define ARM_SMMU_GR0_TLBIALLH 0x6c
90 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
91
92 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
93 #define sTLBGSTATUS_GSACTIVE BIT(0)
94
95 /* Stream mapping registers */
96 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
97 #define SMR_VALID BIT(31)
98 #define SMR_MASK GENMASK(31, 16)
99 #define SMR_ID GENMASK(15, 0)
100
101 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
102 #define S2CR_PRIVCFG GENMASK(25, 24)
103 enum arm_smmu_s2cr_privcfg {
104 S2CR_PRIVCFG_DEFAULT,
105 S2CR_PRIVCFG_DIPAN,
106 S2CR_PRIVCFG_UNPRIV,
107 S2CR_PRIVCFG_PRIV,
108 };
109 #define S2CR_TYPE GENMASK(17, 16)
110 enum arm_smmu_s2cr_type {
111 S2CR_TYPE_TRANS,
112 S2CR_TYPE_BYPASS,
113 S2CR_TYPE_FAULT,
114 };
115 #define S2CR_EXIDVALID BIT(10)
116 #define S2CR_CBNDX GENMASK(7, 0)
117
118 /* Context bank attribute registers */
119 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
120 #define CBAR_IRPTNDX GENMASK(31, 24)
121 #define CBAR_TYPE GENMASK(17, 16)
122 enum arm_smmu_cbar_type {
123 CBAR_TYPE_S2_TRANS,
124 CBAR_TYPE_S1_TRANS_S2_BYPASS,
125 CBAR_TYPE_S1_TRANS_S2_FAULT,
126 CBAR_TYPE_S1_TRANS_S2_TRANS,
127 };
128 #define CBAR_S1_MEMATTR GENMASK(15, 12)
129 #define CBAR_S1_MEMATTR_WB 0xf
130 #define CBAR_S1_BPSHCFG GENMASK(9, 8)
131 #define CBAR_S1_BPSHCFG_NSH 3
132 #define CBAR_VMID GENMASK(7, 0)
133
134 #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
135
136 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
137 #define CBA2R_VMID16 GENMASK(31, 16)
138 #define CBA2R_VA64 BIT(0)
139
140 #define ARM_SMMU_CB_SCTLR 0x0
141 #define SCTLR_S1_ASIDPNE BIT(12)
142 #define SCTLR_CFCFG BIT(7)
143 #define SCTLR_CFIE BIT(6)
144 #define SCTLR_CFRE BIT(5)
145 #define SCTLR_E BIT(4)
146 #define SCTLR_AFE BIT(2)
147 #define SCTLR_TRE BIT(1)
148 #define SCTLR_M BIT(0)
149
150 #define ARM_SMMU_CB_ACTLR 0x4
151
152 #define ARM_SMMU_CB_RESUME 0x8
153 #define RESUME_TERMINATE BIT(0)
154
155 #define ARM_SMMU_CB_TCR2 0x10
156 #define TCR2_SEP GENMASK(17, 15)
157 #define TCR2_SEP_UPSTREAM 0x7
158 #define TCR2_AS BIT(4)
159
160 #define ARM_SMMU_CB_TTBR0 0x20
161 #define ARM_SMMU_CB_TTBR1 0x28
162 #define TTBRn_ASID GENMASK_ULL(63, 48)
163
164 #define ARM_SMMU_CB_TCR 0x30
165 #define ARM_SMMU_CB_CONTEXTIDR 0x34
166 #define ARM_SMMU_CB_S1_MAIR0 0x38
167 #define ARM_SMMU_CB_S1_MAIR1 0x3c
168
169 #define ARM_SMMU_CB_PAR 0x50
170 #define CB_PAR_F BIT(0)
171
172 #define ARM_SMMU_CB_FSR 0x58
173 #define FSR_MULTI BIT(31)
174 #define FSR_SS BIT(30)
175 #define FSR_UUT BIT(8)
176 #define FSR_ASF BIT(7)
177 #define FSR_TLBLKF BIT(6)
178 #define FSR_TLBMCF BIT(5)
179 #define FSR_EF BIT(4)
180 #define FSR_PF BIT(3)
181 #define FSR_AFF BIT(2)
182 #define FSR_TF BIT(1)
183
184 #define FSR_IGN (FSR_AFF | FSR_ASF | \
185 FSR_TLBMCF | FSR_TLBLKF)
186 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
187 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
188
189 #define ARM_SMMU_CB_FAR 0x60
190
191 #define ARM_SMMU_CB_FSYNR0 0x68
192 #define FSYNR0_WNR BIT(4)
193
194 #define ARM_SMMU_CB_S1_TLBIVA 0x600
195 #define ARM_SMMU_CB_S1_TLBIASID 0x610
196 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
197 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
198 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
199 #define ARM_SMMU_CB_TLBSYNC 0x7f0
200 #define ARM_SMMU_CB_TLBSTATUS 0x7f4
201 #define ARM_SMMU_CB_ATS1PR 0x800
202
203 #define ARM_SMMU_CB_ATSR 0x8f0
204 #define ATSR_ACTIVE BIT(0)
205
206
207 /* Maximum number of context banks per SMMU */
208 #define ARM_SMMU_MAX_CBS 128
209
210
211 /* Shared driver definitions */
212 enum arm_smmu_arch_version {
213 ARM_SMMU_V1,
214 ARM_SMMU_V1_64K,
215 ARM_SMMU_V2,
216 };
217
218 enum arm_smmu_implementation {
219 GENERIC_SMMU,
220 ARM_MMU500,
221 CAVIUM_SMMUV2,
222 QCOM_SMMUV2,
223 };
224
225 struct arm_smmu_s2cr {
226 struct iommu_group *group;
227 int count;
228 enum arm_smmu_s2cr_type type;
229 enum arm_smmu_s2cr_privcfg privcfg;
230 u8 cbndx;
231 bool pinned;
232 };
233
234 struct arm_smmu_smr {
235 u16 mask;
236 u16 id;
237 bool valid;
238 };
239
240 struct arm_smmu_device {
241 struct device *dev;
242
243 void __iomem *base;
244 unsigned int numpage;
245 unsigned int pgshift;
246
247 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
248 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
249 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
250 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
251 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
252 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
253 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
254 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
255 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
256 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
257 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
258 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
259 #define ARM_SMMU_FEAT_EXIDS (1 << 12)
260 u32 features;
261
262 enum arm_smmu_arch_version version;
263 enum arm_smmu_implementation model;
264 const struct arm_smmu_impl *impl;
265
266 u32 num_context_banks;
267 u32 num_s2_context_banks;
268 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
269 struct arm_smmu_cb *cbs;
270 atomic_t irptndx;
271
272 u32 num_mapping_groups;
273 u16 streamid_mask;
274 u16 smr_mask_mask;
275 struct arm_smmu_smr *smrs;
276 struct arm_smmu_s2cr *s2crs;
277 struct mutex stream_map_mutex;
278
279 unsigned long va_size;
280 unsigned long ipa_size;
281 unsigned long pa_size;
282 unsigned long pgsize_bitmap;
283
284 u32 num_global_irqs;
285 u32 num_context_irqs;
286 unsigned int *irqs;
287 struct clk_bulk_data *clks;
288 int num_clks;
289
290 spinlock_t global_sync_lock;
291
292 /* IOMMU core code handle */
293 struct iommu_device iommu;
294 };
295
296 enum arm_smmu_context_fmt {
297 ARM_SMMU_CTX_FMT_NONE,
298 ARM_SMMU_CTX_FMT_AARCH64,
299 ARM_SMMU_CTX_FMT_AARCH32_L,
300 ARM_SMMU_CTX_FMT_AARCH32_S,
301 };
302
303 struct arm_smmu_cfg {
304 u8 cbndx;
305 u8 irptndx;
306 union {
307 u16 asid;
308 u16 vmid;
309 };
310 enum arm_smmu_cbar_type cbar;
311 enum arm_smmu_context_fmt fmt;
312 };
313 #define INVALID_IRPTNDX 0xff
314
315 enum arm_smmu_domain_stage {
316 ARM_SMMU_DOMAIN_S1 = 0,
317 ARM_SMMU_DOMAIN_S2,
318 ARM_SMMU_DOMAIN_NESTED,
319 ARM_SMMU_DOMAIN_BYPASS,
320 };
321
322 struct arm_smmu_flush_ops {
323 struct iommu_flush_ops tlb;
324 void (*tlb_inv_range)(unsigned long iova, size_t size, size_t granule,
325 bool leaf, void *cookie);
326 void (*tlb_sync)(void *cookie);
327 };
328
329 struct arm_smmu_domain {
330 struct arm_smmu_device *smmu;
331 struct io_pgtable_ops *pgtbl_ops;
332 const struct arm_smmu_flush_ops *flush_ops;
333 struct arm_smmu_cfg cfg;
334 enum arm_smmu_domain_stage stage;
335 bool non_strict;
336 struct mutex init_mutex; /* Protects smmu pointer */
337 spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
338 struct iommu_domain domain;
339 };
340
341
342 /* Implementation details, yay! */
343 struct arm_smmu_impl {
344 u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
345 void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
346 u32 val);
347 u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
348 void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
349 u64 val);
350 int (*cfg_probe)(struct arm_smmu_device *smmu);
351 int (*reset)(struct arm_smmu_device *smmu);
352 int (*init_context)(struct arm_smmu_domain *smmu_domain);
353 };
354
arm_smmu_page(struct arm_smmu_device * smmu,int n)355 static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
356 {
357 return smmu->base + (n << smmu->pgshift);
358 }
359
arm_smmu_readl(struct arm_smmu_device * smmu,int page,int offset)360 static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
361 {
362 if (smmu->impl && unlikely(smmu->impl->read_reg))
363 return smmu->impl->read_reg(smmu, page, offset);
364 return readl_relaxed(arm_smmu_page(smmu, page) + offset);
365 }
366
arm_smmu_writel(struct arm_smmu_device * smmu,int page,int offset,u32 val)367 static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
368 int offset, u32 val)
369 {
370 if (smmu->impl && unlikely(smmu->impl->write_reg))
371 smmu->impl->write_reg(smmu, page, offset, val);
372 else
373 writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
374 }
375
arm_smmu_readq(struct arm_smmu_device * smmu,int page,int offset)376 static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
377 {
378 if (smmu->impl && unlikely(smmu->impl->read_reg64))
379 return smmu->impl->read_reg64(smmu, page, offset);
380 return readq_relaxed(arm_smmu_page(smmu, page) + offset);
381 }
382
arm_smmu_writeq(struct arm_smmu_device * smmu,int page,int offset,u64 val)383 static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
384 int offset, u64 val)
385 {
386 if (smmu->impl && unlikely(smmu->impl->write_reg64))
387 smmu->impl->write_reg64(smmu, page, offset, val);
388 else
389 writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
390 }
391
392 #define ARM_SMMU_GR0 0
393 #define ARM_SMMU_GR1 1
394 #define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
395
396 #define arm_smmu_gr0_read(s, o) \
397 arm_smmu_readl((s), ARM_SMMU_GR0, (o))
398 #define arm_smmu_gr0_write(s, o, v) \
399 arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
400
401 #define arm_smmu_gr1_read(s, o) \
402 arm_smmu_readl((s), ARM_SMMU_GR1, (o))
403 #define arm_smmu_gr1_write(s, o, v) \
404 arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
405
406 #define arm_smmu_cb_read(s, n, o) \
407 arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
408 #define arm_smmu_cb_write(s, n, o, v) \
409 arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
410 #define arm_smmu_cb_readq(s, n, o) \
411 arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
412 #define arm_smmu_cb_writeq(s, n, o, v) \
413 arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
414
415 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
416 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
417
418 int arm_mmu500_reset(struct arm_smmu_device *smmu);
419
420 #endif /* _ARM_SMMU_H */
421