1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * IOMMU API for ARM architected SMMUv3 implementations.
4 *
5 * Copyright (C) 2015 ARM Limited
6 */
7
8 #ifndef _ARM_SMMU_V3_H
9 #define _ARM_SMMU_V3_H
10
11 #include <linux/delay.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/mmzone.h>
15 #include <linux/sizes.h>
16
17 struct arm_smmu_device;
18
19 #include <asm/arm-smmu-v3-common.h>
20
21 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
22 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
23 #define Q_ENT(q, p) ((q)->base + \
24 Q_IDX(&((q)->llq), p) * \
25 (q)->ent_dwords)
26
27 /* Ensure DMA allocations are naturally aligned */
28 #ifdef CONFIG_CMA_ALIGNMENT
29 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
30 #else
31 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_PAGE_ORDER)
32 #endif
33
34 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
35
36 /* High-level queue structures */
37 #define ARM_SMMU_POLL_TIMEOUT_US 1000000 /* 1s! */
38 #define ARM_SMMU_POLL_SPIN_COUNT 10
39
40 #define MSI_IOVA_BASE 0x8000000
41 #define MSI_IOVA_LENGTH 0x100000
42
43 struct arm_smmu_ll_queue {
44 union {
45 u64 val;
46 struct {
47 u32 prod;
48 u32 cons;
49 };
50 struct {
51 atomic_t prod;
52 atomic_t cons;
53 } atomic;
54 u8 __pad[SMP_CACHE_BYTES];
55 } ____cacheline_aligned_in_smp;
56 u32 max_n_shift;
57 };
58
59 struct arm_smmu_queue {
60 struct arm_smmu_ll_queue llq;
61 int irq; /* Wired interrupt */
62
63 __le64 *base;
64 dma_addr_t base_dma;
65 u64 q_base;
66
67 size_t ent_dwords;
68
69 u32 __iomem *prod_reg;
70 u32 __iomem *cons_reg;
71 };
72
73 struct arm_smmu_queue_poll {
74 ktime_t timeout;
75 unsigned int delay;
76 unsigned int spin_cnt;
77 bool wfe;
78 };
79
80 struct arm_smmu_cmdq {
81 struct arm_smmu_queue q;
82 atomic_long_t *valid_map;
83 atomic_t owner_prod;
84 atomic_t lock;
85 bool (*supports_cmd)(struct arm_smmu_cmdq_ent *ent);
86 };
87
arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq * cmdq,struct arm_smmu_cmdq_ent * ent)88 static inline bool arm_smmu_cmdq_supports_cmd(struct arm_smmu_cmdq *cmdq,
89 struct arm_smmu_cmdq_ent *ent)
90 {
91 return cmdq->supports_cmd ? cmdq->supports_cmd(ent) : true;
92 }
93
94 struct arm_smmu_evtq {
95 struct arm_smmu_queue q;
96 struct iopf_queue *iopf;
97 u32 max_stalls;
98 };
99
100 struct arm_smmu_priq {
101 struct arm_smmu_queue q;
102 };
103
104 /* High-level stream table and context descriptor structures */
105 struct arm_smmu_ctx_desc {
106 u16 asid;
107 };
108
109 struct arm_smmu_ctx_desc_cfg {
110 union {
111 struct {
112 struct arm_smmu_cd *table;
113 unsigned int num_ents;
114 } linear;
115 struct {
116 struct arm_smmu_cdtab_l1 *l1tab;
117 struct arm_smmu_cdtab_l2 **l2ptrs;
118 unsigned int num_l1_ents;
119 } l2;
120 };
121 dma_addr_t cdtab_dma;
122 unsigned int used_ssids;
123 u8 in_ste;
124 u8 s1fmt;
125 /* log2 of the maximum number of CDs supported by this table */
126 u8 s1cdmax;
127 };
128
129 static inline bool
arm_smmu_cdtab_allocated(struct arm_smmu_ctx_desc_cfg * cfg)130 arm_smmu_cdtab_allocated(struct arm_smmu_ctx_desc_cfg *cfg)
131 {
132 return cfg->linear.table || cfg->l2.l1tab;
133 }
134
135 /* True if the cd table has SSIDS > 0 in use. */
arm_smmu_ssids_in_use(struct arm_smmu_ctx_desc_cfg * cd_table)136 static inline bool arm_smmu_ssids_in_use(struct arm_smmu_ctx_desc_cfg *cd_table)
137 {
138 return cd_table->used_ssids;
139 }
140
141 struct arm_smmu_s2_cfg {
142 u16 vmid;
143 };
144
145 struct arm_smmu_impl_ops {
146 int (*device_reset)(struct arm_smmu_device *smmu);
147 void (*device_remove)(struct arm_smmu_device *smmu);
148 int (*init_structures)(struct arm_smmu_device *smmu);
149 struct arm_smmu_cmdq *(*get_secondary_cmdq)(
150 struct arm_smmu_device *smmu, struct arm_smmu_cmdq_ent *ent);
151 };
152
153 /* An SMMUv3 instance */
154 struct arm_smmu_device {
155 struct device *dev;
156 struct device *impl_dev;
157 const struct arm_smmu_impl_ops *impl_ops;
158
159 void __iomem *base;
160 void __iomem *page1;
161
162 /* See arm-smmu-v3-common.h*/
163 u32 features;
164
165 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
166 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
167 #define ARM_SMMU_OPT_MSIPOLL (1 << 2)
168 #define ARM_SMMU_OPT_CMDQ_FORCE_SYNC (1 << 3)
169 #define ARM_SMMU_OPT_TEGRA241_CMDQV (1 << 4)
170 u32 options;
171
172 struct arm_smmu_cmdq cmdq;
173 struct arm_smmu_evtq evtq;
174 struct arm_smmu_priq priq;
175
176 int gerr_irq;
177 int combined_irq;
178
179 unsigned long ias; /* IPA */
180 unsigned long oas; /* PA */
181 unsigned long pgsize_bitmap;
182
183 #define ARM_SMMU_MAX_ASIDS (1 << 16)
184 unsigned int asid_bits;
185
186 #define ARM_SMMU_MAX_VMIDS (1 << 16)
187 unsigned int vmid_bits;
188 struct ida vmid_map;
189
190 unsigned int ssid_bits;
191 unsigned int sid_bits;
192
193 struct arm_smmu_strtab_cfg strtab_cfg;
194
195 /* IOMMU core code handle */
196 struct iommu_device iommu;
197
198 struct rb_root streams;
199 struct mutex streams_mutex;
200 };
201
202 struct arm_smmu_stream {
203 u32 id;
204 struct arm_smmu_master *master;
205 struct rb_node node;
206 };
207
208 struct arm_smmu_event {
209 u8 ssv : 1,
210 read : 1;
211 u8 id;
212 u32 sid;
213 u32 ssid;
214 u64 iova;
215 struct device *dev;
216 };
217
218 /* SMMU private data for each master */
219 struct arm_smmu_master {
220 struct arm_smmu_device *smmu;
221 struct device *dev;
222 struct arm_smmu_stream *streams;
223 /* Locked by the iommu core using the group mutex */
224 struct arm_smmu_ctx_desc_cfg cd_table;
225 unsigned int num_streams;
226 bool ats_enabled : 1;
227 bool ste_ats_enabled : 1;
228 bool stall_enabled;
229 bool sva_enabled;
230 bool iopf_enabled;
231 unsigned int ssid_bits;
232 };
233
234 /* SMMU private data for an IOMMU domain */
235 enum arm_smmu_domain_stage {
236 ARM_SMMU_DOMAIN_S1 = 0,
237 ARM_SMMU_DOMAIN_S2,
238 };
239
240 struct arm_smmu_domain {
241 struct arm_smmu_device *smmu;
242 struct mutex init_mutex; /* Protects smmu pointer */
243
244 struct io_pgtable_ops *pgtbl_ops;
245 atomic_t nr_ats_masters;
246
247 enum arm_smmu_domain_stage stage;
248 union {
249 struct arm_smmu_ctx_desc cd;
250 struct arm_smmu_s2_cfg s2_cfg;
251 };
252
253 struct iommu_domain domain;
254
255 /* List of struct arm_smmu_master_domain */
256 struct list_head devices;
257 spinlock_t devices_lock;
258
259 struct mmu_notifier mmu_notifier;
260 };
261
262 /* The following are exposed for testing purposes. */
263 struct arm_smmu_entry_writer_ops;
264 struct arm_smmu_entry_writer {
265 const struct arm_smmu_entry_writer_ops *ops;
266 struct arm_smmu_master *master;
267 };
268
269 struct arm_smmu_entry_writer_ops {
270 void (*get_used)(const __le64 *entry, __le64 *used);
271 void (*sync)(struct arm_smmu_entry_writer *writer);
272 };
273
274 #if IS_ENABLED(CONFIG_KUNIT)
275 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits);
276 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *cur,
277 const __le64 *target);
278 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits);
279 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target);
280 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
281 struct arm_smmu_ste *target);
282 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
283 struct arm_smmu_master *master, bool ats_enabled,
284 unsigned int s1dss);
285 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
286 struct arm_smmu_master *master,
287 struct arm_smmu_domain *smmu_domain,
288 bool ats_enabled);
289 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
290 struct arm_smmu_master *master, struct mm_struct *mm,
291 u16 asid);
292 #endif
293
294 struct arm_smmu_master_domain {
295 struct list_head devices_elm;
296 struct arm_smmu_master *master;
297 ioasid_t ssid;
298 };
299
to_smmu_domain(struct iommu_domain * dom)300 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
301 {
302 return container_of(dom, struct arm_smmu_domain, domain);
303 }
304
305 extern struct xarray arm_smmu_asid_xa;
306 extern struct mutex arm_smmu_asid_lock;
307
308 struct arm_smmu_domain *arm_smmu_domain_alloc(void);
309
310 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid);
311 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
312 u32 ssid);
313 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
314 struct arm_smmu_master *master,
315 struct arm_smmu_domain *smmu_domain);
316 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
317 struct arm_smmu_cd *cdptr,
318 const struct arm_smmu_cd *target);
319
320 int arm_smmu_set_pasid(struct arm_smmu_master *master,
321 struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
322 struct arm_smmu_cd *cd);
323
324 int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
325 unsigned int reg_off, unsigned int ack_off);
326 int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr);
327 int arm_smmu_device_disable(struct arm_smmu_device *smmu);
328 struct iommu_group *arm_smmu_device_group(struct device *dev);
329 int arm_smmu_of_xlate(struct device *dev, const struct of_phandle_args *args);
330 void arm_smmu_get_resv_regions(struct device *dev,
331 struct list_head *head);
332
333 struct platform_device;
334 int arm_smmu_fw_probe(struct platform_device *pdev,
335 struct arm_smmu_device *smmu);
336 int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu);
337 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
338 struct arm_smmu_queue *q,
339 void __iomem *page,
340 unsigned long prod_off,
341 unsigned long cons_off,
342 size_t dwords, const char *name);
343 int arm_smmu_init_strtab(struct arm_smmu_device *smmu);
344 void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
345 dma_addr_t l2ptr_dma);
346 void arm_smmu_write_strtab(struct arm_smmu_device *smmu);
347
348 void arm_smmu_probe_irq(struct platform_device *pdev,
349 struct arm_smmu_device *smmu);
350 void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu,
351 irqreturn_t evtqirq(int irq, void *dev),
352 irqreturn_t gerrorirq(int irq, void *dev),
353 irqreturn_t priirq(int irq, void *dev));
354
355 int arm_smmu_register_iommu(struct arm_smmu_device *smmu,
356 struct iommu_ops *ops, phys_addr_t ioaddr);
357 void arm_smmu_unregister_iommu(struct arm_smmu_device *smmu);
358
359 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
360 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
361 size_t granule, bool leaf,
362 struct arm_smmu_domain *smmu_domain);
363 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
364 unsigned long iova, size_t size);
365
366 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
367 struct arm_smmu_cmdq *cmdq);
368 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
369 struct arm_smmu_queue *q, void __iomem *page,
370 unsigned long prod_off, unsigned long cons_off,
371 size_t dwords, const char *name);
372 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
373 struct arm_smmu_cmdq *cmdq);
374
375 #ifdef CONFIG_ARM_SMMU_V3_SVA
376 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
377 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
378 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
379 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
380 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
381 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
382 void arm_smmu_sva_notifier_synchronize(void);
383 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
384 struct mm_struct *mm);
385 #else /* CONFIG_ARM_SMMU_V3_SVA */
arm_smmu_sva_supported(struct arm_smmu_device * smmu)386 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
387 {
388 return false;
389 }
390
arm_smmu_master_sva_supported(struct arm_smmu_master * master)391 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
392 {
393 return false;
394 }
395
arm_smmu_master_sva_enabled(struct arm_smmu_master * master)396 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
397 {
398 return false;
399 }
400
arm_smmu_master_enable_sva(struct arm_smmu_master * master)401 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
402 {
403 return -ENODEV;
404 }
405
arm_smmu_master_disable_sva(struct arm_smmu_master * master)406 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
407 {
408 return -ENODEV;
409 }
410
arm_smmu_master_iopf_supported(struct arm_smmu_master * master)411 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
412 {
413 return false;
414 }
415
arm_smmu_sva_notifier_synchronize(void)416 static inline void arm_smmu_sva_notifier_synchronize(void) {}
417
418 #define arm_smmu_sva_domain_alloc NULL
419
420 #endif /* CONFIG_ARM_SMMU_V3_SVA */
421
422 #ifdef CONFIG_TEGRA241_CMDQV
423 struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu);
424 #else /* CONFIG_TEGRA241_CMDQV */
425 static inline struct arm_smmu_device *
tegra241_cmdqv_probe(struct arm_smmu_device * smmu)426 tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
427 {
428 return ERR_PTR(-ENODEV);
429 }
430 #endif /* CONFIG_TEGRA241_CMDQV */
431
432 /* Queue functions shared with common and kernel drivers */
queue_has_space(struct arm_smmu_ll_queue * q,u32 n)433 static bool __maybe_unused queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
434 {
435 u32 space, prod, cons;
436
437 prod = Q_IDX(q, q->prod);
438 cons = Q_IDX(q, q->cons);
439
440 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
441 space = (1 << q->max_n_shift) - (prod - cons);
442 else
443 space = cons - prod;
444
445 return space >= n;
446 }
447
queue_full(struct arm_smmu_ll_queue * q)448 static bool __maybe_unused queue_full(struct arm_smmu_ll_queue *q)
449 {
450 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
451 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
452 }
453
queue_empty(struct arm_smmu_ll_queue * q)454 static bool __maybe_unused queue_empty(struct arm_smmu_ll_queue *q)
455 {
456 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
457 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
458 }
459
queue_consumed(struct arm_smmu_ll_queue * q,u32 prod)460 static bool __maybe_unused queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
461 {
462 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
463 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
464 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
465 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
466 }
467
queue_sync_cons_out(struct arm_smmu_queue * q)468 static void __maybe_unused queue_sync_cons_out(struct arm_smmu_queue *q)
469 {
470 /*
471 * Ensure that all CPU accesses (reads and writes) to the queue
472 * are complete before we update the cons pointer.
473 */
474 __iomb();
475 writel_relaxed(q->llq.cons, q->cons_reg);
476 }
477
queue_inc_cons(struct arm_smmu_ll_queue * q)478 static void __maybe_unused queue_inc_cons(struct arm_smmu_ll_queue *q)
479 {
480 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
481 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
482 }
483
queue_sync_cons_ovf(struct arm_smmu_queue * q)484 static void __maybe_unused queue_sync_cons_ovf(struct arm_smmu_queue *q)
485 {
486 struct arm_smmu_ll_queue *llq = &q->llq;
487
488 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
489 return;
490
491 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
492 Q_IDX(llq, llq->cons);
493 queue_sync_cons_out(q);
494 }
495
queue_sync_prod_in(struct arm_smmu_queue * q)496 static int __maybe_unused queue_sync_prod_in(struct arm_smmu_queue *q)
497 {
498 u32 prod;
499 int ret = 0;
500
501 /*
502 * We can't use the _relaxed() variant here, as we must prevent
503 * speculative reads of the queue before we have determined that
504 * prod has indeed moved.
505 */
506 prod = readl(q->prod_reg);
507
508 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
509 ret = -EOVERFLOW;
510
511 q->llq.prod = prod;
512 return ret;
513 }
514
queue_inc_prod_n(struct arm_smmu_ll_queue * q,int n)515 static u32 __maybe_unused queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
516 {
517 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
518 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
519 }
520
queue_poll_init(struct arm_smmu_device * smmu,struct arm_smmu_queue_poll * qp)521 static void __maybe_unused queue_poll_init(struct arm_smmu_device *smmu,
522 struct arm_smmu_queue_poll *qp)
523 {
524 qp->delay = 1;
525 qp->spin_cnt = 0;
526 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
527 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
528 }
529
queue_poll(struct arm_smmu_queue_poll * qp)530 static int __maybe_unused queue_poll(struct arm_smmu_queue_poll *qp)
531 {
532 if (ktime_compare(ktime_get(), qp->timeout) > 0)
533 return -ETIMEDOUT;
534
535 if (qp->wfe) {
536 wfe();
537 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
538 cpu_relax();
539 } else {
540 udelay(qp->delay);
541 qp->delay *= 2;
542 qp->spin_cnt = 0;
543 }
544
545 return 0;
546 }
547
queue_write(__le64 * dst,u64 * src,size_t n_dwords)548 static void __maybe_unused queue_write(__le64 *dst, u64 *src, size_t n_dwords)
549 {
550 int i;
551
552 for (i = 0; i < n_dwords; ++i)
553 *dst++ = cpu_to_le64(*src++);
554 }
555
queue_read(u64 * dst,__le64 * src,size_t n_dwords)556 static void __maybe_unused queue_read(u64 *dst, __le64 *src, size_t n_dwords)
557 {
558 int i;
559
560 for (i = 0; i < n_dwords; ++i)
561 *dst++ = le64_to_cpu(*src++);
562 }
563
queue_remove_raw(struct arm_smmu_queue * q,u64 * ent)564 static int __maybe_unused queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
565 {
566 if (queue_empty(&q->llq))
567 return -EAGAIN;
568
569 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
570 queue_inc_cons(&q->llq);
571 queue_sync_cons_out(q);
572 return 0;
573 }
574
575 enum arm_smmu_msi_index {
576 EVTQ_MSI_INDEX,
577 GERROR_MSI_INDEX,
578 PRIQ_MSI_INDEX,
579 ARM_SMMU_MAX_MSIS,
580 };
581
582 #endif /* _ARM_SMMU_V3_H */
583