1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2015-2016 MediaTek Inc.
4 * Author: Yong Wu <yong.wu@mediatek.com>
5 */
6 #include <linux/memblock.h>
7 #include <linux/bug.h>
8 #include <linux/clk.h>
9 #include <linux/component.h>
10 #include <linux/device.h>
11 #include <linux/dma-iommu.h>
12 #include <linux/err.h>
13 #include <linux/interrupt.h>
14 #include <linux/io.h>
15 #include <linux/iommu.h>
16 #include <linux/iopoll.h>
17 #include <linux/list.h>
18 #include <linux/of_address.h>
19 #include <linux/of_iommu.h>
20 #include <linux/of_irq.h>
21 #include <linux/of_platform.h>
22 #include <linux/platform_device.h>
23 #include <linux/slab.h>
24 #include <linux/spinlock.h>
25 #include <asm/barrier.h>
26 #include <soc/mediatek/smi.h>
27
28 #include "mtk_iommu.h"
29
30 #define REG_MMU_PT_BASE_ADDR 0x000
31 #define MMU_PT_ADDR_MASK GENMASK(31, 7)
32
33 #define REG_MMU_INVALIDATE 0x020
34 #define F_ALL_INVLD 0x2
35 #define F_MMU_INV_RANGE 0x1
36
37 #define REG_MMU_INVLD_START_A 0x024
38 #define REG_MMU_INVLD_END_A 0x028
39
40 #define REG_MMU_INV_SEL 0x038
41 #define F_INVLD_EN0 BIT(0)
42 #define F_INVLD_EN1 BIT(1)
43
44 #define REG_MMU_STANDARD_AXI_MODE 0x048
45 #define REG_MMU_DCM_DIS 0x050
46
47 #define REG_MMU_CTRL_REG 0x110
48 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4)
49 #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
50 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5)
51
52 #define REG_MMU_IVRP_PADDR 0x114
53
54 #define REG_MMU_VLD_PA_RNG 0x118
55 #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA))
56
57 #define REG_MMU_INT_CONTROL0 0x120
58 #define F_L2_MULIT_HIT_EN BIT(0)
59 #define F_TABLE_WALK_FAULT_INT_EN BIT(1)
60 #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2)
61 #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3)
62 #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5)
63 #define F_MISS_FIFO_ERR_INT_EN BIT(6)
64 #define F_INT_CLR_BIT BIT(12)
65
66 #define REG_MMU_INT_MAIN_CONTROL 0x124
67 /* mmu0 | mmu1 */
68 #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7))
69 #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8))
70 #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9))
71 #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10))
72 #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11))
73 #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12))
74 #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13))
75
76 #define REG_MMU_CPE_DONE 0x12C
77
78 #define REG_MMU_FAULT_ST1 0x134
79 #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0)
80 #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7)
81
82 #define REG_MMU0_FAULT_VA 0x13c
83 #define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
84 #define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
85
86 #define REG_MMU0_INVLD_PA 0x140
87 #define REG_MMU1_FAULT_VA 0x144
88 #define REG_MMU1_INVLD_PA 0x148
89 #define REG_MMU0_INT_ID 0x150
90 #define REG_MMU1_INT_ID 0x154
91 #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
92 #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
93
94 #define MTK_PROTECT_PA_ALIGN 128
95
96 /*
97 * Get the local arbiter ID and the portid within the larb arbiter
98 * from mtk_m4u_id which is defined by MTK_M4U_ID.
99 */
100 #define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0xf)
101 #define MTK_M4U_TO_PORT(id) ((id) & 0x1f)
102
103 struct mtk_iommu_domain {
104 spinlock_t pgtlock; /* lock for page table */
105
106 struct io_pgtable_cfg cfg;
107 struct io_pgtable_ops *iop;
108
109 struct iommu_domain domain;
110 };
111
112 static const struct iommu_ops mtk_iommu_ops;
113
114 /*
115 * In M4U 4GB mode, the physical address is remapped as below:
116 *
117 * CPU Physical address:
118 * ====================
119 *
120 * 0 1G 2G 3G 4G 5G
121 * |---A---|---B---|---C---|---D---|---E---|
122 * +--I/O--+------------Memory-------------+
123 *
124 * IOMMU output physical address:
125 * =============================
126 *
127 * 4G 5G 6G 7G 8G
128 * |---E---|---B---|---C---|---D---|
129 * +------------Memory-------------+
130 *
131 * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the
132 * bit32 of the CPU physical address always is needed to set, and for Region
133 * 'E', the CPU physical address keep as is.
134 * Additionally, The iommu consumers always use the CPU phyiscal address.
135 */
136 #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL
137
138 static LIST_HEAD(m4ulist); /* List all the M4U HWs */
139
140 #define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
141
142 /*
143 * There may be 1 or 2 M4U HWs, But we always expect they are in the same domain
144 * for the performance.
145 *
146 * Here always return the mtk_iommu_data of the first probed M4U where the
147 * iommu domain information is recorded.
148 */
mtk_iommu_get_m4u_data(void)149 static struct mtk_iommu_data *mtk_iommu_get_m4u_data(void)
150 {
151 struct mtk_iommu_data *data;
152
153 for_each_m4u(data)
154 return data;
155
156 return NULL;
157 }
158
to_mtk_domain(struct iommu_domain * dom)159 static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
160 {
161 return container_of(dom, struct mtk_iommu_domain, domain);
162 }
163
mtk_iommu_tlb_flush_all(void * cookie)164 static void mtk_iommu_tlb_flush_all(void *cookie)
165 {
166 struct mtk_iommu_data *data = cookie;
167
168 for_each_m4u(data) {
169 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
170 data->base + REG_MMU_INV_SEL);
171 writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
172 wmb(); /* Make sure the tlb flush all done */
173 }
174 }
175
mtk_iommu_tlb_add_flush_nosync(unsigned long iova,size_t size,size_t granule,bool leaf,void * cookie)176 static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size,
177 size_t granule, bool leaf,
178 void *cookie)
179 {
180 struct mtk_iommu_data *data = cookie;
181
182 for_each_m4u(data) {
183 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
184 data->base + REG_MMU_INV_SEL);
185
186 writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A);
187 writel_relaxed(iova + size - 1,
188 data->base + REG_MMU_INVLD_END_A);
189 writel_relaxed(F_MMU_INV_RANGE,
190 data->base + REG_MMU_INVALIDATE);
191 data->tlb_flush_active = true;
192 }
193 }
194
mtk_iommu_tlb_sync(void * cookie)195 static void mtk_iommu_tlb_sync(void *cookie)
196 {
197 struct mtk_iommu_data *data = cookie;
198 int ret;
199 u32 tmp;
200
201 for_each_m4u(data) {
202 /* Avoid timing out if there's nothing to wait for */
203 if (!data->tlb_flush_active)
204 return;
205
206 ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
207 tmp, tmp != 0, 10, 100000);
208 if (ret) {
209 dev_warn(data->dev,
210 "Partial TLB flush timed out, falling back to full flush\n");
211 mtk_iommu_tlb_flush_all(cookie);
212 }
213 /* Clear the CPE status */
214 writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
215 data->tlb_flush_active = false;
216 }
217 }
218
mtk_iommu_tlb_flush_walk(unsigned long iova,size_t size,size_t granule,void * cookie)219 static void mtk_iommu_tlb_flush_walk(unsigned long iova, size_t size,
220 size_t granule, void *cookie)
221 {
222 struct mtk_iommu_data *data = cookie;
223 unsigned long flags;
224
225 spin_lock_irqsave(&data->tlb_lock, flags);
226 mtk_iommu_tlb_add_flush_nosync(iova, size, granule, false, cookie);
227 mtk_iommu_tlb_sync(cookie);
228 spin_unlock_irqrestore(&data->tlb_lock, flags);
229 }
230
mtk_iommu_tlb_flush_leaf(unsigned long iova,size_t size,size_t granule,void * cookie)231 static void mtk_iommu_tlb_flush_leaf(unsigned long iova, size_t size,
232 size_t granule, void *cookie)
233 {
234 struct mtk_iommu_data *data = cookie;
235 unsigned long flags;
236
237 spin_lock_irqsave(&data->tlb_lock, flags);
238 mtk_iommu_tlb_add_flush_nosync(iova, size, granule, true, cookie);
239 mtk_iommu_tlb_sync(cookie);
240 spin_unlock_irqrestore(&data->tlb_lock, flags);
241 }
242
mtk_iommu_tlb_flush_page_nosync(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)243 static void mtk_iommu_tlb_flush_page_nosync(struct iommu_iotlb_gather *gather,
244 unsigned long iova, size_t granule,
245 void *cookie)
246 {
247 struct mtk_iommu_data *data = cookie;
248 unsigned long flags;
249
250 spin_lock_irqsave(&data->tlb_lock, flags);
251 mtk_iommu_tlb_add_flush_nosync(iova, granule, granule, true, cookie);
252 spin_unlock_irqrestore(&data->tlb_lock, flags);
253 }
254
255 static const struct iommu_flush_ops mtk_iommu_flush_ops = {
256 .tlb_flush_all = mtk_iommu_tlb_flush_all,
257 .tlb_flush_walk = mtk_iommu_tlb_flush_walk,
258 .tlb_flush_leaf = mtk_iommu_tlb_flush_leaf,
259 .tlb_add_page = mtk_iommu_tlb_flush_page_nosync,
260 };
261
mtk_iommu_isr(int irq,void * dev_id)262 static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
263 {
264 struct mtk_iommu_data *data = dev_id;
265 struct mtk_iommu_domain *dom = data->m4u_dom;
266 u32 int_state, regval, fault_iova, fault_pa;
267 unsigned int fault_larb, fault_port;
268 bool layer, write;
269
270 /* Read error info from registers */
271 int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1);
272 if (int_state & F_REG_MMU0_FAULT_MASK) {
273 regval = readl_relaxed(data->base + REG_MMU0_INT_ID);
274 fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA);
275 fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA);
276 } else {
277 regval = readl_relaxed(data->base + REG_MMU1_INT_ID);
278 fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA);
279 fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA);
280 }
281 layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
282 write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
283 fault_larb = F_MMU_INT_ID_LARB_ID(regval);
284 fault_port = F_MMU_INT_ID_PORT_ID(regval);
285
286 fault_larb = data->plat_data->larbid_remap[fault_larb];
287
288 if (report_iommu_fault(&dom->domain, data->dev, fault_iova,
289 write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
290 dev_err_ratelimited(
291 data->dev,
292 "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n",
293 int_state, fault_iova, fault_pa, fault_larb, fault_port,
294 layer, write ? "write" : "read");
295 }
296
297 /* Interrupt clear */
298 regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0);
299 regval |= F_INT_CLR_BIT;
300 writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0);
301
302 mtk_iommu_tlb_flush_all(data);
303
304 return IRQ_HANDLED;
305 }
306
mtk_iommu_config(struct mtk_iommu_data * data,struct device * dev,bool enable)307 static void mtk_iommu_config(struct mtk_iommu_data *data,
308 struct device *dev, bool enable)
309 {
310 struct mtk_smi_larb_iommu *larb_mmu;
311 unsigned int larbid, portid;
312 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
313 int i;
314
315 for (i = 0; i < fwspec->num_ids; ++i) {
316 larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
317 portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
318 larb_mmu = &data->larb_imu[larbid];
319
320 dev_dbg(dev, "%s iommu port: %d\n",
321 enable ? "enable" : "disable", portid);
322
323 if (enable)
324 larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);
325 else
326 larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);
327 }
328 }
329
mtk_iommu_domain_finalise(struct mtk_iommu_domain * dom)330 static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
331 {
332 struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
333
334 spin_lock_init(&dom->pgtlock);
335
336 dom->cfg = (struct io_pgtable_cfg) {
337 .quirks = IO_PGTABLE_QUIRK_ARM_NS |
338 IO_PGTABLE_QUIRK_NO_PERMS |
339 IO_PGTABLE_QUIRK_TLBI_ON_MAP |
340 IO_PGTABLE_QUIRK_ARM_MTK_EXT,
341 .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap,
342 .ias = 32,
343 .oas = 34,
344 .tlb = &mtk_iommu_flush_ops,
345 .iommu_dev = data->dev,
346 };
347
348 dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
349 if (!dom->iop) {
350 dev_err(data->dev, "Failed to alloc io pgtable\n");
351 return -EINVAL;
352 }
353
354 /* Update our support page sizes bitmap */
355 dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
356 return 0;
357 }
358
mtk_iommu_domain_alloc(unsigned type)359 static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
360 {
361 struct mtk_iommu_domain *dom;
362
363 if (type != IOMMU_DOMAIN_DMA)
364 return NULL;
365
366 dom = kzalloc(sizeof(*dom), GFP_KERNEL);
367 if (!dom)
368 return NULL;
369
370 if (iommu_get_dma_cookie(&dom->domain))
371 goto free_dom;
372
373 if (mtk_iommu_domain_finalise(dom))
374 goto put_dma_cookie;
375
376 dom->domain.geometry.aperture_start = 0;
377 dom->domain.geometry.aperture_end = DMA_BIT_MASK(32);
378 dom->domain.geometry.force_aperture = true;
379
380 return &dom->domain;
381
382 put_dma_cookie:
383 iommu_put_dma_cookie(&dom->domain);
384 free_dom:
385 kfree(dom);
386 return NULL;
387 }
388
mtk_iommu_domain_free(struct iommu_domain * domain)389 static void mtk_iommu_domain_free(struct iommu_domain *domain)
390 {
391 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
392
393 free_io_pgtable_ops(dom->iop);
394 iommu_put_dma_cookie(domain);
395 kfree(to_mtk_domain(domain));
396 }
397
mtk_iommu_attach_device(struct iommu_domain * domain,struct device * dev)398 static int mtk_iommu_attach_device(struct iommu_domain *domain,
399 struct device *dev)
400 {
401 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
402 struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv;
403
404 if (!data)
405 return -ENODEV;
406
407 /* Update the pgtable base address register of the M4U HW */
408 if (!data->m4u_dom) {
409 data->m4u_dom = dom;
410 writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
411 data->base + REG_MMU_PT_BASE_ADDR);
412 }
413
414 mtk_iommu_config(data, dev, true);
415 return 0;
416 }
417
mtk_iommu_detach_device(struct iommu_domain * domain,struct device * dev)418 static void mtk_iommu_detach_device(struct iommu_domain *domain,
419 struct device *dev)
420 {
421 struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv;
422
423 if (!data)
424 return;
425
426 mtk_iommu_config(data, dev, false);
427 }
428
mtk_iommu_map(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t size,int prot)429 static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
430 phys_addr_t paddr, size_t size, int prot)
431 {
432 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
433 struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
434 unsigned long flags;
435 int ret;
436
437 /* The "4GB mode" M4U physically can not use the lower remap of Dram. */
438 if (data->enable_4GB)
439 paddr |= BIT_ULL(32);
440
441 spin_lock_irqsave(&dom->pgtlock, flags);
442 ret = dom->iop->map(dom->iop, iova, paddr, size, prot);
443 spin_unlock_irqrestore(&dom->pgtlock, flags);
444
445 return ret;
446 }
447
mtk_iommu_unmap(struct iommu_domain * domain,unsigned long iova,size_t size,struct iommu_iotlb_gather * gather)448 static size_t mtk_iommu_unmap(struct iommu_domain *domain,
449 unsigned long iova, size_t size,
450 struct iommu_iotlb_gather *gather)
451 {
452 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
453 unsigned long flags;
454 size_t unmapsz;
455
456 spin_lock_irqsave(&dom->pgtlock, flags);
457 unmapsz = dom->iop->unmap(dom->iop, iova, size, gather);
458 spin_unlock_irqrestore(&dom->pgtlock, flags);
459
460 return unmapsz;
461 }
462
mtk_iommu_flush_iotlb_all(struct iommu_domain * domain)463 static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
464 {
465 mtk_iommu_tlb_flush_all(mtk_iommu_get_m4u_data());
466 }
467
mtk_iommu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)468 static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
469 struct iommu_iotlb_gather *gather)
470 {
471 struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
472 unsigned long flags;
473
474 spin_lock_irqsave(&data->tlb_lock, flags);
475 mtk_iommu_tlb_sync(data);
476 spin_unlock_irqrestore(&data->tlb_lock, flags);
477 }
478
mtk_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)479 static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
480 dma_addr_t iova)
481 {
482 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
483 struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
484 unsigned long flags;
485 phys_addr_t pa;
486
487 spin_lock_irqsave(&dom->pgtlock, flags);
488 pa = dom->iop->iova_to_phys(dom->iop, iova);
489 spin_unlock_irqrestore(&dom->pgtlock, flags);
490
491 if (data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
492 pa &= ~BIT_ULL(32);
493
494 return pa;
495 }
496
mtk_iommu_add_device(struct device * dev)497 static int mtk_iommu_add_device(struct device *dev)
498 {
499 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
500 struct mtk_iommu_data *data;
501 struct iommu_group *group;
502
503 if (!fwspec || fwspec->ops != &mtk_iommu_ops)
504 return -ENODEV; /* Not a iommu client device */
505
506 data = fwspec->iommu_priv;
507 iommu_device_link(&data->iommu, dev);
508
509 group = iommu_group_get_for_dev(dev);
510 if (IS_ERR(group))
511 return PTR_ERR(group);
512
513 iommu_group_put(group);
514 return 0;
515 }
516
mtk_iommu_remove_device(struct device * dev)517 static void mtk_iommu_remove_device(struct device *dev)
518 {
519 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
520 struct mtk_iommu_data *data;
521
522 if (!fwspec || fwspec->ops != &mtk_iommu_ops)
523 return;
524
525 data = fwspec->iommu_priv;
526 iommu_device_unlink(&data->iommu, dev);
527
528 iommu_group_remove_device(dev);
529 iommu_fwspec_free(dev);
530 }
531
mtk_iommu_device_group(struct device * dev)532 static struct iommu_group *mtk_iommu_device_group(struct device *dev)
533 {
534 struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
535
536 if (!data)
537 return ERR_PTR(-ENODEV);
538
539 /* All the client devices are in the same m4u iommu-group */
540 if (!data->m4u_group) {
541 data->m4u_group = iommu_group_alloc();
542 if (IS_ERR(data->m4u_group))
543 dev_err(dev, "Failed to allocate M4U IOMMU group\n");
544 } else {
545 iommu_group_ref_get(data->m4u_group);
546 }
547 return data->m4u_group;
548 }
549
mtk_iommu_of_xlate(struct device * dev,struct of_phandle_args * args)550 static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
551 {
552 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
553 struct platform_device *m4updev;
554
555 if (args->args_count != 1) {
556 dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",
557 args->args_count);
558 return -EINVAL;
559 }
560
561 if (!fwspec->iommu_priv) {
562 /* Get the m4u device */
563 m4updev = of_find_device_by_node(args->np);
564 if (WARN_ON(!m4updev))
565 return -EINVAL;
566
567 fwspec->iommu_priv = platform_get_drvdata(m4updev);
568 }
569
570 return iommu_fwspec_add_ids(dev, args->args, 1);
571 }
572
573 static const struct iommu_ops mtk_iommu_ops = {
574 .domain_alloc = mtk_iommu_domain_alloc,
575 .domain_free = mtk_iommu_domain_free,
576 .attach_dev = mtk_iommu_attach_device,
577 .detach_dev = mtk_iommu_detach_device,
578 .map = mtk_iommu_map,
579 .unmap = mtk_iommu_unmap,
580 .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
581 .iotlb_sync = mtk_iommu_iotlb_sync,
582 .iova_to_phys = mtk_iommu_iova_to_phys,
583 .add_device = mtk_iommu_add_device,
584 .remove_device = mtk_iommu_remove_device,
585 .device_group = mtk_iommu_device_group,
586 .of_xlate = mtk_iommu_of_xlate,
587 .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
588 };
589
mtk_iommu_hw_init(const struct mtk_iommu_data * data)590 static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
591 {
592 u32 regval;
593 int ret;
594
595 ret = clk_prepare_enable(data->bclk);
596 if (ret) {
597 dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);
598 return ret;
599 }
600
601 if (data->plat_data->m4u_plat == M4U_MT8173)
602 regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
603 F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173;
604 else
605 regval = F_MMU_TF_PROT_TO_PROGRAM_ADDR;
606 writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
607
608 regval = F_L2_MULIT_HIT_EN |
609 F_TABLE_WALK_FAULT_INT_EN |
610 F_PREETCH_FIFO_OVERFLOW_INT_EN |
611 F_MISS_FIFO_OVERFLOW_INT_EN |
612 F_PREFETCH_FIFO_ERR_INT_EN |
613 F_MISS_FIFO_ERR_INT_EN;
614 writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0);
615
616 regval = F_INT_TRANSLATION_FAULT |
617 F_INT_MAIN_MULTI_HIT_FAULT |
618 F_INT_INVALID_PA_FAULT |
619 F_INT_ENTRY_REPLACEMENT_FAULT |
620 F_INT_TLB_MISS_FAULT |
621 F_INT_MISS_TRANSACTION_FIFO_FAULT |
622 F_INT_PRETETCH_TRANSATION_FIFO_FAULT;
623 writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL);
624
625 if (data->plat_data->m4u_plat == M4U_MT8173)
626 regval = (data->protect_base >> 1) | (data->enable_4GB << 31);
627 else
628 regval = lower_32_bits(data->protect_base) |
629 upper_32_bits(data->protect_base);
630 writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR);
631
632 if (data->enable_4GB && data->plat_data->has_vld_pa_rng) {
633 /*
634 * If 4GB mode is enabled, the validate PA range is from
635 * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
636 */
637 regval = F_MMU_VLD_PA_RNG(7, 4);
638 writel_relaxed(regval, data->base + REG_MMU_VLD_PA_RNG);
639 }
640 writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
641
642 if (data->plat_data->reset_axi)
643 writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
644
645 if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
646 dev_name(data->dev), (void *)data)) {
647 writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);
648 clk_disable_unprepare(data->bclk);
649 dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);
650 return -ENODEV;
651 }
652
653 return 0;
654 }
655
656 static const struct component_master_ops mtk_iommu_com_ops = {
657 .bind = mtk_iommu_bind,
658 .unbind = mtk_iommu_unbind,
659 };
660
mtk_iommu_probe(struct platform_device * pdev)661 static int mtk_iommu_probe(struct platform_device *pdev)
662 {
663 struct mtk_iommu_data *data;
664 struct device *dev = &pdev->dev;
665 struct resource *res;
666 resource_size_t ioaddr;
667 struct component_match *match = NULL;
668 void *protect;
669 int i, larb_nr, ret;
670
671 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
672 if (!data)
673 return -ENOMEM;
674 data->dev = dev;
675 data->plat_data = of_device_get_match_data(dev);
676
677 /* Protect memory. HW will access here while translation fault.*/
678 protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
679 if (!protect)
680 return -ENOMEM;
681 data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
682
683 /* Whether the current dram is over 4GB */
684 data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
685 if (!data->plat_data->has_4gb_mode)
686 data->enable_4GB = false;
687
688 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
689 data->base = devm_ioremap_resource(dev, res);
690 if (IS_ERR(data->base))
691 return PTR_ERR(data->base);
692 ioaddr = res->start;
693
694 data->irq = platform_get_irq(pdev, 0);
695 if (data->irq < 0)
696 return data->irq;
697
698 if (data->plat_data->has_bclk) {
699 data->bclk = devm_clk_get(dev, "bclk");
700 if (IS_ERR(data->bclk))
701 return PTR_ERR(data->bclk);
702 }
703
704 larb_nr = of_count_phandle_with_args(dev->of_node,
705 "mediatek,larbs", NULL);
706 if (larb_nr < 0)
707 return larb_nr;
708
709 for (i = 0; i < larb_nr; i++) {
710 struct device_node *larbnode;
711 struct platform_device *plarbdev;
712 u32 id;
713
714 larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
715 if (!larbnode)
716 return -EINVAL;
717
718 if (!of_device_is_available(larbnode)) {
719 of_node_put(larbnode);
720 continue;
721 }
722
723 ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
724 if (ret)/* The id is consecutive if there is no this property */
725 id = i;
726
727 plarbdev = of_find_device_by_node(larbnode);
728 if (!plarbdev) {
729 of_node_put(larbnode);
730 return -EPROBE_DEFER;
731 }
732 data->larb_imu[id].dev = &plarbdev->dev;
733
734 component_match_add_release(dev, &match, release_of,
735 compare_of, larbnode);
736 }
737
738 platform_set_drvdata(pdev, data);
739
740 ret = mtk_iommu_hw_init(data);
741 if (ret)
742 return ret;
743
744 ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,
745 "mtk-iommu.%pa", &ioaddr);
746 if (ret)
747 return ret;
748
749 iommu_device_set_ops(&data->iommu, &mtk_iommu_ops);
750 iommu_device_set_fwnode(&data->iommu, &pdev->dev.of_node->fwnode);
751
752 ret = iommu_device_register(&data->iommu);
753 if (ret)
754 return ret;
755
756 spin_lock_init(&data->tlb_lock);
757 list_add_tail(&data->list, &m4ulist);
758
759 if (!iommu_present(&platform_bus_type))
760 bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
761
762 return component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
763 }
764
mtk_iommu_remove(struct platform_device * pdev)765 static int mtk_iommu_remove(struct platform_device *pdev)
766 {
767 struct mtk_iommu_data *data = platform_get_drvdata(pdev);
768
769 iommu_device_sysfs_remove(&data->iommu);
770 iommu_device_unregister(&data->iommu);
771
772 if (iommu_present(&platform_bus_type))
773 bus_set_iommu(&platform_bus_type, NULL);
774
775 clk_disable_unprepare(data->bclk);
776 devm_free_irq(&pdev->dev, data->irq, data);
777 component_master_del(&pdev->dev, &mtk_iommu_com_ops);
778 return 0;
779 }
780
mtk_iommu_suspend(struct device * dev)781 static int __maybe_unused mtk_iommu_suspend(struct device *dev)
782 {
783 struct mtk_iommu_data *data = dev_get_drvdata(dev);
784 struct mtk_iommu_suspend_reg *reg = &data->reg;
785 void __iomem *base = data->base;
786
787 reg->standard_axi_mode = readl_relaxed(base +
788 REG_MMU_STANDARD_AXI_MODE);
789 reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS);
790 reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
791 reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
792 reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
793 reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR);
794 reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG);
795 clk_disable_unprepare(data->bclk);
796 return 0;
797 }
798
mtk_iommu_resume(struct device * dev)799 static int __maybe_unused mtk_iommu_resume(struct device *dev)
800 {
801 struct mtk_iommu_data *data = dev_get_drvdata(dev);
802 struct mtk_iommu_suspend_reg *reg = &data->reg;
803 struct mtk_iommu_domain *m4u_dom = data->m4u_dom;
804 void __iomem *base = data->base;
805 int ret;
806
807 ret = clk_prepare_enable(data->bclk);
808 if (ret) {
809 dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
810 return ret;
811 }
812 writel_relaxed(reg->standard_axi_mode,
813 base + REG_MMU_STANDARD_AXI_MODE);
814 writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
815 writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);
816 writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0);
817 writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
818 writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
819 writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
820 if (m4u_dom)
821 writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
822 base + REG_MMU_PT_BASE_ADDR);
823 return 0;
824 }
825
826 static const struct dev_pm_ops mtk_iommu_pm_ops = {
827 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
828 };
829
830 static const struct mtk_iommu_plat_data mt2712_data = {
831 .m4u_plat = M4U_MT2712,
832 .has_4gb_mode = true,
833 .has_bclk = true,
834 .has_vld_pa_rng = true,
835 .larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
836 };
837
838 static const struct mtk_iommu_plat_data mt8173_data = {
839 .m4u_plat = M4U_MT8173,
840 .has_4gb_mode = true,
841 .has_bclk = true,
842 .reset_axi = true,
843 .larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */
844 };
845
846 static const struct mtk_iommu_plat_data mt8183_data = {
847 .m4u_plat = M4U_MT8183,
848 .reset_axi = true,
849 .larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1},
850 };
851
852 static const struct of_device_id mtk_iommu_of_ids[] = {
853 { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
854 { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data},
855 { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data},
856 {}
857 };
858
859 static struct platform_driver mtk_iommu_driver = {
860 .probe = mtk_iommu_probe,
861 .remove = mtk_iommu_remove,
862 .driver = {
863 .name = "mtk-iommu",
864 .of_match_table = of_match_ptr(mtk_iommu_of_ids),
865 .pm = &mtk_iommu_pm_ops,
866 }
867 };
868
mtk_iommu_init(void)869 static int __init mtk_iommu_init(void)
870 {
871 int ret;
872
873 ret = platform_driver_register(&mtk_iommu_driver);
874 if (ret != 0)
875 pr_err("Failed to register MTK IOMMU driver\n");
876
877 return ret;
878 }
879
880 subsys_initcall(mtk_iommu_init)
881