1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37
38 #include <linux/fsl/mc.h>
39
40 #include "arm-smmu.h"
41 #include "../../dma-iommu.h"
42
43 /*
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
49 */
50 #define QCOM_DUMMY_VAL -1
51
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
54
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
67 }
68
69 static bool using_legacy_binding, using_generic_binding;
70
arm_smmu_rpm_get(struct arm_smmu_device * smmu)71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72 {
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
75
76 return 0;
77 }
78
arm_smmu_rpm_put(struct arm_smmu_device * smmu)79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80 {
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
83 }
84
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)85 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
86 {
87 /*
88 * Setup an autosuspend delay to avoid bouncing runpm state.
89 * Otherwise, if a driver for a suspended consumer device
90 * unmaps buffers, it will runpm resume/suspend for each one.
91 *
92 * For example, when used by a GPU device, when an application
93 * or game exits, it can trigger unmapping 100s or 1000s of
94 * buffers. With a runpm cycle for each buffer, that adds up
95 * to 5-10sec worth of reprogramming the context bank, while
96 * the system appears to be locked up to the user.
97 */
98 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
99 pm_runtime_use_autosuspend(smmu->dev);
100 }
101
to_smmu_domain(struct iommu_domain * dom)102 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
103 {
104 return container_of(dom, struct arm_smmu_domain, domain);
105 }
106
107 static struct platform_driver arm_smmu_driver;
108 static struct iommu_ops arm_smmu_ops;
109
110 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)111 static struct device_node *dev_get_dev_node(struct device *dev)
112 {
113 if (dev_is_pci(dev)) {
114 struct pci_bus *bus = to_pci_dev(dev)->bus;
115
116 while (!pci_is_root_bus(bus))
117 bus = bus->parent;
118 return of_node_get(bus->bridge->parent->of_node);
119 }
120
121 return of_node_get(dev->of_node);
122 }
123
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)124 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
125 {
126 *((__be32 *)data) = cpu_to_be32(alias);
127 return 0; /* Continue walking */
128 }
129
__find_legacy_master_phandle(struct device * dev,void * data)130 static int __find_legacy_master_phandle(struct device *dev, void *data)
131 {
132 struct of_phandle_iterator *it = *(void **)data;
133 struct device_node *np = it->node;
134 int err;
135
136 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
137 "#stream-id-cells", -1)
138 if (it->node == np) {
139 *(void **)data = dev;
140 return 1;
141 }
142 it->node = np;
143 return err == -ENOENT ? 0 : err;
144 }
145
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)146 static int arm_smmu_register_legacy_master(struct device *dev,
147 struct arm_smmu_device **smmu)
148 {
149 struct device *smmu_dev;
150 struct device_node *np;
151 struct of_phandle_iterator it;
152 void *data = ⁢
153 u32 *sids;
154 __be32 pci_sid;
155 int err;
156
157 np = dev_get_dev_node(dev);
158 if (!np || !of_property_present(np, "#stream-id-cells")) {
159 of_node_put(np);
160 return -ENODEV;
161 }
162
163 it.node = np;
164 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
165 __find_legacy_master_phandle);
166 smmu_dev = data;
167 of_node_put(np);
168 if (err == 0)
169 return -ENODEV;
170 if (err < 0)
171 return err;
172
173 if (dev_is_pci(dev)) {
174 /* "mmu-masters" assumes Stream ID == Requester ID */
175 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
176 &pci_sid);
177 it.cur = &pci_sid;
178 it.cur_count = 1;
179 }
180
181 err = iommu_fwspec_init(dev, NULL);
182 if (err)
183 return err;
184
185 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
186 if (!sids)
187 return -ENOMEM;
188
189 *smmu = dev_get_drvdata(smmu_dev);
190 of_phandle_iterator_args(&it, sids, it.cur_count);
191 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
192 kfree(sids);
193 return err;
194 }
195 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)196 static int arm_smmu_register_legacy_master(struct device *dev,
197 struct arm_smmu_device **smmu)
198 {
199 return -ENODEV;
200 }
201 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
202
__arm_smmu_free_bitmap(unsigned long * map,int idx)203 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
204 {
205 clear_bit(idx, map);
206 }
207
208 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)209 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
210 int sync, int status)
211 {
212 unsigned int spin_cnt, delay;
213 u32 reg;
214
215 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
216 return smmu->impl->tlb_sync(smmu, page, sync, status);
217
218 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
219 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
220 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
221 reg = arm_smmu_readl(smmu, page, status);
222 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
223 return;
224 cpu_relax();
225 }
226 udelay(delay);
227 }
228 dev_err_ratelimited(smmu->dev,
229 "TLB sync timed out -- SMMU may be deadlocked\n");
230 }
231
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)232 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
233 {
234 unsigned long flags;
235
236 spin_lock_irqsave(&smmu->global_sync_lock, flags);
237 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
238 ARM_SMMU_GR0_sTLBGSTATUS);
239 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
240 }
241
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)242 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
243 {
244 struct arm_smmu_device *smmu = smmu_domain->smmu;
245 unsigned long flags;
246
247 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
248 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
249 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
250 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
251 }
252
arm_smmu_tlb_inv_context_s1(void * cookie)253 static void arm_smmu_tlb_inv_context_s1(void *cookie)
254 {
255 struct arm_smmu_domain *smmu_domain = cookie;
256 /*
257 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
258 * current CPU are visible beforehand.
259 */
260 wmb();
261 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
262 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
263 arm_smmu_tlb_sync_context(smmu_domain);
264 }
265
arm_smmu_tlb_inv_context_s2(void * cookie)266 static void arm_smmu_tlb_inv_context_s2(void *cookie)
267 {
268 struct arm_smmu_domain *smmu_domain = cookie;
269 struct arm_smmu_device *smmu = smmu_domain->smmu;
270
271 /* See above */
272 wmb();
273 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
274 arm_smmu_tlb_sync_global(smmu);
275 }
276
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)277 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
278 size_t granule, void *cookie, int reg)
279 {
280 struct arm_smmu_domain *smmu_domain = cookie;
281 struct arm_smmu_device *smmu = smmu_domain->smmu;
282 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
283 int idx = cfg->cbndx;
284
285 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
286 wmb();
287
288 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
289 iova = (iova >> 12) << 12;
290 iova |= cfg->asid;
291 do {
292 arm_smmu_cb_write(smmu, idx, reg, iova);
293 iova += granule;
294 } while (size -= granule);
295 } else {
296 iova >>= 12;
297 iova |= (u64)cfg->asid << 48;
298 do {
299 arm_smmu_cb_writeq(smmu, idx, reg, iova);
300 iova += granule >> 12;
301 } while (size -= granule);
302 }
303 }
304
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)305 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
306 size_t granule, void *cookie, int reg)
307 {
308 struct arm_smmu_domain *smmu_domain = cookie;
309 struct arm_smmu_device *smmu = smmu_domain->smmu;
310 int idx = smmu_domain->cfg.cbndx;
311
312 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
313 wmb();
314
315 iova >>= 12;
316 do {
317 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
318 arm_smmu_cb_writeq(smmu, idx, reg, iova);
319 else
320 arm_smmu_cb_write(smmu, idx, reg, iova);
321 iova += granule >> 12;
322 } while (size -= granule);
323 }
324
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)325 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
326 size_t granule, void *cookie)
327 {
328 struct arm_smmu_domain *smmu_domain = cookie;
329 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
330
331 if (cfg->flush_walk_prefer_tlbiasid) {
332 arm_smmu_tlb_inv_context_s1(cookie);
333 } else {
334 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
335 ARM_SMMU_CB_S1_TLBIVA);
336 arm_smmu_tlb_sync_context(cookie);
337 }
338 }
339
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)340 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
341 unsigned long iova, size_t granule,
342 void *cookie)
343 {
344 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
345 ARM_SMMU_CB_S1_TLBIVAL);
346 }
347
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)348 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
349 size_t granule, void *cookie)
350 {
351 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
352 ARM_SMMU_CB_S2_TLBIIPAS2);
353 arm_smmu_tlb_sync_context(cookie);
354 }
355
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)356 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
357 unsigned long iova, size_t granule,
358 void *cookie)
359 {
360 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
361 ARM_SMMU_CB_S2_TLBIIPAS2L);
362 }
363
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)364 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
365 size_t granule, void *cookie)
366 {
367 arm_smmu_tlb_inv_context_s2(cookie);
368 }
369 /*
370 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
371 * almost negligible, but the benefit of getting the first one in as far ahead
372 * of the sync as possible is significant, hence we don't just make this a
373 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
374 * think.
375 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)376 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
377 unsigned long iova, size_t granule,
378 void *cookie)
379 {
380 struct arm_smmu_domain *smmu_domain = cookie;
381 struct arm_smmu_device *smmu = smmu_domain->smmu;
382
383 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
384 wmb();
385
386 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
387 }
388
389 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
390 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
391 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
392 .tlb_add_page = arm_smmu_tlb_add_page_s1,
393 };
394
395 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
396 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
397 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
398 .tlb_add_page = arm_smmu_tlb_add_page_s2,
399 };
400
401 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
402 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
403 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
404 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
405 };
406
407
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)408 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
409 struct arm_smmu_context_fault_info *cfi)
410 {
411 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
412 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
413 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
414 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
415 }
416
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)417 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
418 const struct arm_smmu_context_fault_info *cfi)
419 {
420 dev_err(smmu->dev,
421 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
422 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
423
424 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
425 cfi->fsr,
426 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
427 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
428 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
429 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
430 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
432 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
433 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
437 cfi->cbfrsynra);
438
439 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
440 cfi->fsynr,
441 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
442 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
443 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
444 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
445 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
448 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
449 }
450
arm_smmu_context_fault(int irq,void * dev)451 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
452 {
453 struct arm_smmu_context_fault_info cfi;
454 struct arm_smmu_domain *smmu_domain = dev;
455 struct arm_smmu_device *smmu = smmu_domain->smmu;
456 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
457 DEFAULT_RATELIMIT_BURST);
458 int idx = smmu_domain->cfg.cbndx;
459 int ret;
460
461 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
462
463 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
464 return IRQ_NONE;
465
466 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
467 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
468
469 if (ret == -ENOSYS && __ratelimit(&rs))
470 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
471
472 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
473 return IRQ_HANDLED;
474 }
475
arm_smmu_global_fault(int irq,void * dev)476 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
477 {
478 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
479 struct arm_smmu_device *smmu = dev;
480 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
481 DEFAULT_RATELIMIT_BURST);
482
483 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
484 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
485 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
486 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
487
488 if (!gfsr)
489 return IRQ_NONE;
490
491 if (__ratelimit(&rs)) {
492 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
493 (gfsr & ARM_SMMU_sGFSR_USF))
494 dev_err(smmu->dev,
495 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
496 (u16)gfsynr1);
497 else
498 dev_err(smmu->dev,
499 "Unexpected global fault, this could be serious\n");
500 dev_err(smmu->dev,
501 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
502 gfsr, gfsynr0, gfsynr1, gfsynr2);
503 }
504
505 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
506 return IRQ_HANDLED;
507 }
508
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)509 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
510 struct io_pgtable_cfg *pgtbl_cfg)
511 {
512 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
513 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
514 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
515
516 cb->cfg = cfg;
517
518 /* TCR */
519 if (stage1) {
520 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
521 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
522 } else {
523 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
524 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
525 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
526 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
527 else
528 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
529 }
530 } else {
531 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
532 }
533
534 /* TTBRs */
535 if (stage1) {
536 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
537 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
538 cb->ttbr[1] = 0;
539 } else {
540 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
541 cfg->asid);
542 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
543 cfg->asid);
544
545 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
546 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
547 else
548 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
549 }
550 } else {
551 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
552 }
553
554 /* MAIRs (stage-1 only) */
555 if (stage1) {
556 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
557 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
558 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
559 } else {
560 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
561 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
562 }
563 }
564 }
565
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)566 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
567 {
568 u32 reg;
569 bool stage1;
570 struct arm_smmu_cb *cb = &smmu->cbs[idx];
571 struct arm_smmu_cfg *cfg = cb->cfg;
572
573 /* Unassigned context banks only need disabling */
574 if (!cfg) {
575 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
576 return;
577 }
578
579 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
580
581 /* CBA2R */
582 if (smmu->version > ARM_SMMU_V1) {
583 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
584 reg = ARM_SMMU_CBA2R_VA64;
585 else
586 reg = 0;
587 /* 16-bit VMIDs live in CBA2R */
588 if (smmu->features & ARM_SMMU_FEAT_VMID16)
589 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
590
591 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
592 }
593
594 /* CBAR */
595 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
596 if (smmu->version < ARM_SMMU_V2)
597 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
598
599 /*
600 * Use the weakest shareability/memory types, so they are
601 * overridden by the ttbcr/pte.
602 */
603 if (stage1) {
604 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
605 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
606 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
607 ARM_SMMU_CBAR_S1_MEMATTR_WB);
608 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
609 /* 8-bit VMIDs live in CBAR */
610 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
611 }
612 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
613
614 /*
615 * TCR
616 * We must write this before the TTBRs, since it determines the
617 * access behaviour of some fields (in particular, ASID[15:8]).
618 */
619 if (stage1 && smmu->version > ARM_SMMU_V1)
620 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
621 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
622
623 /* TTBRs */
624 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
625 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
626 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
627 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
628 } else {
629 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
630 if (stage1)
631 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
632 cb->ttbr[1]);
633 }
634
635 /* MAIRs (stage-1 only) */
636 if (stage1) {
637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
638 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
639 }
640
641 /* SCTLR */
642 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
643 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
644 if (stage1)
645 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
646 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
647 reg |= ARM_SMMU_SCTLR_E;
648
649 if (smmu->impl && smmu->impl->write_sctlr)
650 smmu->impl->write_sctlr(smmu, idx, reg);
651 else
652 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
653 }
654
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)655 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
656 struct arm_smmu_device *smmu,
657 struct device *dev, unsigned int start)
658 {
659 if (smmu->impl && smmu->impl->alloc_context_bank)
660 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
661
662 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
663 }
664
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)665 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
666 struct arm_smmu_device *smmu,
667 struct device *dev)
668 {
669 int irq, start, ret = 0;
670 unsigned long ias, oas;
671 struct io_pgtable_ops *pgtbl_ops;
672 struct io_pgtable_cfg pgtbl_cfg;
673 enum io_pgtable_fmt fmt;
674 struct iommu_domain *domain = &smmu_domain->domain;
675 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
676 irqreturn_t (*context_fault)(int irq, void *dev);
677
678 mutex_lock(&smmu_domain->init_mutex);
679 if (smmu_domain->smmu)
680 goto out_unlock;
681
682 /*
683 * Mapping the requested stage onto what we support is surprisingly
684 * complicated, mainly because the spec allows S1+S2 SMMUs without
685 * support for nested translation. That means we end up with the
686 * following table:
687 *
688 * Requested Supported Actual
689 * S1 N S1
690 * S1 S1+S2 S1
691 * S1 S2 S2
692 * S1 S1 S1
693 * N N N
694 * N S1+S2 S2
695 * N S2 S2
696 * N S1 S1
697 *
698 * Note that you can't actually request stage-2 mappings.
699 */
700 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
701 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
702 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
703 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
704
705 /*
706 * Choosing a suitable context format is even more fiddly. Until we
707 * grow some way for the caller to express a preference, and/or move
708 * the decision into the io-pgtable code where it arguably belongs,
709 * just aim for the closest thing to the rest of the system, and hope
710 * that the hardware isn't esoteric enough that we can't assume AArch64
711 * support to be a superset of AArch32 support...
712 */
713 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
714 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
715 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
716 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
717 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
718 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
719 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
720 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
721 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
722 ARM_SMMU_FEAT_FMT_AARCH64_16K |
723 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
725
726 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
727 ret = -EINVAL;
728 goto out_unlock;
729 }
730
731 switch (smmu_domain->stage) {
732 case ARM_SMMU_DOMAIN_S1:
733 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
734 start = smmu->num_s2_context_banks;
735 ias = smmu->va_size;
736 oas = smmu->ipa_size;
737 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
738 fmt = ARM_64_LPAE_S1;
739 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
740 fmt = ARM_32_LPAE_S1;
741 ias = min(ias, 32UL);
742 oas = min(oas, 40UL);
743 } else {
744 fmt = ARM_V7S;
745 ias = min(ias, 32UL);
746 oas = min(oas, 32UL);
747 }
748 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
749 break;
750 case ARM_SMMU_DOMAIN_NESTED:
751 /*
752 * We will likely want to change this if/when KVM gets
753 * involved.
754 */
755 case ARM_SMMU_DOMAIN_S2:
756 cfg->cbar = CBAR_TYPE_S2_TRANS;
757 start = 0;
758 ias = smmu->ipa_size;
759 oas = smmu->pa_size;
760 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
761 fmt = ARM_64_LPAE_S2;
762 } else {
763 fmt = ARM_32_LPAE_S2;
764 ias = min(ias, 40UL);
765 oas = min(oas, 40UL);
766 }
767 if (smmu->version == ARM_SMMU_V2)
768 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
769 else
770 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
771 break;
772 default:
773 ret = -EINVAL;
774 goto out_unlock;
775 }
776
777 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
778 if (ret < 0) {
779 goto out_unlock;
780 }
781
782 smmu_domain->smmu = smmu;
783
784 cfg->cbndx = ret;
785 if (smmu->version < ARM_SMMU_V2) {
786 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
787 cfg->irptndx %= smmu->num_context_irqs;
788 } else {
789 cfg->irptndx = cfg->cbndx;
790 }
791
792 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
793 cfg->vmid = cfg->cbndx + 1;
794 else
795 cfg->asid = cfg->cbndx;
796
797 pgtbl_cfg = (struct io_pgtable_cfg) {
798 .pgsize_bitmap = smmu->pgsize_bitmap,
799 .ias = ias,
800 .oas = oas,
801 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
802 .tlb = smmu_domain->flush_ops,
803 .iommu_dev = smmu->dev,
804 };
805
806 if (smmu->impl && smmu->impl->init_context) {
807 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
808 if (ret)
809 goto out_clear_smmu;
810 }
811
812 if (smmu_domain->pgtbl_quirks)
813 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
814
815 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
816 if (!pgtbl_ops) {
817 ret = -ENOMEM;
818 goto out_clear_smmu;
819 }
820
821 /* Update the domain's page sizes to reflect the page table format */
822 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
823
824 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
825 domain->geometry.aperture_start = ~0UL << ias;
826 domain->geometry.aperture_end = ~0UL;
827 } else {
828 domain->geometry.aperture_end = (1UL << ias) - 1;
829 }
830
831 domain->geometry.force_aperture = true;
832
833 /* Initialise the context bank with our page table cfg */
834 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
835 arm_smmu_write_context_bank(smmu, cfg->cbndx);
836
837 /*
838 * Request context fault interrupt. Do this last to avoid the
839 * handler seeing a half-initialised domain state.
840 */
841 irq = smmu->irqs[cfg->irptndx];
842
843 if (smmu->impl && smmu->impl->context_fault)
844 context_fault = smmu->impl->context_fault;
845 else
846 context_fault = arm_smmu_context_fault;
847
848 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
849 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
850 context_fault,
851 IRQF_ONESHOT | IRQF_SHARED,
852 "arm-smmu-context-fault",
853 smmu_domain);
854 else
855 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
856 "arm-smmu-context-fault", smmu_domain);
857
858 if (ret < 0) {
859 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
860 cfg->irptndx, irq);
861 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
862 }
863
864 mutex_unlock(&smmu_domain->init_mutex);
865
866 /* Publish page table ops for map/unmap */
867 smmu_domain->pgtbl_ops = pgtbl_ops;
868 return 0;
869
870 out_clear_smmu:
871 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
872 smmu_domain->smmu = NULL;
873 out_unlock:
874 mutex_unlock(&smmu_domain->init_mutex);
875 return ret;
876 }
877
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)878 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
879 {
880 struct arm_smmu_device *smmu = smmu_domain->smmu;
881 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
882 int ret, irq;
883
884 if (!smmu)
885 return;
886
887 ret = arm_smmu_rpm_get(smmu);
888 if (ret < 0)
889 return;
890
891 /*
892 * Disable the context bank and free the page tables before freeing
893 * it.
894 */
895 smmu->cbs[cfg->cbndx].cfg = NULL;
896 arm_smmu_write_context_bank(smmu, cfg->cbndx);
897
898 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
899 irq = smmu->irqs[cfg->irptndx];
900 devm_free_irq(smmu->dev, irq, smmu_domain);
901 }
902
903 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
904 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
905
906 arm_smmu_rpm_put(smmu);
907 }
908
arm_smmu_domain_alloc_paging(struct device * dev)909 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
910 {
911 struct arm_smmu_domain *smmu_domain;
912
913 /*
914 * Allocate the domain and initialise some of its data structures.
915 * We can't really do anything meaningful until we've added a
916 * master.
917 */
918 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
919 if (!smmu_domain)
920 return NULL;
921
922 mutex_init(&smmu_domain->init_mutex);
923 spin_lock_init(&smmu_domain->cb_lock);
924
925 return &smmu_domain->domain;
926 }
927
arm_smmu_domain_free(struct iommu_domain * domain)928 static void arm_smmu_domain_free(struct iommu_domain *domain)
929 {
930 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
931
932 /*
933 * Free the domain resources. We assume that all devices have
934 * already been detached.
935 */
936 arm_smmu_destroy_domain_context(smmu_domain);
937 kfree(smmu_domain);
938 }
939
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)940 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
941 {
942 struct arm_smmu_smr *smr = smmu->smrs + idx;
943 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
944 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
945
946 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
947 reg |= ARM_SMMU_SMR_VALID;
948 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
949 }
950
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)951 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
952 {
953 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
954 u32 reg;
955
956 if (smmu->impl && smmu->impl->write_s2cr) {
957 smmu->impl->write_s2cr(smmu, idx);
958 return;
959 }
960
961 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
962 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
963 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
964
965 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
966 smmu->smrs[idx].valid)
967 reg |= ARM_SMMU_S2CR_EXIDVALID;
968 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
969 }
970
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)971 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
972 {
973 arm_smmu_write_s2cr(smmu, idx);
974 if (smmu->smrs)
975 arm_smmu_write_smr(smmu, idx);
976 }
977
978 /*
979 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
980 * should be called after sCR0 is written.
981 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)982 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
983 {
984 u32 smr;
985 int i;
986
987 if (!smmu->smrs)
988 return;
989 /*
990 * If we've had to accommodate firmware memory regions, we may
991 * have live SMRs by now; tread carefully...
992 *
993 * Somewhat perversely, not having a free SMR for this test implies we
994 * can get away without it anyway, as we'll only be able to 'allocate'
995 * these SMRs for the ID/mask values we're already trusting to be OK.
996 */
997 for (i = 0; i < smmu->num_mapping_groups; i++)
998 if (!smmu->smrs[i].valid)
999 goto smr_ok;
1000 return;
1001 smr_ok:
1002 /*
1003 * SMR.ID bits may not be preserved if the corresponding MASK
1004 * bits are set, so check each one separately. We can reject
1005 * masters later if they try to claim IDs outside these masks.
1006 */
1007 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1008 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1009 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1010 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1011
1012 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1013 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1014 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1015 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1016 }
1017
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1018 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1019 {
1020 struct arm_smmu_smr *smrs = smmu->smrs;
1021 int i, free_idx = -ENOSPC;
1022
1023 /* Stream indexing is blissfully easy */
1024 if (!smrs)
1025 return id;
1026
1027 /* Validating SMRs is... less so */
1028 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1029 if (!smrs[i].valid) {
1030 /*
1031 * Note the first free entry we come across, which
1032 * we'll claim in the end if nothing else matches.
1033 */
1034 if (free_idx < 0)
1035 free_idx = i;
1036 continue;
1037 }
1038 /*
1039 * If the new entry is _entirely_ matched by an existing entry,
1040 * then reuse that, with the guarantee that there also cannot
1041 * be any subsequent conflicting entries. In normal use we'd
1042 * expect simply identical entries for this case, but there's
1043 * no harm in accommodating the generalisation.
1044 */
1045 if ((mask & smrs[i].mask) == mask &&
1046 !((id ^ smrs[i].id) & ~smrs[i].mask))
1047 return i;
1048 /*
1049 * If the new entry has any other overlap with an existing one,
1050 * though, then there always exists at least one stream ID
1051 * which would cause a conflict, and we can't allow that risk.
1052 */
1053 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1054 return -EINVAL;
1055 }
1056
1057 return free_idx;
1058 }
1059
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1060 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1061 {
1062 if (--smmu->s2crs[idx].count)
1063 return false;
1064
1065 smmu->s2crs[idx] = s2cr_init_val;
1066 if (smmu->smrs)
1067 smmu->smrs[idx].valid = false;
1068
1069 return true;
1070 }
1071
arm_smmu_master_alloc_smes(struct device * dev)1072 static int arm_smmu_master_alloc_smes(struct device *dev)
1073 {
1074 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1075 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1076 struct arm_smmu_device *smmu = cfg->smmu;
1077 struct arm_smmu_smr *smrs = smmu->smrs;
1078 int i, idx, ret;
1079
1080 mutex_lock(&smmu->stream_map_mutex);
1081 /* Figure out a viable stream map entry allocation */
1082 for_each_cfg_sme(cfg, fwspec, i, idx) {
1083 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1084 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1085
1086 if (idx != INVALID_SMENDX) {
1087 ret = -EEXIST;
1088 goto out_err;
1089 }
1090
1091 ret = arm_smmu_find_sme(smmu, sid, mask);
1092 if (ret < 0)
1093 goto out_err;
1094
1095 idx = ret;
1096 if (smrs && smmu->s2crs[idx].count == 0) {
1097 smrs[idx].id = sid;
1098 smrs[idx].mask = mask;
1099 smrs[idx].valid = true;
1100 }
1101 smmu->s2crs[idx].count++;
1102 cfg->smendx[i] = (s16)idx;
1103 }
1104
1105 /* It worked! Now, poke the actual hardware */
1106 for_each_cfg_sme(cfg, fwspec, i, idx)
1107 arm_smmu_write_sme(smmu, idx);
1108
1109 mutex_unlock(&smmu->stream_map_mutex);
1110 return 0;
1111
1112 out_err:
1113 while (i--) {
1114 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1115 cfg->smendx[i] = INVALID_SMENDX;
1116 }
1117 mutex_unlock(&smmu->stream_map_mutex);
1118 return ret;
1119 }
1120
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1121 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1122 struct iommu_fwspec *fwspec)
1123 {
1124 struct arm_smmu_device *smmu = cfg->smmu;
1125 int i, idx;
1126
1127 mutex_lock(&smmu->stream_map_mutex);
1128 for_each_cfg_sme(cfg, fwspec, i, idx) {
1129 if (arm_smmu_free_sme(smmu, idx))
1130 arm_smmu_write_sme(smmu, idx);
1131 cfg->smendx[i] = INVALID_SMENDX;
1132 }
1133 mutex_unlock(&smmu->stream_map_mutex);
1134 }
1135
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1136 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1137 enum arm_smmu_s2cr_type type,
1138 u8 cbndx, struct iommu_fwspec *fwspec)
1139 {
1140 struct arm_smmu_device *smmu = cfg->smmu;
1141 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1142 int i, idx;
1143
1144 for_each_cfg_sme(cfg, fwspec, i, idx) {
1145 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1146 continue;
1147
1148 s2cr[idx].type = type;
1149 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1150 s2cr[idx].cbndx = cbndx;
1151 arm_smmu_write_s2cr(smmu, idx);
1152 }
1153 }
1154
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1155 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1156 {
1157 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1158 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1159 struct arm_smmu_master_cfg *cfg;
1160 struct arm_smmu_device *smmu;
1161 int ret;
1162
1163 /*
1164 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1165 * domains between of_xlate() and probe_device() - we have no way to cope
1166 * with that, so until ARM gets converted to rely on groups and default
1167 * domains, just say no (but more politely than by dereferencing NULL).
1168 * This should be at least a WARN_ON once that's sorted.
1169 */
1170 cfg = dev_iommu_priv_get(dev);
1171 if (!cfg)
1172 return -ENODEV;
1173
1174 smmu = cfg->smmu;
1175
1176 ret = arm_smmu_rpm_get(smmu);
1177 if (ret < 0)
1178 return ret;
1179
1180 /* Ensure that the domain is finalised */
1181 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1182 if (ret < 0)
1183 goto rpm_put;
1184
1185 /*
1186 * Sanity check the domain. We don't support domains across
1187 * different SMMUs.
1188 */
1189 if (smmu_domain->smmu != smmu) {
1190 ret = -EINVAL;
1191 goto rpm_put;
1192 }
1193
1194 /* Looks ok, so add the device to the domain */
1195 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1196 smmu_domain->cfg.cbndx, fwspec);
1197 arm_smmu_rpm_use_autosuspend(smmu);
1198 rpm_put:
1199 arm_smmu_rpm_put(smmu);
1200 return ret;
1201 }
1202
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1203 static int arm_smmu_attach_dev_type(struct device *dev,
1204 enum arm_smmu_s2cr_type type)
1205 {
1206 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1207 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1208 struct arm_smmu_device *smmu;
1209 int ret;
1210
1211 if (!cfg)
1212 return -ENODEV;
1213 smmu = cfg->smmu;
1214
1215 ret = arm_smmu_rpm_get(smmu);
1216 if (ret < 0)
1217 return ret;
1218
1219 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1220 arm_smmu_rpm_use_autosuspend(smmu);
1221 arm_smmu_rpm_put(smmu);
1222 return 0;
1223 }
1224
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1225 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1226 struct device *dev)
1227 {
1228 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1229 }
1230
1231 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1232 .attach_dev = arm_smmu_attach_dev_identity,
1233 };
1234
1235 static struct iommu_domain arm_smmu_identity_domain = {
1236 .type = IOMMU_DOMAIN_IDENTITY,
1237 .ops = &arm_smmu_identity_ops,
1238 };
1239
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1240 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1241 struct device *dev)
1242 {
1243 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1244 }
1245
1246 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1247 .attach_dev = arm_smmu_attach_dev_blocked,
1248 };
1249
1250 static struct iommu_domain arm_smmu_blocked_domain = {
1251 .type = IOMMU_DOMAIN_BLOCKED,
1252 .ops = &arm_smmu_blocked_ops,
1253 };
1254
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1255 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1256 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1257 int prot, gfp_t gfp, size_t *mapped)
1258 {
1259 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1260 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1261 int ret;
1262
1263 if (!ops)
1264 return -ENODEV;
1265
1266 arm_smmu_rpm_get(smmu);
1267 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1268 arm_smmu_rpm_put(smmu);
1269
1270 return ret;
1271 }
1272
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1273 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1274 size_t pgsize, size_t pgcount,
1275 struct iommu_iotlb_gather *iotlb_gather)
1276 {
1277 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1278 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1279 size_t ret;
1280
1281 if (!ops)
1282 return 0;
1283
1284 arm_smmu_rpm_get(smmu);
1285 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1286 arm_smmu_rpm_put(smmu);
1287
1288 return ret;
1289 }
1290
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1291 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1292 {
1293 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1294 struct arm_smmu_device *smmu = smmu_domain->smmu;
1295
1296 if (smmu_domain->flush_ops) {
1297 arm_smmu_rpm_get(smmu);
1298 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1299 arm_smmu_rpm_put(smmu);
1300 }
1301 }
1302
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1303 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1304 struct iommu_iotlb_gather *gather)
1305 {
1306 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1307 struct arm_smmu_device *smmu = smmu_domain->smmu;
1308
1309 if (!smmu)
1310 return;
1311
1312 arm_smmu_rpm_get(smmu);
1313 if (smmu->version == ARM_SMMU_V2 ||
1314 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1315 arm_smmu_tlb_sync_context(smmu_domain);
1316 else
1317 arm_smmu_tlb_sync_global(smmu);
1318 arm_smmu_rpm_put(smmu);
1319 }
1320
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1321 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1322 dma_addr_t iova)
1323 {
1324 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1325 struct arm_smmu_device *smmu = smmu_domain->smmu;
1326 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1327 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1328 struct device *dev = smmu->dev;
1329 void __iomem *reg;
1330 u32 tmp;
1331 u64 phys;
1332 unsigned long va, flags;
1333 int ret, idx = cfg->cbndx;
1334 phys_addr_t addr = 0;
1335
1336 ret = arm_smmu_rpm_get(smmu);
1337 if (ret < 0)
1338 return 0;
1339
1340 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1341 va = iova & ~0xfffUL;
1342 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1343 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1344 else
1345 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1346
1347 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1348 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1349 5, 50)) {
1350 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1351 dev_err(dev,
1352 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1353 &iova);
1354 arm_smmu_rpm_put(smmu);
1355 return ops->iova_to_phys(ops, iova);
1356 }
1357
1358 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1359 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1360 if (phys & ARM_SMMU_CB_PAR_F) {
1361 dev_err(dev, "translation fault!\n");
1362 dev_err(dev, "PAR = 0x%llx\n", phys);
1363 goto out;
1364 }
1365
1366 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1367 out:
1368 arm_smmu_rpm_put(smmu);
1369
1370 return addr;
1371 }
1372
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1373 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1374 dma_addr_t iova)
1375 {
1376 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1377 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1378
1379 if (!ops)
1380 return 0;
1381
1382 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1383 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1384 return arm_smmu_iova_to_phys_hard(domain, iova);
1385
1386 return ops->iova_to_phys(ops, iova);
1387 }
1388
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1389 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1390 {
1391 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1392
1393 switch (cap) {
1394 case IOMMU_CAP_CACHE_COHERENCY:
1395 /*
1396 * It's overwhelmingly the case in practice that when the pagetable
1397 * walk interface is connected to a coherent interconnect, all the
1398 * translation interfaces are too. Furthermore if the device is
1399 * natively coherent, then its translation interface must also be.
1400 */
1401 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1402 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1403 case IOMMU_CAP_NOEXEC:
1404 case IOMMU_CAP_DEFERRED_FLUSH:
1405 return true;
1406 default:
1407 return false;
1408 }
1409 }
1410
1411 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1412 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1413 {
1414 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1415 fwnode);
1416 put_device(dev);
1417 return dev ? dev_get_drvdata(dev) : NULL;
1418 }
1419
arm_smmu_probe_device(struct device * dev)1420 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1421 {
1422 struct arm_smmu_device *smmu = NULL;
1423 struct arm_smmu_master_cfg *cfg;
1424 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1425 int i, ret;
1426
1427 if (using_legacy_binding) {
1428 ret = arm_smmu_register_legacy_master(dev, &smmu);
1429
1430 /*
1431 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1432 * will allocate/initialise a new one. Thus we need to update fwspec for
1433 * later use.
1434 */
1435 fwspec = dev_iommu_fwspec_get(dev);
1436 if (ret)
1437 goto out_free;
1438 } else {
1439 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1440
1441 /*
1442 * Defer probe if the relevant SMMU instance hasn't finished
1443 * probing yet. This is a fragile hack and we'd ideally
1444 * avoid this race in the core code. Until that's ironed
1445 * out, however, this is the most pragmatic option on the
1446 * table.
1447 */
1448 if (!smmu)
1449 return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER,
1450 "smmu dev has not bound yet\n"));
1451 }
1452
1453 ret = -EINVAL;
1454 for (i = 0; i < fwspec->num_ids; i++) {
1455 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1456 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1457
1458 if (sid & ~smmu->streamid_mask) {
1459 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1460 sid, smmu->streamid_mask);
1461 goto out_free;
1462 }
1463 if (mask & ~smmu->smr_mask_mask) {
1464 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1465 mask, smmu->smr_mask_mask);
1466 goto out_free;
1467 }
1468 }
1469
1470 ret = -ENOMEM;
1471 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1472 GFP_KERNEL);
1473 if (!cfg)
1474 goto out_free;
1475
1476 cfg->smmu = smmu;
1477 dev_iommu_priv_set(dev, cfg);
1478 while (i--)
1479 cfg->smendx[i] = INVALID_SMENDX;
1480
1481 ret = arm_smmu_rpm_get(smmu);
1482 if (ret < 0)
1483 goto out_cfg_free;
1484
1485 ret = arm_smmu_master_alloc_smes(dev);
1486 arm_smmu_rpm_put(smmu);
1487
1488 if (ret)
1489 goto out_cfg_free;
1490
1491 device_link_add(dev, smmu->dev,
1492 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1493
1494 return &smmu->iommu;
1495
1496 out_cfg_free:
1497 kfree(cfg);
1498 out_free:
1499 iommu_fwspec_free(dev);
1500 return ERR_PTR(ret);
1501 }
1502
arm_smmu_release_device(struct device * dev)1503 static void arm_smmu_release_device(struct device *dev)
1504 {
1505 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1506 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1507 int ret;
1508
1509 ret = arm_smmu_rpm_get(cfg->smmu);
1510 if (ret < 0)
1511 return;
1512
1513 arm_smmu_master_free_smes(cfg, fwspec);
1514
1515 arm_smmu_rpm_put(cfg->smmu);
1516
1517 kfree(cfg);
1518 }
1519
arm_smmu_probe_finalize(struct device * dev)1520 static void arm_smmu_probe_finalize(struct device *dev)
1521 {
1522 struct arm_smmu_master_cfg *cfg;
1523 struct arm_smmu_device *smmu;
1524
1525 cfg = dev_iommu_priv_get(dev);
1526 smmu = cfg->smmu;
1527
1528 if (smmu->impl && smmu->impl->probe_finalize)
1529 smmu->impl->probe_finalize(smmu, dev);
1530 }
1531
arm_smmu_device_group(struct device * dev)1532 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1533 {
1534 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1535 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1536 struct arm_smmu_device *smmu = cfg->smmu;
1537 struct iommu_group *group = NULL;
1538 int i, idx;
1539
1540 mutex_lock(&smmu->stream_map_mutex);
1541 for_each_cfg_sme(cfg, fwspec, i, idx) {
1542 if (group && smmu->s2crs[idx].group &&
1543 group != smmu->s2crs[idx].group) {
1544 mutex_unlock(&smmu->stream_map_mutex);
1545 return ERR_PTR(-EINVAL);
1546 }
1547
1548 group = smmu->s2crs[idx].group;
1549 }
1550
1551 if (group) {
1552 mutex_unlock(&smmu->stream_map_mutex);
1553 return iommu_group_ref_get(group);
1554 }
1555
1556 if (dev_is_pci(dev))
1557 group = pci_device_group(dev);
1558 else if (dev_is_fsl_mc(dev))
1559 group = fsl_mc_device_group(dev);
1560 else
1561 group = generic_device_group(dev);
1562
1563 /* Remember group for faster lookups */
1564 if (!IS_ERR(group))
1565 for_each_cfg_sme(cfg, fwspec, i, idx)
1566 smmu->s2crs[idx].group = group;
1567
1568 mutex_unlock(&smmu->stream_map_mutex);
1569 return group;
1570 }
1571
arm_smmu_enable_nesting(struct iommu_domain * domain)1572 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1573 {
1574 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1575 int ret = 0;
1576
1577 mutex_lock(&smmu_domain->init_mutex);
1578 if (smmu_domain->smmu)
1579 ret = -EPERM;
1580 else
1581 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1582 mutex_unlock(&smmu_domain->init_mutex);
1583
1584 return ret;
1585 }
1586
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1587 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1588 unsigned long quirks)
1589 {
1590 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1591 int ret = 0;
1592
1593 mutex_lock(&smmu_domain->init_mutex);
1594 if (smmu_domain->smmu)
1595 ret = -EPERM;
1596 else
1597 smmu_domain->pgtbl_quirks = quirks;
1598 mutex_unlock(&smmu_domain->init_mutex);
1599
1600 return ret;
1601 }
1602
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1603 static int arm_smmu_of_xlate(struct device *dev,
1604 const struct of_phandle_args *args)
1605 {
1606 u32 mask, fwid = 0;
1607
1608 if (args->args_count > 0)
1609 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1610
1611 if (args->args_count > 1)
1612 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1613 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1614 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1615
1616 return iommu_fwspec_add_ids(dev, &fwid, 1);
1617 }
1618
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1619 static void arm_smmu_get_resv_regions(struct device *dev,
1620 struct list_head *head)
1621 {
1622 struct iommu_resv_region *region;
1623 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1624
1625 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1626 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1627 if (!region)
1628 return;
1629
1630 list_add_tail(®ion->list, head);
1631
1632 iommu_dma_get_resv_regions(dev, head);
1633 }
1634
arm_smmu_def_domain_type(struct device * dev)1635 static int arm_smmu_def_domain_type(struct device *dev)
1636 {
1637 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1638 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1639
1640 if (using_legacy_binding)
1641 return IOMMU_DOMAIN_IDENTITY;
1642
1643 if (impl && impl->def_domain_type)
1644 return impl->def_domain_type(dev);
1645
1646 return 0;
1647 }
1648
1649 static struct iommu_ops arm_smmu_ops = {
1650 .identity_domain = &arm_smmu_identity_domain,
1651 .blocked_domain = &arm_smmu_blocked_domain,
1652 .capable = arm_smmu_capable,
1653 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1654 .probe_device = arm_smmu_probe_device,
1655 .release_device = arm_smmu_release_device,
1656 .probe_finalize = arm_smmu_probe_finalize,
1657 .device_group = arm_smmu_device_group,
1658 .of_xlate = arm_smmu_of_xlate,
1659 .get_resv_regions = arm_smmu_get_resv_regions,
1660 .def_domain_type = arm_smmu_def_domain_type,
1661 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1662 .owner = THIS_MODULE,
1663 .default_domain_ops = &(const struct iommu_domain_ops) {
1664 .attach_dev = arm_smmu_attach_dev,
1665 .map_pages = arm_smmu_map_pages,
1666 .unmap_pages = arm_smmu_unmap_pages,
1667 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1668 .iotlb_sync = arm_smmu_iotlb_sync,
1669 .iova_to_phys = arm_smmu_iova_to_phys,
1670 .enable_nesting = arm_smmu_enable_nesting,
1671 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1672 .free = arm_smmu_domain_free,
1673 }
1674 };
1675
arm_smmu_device_reset(struct arm_smmu_device * smmu)1676 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1677 {
1678 int i;
1679 u32 reg;
1680
1681 /* clear global FSR */
1682 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1683 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1684
1685 /*
1686 * Reset stream mapping groups: Initial values mark all SMRn as
1687 * invalid and all S2CRn as bypass unless overridden.
1688 */
1689 for (i = 0; i < smmu->num_mapping_groups; ++i)
1690 arm_smmu_write_sme(smmu, i);
1691
1692 /* Make sure all context banks are disabled and clear CB_FSR */
1693 for (i = 0; i < smmu->num_context_banks; ++i) {
1694 arm_smmu_write_context_bank(smmu, i);
1695 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1696 }
1697
1698 /* Invalidate the TLB, just in case */
1699 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1700 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1701
1702 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1703
1704 /* Enable fault reporting */
1705 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1706 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1707
1708 /* Disable TLB broadcasting. */
1709 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1710
1711 /* Enable client access, handling unmatched streams as appropriate */
1712 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1713 if (disable_bypass)
1714 reg |= ARM_SMMU_sCR0_USFCFG;
1715 else
1716 reg &= ~ARM_SMMU_sCR0_USFCFG;
1717
1718 /* Disable forced broadcasting */
1719 reg &= ~ARM_SMMU_sCR0_FB;
1720
1721 /* Don't upgrade barriers */
1722 reg &= ~(ARM_SMMU_sCR0_BSU);
1723
1724 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1725 reg |= ARM_SMMU_sCR0_VMID16EN;
1726
1727 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1728 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1729
1730 if (smmu->impl && smmu->impl->reset)
1731 smmu->impl->reset(smmu);
1732
1733 /* Push the button */
1734 arm_smmu_tlb_sync_global(smmu);
1735 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1736 }
1737
arm_smmu_id_size_to_bits(int size)1738 static int arm_smmu_id_size_to_bits(int size)
1739 {
1740 switch (size) {
1741 case 0:
1742 return 32;
1743 case 1:
1744 return 36;
1745 case 2:
1746 return 40;
1747 case 3:
1748 return 42;
1749 case 4:
1750 return 44;
1751 case 5:
1752 default:
1753 return 48;
1754 }
1755 }
1756
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1757 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1758 {
1759 unsigned int size;
1760 u32 id;
1761 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1762 int i, ret;
1763
1764 dev_notice(smmu->dev, "probing hardware configuration...\n");
1765 dev_notice(smmu->dev, "SMMUv%d with:\n",
1766 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1767
1768 /* ID0 */
1769 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1770
1771 /* Restrict available stages based on module parameter */
1772 if (force_stage == 1)
1773 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1774 else if (force_stage == 2)
1775 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1776
1777 if (id & ARM_SMMU_ID0_S1TS) {
1778 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1779 dev_notice(smmu->dev, "\tstage 1 translation\n");
1780 }
1781
1782 if (id & ARM_SMMU_ID0_S2TS) {
1783 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1784 dev_notice(smmu->dev, "\tstage 2 translation\n");
1785 }
1786
1787 if (id & ARM_SMMU_ID0_NTS) {
1788 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1789 dev_notice(smmu->dev, "\tnested translation\n");
1790 }
1791
1792 if (!(smmu->features &
1793 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1794 dev_err(smmu->dev, "\tno translation support!\n");
1795 return -ENODEV;
1796 }
1797
1798 if ((id & ARM_SMMU_ID0_S1TS) &&
1799 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1800 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1801 dev_notice(smmu->dev, "\taddress translation ops\n");
1802 }
1803
1804 /*
1805 * In order for DMA API calls to work properly, we must defer to what
1806 * the FW says about coherency, regardless of what the hardware claims.
1807 * Fortunately, this also opens up a workaround for systems where the
1808 * ID register value has ended up configured incorrectly.
1809 */
1810 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1811 if (cttw_fw || cttw_reg)
1812 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1813 cttw_fw ? "" : "non-");
1814 if (cttw_fw != cttw_reg)
1815 dev_notice(smmu->dev,
1816 "\t(IDR0.CTTW overridden by FW configuration)\n");
1817
1818 /* Max. number of entries we have for stream matching/indexing */
1819 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1820 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1821 size = 1 << 16;
1822 } else {
1823 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1824 }
1825 smmu->streamid_mask = size - 1;
1826 if (id & ARM_SMMU_ID0_SMS) {
1827 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1828 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1829 if (size == 0) {
1830 dev_err(smmu->dev,
1831 "stream-matching supported, but no SMRs present!\n");
1832 return -ENODEV;
1833 }
1834
1835 /* Zero-initialised to mark as invalid */
1836 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1837 GFP_KERNEL);
1838 if (!smmu->smrs)
1839 return -ENOMEM;
1840
1841 dev_notice(smmu->dev,
1842 "\tstream matching with %u register groups", size);
1843 }
1844 /* s2cr->type == 0 means translation, so initialise explicitly */
1845 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1846 GFP_KERNEL);
1847 if (!smmu->s2crs)
1848 return -ENOMEM;
1849 for (i = 0; i < size; i++)
1850 smmu->s2crs[i] = s2cr_init_val;
1851
1852 smmu->num_mapping_groups = size;
1853 mutex_init(&smmu->stream_map_mutex);
1854 spin_lock_init(&smmu->global_sync_lock);
1855
1856 if (smmu->version < ARM_SMMU_V2 ||
1857 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1858 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1859 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1860 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1861 }
1862
1863 /* ID1 */
1864 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1865 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1866
1867 /* Check for size mismatch of SMMU address space from mapped region */
1868 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1869 if (smmu->numpage != 2 * size << smmu->pgshift)
1870 dev_warn(smmu->dev,
1871 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1872 2 * size << smmu->pgshift, smmu->numpage);
1873 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1874 smmu->numpage = size;
1875
1876 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1877 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1878 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1879 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1880 return -ENODEV;
1881 }
1882 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1883 smmu->num_context_banks, smmu->num_s2_context_banks);
1884 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1885 sizeof(*smmu->cbs), GFP_KERNEL);
1886 if (!smmu->cbs)
1887 return -ENOMEM;
1888
1889 /* ID2 */
1890 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1891 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1892 smmu->ipa_size = size;
1893
1894 /* The output mask is also applied for bypass */
1895 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1896 smmu->pa_size = size;
1897
1898 if (id & ARM_SMMU_ID2_VMID16)
1899 smmu->features |= ARM_SMMU_FEAT_VMID16;
1900
1901 /*
1902 * What the page table walker can address actually depends on which
1903 * descriptor format is in use, but since a) we don't know that yet,
1904 * and b) it can vary per context bank, this will have to do...
1905 */
1906 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1907 dev_warn(smmu->dev,
1908 "failed to set DMA mask for table walker\n");
1909
1910 if (smmu->version < ARM_SMMU_V2) {
1911 smmu->va_size = smmu->ipa_size;
1912 if (smmu->version == ARM_SMMU_V1_64K)
1913 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1914 } else {
1915 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1916 smmu->va_size = arm_smmu_id_size_to_bits(size);
1917 if (id & ARM_SMMU_ID2_PTFS_4K)
1918 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1919 if (id & ARM_SMMU_ID2_PTFS_16K)
1920 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1921 if (id & ARM_SMMU_ID2_PTFS_64K)
1922 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1923 }
1924
1925 if (smmu->impl && smmu->impl->cfg_probe) {
1926 ret = smmu->impl->cfg_probe(smmu);
1927 if (ret)
1928 return ret;
1929 }
1930
1931 /* Now we've corralled the various formats, what'll it do? */
1932 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1933 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1934 if (smmu->features &
1935 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1936 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1937 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1938 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1939 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1940 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1941
1942 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1943 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1944 else
1945 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1946 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1947 smmu->pgsize_bitmap);
1948
1949
1950 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1951 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1952 smmu->va_size, smmu->ipa_size);
1953
1954 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1955 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1956 smmu->ipa_size, smmu->pa_size);
1957
1958 return 0;
1959 }
1960
1961 struct arm_smmu_match_data {
1962 enum arm_smmu_arch_version version;
1963 enum arm_smmu_implementation model;
1964 };
1965
1966 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1967 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1968
1969 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1970 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1971 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1972 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1973 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1974 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1975
1976 static const struct of_device_id arm_smmu_of_match[] = {
1977 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1978 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1979 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1980 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1981 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1982 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1983 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1984 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1985 { },
1986 };
1987 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1988
1989 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1990 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1991 {
1992 int ret = 0;
1993
1994 switch (model) {
1995 case ACPI_IORT_SMMU_V1:
1996 case ACPI_IORT_SMMU_CORELINK_MMU400:
1997 smmu->version = ARM_SMMU_V1;
1998 smmu->model = GENERIC_SMMU;
1999 break;
2000 case ACPI_IORT_SMMU_CORELINK_MMU401:
2001 smmu->version = ARM_SMMU_V1_64K;
2002 smmu->model = GENERIC_SMMU;
2003 break;
2004 case ACPI_IORT_SMMU_V2:
2005 smmu->version = ARM_SMMU_V2;
2006 smmu->model = GENERIC_SMMU;
2007 break;
2008 case ACPI_IORT_SMMU_CORELINK_MMU500:
2009 smmu->version = ARM_SMMU_V2;
2010 smmu->model = ARM_MMU500;
2011 break;
2012 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2013 smmu->version = ARM_SMMU_V2;
2014 smmu->model = CAVIUM_SMMUV2;
2015 break;
2016 default:
2017 ret = -ENODEV;
2018 }
2019
2020 return ret;
2021 }
2022
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2023 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2024 u32 *global_irqs, u32 *pmu_irqs)
2025 {
2026 struct device *dev = smmu->dev;
2027 struct acpi_iort_node *node =
2028 *(struct acpi_iort_node **)dev_get_platdata(dev);
2029 struct acpi_iort_smmu *iort_smmu;
2030 int ret;
2031
2032 /* Retrieve SMMU1/2 specific data */
2033 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2034
2035 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2036 if (ret < 0)
2037 return ret;
2038
2039 /* Ignore the configuration access interrupt */
2040 *global_irqs = 1;
2041 *pmu_irqs = 0;
2042
2043 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2044 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2045
2046 return 0;
2047 }
2048 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2049 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2050 u32 *global_irqs, u32 *pmu_irqs)
2051 {
2052 return -ENODEV;
2053 }
2054 #endif
2055
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2056 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2057 u32 *global_irqs, u32 *pmu_irqs)
2058 {
2059 const struct arm_smmu_match_data *data;
2060 struct device *dev = smmu->dev;
2061 bool legacy_binding;
2062
2063 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2064 return dev_err_probe(dev, -ENODEV,
2065 "missing #global-interrupts property\n");
2066 *pmu_irqs = 0;
2067
2068 data = of_device_get_match_data(dev);
2069 smmu->version = data->version;
2070 smmu->model = data->model;
2071
2072 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2073 if (legacy_binding && !using_generic_binding) {
2074 if (!using_legacy_binding) {
2075 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2076 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2077 }
2078 using_legacy_binding = true;
2079 } else if (!legacy_binding && !using_legacy_binding) {
2080 using_generic_binding = true;
2081 } else {
2082 dev_err(dev, "not probing due to mismatched DT properties\n");
2083 return -ENODEV;
2084 }
2085
2086 if (of_dma_is_coherent(dev->of_node))
2087 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2088
2089 return 0;
2090 }
2091
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2092 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2093 {
2094 struct list_head rmr_list;
2095 struct iommu_resv_region *e;
2096 int idx, cnt = 0;
2097 u32 reg;
2098
2099 INIT_LIST_HEAD(&rmr_list);
2100 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2101
2102 /*
2103 * Rather than trying to look at existing mappings that
2104 * are setup by the firmware and then invalidate the ones
2105 * that do no have matching RMR entries, just disable the
2106 * SMMU until it gets enabled again in the reset routine.
2107 */
2108 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2109 reg |= ARM_SMMU_sCR0_CLIENTPD;
2110 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2111
2112 list_for_each_entry(e, &rmr_list, list) {
2113 struct iommu_iort_rmr_data *rmr;
2114 int i;
2115
2116 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2117 for (i = 0; i < rmr->num_sids; i++) {
2118 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2119 if (idx < 0)
2120 continue;
2121
2122 if (smmu->s2crs[idx].count == 0) {
2123 smmu->smrs[idx].id = rmr->sids[i];
2124 smmu->smrs[idx].mask = 0;
2125 smmu->smrs[idx].valid = true;
2126 }
2127 smmu->s2crs[idx].count++;
2128 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2129 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2130
2131 cnt++;
2132 }
2133 }
2134
2135 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2136 cnt == 1 ? "" : "s");
2137 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2138 }
2139
arm_smmu_device_probe(struct platform_device * pdev)2140 static int arm_smmu_device_probe(struct platform_device *pdev)
2141 {
2142 struct resource *res;
2143 struct arm_smmu_device *smmu;
2144 struct device *dev = &pdev->dev;
2145 int num_irqs, i, err;
2146 u32 global_irqs, pmu_irqs;
2147 irqreturn_t (*global_fault)(int irq, void *dev);
2148
2149 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2150 if (!smmu) {
2151 dev_err(dev, "failed to allocate arm_smmu_device\n");
2152 return -ENOMEM;
2153 }
2154 smmu->dev = dev;
2155
2156 if (dev->of_node)
2157 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2158 else
2159 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2160 if (err)
2161 return err;
2162
2163 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2164 if (IS_ERR(smmu->base))
2165 return PTR_ERR(smmu->base);
2166 smmu->ioaddr = res->start;
2167
2168 /*
2169 * The resource size should effectively match the value of SMMU_TOP;
2170 * stash that temporarily until we know PAGESIZE to validate it with.
2171 */
2172 smmu->numpage = resource_size(res);
2173
2174 smmu = arm_smmu_impl_init(smmu);
2175 if (IS_ERR(smmu))
2176 return PTR_ERR(smmu);
2177
2178 num_irqs = platform_irq_count(pdev);
2179
2180 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2181 if (smmu->num_context_irqs <= 0)
2182 return dev_err_probe(dev, -ENODEV,
2183 "found %d interrupts but expected at least %d\n",
2184 num_irqs, global_irqs + pmu_irqs + 1);
2185
2186 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2187 sizeof(*smmu->irqs), GFP_KERNEL);
2188 if (!smmu->irqs)
2189 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2190 smmu->num_context_irqs);
2191
2192 for (i = 0; i < smmu->num_context_irqs; i++) {
2193 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2194
2195 if (irq < 0)
2196 return irq;
2197 smmu->irqs[i] = irq;
2198 }
2199
2200 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2201 if (err < 0) {
2202 dev_err(dev, "failed to get clocks %d\n", err);
2203 return err;
2204 }
2205 smmu->num_clks = err;
2206
2207 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2208 if (err)
2209 return err;
2210
2211 err = arm_smmu_device_cfg_probe(smmu);
2212 if (err)
2213 return err;
2214
2215 if (smmu->version == ARM_SMMU_V2) {
2216 if (smmu->num_context_banks > smmu->num_context_irqs) {
2217 dev_err(dev,
2218 "found only %d context irq(s) but %d required\n",
2219 smmu->num_context_irqs, smmu->num_context_banks);
2220 return -ENODEV;
2221 }
2222
2223 /* Ignore superfluous interrupts */
2224 smmu->num_context_irqs = smmu->num_context_banks;
2225 }
2226
2227 if (smmu->impl && smmu->impl->global_fault)
2228 global_fault = smmu->impl->global_fault;
2229 else
2230 global_fault = arm_smmu_global_fault;
2231
2232 for (i = 0; i < global_irqs; i++) {
2233 int irq = platform_get_irq(pdev, i);
2234
2235 if (irq < 0)
2236 return irq;
2237
2238 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2239 "arm-smmu global fault", smmu);
2240 if (err)
2241 return dev_err_probe(dev, err,
2242 "failed to request global IRQ %d (%u)\n",
2243 i, irq);
2244 }
2245
2246 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2247 "smmu.%pa", &smmu->ioaddr);
2248 if (err) {
2249 dev_err(dev, "Failed to register iommu in sysfs\n");
2250 return err;
2251 }
2252
2253 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2254 using_legacy_binding ? NULL : dev);
2255 if (err) {
2256 dev_err(dev, "Failed to register iommu\n");
2257 iommu_device_sysfs_remove(&smmu->iommu);
2258 return err;
2259 }
2260
2261 platform_set_drvdata(pdev, smmu);
2262
2263 /* Check for RMRs and install bypass SMRs if any */
2264 arm_smmu_rmr_install_bypass_smr(smmu);
2265
2266 arm_smmu_device_reset(smmu);
2267 arm_smmu_test_smr_masks(smmu);
2268
2269 /*
2270 * We want to avoid touching dev->power.lock in fastpaths unless
2271 * it's really going to do something useful - pm_runtime_enabled()
2272 * can serve as an ideal proxy for that decision. So, conditionally
2273 * enable pm_runtime.
2274 */
2275 if (dev->pm_domain) {
2276 pm_runtime_set_active(dev);
2277 pm_runtime_enable(dev);
2278 }
2279
2280 return 0;
2281 }
2282
arm_smmu_device_shutdown(struct platform_device * pdev)2283 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2284 {
2285 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2286
2287 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2288 dev_notice(&pdev->dev, "disabling translation\n");
2289
2290 arm_smmu_rpm_get(smmu);
2291 /* Turn the thing off */
2292 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2293 arm_smmu_rpm_put(smmu);
2294
2295 if (pm_runtime_enabled(smmu->dev))
2296 pm_runtime_force_suspend(smmu->dev);
2297 else
2298 clk_bulk_disable(smmu->num_clks, smmu->clks);
2299
2300 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2301 }
2302
arm_smmu_device_remove(struct platform_device * pdev)2303 static void arm_smmu_device_remove(struct platform_device *pdev)
2304 {
2305 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2306
2307 iommu_device_unregister(&smmu->iommu);
2308 iommu_device_sysfs_remove(&smmu->iommu);
2309
2310 arm_smmu_device_shutdown(pdev);
2311 }
2312
arm_smmu_runtime_resume(struct device * dev)2313 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2314 {
2315 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2316 int ret;
2317
2318 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2319 if (ret)
2320 return ret;
2321
2322 arm_smmu_device_reset(smmu);
2323
2324 return 0;
2325 }
2326
arm_smmu_runtime_suspend(struct device * dev)2327 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2328 {
2329 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2330
2331 clk_bulk_disable(smmu->num_clks, smmu->clks);
2332
2333 return 0;
2334 }
2335
arm_smmu_pm_resume(struct device * dev)2336 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2337 {
2338 int ret;
2339 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2340
2341 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2342 if (ret)
2343 return ret;
2344
2345 if (pm_runtime_suspended(dev))
2346 return 0;
2347
2348 ret = arm_smmu_runtime_resume(dev);
2349 if (ret)
2350 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2351
2352 return ret;
2353 }
2354
arm_smmu_pm_suspend(struct device * dev)2355 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2356 {
2357 int ret = 0;
2358 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2359
2360 if (pm_runtime_suspended(dev))
2361 goto clk_unprepare;
2362
2363 ret = arm_smmu_runtime_suspend(dev);
2364 if (ret)
2365 return ret;
2366
2367 clk_unprepare:
2368 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2369 return ret;
2370 }
2371
2372 static const struct dev_pm_ops arm_smmu_pm_ops = {
2373 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2374 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2375 arm_smmu_runtime_resume, NULL)
2376 };
2377
2378 static struct platform_driver arm_smmu_driver = {
2379 .driver = {
2380 .name = "arm-smmu",
2381 .of_match_table = arm_smmu_of_match,
2382 .pm = &arm_smmu_pm_ops,
2383 .suppress_bind_attrs = true,
2384 },
2385 .probe = arm_smmu_device_probe,
2386 .remove_new = arm_smmu_device_remove,
2387 .shutdown = arm_smmu_device_shutdown,
2388 };
2389 module_platform_driver(arm_smmu_driver);
2390
2391 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2392 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2393 MODULE_ALIAS("platform:arm-smmu");
2394 MODULE_LICENSE("GPL v2");
2395