Lines Matching +full:ipmmu +full:- +full:vmsa
2 * IPMMU VMSA
13 #include <linux/dma-iommu.h>
14 #include <linux/dma-mapping.h>
31 #include <asm/dma-iommu.h>
35 #define arm_iommu_attach_device(...) -ENODEV
40 #include "io-pgtable.h"
87 return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL; in to_ipmmu()
92 /* -----------------------------------------------------------------------------
200 #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16))
212 #define IMUASID32(n) (0x0608 + (((n) - 32) * 16))
218 /* -----------------------------------------------------------------------------
226 return mmu->root == mmu; in ipmmu_is_root()
248 /* -----------------------------------------------------------------------------
254 return ioread32(mmu->base + offset); in ipmmu_read()
260 iowrite32(data, mmu->base + offset); in ipmmu_write()
266 return ipmmu_read(domain->mmu->root, in ipmmu_ctx_read_root()
267 domain->context_id * IM_CTX_SIZE + reg); in ipmmu_ctx_read_root()
273 ipmmu_write(domain->mmu->root, in ipmmu_ctx_write_root()
274 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_root()
280 if (domain->mmu != domain->mmu->root) in ipmmu_ctx_write_all()
281 ipmmu_write(domain->mmu, in ipmmu_ctx_write_all()
282 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_all()
284 ipmmu_write(domain->mmu->root, in ipmmu_ctx_write_all()
285 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_all()
288 /* -----------------------------------------------------------------------------
300 dev_err_ratelimited(domain->mmu->dev, in ipmmu_tlb_sync()
301 "TLB sync timed out -- MMU may be deadlocked\n"); in ipmmu_tlb_sync()
325 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_enable()
328 * TODO: Reference-count the microTLB as several bus masters can be in ipmmu_utlb_enable()
336 IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | in ipmmu_utlb_enable()
346 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_disable()
370 /* -----------------------------------------------------------------------------
380 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_allocate_context()
382 ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); in ipmmu_domain_allocate_context()
383 if (ret != mmu->num_ctx) { in ipmmu_domain_allocate_context()
384 mmu->domains[ret] = domain; in ipmmu_domain_allocate_context()
385 set_bit(ret, mmu->ctx); in ipmmu_domain_allocate_context()
387 ret = -EBUSY; in ipmmu_domain_allocate_context()
389 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_allocate_context()
399 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_free_context()
401 clear_bit(context_id, mmu->ctx); in ipmmu_domain_free_context()
402 mmu->domains[context_id] = NULL; in ipmmu_domain_free_context()
404 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_free_context()
416 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory in ipmmu_domain_init_context()
417 * access, Long-descriptor format" that the NStable bit being set in a in ipmmu_domain_init_context()
419 * entries being ignored and considered as being set. The IPMMU seems in ipmmu_domain_init_context()
422 * non-secure mode. in ipmmu_domain_init_context()
424 domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; in ipmmu_domain_init_context()
425 domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; in ipmmu_domain_init_context()
426 domain->cfg.ias = 32; in ipmmu_domain_init_context()
427 domain->cfg.oas = 40; in ipmmu_domain_init_context()
428 domain->cfg.tlb = &ipmmu_gather_ops; in ipmmu_domain_init_context()
429 domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); in ipmmu_domain_init_context()
430 domain->io_domain.geometry.force_aperture = true; in ipmmu_domain_init_context()
433 * cache handling. For now, delegate it to the io-pgtable code. in ipmmu_domain_init_context()
435 domain->cfg.iommu_dev = domain->mmu->root->dev; in ipmmu_domain_init_context()
440 ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); in ipmmu_domain_init_context()
444 domain->context_id = ret; in ipmmu_domain_init_context()
446 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, in ipmmu_domain_init_context()
448 if (!domain->iop) { in ipmmu_domain_init_context()
449 ipmmu_domain_free_context(domain->mmu->root, in ipmmu_domain_init_context()
450 domain->context_id); in ipmmu_domain_init_context()
451 return -EINVAL; in ipmmu_domain_init_context()
455 ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; in ipmmu_domain_init_context()
461 * We use long descriptors with inner-shareable WBWA tables and allocate in ipmmu_domain_init_context()
462 * the whole 32-bit VA space to TTBR0. in ipmmu_domain_init_context()
464 if (domain->mmu->features->twobit_imttbcr_sl0) in ipmmu_domain_init_context()
475 domain->cfg.arm_lpae_s1_cfg.mair[0]); in ipmmu_domain_init_context()
478 if (domain->mmu->features->setup_imbuscr) in ipmmu_domain_init_context()
491 * Enable the MMU and interrupt generation. The long-descriptor in ipmmu_domain_init_context()
504 if (!domain->mmu) in ipmmu_domain_destroy_context()
515 ipmmu_domain_free_context(domain->mmu->root, domain->context_id); in ipmmu_domain_destroy_context()
518 /* -----------------------------------------------------------------------------
525 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_domain_irq()
545 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n", in ipmmu_domain_irq()
548 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n", in ipmmu_domain_irq()
560 if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) in ipmmu_domain_irq()
563 dev_err_ratelimited(mmu->dev, in ipmmu_domain_irq()
577 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_irq()
582 for (i = 0; i < mmu->num_ctx; i++) { in ipmmu_irq()
583 if (!mmu->domains[i]) in ipmmu_irq()
585 if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) in ipmmu_irq()
589 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_irq()
594 /* -----------------------------------------------------------------------------
606 mutex_init(&domain->mutex); in __ipmmu_domain_alloc()
608 return &domain->io_domain; in __ipmmu_domain_alloc()
642 free_io_pgtable_ops(domain->iop); in ipmmu_domain_free()
649 struct iommu_fwspec *fwspec = dev->iommu_fwspec; in ipmmu_attach_device()
656 dev_err(dev, "Cannot attach to IPMMU\n"); in ipmmu_attach_device()
657 return -ENXIO; in ipmmu_attach_device()
660 mutex_lock(&domain->mutex); in ipmmu_attach_device()
662 if (!domain->mmu) { in ipmmu_attach_device()
664 domain->mmu = mmu; in ipmmu_attach_device()
667 dev_err(dev, "Unable to initialize IPMMU context\n"); in ipmmu_attach_device()
668 domain->mmu = NULL; in ipmmu_attach_device()
670 dev_info(dev, "Using IPMMU context %u\n", in ipmmu_attach_device()
671 domain->context_id); in ipmmu_attach_device()
673 } else if (domain->mmu != mmu) { in ipmmu_attach_device()
678 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", in ipmmu_attach_device()
679 dev_name(mmu->dev), dev_name(domain->mmu->dev)); in ipmmu_attach_device()
680 ret = -EINVAL; in ipmmu_attach_device()
682 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); in ipmmu_attach_device()
684 mutex_unlock(&domain->mutex); in ipmmu_attach_device()
689 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_attach_device()
690 ipmmu_utlb_enable(domain, fwspec->ids[i]); in ipmmu_attach_device()
698 struct iommu_fwspec *fwspec = dev->iommu_fwspec; in ipmmu_detach_device()
702 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_detach_device()
703 ipmmu_utlb_disable(domain, fwspec->ids[i]); in ipmmu_detach_device()
716 return -ENODEV; in ipmmu_map()
718 return domain->iop->map(domain->iop, iova, paddr, size, prot); in ipmmu_map()
726 return domain->iop->unmap(domain->iop, iova, size); in ipmmu_unmap()
733 if (domain->mmu) in ipmmu_iotlb_sync()
744 return domain->iop->iova_to_phys(domain->iop, iova); in ipmmu_iova_to_phys()
752 ipmmu_pdev = of_find_device_by_node(args->np); in ipmmu_init_platform_device()
754 return -ENODEV; in ipmmu_init_platform_device()
756 dev->iommu_fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); in ipmmu_init_platform_device()
762 /* By default, do not allow use of IPMMU */ in ipmmu_slave_whitelist()
778 /* For R-Car Gen3 use a white list to opt-in slave devices */ in ipmmu_of_xlate()
780 return -ENODEV; in ipmmu_of_xlate()
782 iommu_fwspec_add_ids(dev, spec->args, 1); in ipmmu_of_xlate()
784 /* Initialize once - xlate() will call multiple times */ in ipmmu_of_xlate()
808 dev_err(dev, "Failed to add device to IPMMU group\n"); in ipmmu_init_arm_mapping()
817 * - Create one mapping per context (TLB). in ipmmu_init_arm_mapping()
818 * - Make the mapping size configurable ? We currently use a 2GB mapping in ipmmu_init_arm_mapping()
821 if (!mmu->mapping) { in ipmmu_init_arm_mapping()
827 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); in ipmmu_init_arm_mapping()
832 mmu->mapping = mapping; in ipmmu_init_arm_mapping()
836 ret = arm_iommu_attach_device(dev, mmu->mapping); in ipmmu_init_arm_mapping()
846 if (mmu->mapping) in ipmmu_init_arm_mapping()
847 arm_iommu_release_mapping(mmu->mapping); in ipmmu_init_arm_mapping()
860 return -ENODEV; in ipmmu_add_device()
884 if (mmu->group) in ipmmu_find_group()
885 return iommu_group_ref_get(mmu->group); in ipmmu_find_group()
889 mmu->group = group; in ipmmu_find_group()
911 /* -----------------------------------------------------------------------------
920 for (i = 0; i < mmu->num_ctx; ++i) in ipmmu_device_reset()
944 .compatible = "renesas,ipmmu-vmsa",
947 .compatible = "renesas,ipmmu-r8a7795",
950 .compatible = "renesas,ipmmu-r8a7796",
953 .compatible = "renesas,ipmmu-r8a77965",
956 .compatible = "renesas,ipmmu-r8a77970",
959 .compatible = "renesas,ipmmu-r8a77995",
975 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); in ipmmu_probe()
977 dev_err(&pdev->dev, "cannot allocate device data\n"); in ipmmu_probe()
978 return -ENOMEM; in ipmmu_probe()
981 mmu->dev = &pdev->dev; in ipmmu_probe()
982 mmu->num_utlbs = 48; in ipmmu_probe()
983 spin_lock_init(&mmu->lock); in ipmmu_probe()
984 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); in ipmmu_probe()
985 mmu->features = of_device_get_match_data(&pdev->dev); in ipmmu_probe()
986 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); in ipmmu_probe()
990 mmu->base = devm_ioremap_resource(&pdev->dev, res); in ipmmu_probe()
991 if (IS_ERR(mmu->base)) in ipmmu_probe()
992 return PTR_ERR(mmu->base); in ipmmu_probe()
995 * The IPMMU has two register banks, for secure and non-secure modes. in ipmmu_probe()
996 * The bank mapped at the beginning of the IPMMU address space in ipmmu_probe()
998 * mode the non-secure register bank is also available at an offset. in ipmmu_probe()
1002 * non-secure operation with the main register bank were not successful. in ipmmu_probe()
1003 * Offset the registers base unconditionally to point to the non-secure in ipmmu_probe()
1006 if (mmu->features->use_ns_alias_offset) in ipmmu_probe()
1007 mmu->base += IM_NS_ALIAS_OFFSET; in ipmmu_probe()
1009 mmu->num_ctx = min_t(unsigned int, IPMMU_CTX_MAX, in ipmmu_probe()
1010 mmu->features->number_of_contexts); in ipmmu_probe()
1015 * Determine if this IPMMU instance is a root device by checking for in ipmmu_probe()
1016 * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. in ipmmu_probe()
1018 if (!mmu->features->has_cache_leaf_nodes || in ipmmu_probe()
1019 !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) in ipmmu_probe()
1020 mmu->root = mmu; in ipmmu_probe()
1022 mmu->root = ipmmu_find_root(); in ipmmu_probe()
1027 if (!mmu->root) in ipmmu_probe()
1028 return -EPROBE_DEFER; in ipmmu_probe()
1033 dev_err(&pdev->dev, "no IRQ found\n"); in ipmmu_probe()
1037 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, in ipmmu_probe()
1038 dev_name(&pdev->dev), mmu); in ipmmu_probe()
1040 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); in ipmmu_probe()
1046 if (mmu->features->reserved_context) { in ipmmu_probe()
1047 dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); in ipmmu_probe()
1048 set_bit(0, mmu->ctx); in ipmmu_probe()
1053 * Register the IPMMU to the IOMMU subsystem in the following cases: in ipmmu_probe()
1054 * - R-Car Gen2 IPMMU (all devices registered) in ipmmu_probe()
1055 * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) in ipmmu_probe()
1057 if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { in ipmmu_probe()
1058 ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, in ipmmu_probe()
1059 dev_name(&pdev->dev)); in ipmmu_probe()
1063 iommu_device_set_ops(&mmu->iommu, &ipmmu_ops); in ipmmu_probe()
1064 iommu_device_set_fwnode(&mmu->iommu, in ipmmu_probe()
1065 &pdev->dev.of_node->fwnode); in ipmmu_probe()
1067 ret = iommu_device_register(&mmu->iommu); in ipmmu_probe()
1092 iommu_device_sysfs_remove(&mmu->iommu); in ipmmu_remove()
1093 iommu_device_unregister(&mmu->iommu); in ipmmu_remove()
1095 arm_iommu_release_mapping(mmu->mapping); in ipmmu_remove()
1104 .name = "ipmmu-vmsa",
1147 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");