72 	struct ipmmu_vmsa_device *mmu;  member
150 static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)  in ipmmu_is_root()  argument
152 	return mmu->root == mmu;  in ipmmu_is_root()
157 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in __ipmmu_check_device()  local
160 	if (ipmmu_is_root(mmu))  in __ipmmu_check_device()
161 		*rootp = mmu;  in __ipmmu_check_device()
178 static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)  in ipmmu_read()  argument
180 	return ioread32(mmu->base + offset);  in ipmmu_read()
183 static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,  in ipmmu_write()  argument
186 	iowrite32(data, mmu->base + offset);  in ipmmu_write()
189 static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_reg()  argument
192 	return mmu->features->ctx_offset_base +  in ipmmu_ctx_reg()
193 	       context_id * mmu->features->ctx_offset_stride + reg;  in ipmmu_ctx_reg()
196 static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_read()  argument
199 	return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));  in ipmmu_ctx_read()
202 static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_ctx_write()  argument
205 	ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);  in ipmmu_ctx_write()
211 	return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);  in ipmmu_ctx_read_root()
217 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_root()
223 	if (domain->mmu != domain->mmu->root)  in ipmmu_ctx_write_all()
224 		ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
226 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
229 static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)  in ipmmu_utlb_reg()  argument
231 	return mmu->features->utlb_offset_base + reg;  in ipmmu_utlb_reg()
234 static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuasid_write()  argument
237 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);  in ipmmu_imuasid_write()
240 static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,  in ipmmu_imuctr_write()  argument
243 	ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);  in ipmmu_imuctr_write()
258 			dev_err_ratelimited(domain->mmu->dev,  in ipmmu_tlb_sync()
283 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_enable()  local
291 	ipmmu_imuasid_write(mmu, utlb, 0);  in ipmmu_utlb_enable()
293 	ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |  in ipmmu_utlb_enable()
295 	mmu->utlb_ctx[utlb] = domain->context_id;  in ipmmu_utlb_enable()
304 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_disable()  local
306 	ipmmu_imuctr_write(mmu, utlb, 0);  in ipmmu_utlb_disable()
307 	mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;  in ipmmu_utlb_disable()
332 static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_allocate_context()  argument
338 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
340 	ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);  in ipmmu_domain_allocate_context()
341 	if (ret != mmu->num_ctx) {  in ipmmu_domain_allocate_context()
342 		mmu->domains[ret] = domain;  in ipmmu_domain_allocate_context()
343 		set_bit(ret, mmu->ctx);  in ipmmu_domain_allocate_context()
347 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_allocate_context()
352 static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,  in ipmmu_domain_free_context()  argument
357 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_domain_free_context()
359 	clear_bit(context_id, mmu->ctx);  in ipmmu_domain_free_context()
360 	mmu->domains[context_id] = NULL;  in ipmmu_domain_free_context()
362 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_domain_free_context()
380 	if (domain->mmu->features->twobit_imttbcr_sl0)  in ipmmu_domain_setup_context()
385 	if (domain->mmu->features->cache_snoop)  in ipmmu_domain_setup_context()
396 	if (domain->mmu->features->setup_imbuscr)  in ipmmu_domain_setup_context()
445 	domain->cfg.iommu_dev = domain->mmu->root->dev;  in ipmmu_domain_init_context()
450 	ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);  in ipmmu_domain_init_context()
459 		ipmmu_domain_free_context(domain->mmu->root,  in ipmmu_domain_init_context()
470 	if (!domain->mmu)  in ipmmu_domain_destroy_context()
481 	ipmmu_domain_free_context(domain->mmu->root, domain->context_id);  in ipmmu_domain_destroy_context()
491 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_domain_irq()  local
513 		dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",  in ipmmu_domain_irq()
516 		dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",  in ipmmu_domain_irq()
528 	if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))  in ipmmu_domain_irq()
531 	dev_err_ratelimited(mmu->dev,  in ipmmu_domain_irq()
540 	struct ipmmu_vmsa_device *mmu = dev;  in ipmmu_irq()  local
545 	spin_lock_irqsave(&mmu->lock, flags);  in ipmmu_irq()
550 	for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_irq()
551 		if (!mmu->domains[i])  in ipmmu_irq()
553 		if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)  in ipmmu_irq()
557 	spin_unlock_irqrestore(&mmu->lock, flags);  in ipmmu_irq()
599 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_attach_device()  local
604 	if (!mmu) {  in ipmmu_attach_device()
611 	if (!domain->mmu) {  in ipmmu_attach_device()
613 		domain->mmu = mmu;  in ipmmu_attach_device()
617 			domain->mmu = NULL;  in ipmmu_attach_device()
622 	} else if (domain->mmu != mmu) {  in ipmmu_attach_device()
628 			dev_name(mmu->dev), dev_name(domain->mmu->dev));  in ipmmu_attach_device()
682 	if (domain->mmu)  in ipmmu_flush_iotlb_all()
779 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_init_arm_mapping()  local
791 	if (!mmu->mapping) {  in ipmmu_init_arm_mapping()
797 			dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");  in ipmmu_init_arm_mapping()
802 		mmu->mapping = mapping;  in ipmmu_init_arm_mapping()
806 	ret = arm_iommu_attach_device(dev, mmu->mapping);  in ipmmu_init_arm_mapping()
815 	if (mmu->mapping)  in ipmmu_init_arm_mapping()
816 		arm_iommu_release_mapping(mmu->mapping);  in ipmmu_init_arm_mapping()
823 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_probe_device()  local
828 	if (!mmu)  in ipmmu_probe_device()
831 	return &mmu->iommu;  in ipmmu_probe_device()
852 	struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);  in ipmmu_find_group()  local
855 	if (mmu->group)  in ipmmu_find_group()
856 		return iommu_group_ref_get(mmu->group);  in ipmmu_find_group()
860 		mmu->group = group;  in ipmmu_find_group()
888 static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)  in ipmmu_device_reset()  argument
893 	for (i = 0; i < mmu->num_ctx; ++i)  in ipmmu_device_reset()
894 		ipmmu_ctx_write(mmu, i, IMCTR, 0);  in ipmmu_device_reset()
969 	struct ipmmu_vmsa_device *mmu;  in ipmmu_probe()  local
974 	mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);  in ipmmu_probe()
975 	if (!mmu) {  in ipmmu_probe()
980 	mmu->dev = &pdev->dev;  in ipmmu_probe()
981 	spin_lock_init(&mmu->lock);  in ipmmu_probe()
982 	bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);  in ipmmu_probe()
983 	mmu->features = of_device_get_match_data(&pdev->dev);  in ipmmu_probe()
984 	memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);  in ipmmu_probe()
991 	mmu->base = devm_ioremap_resource(&pdev->dev, res);  in ipmmu_probe()
992 	if (IS_ERR(mmu->base))  in ipmmu_probe()
993 		return PTR_ERR(mmu->base);  in ipmmu_probe()
1007 	if (mmu->features->use_ns_alias_offset)  in ipmmu_probe()
1008 		mmu->base += IM_NS_ALIAS_OFFSET;  in ipmmu_probe()
1010 	mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);  in ipmmu_probe()
1016 	if (!mmu->features->has_cache_leaf_nodes ||  in ipmmu_probe()
1018 		mmu->root = mmu;  in ipmmu_probe()
1020 		mmu->root = ipmmu_find_root();  in ipmmu_probe()
1025 	if (!mmu->root)  in ipmmu_probe()
1029 	if (ipmmu_is_root(mmu)) {  in ipmmu_probe()
1035 				       dev_name(&pdev->dev), mmu);  in ipmmu_probe()
1041 		ipmmu_device_reset(mmu);  in ipmmu_probe()
1043 		if (mmu->features->reserved_context) {  in ipmmu_probe()
1045 			set_bit(0, mmu->ctx);  in ipmmu_probe()
1054 	if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {  in ipmmu_probe()
1055 		ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,  in ipmmu_probe()
1060 		ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);  in ipmmu_probe()
1076 	platform_set_drvdata(pdev, mmu);  in ipmmu_probe()
1083 	struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);  in ipmmu_remove()  local
1085 	iommu_device_sysfs_remove(&mmu->iommu);  in ipmmu_remove()
1086 	iommu_device_unregister(&mmu->iommu);  in ipmmu_remove()
1088 	arm_iommu_release_mapping(mmu->mapping);  in ipmmu_remove()
1090 	ipmmu_device_reset(mmu);  in ipmmu_remove()
1098 	struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);  in ipmmu_resume_noirq()  local
1102 	if (ipmmu_is_root(mmu)) {  in ipmmu_resume_noirq()
1103 		ipmmu_device_reset(mmu);  in ipmmu_resume_noirq()
1105 		for (i = 0; i < mmu->num_ctx; i++) {  in ipmmu_resume_noirq()
1106 			if (!mmu->domains[i])  in ipmmu_resume_noirq()
1109 			ipmmu_domain_setup_context(mmu->domains[i]);  in ipmmu_resume_noirq()
1114 	for (i = 0; i < mmu->features->num_utlbs; i++) {  in ipmmu_resume_noirq()
1115 		if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)  in ipmmu_resume_noirq()
1118 		ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);  in ipmmu_resume_noirq()