210 static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain,  in ipmmu_ctx_read_root()  argument
213 	return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);  in ipmmu_ctx_read_root()
216 static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain,  in ipmmu_ctx_write_root()  argument
219 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_root()
222 static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain,  in ipmmu_ctx_write_all()  argument
225 	if (domain->mmu != domain->mmu->root)  in ipmmu_ctx_write_all()
226 		ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
228 	ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);  in ipmmu_ctx_write_all()
253 static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)  in ipmmu_tlb_sync()  argument
257 	while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) {  in ipmmu_tlb_sync()
260 			dev_err_ratelimited(domain->mmu->dev,  in ipmmu_tlb_sync()
268 static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)  in ipmmu_tlb_invalidate()  argument
272 	reg = ipmmu_ctx_read_root(domain, IMCTR);  in ipmmu_tlb_invalidate()
274 	ipmmu_ctx_write_all(domain, IMCTR, reg);  in ipmmu_tlb_invalidate()
276 	ipmmu_tlb_sync(domain);  in ipmmu_tlb_invalidate()
282 static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,  in ipmmu_utlb_enable()  argument
285 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_enable()
295 	ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |  in ipmmu_utlb_enable()
297 	mmu->utlb_ctx[utlb] = domain->context_id;  in ipmmu_utlb_enable()
303 static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,  in ipmmu_utlb_disable()  argument
306 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_utlb_disable()
314 	struct ipmmu_vmsa_domain *domain = cookie;  in ipmmu_tlb_flush_all()  local
316 	ipmmu_tlb_invalidate(domain);  in ipmmu_tlb_flush_all()
332  * Domain/Context Management
336 					 struct ipmmu_vmsa_domain *domain)  in ipmmu_domain_allocate_context()  argument
345 		mmu->domains[ret] = domain;  in ipmmu_domain_allocate_context()
368 static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain)  in ipmmu_domain_setup_context()  argument
374 	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;  in ipmmu_domain_setup_context()
375 	ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);  in ipmmu_domain_setup_context()
376 	ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);  in ipmmu_domain_setup_context()
383 	if (domain->mmu->features->twobit_imttbcr_sl0)  in ipmmu_domain_setup_context()
388 	if (domain->mmu->features->cache_snoop)  in ipmmu_domain_setup_context()
392 	ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp);  in ipmmu_domain_setup_context()
395 	ipmmu_ctx_write_root(domain, IMMAIR0,  in ipmmu_domain_setup_context()
396 			     domain->cfg.arm_lpae_s1_cfg.mair);  in ipmmu_domain_setup_context()
399 	if (domain->mmu->features->setup_imbuscr)  in ipmmu_domain_setup_context()
400 		ipmmu_ctx_write_root(domain, IMBUSCR,  in ipmmu_domain_setup_context()
401 				     ipmmu_ctx_read_root(domain, IMBUSCR) &  in ipmmu_domain_setup_context()
408 	ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR));  in ipmmu_domain_setup_context()
417 	ipmmu_ctx_write_all(domain, IMCTR,  in ipmmu_domain_setup_context()
421 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)  in ipmmu_domain_init_context()  argument
436 	domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS;  in ipmmu_domain_init_context()
437 	domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K;  in ipmmu_domain_init_context()
438 	domain->cfg.ias = 32;  in ipmmu_domain_init_context()
439 	domain->cfg.oas = 40;  in ipmmu_domain_init_context()
440 	domain->cfg.tlb = &ipmmu_flush_ops;  in ipmmu_domain_init_context()
441 	domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32);  in ipmmu_domain_init_context()
442 	domain->io_domain.geometry.force_aperture = true;  in ipmmu_domain_init_context()
447 	domain->cfg.coherent_walk = false;  in ipmmu_domain_init_context()
448 	domain->cfg.iommu_dev = domain->mmu->root->dev;  in ipmmu_domain_init_context()
453 	ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);  in ipmmu_domain_init_context()
457 	domain->context_id = ret;  in ipmmu_domain_init_context()
459 	domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,  in ipmmu_domain_init_context()
460 					   domain);  in ipmmu_domain_init_context()
461 	if (!domain->iop) {  in ipmmu_domain_init_context()
462 		ipmmu_domain_free_context(domain->mmu->root,  in ipmmu_domain_init_context()
463 					  domain->context_id);  in ipmmu_domain_init_context()
467 	ipmmu_domain_setup_context(domain);  in ipmmu_domain_init_context()
471 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)  in ipmmu_domain_destroy_context()  argument
473 	if (!domain->mmu)  in ipmmu_domain_destroy_context()
482 	ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH);  in ipmmu_domain_destroy_context()
483 	ipmmu_tlb_sync(domain);  in ipmmu_domain_destroy_context()
484 	ipmmu_domain_free_context(domain->mmu->root, domain->context_id);  in ipmmu_domain_destroy_context()
491 static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)  in ipmmu_domain_irq()  argument
494 	struct ipmmu_vmsa_device *mmu = domain->mmu;  in ipmmu_domain_irq()
498 	status = ipmmu_ctx_read_root(domain, IMSTR);  in ipmmu_domain_irq()
502 	iova = ipmmu_ctx_read_root(domain, IMELAR);  in ipmmu_domain_irq()
504 		iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32;  in ipmmu_domain_irq()
512 	ipmmu_ctx_write_root(domain, IMSTR, 0);  in ipmmu_domain_irq()
531 	if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))  in ipmmu_domain_irq()
571 	struct ipmmu_vmsa_domain *domain;  in __ipmmu_domain_alloc()  local
573 	domain = kzalloc(sizeof(*domain), GFP_KERNEL);  in __ipmmu_domain_alloc()
574 	if (!domain)  in __ipmmu_domain_alloc()
577 	mutex_init(&domain->mutex);  in __ipmmu_domain_alloc()
579 	return &domain->io_domain;  in __ipmmu_domain_alloc()
605 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_domain_free()  local
608 	 * Free the domain resources. We assume that all devices have already  in ipmmu_domain_free()
612 	ipmmu_domain_destroy_context(domain);  in ipmmu_domain_free()
613 	free_io_pgtable_ops(domain->iop);  in ipmmu_domain_free()
614 	kfree(domain);  in ipmmu_domain_free()
622 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_attach_device()  local
631 	mutex_lock(&domain->mutex);  in ipmmu_attach_device()
633 	if (!domain->mmu) {  in ipmmu_attach_device()
634 		/* The domain hasn't been used yet, initialize it. */  in ipmmu_attach_device()
635 		domain->mmu = mmu;  in ipmmu_attach_device()
636 		ret = ipmmu_domain_init_context(domain);  in ipmmu_attach_device()
639 			domain->mmu = NULL;  in ipmmu_attach_device()
642 				 domain->context_id);  in ipmmu_attach_device()
644 	} else if (domain->mmu != mmu) {  in ipmmu_attach_device()
647 		 * different IOMMUs to the same domain.  in ipmmu_attach_device()
649 		dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",  in ipmmu_attach_device()
650 			dev_name(mmu->dev), dev_name(domain->mmu->dev));  in ipmmu_attach_device()
653 		dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);  in ipmmu_attach_device()
655 	mutex_unlock(&domain->mutex);  in ipmmu_attach_device()
661 		ipmmu_utlb_enable(domain, fwspec->ids[i]);  in ipmmu_attach_device()
670 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_detach_device()  local
674 		ipmmu_utlb_disable(domain, fwspec->ids[i]);  in ipmmu_detach_device()
684 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_map()  local
686 	if (!domain)  in ipmmu_map()
689 	return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp);  in ipmmu_map()
695 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_unmap()  local
697 	return domain->iop->unmap(domain->iop, iova, size, gather);  in ipmmu_unmap()
702 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_flush_iotlb_all()  local
704 	if (domain->mmu)  in ipmmu_flush_iotlb_all()
705 		ipmmu_tlb_flush_all(domain);  in ipmmu_flush_iotlb_all()
717 	struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);  in ipmmu_iova_to_phys()  local
721 	return domain->iop->iova_to_phys(domain->iop, iova);  in ipmmu_iova_to_phys()
815 	 * VAs. This will allocate a corresponding IOMMU domain.  in ipmmu_init_arm_mapping()