78 int iommu_get_dma_cookie(struct iommu_domain *domain)  in iommu_get_dma_cookie()  argument
80 	if (domain->iova_cookie)  in iommu_get_dma_cookie()
83 	domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);  in iommu_get_dma_cookie()
84 	if (!domain->iova_cookie)  in iommu_get_dma_cookie()
103 int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)  in iommu_get_msi_cookie()  argument
107 	if (domain->type != IOMMU_DOMAIN_UNMANAGED)  in iommu_get_msi_cookie()
110 	if (domain->iova_cookie)  in iommu_get_msi_cookie()
118 	domain->iova_cookie = cookie;  in iommu_get_msi_cookie()
130 void iommu_put_dma_cookie(struct iommu_domain *domain)  in iommu_put_dma_cookie()  argument
132 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_put_dma_cookie()
146 	domain->iova_cookie = NULL;  in iommu_put_dma_cookie()
238 		struct iommu_domain *domain)  in iova_reserve_iommu_regions()  argument
240 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iova_reserve_iommu_regions()
278 	struct iommu_domain *domain;  in iommu_dma_flush_iotlb_all()  local
281 	domain = cookie->fq_domain;  in iommu_dma_flush_iotlb_all()
286 	domain->ops->flush_iotlb_all(domain);  in iommu_dma_flush_iotlb_all()
301 static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,  in iommu_dma_init_domain()  argument
304 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_init_domain()
315 	order = __ffs(domain->pgsize_bitmap);  in iommu_dma_init_domain()
319 	if (domain->geometry.force_aperture) {  in iommu_dma_init_domain()
320 		if (base > domain->geometry.aperture_end ||  in iommu_dma_init_domain()
321 		    base + size <= domain->geometry.aperture_start) {  in iommu_dma_init_domain()
327 				domain->geometry.aperture_start >> order);  in iommu_dma_init_domain()
343 	if (!cookie->fq_domain && !iommu_domain_get_attr(domain,  in iommu_dma_init_domain()
345 		cookie->fq_domain = domain;  in iommu_dma_init_domain()
352 	return iova_reserve_iommu_regions(dev, domain);  in iommu_dma_init_domain()
384 static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,  in iommu_dma_alloc_iova()  argument
387 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_iova()
410 	if (domain->geometry.force_aperture)  in iommu_dma_alloc_iova()
411 		dma_limit = min(dma_limit, domain->geometry.aperture_end);  in iommu_dma_alloc_iova()
444 	struct iommu_domain *domain = iommu_get_dma_domain(dev);  in __iommu_dma_unmap()  local
445 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_unmap()
455 	unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);  in __iommu_dma_unmap()
459 		iommu_tlb_sync(domain, &iotlb_gather);  in __iommu_dma_unmap()
466 	struct iommu_domain *domain = iommu_get_dma_domain(dev);  in __iommu_dma_map()  local
467 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_map()
474 	iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);  in __iommu_dma_map()
478 	if (iommu_map(domain, iova, phys - iova_off, size, prot)) {  in __iommu_dma_map()
567 	struct iommu_domain *domain = iommu_get_dma_domain(dev);  in iommu_dma_alloc_remap()  local
568 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_remap()
573 	unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;  in iommu_dma_alloc_remap()
598 	iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);  in iommu_dma_alloc_remap()
613 	if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot)  in iommu_dma_alloc_remap()
813 	struct iommu_domain *domain = iommu_get_dma_domain(dev);  in iommu_dma_map_sg()  local
814 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_map_sg()
865 	iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);  in iommu_dma_map_sg()
873 	if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)  in iommu_dma_map_sg()
1088 	struct iommu_domain *domain = iommu_get_dma_domain(dev);  in iommu_dma_get_merge_boundary()  local
1090 	return (1UL << __ffs(domain->pgsize_bitmap)) - 1;  in iommu_dma_get_merge_boundary()
1117 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);  in iommu_setup_dma_ops()  local
1119 	if (!domain)  in iommu_setup_dma_ops()
1126 	if (domain->type == IOMMU_DOMAIN_DMA) {  in iommu_setup_dma_ops()
1127 		if (iommu_dma_init_domain(domain, dma_base, size, dev))  in iommu_setup_dma_ops()
1139 		phys_addr_t msi_addr, struct iommu_domain *domain)  in iommu_dma_get_msi_page()  argument
1141 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_get_msi_page()
1156 	iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);  in iommu_dma_get_msi_page()
1160 	if (iommu_map(domain, iova, msi_addr, size, prot))  in iommu_dma_get_msi_page()
1179 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);  in iommu_dma_prepare_msi()  local
1183 	if (!domain || !domain->iova_cookie) {  in iommu_dma_prepare_msi()
1194 	msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);  in iommu_dma_prepare_msi()
1208 	const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);  in iommu_dma_compose_msi_msg()  local
1213 	if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))  in iommu_dma_compose_msi_msg()
1217 	msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;  in iommu_dma_compose_msi_msg()