42 		struct iova_domain	iovad;  member
55 		return cookie->iovad.granule;  in cookie_msi_granule()
138 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)  in iommu_put_dma_cookie()
139 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
172 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()  local
176 	start -= iova_offset(iovad, start);  in cookie_init_hw_msi_region()
177 	num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);  in cookie_init_hw_msi_region()
188 		start += iovad->granule;  in cookie_init_hw_msi_region()
195 		struct iova_domain *iovad)  in iova_reserve_pci_windows()  argument
206 		lo = iova_pfn(iovad, window->res->start - window->offset);  in iova_reserve_pci_windows()
207 		hi = iova_pfn(iovad, window->res->end - window->offset);  in iova_reserve_pci_windows()
208 		reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
216 			lo = iova_pfn(iovad, start);  in iova_reserve_pci_windows()
217 			hi = iova_pfn(iovad, end);  in iova_reserve_pci_windows()
218 			reserve_iova(iovad, lo, hi);  in iova_reserve_pci_windows()
241 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()  local
247 		ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);  in iova_reserve_iommu_regions()
260 		lo = iova_pfn(iovad, region->start);  in iova_reserve_iommu_regions()
261 		hi = iova_pfn(iovad, region->start + region->length - 1);  in iova_reserve_iommu_regions()
262 		reserve_iova(iovad, lo, hi);  in iova_reserve_iommu_regions()
275 static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)  in iommu_dma_flush_iotlb_all()  argument
280 	cookie = container_of(iovad, struct iommu_dma_cookie, iovad);  in iommu_dma_flush_iotlb_all()
306 	struct iova_domain *iovad;  in iommu_dma_init_domain()  local
312 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
331 	if (iovad->start_pfn) {  in iommu_dma_init_domain()
332 		if (1UL << order != iovad->granule ||  in iommu_dma_init_domain()
333 		    base_pfn != iovad->start_pfn) {  in iommu_dma_init_domain()
341 	init_iova_domain(iovad, 1UL << order, base_pfn);  in iommu_dma_init_domain()
346 		init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);  in iommu_dma_init_domain()
388 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()  local
396 	shift = iova_shift(iovad);  in iommu_dma_alloc_iova()
415 		iova = alloc_iova_fast(iovad, iova_len,  in iommu_dma_alloc_iova()
419 		iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,  in iommu_dma_alloc_iova()
428 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()  local
434 		queue_iova(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
435 				size >> iova_shift(iovad), 0);  in iommu_dma_free_iova()
437 		free_iova_fast(iovad, iova_pfn(iovad, iova),  in iommu_dma_free_iova()
438 				size >> iova_shift(iovad));  in iommu_dma_free_iova()
446 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()  local
447 	size_t iova_off = iova_offset(iovad, dma_addr);  in __iommu_dma_unmap()
452 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_unmap()
468 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()  local
469 	size_t iova_off = iova_offset(iovad, phys);  in __iommu_dma_map()
472 	size = iova_align(iovad, size + iova_off);  in __iommu_dma_map()
569 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_remap()  local
597 	size = iova_align(iovad, size);  in iommu_dma_alloc_remap()
815 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()  local
833 		size_t s_iova_off = iova_offset(iovad, s->offset);  in iommu_dma_map_sg()
840 		s_length = iova_align(iovad, s_length + s_iova_off);  in iommu_dma_map_sg()