109 static phys_addr_t __arm64_noalias_map(struct device *dev, phys_addr_t phys,  in __arm64_noalias_map()  argument
113 	bounce = bounce || (phys | size) & ~PAGE_MASK;  in __arm64_noalias_map()
115 		phys = swiotlb_tbl_map_single(dev, phys, size, PAGE_ALIGN(size),  in __arm64_noalias_map()
117 		if (phys == DMA_MAPPING_ERROR)  in __arm64_noalias_map()
120 	if (set_nc(phys_to_virt(phys & PAGE_MASK), PAGE_ALIGN(size)))  in __arm64_noalias_map()
123 	return phys;  in __arm64_noalias_map()
126 		swiotlb_tbl_unmap_single(dev, phys, size, PAGE_ALIGN(size), dir,  in __arm64_noalias_map()
132 static void __arm64_noalias_unmap(struct device *dev, phys_addr_t phys, size_t size,  in __arm64_noalias_unmap()  argument
135 	clear_nc(phys_to_virt(phys & PAGE_MASK), PAGE_ALIGN(size));  in __arm64_noalias_unmap()
136 	if (is_swiotlb_buffer(phys))  in __arm64_noalias_unmap()
137 		swiotlb_tbl_unmap_single(dev, phys, size, PAGE_ALIGN(size), dir, attrs);  in __arm64_noalias_unmap()
140 static void __arm64_noalias_sync_for_device(struct device *dev, phys_addr_t phys,  in __arm64_noalias_sync_for_device()  argument
143 	if (is_swiotlb_buffer(phys))  in __arm64_noalias_sync_for_device()
144 		swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_DEVICE);  in __arm64_noalias_sync_for_device()
146 		arch_sync_dma_for_device(phys, size, dir);  in __arm64_noalias_sync_for_device()
149 static void __arm64_noalias_sync_for_cpu(struct device *dev, phys_addr_t phys,  in __arm64_noalias_sync_for_cpu()  argument
152 	if (is_swiotlb_buffer(phys))  in __arm64_noalias_sync_for_cpu()
153 		swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_CPU);  in __arm64_noalias_sync_for_cpu()
155 		arch_sync_dma_for_cpu(phys, size, dir);  in __arm64_noalias_sync_for_cpu()
192 	phys_addr_t phys = page_to_phys(page) + offset;  in arm64_noalias_map_page()  local
193 	bool bounce = !dma_capable(dev, phys_to_dma(dev, phys), size, true);  in arm64_noalias_map_page()
196 		arch_sync_dma_for_device(phys, size, dir);  in arm64_noalias_map_page()
197 		return phys_to_dma(dev, phys);  in arm64_noalias_map_page()
201 	phys = __arm64_noalias_map(dev, phys, size, dir, attrs, bounce);  in arm64_noalias_map_page()
202 	if (phys == DMA_MAPPING_ERROR)  in arm64_noalias_map_page()
205 	return phys_to_dma(dev, phys);  in arm64_noalias_map_page()
354 	phys_addr_t phys = page_to_phys(page) + offset;  in arm64_iommu_map_page()  local
360 	phys = __arm64_noalias_map(dev, phys, size, dir, attrs, page_mapped(page));  in arm64_iommu_map_page()
361 	if (phys == DMA_MAPPING_ERROR)  in arm64_iommu_map_page()
365 	ret = iommu_dma_ops->map_page(dev, phys_to_page(phys), offset_in_page(phys),  in arm64_iommu_map_page()
368 		__arm64_noalias_unmap(dev, phys, size, dir, attrs);  in arm64_iommu_map_page()
375 	phys_addr_t phys;  in arm64_iommu_unmap_page()  local
380 	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);  in arm64_iommu_unmap_page()
382 	__arm64_noalias_unmap(dev, phys, size, dir, attrs);  in arm64_iommu_unmap_page()
400 		phys_addr_t phys = sg_phys(sg);  in arm64_iommu_map_sg()  local
410 		phys = __arm64_noalias_map(dev, phys, sg->length, dir, attrs, false);  in arm64_iommu_map_sg()
411 		if (phys == DMA_MAPPING_ERROR)  in arm64_iommu_map_sg()
415 		sg_assign_page(sg, phys_to_page(phys));  in arm64_iommu_map_sg()
416 		sg->offset = offset_in_page(phys);  in arm64_iommu_map_sg()
455 		phys_addr_t phys = iommu_iova_to_phys(domain, iova);  in arm64_iommu_unmap_sg()  local
457 		__arm64_noalias_unmap(dev, phys, sg->length, dir, attrs);  in arm64_iommu_unmap_sg()
470 	phys_addr_t phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);  in arm64_iommu_sync_single_for_device()  local
472 	__arm64_noalias_sync_for_device(dev, phys, size, dir);  in arm64_iommu_sync_single_for_device()
478 	phys_addr_t phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), addr);  in arm64_iommu_sync_single_for_cpu()  local
480 	__arm64_noalias_sync_for_cpu(dev, phys, size, dir);  in arm64_iommu_sync_single_for_cpu()
492 		phys_addr_t phys = iommu_iova_to_phys(domain, iova);  in arm64_iommu_sync_sg_for_device()  local
494 		__arm64_noalias_sync_for_device(dev, phys, sg->length, dir);  in arm64_iommu_sync_sg_for_device()
512 		phys_addr_t phys = iommu_iova_to_phys(domain, iova);  in arm64_iommu_sync_sg_for_cpu()  local
514 		__arm64_noalias_sync_for_cpu(dev, phys, sg->length, dir);  in arm64_iommu_sync_sg_for_cpu()