70 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)  in dma_coherent_ok()  argument
76 	return dma_addr + size - 1 <=  in dma_coherent_ok()
80 static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)  in dma_set_decrypted()  argument
84 	return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));  in dma_set_decrypted()
87 static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)  in dma_set_encrypted()  argument
93 	ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));  in dma_set_encrypted()
100 				    size_t size)  in __dma_direct_free_pages()  argument
102 	if (swiotlb_free(dev, page, size))  in __dma_direct_free_pages()
104 	dma_free_contiguous(dev, page, size);  in __dma_direct_free_pages()
107 static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)  in dma_direct_alloc_swiotlb()  argument
109 	struct page *page = swiotlb_alloc(dev, size);  in dma_direct_alloc_swiotlb()
111 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in dma_direct_alloc_swiotlb()
112 		swiotlb_free(dev, page, size);  in dma_direct_alloc_swiotlb()
119 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,  in __dma_direct_alloc_pages()  argument
126 	WARN_ON_ONCE(!PAGE_ALIGNED(size));  in __dma_direct_alloc_pages()
129 		return dma_direct_alloc_swiotlb(dev, size);  in __dma_direct_alloc_pages()
133 	page = dma_alloc_contiguous(dev, size, gfp);  in __dma_direct_alloc_pages()
135 		if (!dma_coherent_ok(dev, page_to_phys(page), size) ||  in __dma_direct_alloc_pages()
137 			dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
143 		page = alloc_pages_node(node, gfp, get_order(size));  in __dma_direct_alloc_pages()
144 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in __dma_direct_alloc_pages()
145 		dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
174 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,  in dma_direct_alloc_from_pool()  argument
186 	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);  in dma_direct_alloc_from_pool()
193 static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,  in dma_direct_alloc_no_mapping()  argument
198 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);  in dma_direct_alloc_no_mapping()
204 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc_no_mapping()
211 void *dma_direct_alloc(struct device *dev, size_t size,  in dma_direct_alloc()  argument
218 	size = PAGE_ALIGN(size);  in dma_direct_alloc()
224 		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);  in dma_direct_alloc()
235 			return arch_dma_alloc(dev, size, dma_handle, gfp,  in dma_direct_alloc()
243 			return dma_alloc_from_global_coherent(dev, size,  in dma_direct_alloc()
254 				return dma_direct_alloc_from_pool(dev, size,  in dma_direct_alloc()
268 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);  in dma_direct_alloc()
271 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);  in dma_direct_alloc()
292 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
295 		ret = dma_common_contiguous_remap(page, size, prot,  in dma_direct_alloc()
301 		if (dma_set_decrypted(dev, ret, size))  in dma_direct_alloc()
305 	memset(ret, 0, size);  in dma_direct_alloc()
308 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
309 		ret = arch_dma_set_uncached(ret, size);  in dma_direct_alloc()
318 	if (dma_set_encrypted(dev, page_address(page), size))  in dma_direct_alloc()
321 	__dma_direct_free_pages(dev, page, size);  in dma_direct_alloc()
326 void dma_direct_free(struct device *dev, size_t size,  in dma_direct_free()  argument
329 	unsigned int page_order = get_order(size);  in dma_direct_free()
334 		dma_free_contiguous(dev, cpu_addr, size);  in dma_direct_free()
343 		arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);  in dma_direct_free()
356 	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))  in dma_direct_free()
363 			arch_dma_clear_uncached(cpu_addr, size);  in dma_direct_free()
364 		if (dma_set_encrypted(dev, cpu_addr, size))  in dma_direct_free()
368 	__dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);  in dma_direct_free()
372 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,  in dma_direct_alloc_pages()  argument
379 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);  in dma_direct_alloc_pages()
381 	page = __dma_direct_alloc_pages(dev, size, gfp, false);  in dma_direct_alloc_pages()
386 	if (dma_set_decrypted(dev, ret, size))  in dma_direct_alloc_pages()
388 	memset(ret, 0, size);  in dma_direct_alloc_pages()
392 	__dma_direct_free_pages(dev, page, size);  in dma_direct_alloc_pages()
396 void dma_direct_free_pages(struct device *dev, size_t size,  in dma_direct_free_pages()  argument
404 	    dma_free_from_pool(dev, vaddr, size))  in dma_direct_free_pages()
407 	if (dma_set_encrypted(dev, vaddr, size))  in dma_direct_free_pages()
409 	__dma_direct_free_pages(dev, page, size);  in dma_direct_free_pages()
526 		size_t size, enum dma_data_direction dir, unsigned long attrs)  in dma_direct_map_resource()  argument
530 	if (unlikely(!dma_capable(dev, dma_addr, size, false))) {  in dma_direct_map_resource()
533 			     &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);  in dma_direct_map_resource()
542 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_direct_get_sgtable()  argument
550 		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in dma_direct_get_sgtable()
561 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_direct_mmap()  argument
565 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_direct_mmap()
573 	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))  in dma_direct_mmap()
575 	if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))  in dma_direct_mmap()
639 			 dma_addr_t dma_start, u64 size)  in dma_direct_set_offset()  argument
658 	map[0].size = size;  in dma_direct_set_offset()