33 static inline struct page *dma_direct_to_page(struct device *dev,  in dma_direct_to_page()
79 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,  in __dma_direct_alloc_pages()
83 	struct page *page = NULL;  in __dma_direct_alloc_pages()  local
90 	page = dma_alloc_contiguous(dev, size, gfp);  in __dma_direct_alloc_pages()
91 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in __dma_direct_alloc_pages()
92 		dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
93 		page = NULL;  in __dma_direct_alloc_pages()
96 	if (!page)  in __dma_direct_alloc_pages()
97 		page = alloc_pages_node(node, gfp, get_order(size));  in __dma_direct_alloc_pages()
98 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in __dma_direct_alloc_pages()
99 		dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
100 		page = NULL;  in __dma_direct_alloc_pages()
116 	return page;  in __dma_direct_alloc_pages()
122 	struct page *page;  in dma_direct_alloc_from_pool()  local
128 	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);  in dma_direct_alloc_from_pool()
129 	if (!page)  in dma_direct_alloc_from_pool()
131 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc_from_pool()
138 	struct page *page;  in dma_direct_alloc()  local
148 		page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);  in dma_direct_alloc()
149 		if (!page)  in dma_direct_alloc()
152 		if (!PageHighMem(page))  in dma_direct_alloc()
153 			arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
154 		*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc()
156 		return page;  in dma_direct_alloc()
175 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO);  in dma_direct_alloc()
176 	if (!page)  in dma_direct_alloc()
181 	    (IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) {  in dma_direct_alloc()
183 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
186 		ret = dma_common_contiguous_remap(page, size,  in dma_direct_alloc()
201 	if (PageHighMem(page)) {  in dma_direct_alloc()
212 	ret = page_address(page);  in dma_direct_alloc()
224 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
230 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc()
235 		err = set_memory_encrypted((unsigned long)page_address(page),  in dma_direct_alloc()
242 	dma_free_contiguous(dev, page, size);  in dma_direct_alloc()
279 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,  in dma_direct_alloc_pages()
282 	struct page *page;  in dma_direct_alloc_pages()  local
289 	page = __dma_direct_alloc_pages(dev, size, gfp);  in dma_direct_alloc_pages()
290 	if (!page)  in dma_direct_alloc_pages()
292 	if (PageHighMem(page)) {  in dma_direct_alloc_pages()
303 	ret = page_address(page);  in dma_direct_alloc_pages()
309 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc_pages()
310 	return page;  in dma_direct_alloc_pages()
312 	dma_free_contiguous(dev, page, size);  in dma_direct_alloc_pages()
317 		struct page *page, dma_addr_t dma_addr,  in dma_direct_free_pages()  argument
320 	void *vaddr = page_address(page);  in dma_direct_free_pages()
330 	dma_free_contiguous(dev, page, size);  in dma_direct_free_pages()
435 	struct page *page = dma_direct_to_page(dev, dma_addr);  in dma_direct_get_sgtable()  local
440 		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in dma_direct_get_sgtable()