96 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
97 void debug_dma_map_single(struct device *dev, const void *addr,
100 static inline void debug_dma_mapping_error(struct device *dev,  in debug_dma_mapping_error()  argument
104 static inline void debug_dma_map_single(struct device *dev, const void *addr,  in debug_dma_map_single()  argument
111 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)  in dma_mapping_error()  argument
113 	debug_dma_mapping_error(dev, dma_addr);  in dma_mapping_error()
120 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
123 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
125 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
127 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
130 int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
132 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
134 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
136 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
138 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
140 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
142 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
144 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
147 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
150 bool dma_can_mmap(struct device *dev);
151 bool dma_pci_p2pdma_supported(struct device *dev);
152 int dma_set_mask(struct device *dev, u64 mask);
153 int dma_set_coherent_mask(struct device *dev, u64 mask);
154 u64 dma_get_required_mask(struct device *dev);
155 bool dma_addressing_limited(struct device *dev);
156 size_t dma_max_mapping_size(struct device *dev);
157 size_t dma_opt_mapping_size(struct device *dev);
158 unsigned long dma_get_merge_boundary(struct device *dev);
159 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
161 void dma_free_noncontiguous(struct device *dev, size_t size,
163 void *dma_vmap_noncontiguous(struct device *dev, size_t size,
165 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr);
166 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma,
169 static inline dma_addr_t dma_map_page_attrs(struct device *dev,  in dma_map_page_attrs()  argument
175 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,  in dma_unmap_page_attrs()  argument
179 static inline unsigned int dma_map_sg_attrs(struct device *dev,  in dma_map_sg_attrs()  argument
185 static inline void dma_unmap_sg_attrs(struct device *dev,  in dma_unmap_sg_attrs()  argument
190 static inline int dma_map_sgtable(struct device *dev, struct sg_table *sgt,  in dma_map_sgtable()  argument
195 static inline dma_addr_t dma_map_resource(struct device *dev,  in dma_map_resource()  argument
201 static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,  in dma_unmap_resource()  argument
205 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)  in dma_mapping_error()  argument
209 static inline void *dma_alloc_attrs(struct device *dev, size_t size,  in dma_alloc_attrs()  argument
214 static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,  in dma_free_attrs()  argument
218 static inline void *dmam_alloc_attrs(struct device *dev, size_t size,  in dmam_alloc_attrs()  argument
223 static inline void dmam_free_coherent(struct device *dev, size_t size,  in dmam_free_coherent()  argument
227 static inline int dma_get_sgtable_attrs(struct device *dev,  in dma_get_sgtable_attrs()  argument
233 static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,  in dma_mmap_attrs()  argument
239 static inline bool dma_can_mmap(struct device *dev)  in dma_can_mmap()  argument
243 static inline bool dma_pci_p2pdma_supported(struct device *dev)  in dma_pci_p2pdma_supported()  argument
247 static inline int dma_set_mask(struct device *dev, u64 mask)  in dma_set_mask()  argument
251 static inline int dma_set_coherent_mask(struct device *dev, u64 mask)  in dma_set_coherent_mask()  argument
255 static inline u64 dma_get_required_mask(struct device *dev)  in dma_get_required_mask()  argument
259 static inline bool dma_addressing_limited(struct device *dev)  in dma_addressing_limited()  argument
263 static inline size_t dma_max_mapping_size(struct device *dev)  in dma_max_mapping_size()  argument
267 static inline size_t dma_opt_mapping_size(struct device *dev)  in dma_opt_mapping_size()  argument
271 static inline unsigned long dma_get_merge_boundary(struct device *dev)  in dma_get_merge_boundary()  argument
275 static inline struct sg_table *dma_alloc_noncontiguous(struct device *dev,  in dma_alloc_noncontiguous()  argument
281 static inline void dma_free_noncontiguous(struct device *dev, size_t size,  in dma_free_noncontiguous()  argument
285 static inline void *dma_vmap_noncontiguous(struct device *dev, size_t size,  in dma_vmap_noncontiguous()  argument
290 static inline void dma_vunmap_noncontiguous(struct device *dev, void *vaddr)  in dma_vunmap_noncontiguous()  argument
293 static inline int dma_mmap_noncontiguous(struct device *dev,  in dma_mmap_noncontiguous()  argument
313 bool dma_iova_try_alloc(struct device *dev, struct dma_iova_state *state,
315 void dma_iova_free(struct device *dev, struct dma_iova_state *state);
316 void dma_iova_destroy(struct device *dev, struct dma_iova_state *state,
319 int dma_iova_sync(struct device *dev, struct dma_iova_state *state,
321 int dma_iova_link(struct device *dev, struct dma_iova_state *state,
324 void dma_iova_unlink(struct device *dev, struct dma_iova_state *state,
332 static inline bool dma_iova_try_alloc(struct device *dev,  in dma_iova_try_alloc()  argument
337 static inline void dma_iova_free(struct device *dev,  in dma_iova_free()  argument
341 static inline void dma_iova_destroy(struct device *dev,  in dma_iova_destroy()  argument
346 static inline int dma_iova_sync(struct device *dev,  in dma_iova_sync()  argument
351 static inline int dma_iova_link(struct device *dev,  in dma_iova_link()  argument
357 static inline void dma_iova_unlink(struct device *dev,  in dma_iova_unlink()  argument
365 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
367 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
369 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
371 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
373 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr);
375 static inline bool dma_dev_need_sync(const struct device *dev)  in dma_dev_need_sync()  argument
378 	return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);  in dma_dev_need_sync()
381 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,  in dma_sync_single_for_cpu()  argument
384 	if (dma_dev_need_sync(dev))  in dma_sync_single_for_cpu()
385 		__dma_sync_single_for_cpu(dev, addr, size, dir);  in dma_sync_single_for_cpu()
388 static inline void dma_sync_single_for_device(struct device *dev,  in dma_sync_single_for_device()  argument
391 	if (dma_dev_need_sync(dev))  in dma_sync_single_for_device()
392 		__dma_sync_single_for_device(dev, addr, size, dir);  in dma_sync_single_for_device()
395 static inline void dma_sync_sg_for_cpu(struct device *dev,  in dma_sync_sg_for_cpu()  argument
398 	if (dma_dev_need_sync(dev))  in dma_sync_sg_for_cpu()
399 		__dma_sync_sg_for_cpu(dev, sg, nelems, dir);  in dma_sync_sg_for_cpu()
402 static inline void dma_sync_sg_for_device(struct device *dev,  in dma_sync_sg_for_device()  argument
405 	if (dma_dev_need_sync(dev))  in dma_sync_sg_for_device()
406 		__dma_sync_sg_for_device(dev, sg, nelems, dir);  in dma_sync_sg_for_device()
409 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)  in dma_need_sync()  argument
411 	return dma_dev_need_sync(dev) ? __dma_need_sync(dev, dma_addr) : false;  in dma_need_sync()
413 bool dma_need_unmap(struct device *dev);
415 static inline bool dma_dev_need_sync(const struct device *dev)  in dma_dev_need_sync()  argument
419 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,  in dma_sync_single_for_cpu()  argument
423 static inline void dma_sync_single_for_device(struct device *dev,  in dma_sync_single_for_device()  argument
427 static inline void dma_sync_sg_for_cpu(struct device *dev,  in dma_sync_sg_for_cpu()  argument
431 static inline void dma_sync_sg_for_device(struct device *dev,  in dma_sync_sg_for_device()  argument
435 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)  in dma_need_sync()  argument
439 static inline bool dma_need_unmap(struct device *dev)  in dma_need_unmap()  argument
445 struct page *dma_alloc_pages(struct device *dev, size_t size,
447 void dma_free_pages(struct device *dev, size_t size, struct page *page,
449 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma,
452 static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,  in dma_alloc_noncoherent()  argument
455 	struct page *page = dma_alloc_pages(dev, size, dma_handle, dir, gfp);  in dma_alloc_noncoherent()
459 static inline void dma_free_noncoherent(struct device *dev, size_t size,  in dma_free_noncoherent()  argument
462 	dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir);  in dma_free_noncoherent()
465 static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,  in dma_map_single_attrs()  argument
469 	if (dev_WARN_ONCE(dev, is_vmalloc_addr(ptr),  in dma_map_single_attrs()
472 	debug_dma_map_single(dev, ptr, size);  in dma_map_single_attrs()
473 	return dma_map_page_attrs(dev, virt_to_page(ptr), offset_in_page(ptr),  in dma_map_single_attrs()
477 static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,  in dma_unmap_single_attrs()  argument
480 	return dma_unmap_page_attrs(dev, addr, size, dir, attrs);  in dma_unmap_single_attrs()
483 static inline void dma_sync_single_range_for_cpu(struct device *dev,  in dma_sync_single_range_for_cpu()  argument
487 	return dma_sync_single_for_cpu(dev, addr + offset, size, dir);  in dma_sync_single_range_for_cpu()
490 static inline void dma_sync_single_range_for_device(struct device *dev,  in dma_sync_single_range_for_device()  argument
494 	return dma_sync_single_for_device(dev, addr + offset, size, dir);  in dma_sync_single_range_for_device()
508 static inline void dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,  in dma_unmap_sgtable()  argument
511 	dma_unmap_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);  in dma_unmap_sgtable()
526 static inline void dma_sync_sgtable_for_cpu(struct device *dev,  in dma_sync_sgtable_for_cpu()  argument
529 	dma_sync_sg_for_cpu(dev, sgt->sgl, sgt->orig_nents, dir);  in dma_sync_sgtable_for_cpu()
543 static inline void dma_sync_sgtable_for_device(struct device *dev,  in dma_sync_sgtable_for_device()  argument
546 	dma_sync_sg_for_device(dev, sgt->sgl, sgt->orig_nents, dir);  in dma_sync_sgtable_for_device()
558 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
560 static inline void *dma_alloc_coherent(struct device *dev, size_t size,  in dma_alloc_coherent()  argument
563 	return dma_alloc_attrs(dev, size, dma_handle, gfp,  in dma_alloc_coherent()
567 static inline void dma_free_coherent(struct device *dev, size_t size,  in dma_free_coherent()  argument
570 	return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);  in dma_free_coherent()
574 static inline u64 dma_get_mask(struct device *dev)  in dma_get_mask()  argument
576 	if (dev->dma_mask && *dev->dma_mask)  in dma_get_mask()
577 		return *dev->dma_mask;  in dma_get_mask()
587 static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)  in dma_set_mask_and_coherent()  argument
589 	int rc = dma_set_mask(dev, mask);  in dma_set_mask_and_coherent()
591 		dma_set_coherent_mask(dev, mask);  in dma_set_mask_and_coherent()
599 static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)  in dma_coerce_mask_and_coherent()  argument
601 	dev->dma_mask = &dev->coherent_dma_mask;  in dma_coerce_mask_and_coherent()
602 	return dma_set_mask_and_coherent(dev, mask);  in dma_coerce_mask_and_coherent()
605 static inline unsigned int dma_get_max_seg_size(struct device *dev)  in dma_get_max_seg_size()  argument
607 	if (dev->dma_parms && dev->dma_parms->max_segment_size)  in dma_get_max_seg_size()
608 		return dev->dma_parms->max_segment_size;  in dma_get_max_seg_size()
612 static inline void dma_set_max_seg_size(struct device *dev, unsigned int size)  in dma_set_max_seg_size()  argument
614 	if (WARN_ON_ONCE(!dev->dma_parms))  in dma_set_max_seg_size()
616 	dev->dma_parms->max_segment_size = size;  in dma_set_max_seg_size()
619 static inline unsigned long dma_get_seg_boundary(struct device *dev)  in dma_get_seg_boundary()  argument
621 	if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)  in dma_get_seg_boundary()
622 		return dev->dma_parms->segment_boundary_mask;  in dma_get_seg_boundary()
637 static inline unsigned long dma_get_seg_boundary_nr_pages(struct device *dev,  in dma_get_seg_boundary_nr_pages()  argument
640 	if (!dev)  in dma_get_seg_boundary_nr_pages()
642 	return (dma_get_seg_boundary(dev) >> page_shift) + 1;  in dma_get_seg_boundary_nr_pages()
645 static inline void dma_set_seg_boundary(struct device *dev, unsigned long mask)  in dma_set_seg_boundary()  argument
647 	if (WARN_ON_ONCE(!dev->dma_parms))  in dma_set_seg_boundary()
649 	dev->dma_parms->segment_boundary_mask = mask;  in dma_set_seg_boundary()
652 static inline unsigned int dma_get_min_align_mask(struct device *dev)  in dma_get_min_align_mask()  argument
654 	if (dev->dma_parms)  in dma_get_min_align_mask()
655 		return dev->dma_parms->min_align_mask;  in dma_get_min_align_mask()
659 static inline void dma_set_min_align_mask(struct device *dev,  in dma_set_min_align_mask()  argument
662 	if (WARN_ON_ONCE(!dev->dma_parms))  in dma_set_min_align_mask()
664 	dev->dma_parms->min_align_mask = min_align_mask;  in dma_set_min_align_mask()
677 static inline void *dmam_alloc_coherent(struct device *dev, size_t size,  in dmam_alloc_coherent()  argument
680 	return dmam_alloc_attrs(dev, size, dma_handle, gfp,  in dmam_alloc_coherent()
684 static inline void *dma_alloc_wc(struct device *dev, size_t size,  in dma_alloc_wc()  argument
692 	return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);  in dma_alloc_wc()
695 static inline void dma_free_wc(struct device *dev, size_t size,  in dma_free_wc()  argument
698 	return dma_free_attrs(dev, size, cpu_addr, dma_addr,  in dma_free_wc()
702 static inline int dma_mmap_wc(struct device *dev,  in dma_mmap_wc()  argument
707 	return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_wc()