• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_DMA_MAPPING_H
3 #define _LINUX_DMA_MAPPING_H
4 
5 #include <linux/sizes.h>
6 #include <linux/string.h>
7 #include <linux/device.h>
8 #include <linux/err.h>
9 #include <linux/dma-debug.h>
10 #include <linux/dma-direction.h>
11 #include <linux/scatterlist.h>
12 #include <linux/bug.h>
13 #include <linux/mem_encrypt.h>
14 #include <linux/android_kabi.h>
15 
16 /**
17  * List of possible attributes associated with a DMA mapping. The semantics
18  * of each attribute should be defined in Documentation/DMA-attributes.txt.
19  *
20  * DMA_ATTR_WRITE_BARRIER: DMA to a memory region with this attribute
21  * forces all pending DMA writes to complete.
22  */
23 #define DMA_ATTR_WRITE_BARRIER		(1UL << 0)
24 /*
25  * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
26  * may be weakly ordered, that is that reads and writes may pass each other.
27  */
28 #define DMA_ATTR_WEAK_ORDERING		(1UL << 1)
29 /*
30  * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
31  * buffered to improve performance.
32  */
33 #define DMA_ATTR_WRITE_COMBINE		(1UL << 2)
34 /*
35  * DMA_ATTR_NON_CONSISTENT: Lets the platform to choose to return either
36  * consistent or non-consistent memory as it sees fit.
37  */
38 #define DMA_ATTR_NON_CONSISTENT		(1UL << 3)
39 /*
40  * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
41  * virtual mapping for the allocated buffer.
42  */
43 #define DMA_ATTR_NO_KERNEL_MAPPING	(1UL << 4)
44 /*
45  * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
46  * the CPU cache for the given buffer assuming that it has been already
47  * transferred to 'device' domain.
48  */
49 #define DMA_ATTR_SKIP_CPU_SYNC		(1UL << 5)
50 /*
51  * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
52  * in physical memory.
53  */
54 #define DMA_ATTR_FORCE_CONTIGUOUS	(1UL << 6)
55 /*
56  * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
57  * that it's probably not worth the time to try to allocate memory to in a way
58  * that gives better TLB efficiency.
59  */
60 #define DMA_ATTR_ALLOC_SINGLE_PAGES	(1UL << 7)
61 /*
62  * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress
63  * allocation failure reports (similarly to __GFP_NOWARN).
64  */
65 #define DMA_ATTR_NO_WARN	(1UL << 8)
66 
67 /*
68  * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully
69  * accessible at an elevated privilege level (and ideally inaccessible or
70  * at least read-only at lesser-privileged levels).
71  */
72 #define DMA_ATTR_PRIVILEGED		(1UL << 9)
73 
74 /*
75  * A dma_addr_t can hold any valid DMA or bus address for the platform.
76  * It can be given to a device to use as a DMA source or target.  A CPU cannot
77  * reference a dma_addr_t directly because there may be translation between
78  * its physical address space and the bus address space.
79  */
80 struct dma_map_ops {
81 	void* (*alloc)(struct device *dev, size_t size,
82 				dma_addr_t *dma_handle, gfp_t gfp,
83 				unsigned long attrs);
84 	void (*free)(struct device *dev, size_t size,
85 			      void *vaddr, dma_addr_t dma_handle,
86 			      unsigned long attrs);
87 	int (*mmap)(struct device *, struct vm_area_struct *,
88 			  void *, dma_addr_t, size_t,
89 			  unsigned long attrs);
90 
91 	int (*get_sgtable)(struct device *dev, struct sg_table *sgt, void *,
92 			   dma_addr_t, size_t, unsigned long attrs);
93 
94 	dma_addr_t (*map_page)(struct device *dev, struct page *page,
95 			       unsigned long offset, size_t size,
96 			       enum dma_data_direction dir,
97 			       unsigned long attrs);
98 	void (*unmap_page)(struct device *dev, dma_addr_t dma_handle,
99 			   size_t size, enum dma_data_direction dir,
100 			   unsigned long attrs);
101 	/*
102 	 * map_sg returns 0 on error and a value > 0 on success.
103 	 * It should never return a value < 0.
104 	 */
105 	int (*map_sg)(struct device *dev, struct scatterlist *sg,
106 		      int nents, enum dma_data_direction dir,
107 		      unsigned long attrs);
108 	void (*unmap_sg)(struct device *dev,
109 			 struct scatterlist *sg, int nents,
110 			 enum dma_data_direction dir,
111 			 unsigned long attrs);
112 	dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr,
113 			       size_t size, enum dma_data_direction dir,
114 			       unsigned long attrs);
115 	void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle,
116 			   size_t size, enum dma_data_direction dir,
117 			   unsigned long attrs);
118 	void (*sync_single_for_cpu)(struct device *dev,
119 				    dma_addr_t dma_handle, size_t size,
120 				    enum dma_data_direction dir);
121 	void (*sync_single_for_device)(struct device *dev,
122 				       dma_addr_t dma_handle, size_t size,
123 				       enum dma_data_direction dir);
124 	void (*sync_sg_for_cpu)(struct device *dev,
125 				struct scatterlist *sg, int nents,
126 				enum dma_data_direction dir);
127 	void (*sync_sg_for_device)(struct device *dev,
128 				   struct scatterlist *sg, int nents,
129 				   enum dma_data_direction dir);
130 	void (*cache_sync)(struct device *dev, void *vaddr, size_t size,
131 			enum dma_data_direction direction);
132 	int (*dma_supported)(struct device *dev, u64 mask);
133 	u64 (*get_required_mask)(struct device *dev);
134 	size_t (*max_mapping_size)(struct device *dev);
135 	unsigned long (*get_merge_boundary)(struct device *dev);
136 
137 	ANDROID_KABI_RESERVE(1);
138 	ANDROID_KABI_RESERVE(2);
139 	ANDROID_KABI_RESERVE(3);
140 	ANDROID_KABI_RESERVE(4);
141 };
142 
143 #define DMA_MAPPING_ERROR		(~(dma_addr_t)0)
144 
145 extern const struct dma_map_ops dma_virt_ops;
146 extern const struct dma_map_ops dma_dummy_ops;
147 
148 #define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
149 
150 #define DMA_MASK_NONE	0x0ULL
151 
valid_dma_direction(int dma_direction)152 static inline int valid_dma_direction(int dma_direction)
153 {
154 	return ((dma_direction == DMA_BIDIRECTIONAL) ||
155 		(dma_direction == DMA_TO_DEVICE) ||
156 		(dma_direction == DMA_FROM_DEVICE));
157 }
158 
159 #ifdef CONFIG_DMA_DECLARE_COHERENT
160 /*
161  * These three functions are only for dma allocator.
162  * Don't use them in device drivers.
163  */
164 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
165 				       dma_addr_t *dma_handle, void **ret);
166 int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr);
167 
168 int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
169 			    void *cpu_addr, size_t size, int *ret);
170 
171 void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size, dma_addr_t *dma_handle);
172 int dma_release_from_global_coherent(int order, void *vaddr);
173 int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *cpu_addr,
174 				  size_t size, int *ret);
175 
176 #else
177 #define dma_alloc_from_dev_coherent(dev, size, handle, ret) (0)
178 #define dma_release_from_dev_coherent(dev, order, vaddr) (0)
179 #define dma_mmap_from_dev_coherent(dev, vma, vaddr, order, ret) (0)
180 
dma_alloc_from_global_coherent(struct device * dev,ssize_t size,dma_addr_t * dma_handle)181 static inline void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size,
182 						   dma_addr_t *dma_handle)
183 {
184 	return NULL;
185 }
186 
dma_release_from_global_coherent(int order,void * vaddr)187 static inline int dma_release_from_global_coherent(int order, void *vaddr)
188 {
189 	return 0;
190 }
191 
dma_mmap_from_global_coherent(struct vm_area_struct * vma,void * cpu_addr,size_t size,int * ret)192 static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
193 						void *cpu_addr, size_t size,
194 						int *ret)
195 {
196 	return 0;
197 }
198 #endif /* CONFIG_DMA_DECLARE_COHERENT */
199 
dma_is_direct(const struct dma_map_ops * ops)200 static inline bool dma_is_direct(const struct dma_map_ops *ops)
201 {
202 	return likely(!ops);
203 }
204 
205 /*
206  * All the dma_direct_* declarations are here just for the indirect call bypass,
207  * and must not be used directly drivers!
208  */
209 dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
210 		unsigned long offset, size_t size, enum dma_data_direction dir,
211 		unsigned long attrs);
212 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
213 		enum dma_data_direction dir, unsigned long attrs);
214 dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
215 		size_t size, enum dma_data_direction dir, unsigned long attrs);
216 
217 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
218     defined(CONFIG_SWIOTLB)
219 void dma_direct_sync_single_for_device(struct device *dev,
220 		dma_addr_t addr, size_t size, enum dma_data_direction dir);
221 void dma_direct_sync_sg_for_device(struct device *dev,
222 		struct scatterlist *sgl, int nents, enum dma_data_direction dir);
223 #else
dma_direct_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)224 static inline void dma_direct_sync_single_for_device(struct device *dev,
225 		dma_addr_t addr, size_t size, enum dma_data_direction dir)
226 {
227 }
dma_direct_sync_sg_for_device(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir)228 static inline void dma_direct_sync_sg_for_device(struct device *dev,
229 		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
230 {
231 }
232 #endif
233 
234 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
235     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
236     defined(CONFIG_SWIOTLB)
237 void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
238 		size_t size, enum dma_data_direction dir, unsigned long attrs);
239 void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
240 		int nents, enum dma_data_direction dir, unsigned long attrs);
241 void dma_direct_sync_single_for_cpu(struct device *dev,
242 		dma_addr_t addr, size_t size, enum dma_data_direction dir);
243 void dma_direct_sync_sg_for_cpu(struct device *dev,
244 		struct scatterlist *sgl, int nents, enum dma_data_direction dir);
245 #else
dma_direct_unmap_page(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)246 static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
247 		size_t size, enum dma_data_direction dir, unsigned long attrs)
248 {
249 }
dma_direct_unmap_sg(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir,unsigned long attrs)250 static inline void dma_direct_unmap_sg(struct device *dev,
251 		struct scatterlist *sgl, int nents, enum dma_data_direction dir,
252 		unsigned long attrs)
253 {
254 }
dma_direct_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)255 static inline void dma_direct_sync_single_for_cpu(struct device *dev,
256 		dma_addr_t addr, size_t size, enum dma_data_direction dir)
257 {
258 }
dma_direct_sync_sg_for_cpu(struct device * dev,struct scatterlist * sgl,int nents,enum dma_data_direction dir)259 static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
260 		struct scatterlist *sgl, int nents, enum dma_data_direction dir)
261 {
262 }
263 #endif
264 
265 size_t dma_direct_max_mapping_size(struct device *dev);
266 
267 #ifdef CONFIG_HAS_DMA
268 #include <asm/dma-mapping.h>
269 
get_dma_ops(struct device * dev)270 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
271 {
272 	if (dev->dma_ops)
273 		return dev->dma_ops;
274 	return get_arch_dma_ops(dev->bus);
275 }
276 
set_dma_ops(struct device * dev,const struct dma_map_ops * dma_ops)277 static inline void set_dma_ops(struct device *dev,
278 			       const struct dma_map_ops *dma_ops)
279 {
280 	dev->dma_ops = dma_ops;
281 }
282 
dma_map_page_attrs(struct device * dev,struct page * page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)283 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
284 		struct page *page, size_t offset, size_t size,
285 		enum dma_data_direction dir, unsigned long attrs)
286 {
287 	const struct dma_map_ops *ops = get_dma_ops(dev);
288 	dma_addr_t addr;
289 
290 	BUG_ON(!valid_dma_direction(dir));
291 	if (dma_is_direct(ops))
292 		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
293 	else
294 		addr = ops->map_page(dev, page, offset, size, dir, attrs);
295 	debug_dma_map_page(dev, page, offset, size, dir, addr);
296 
297 	return addr;
298 }
299 
dma_unmap_page_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)300 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
301 		size_t size, enum dma_data_direction dir, unsigned long attrs)
302 {
303 	const struct dma_map_ops *ops = get_dma_ops(dev);
304 
305 	BUG_ON(!valid_dma_direction(dir));
306 	if (dma_is_direct(ops))
307 		dma_direct_unmap_page(dev, addr, size, dir, attrs);
308 	else if (ops->unmap_page)
309 		ops->unmap_page(dev, addr, size, dir, attrs);
310 	debug_dma_unmap_page(dev, addr, size, dir);
311 }
312 
313 /*
314  * dma_maps_sg_attrs returns 0 on error and > 0 on success.
315  * It should never return a value < 0.
316  */
dma_map_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)317 static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
318 				   int nents, enum dma_data_direction dir,
319 				   unsigned long attrs)
320 {
321 	const struct dma_map_ops *ops = get_dma_ops(dev);
322 	int ents;
323 
324 	BUG_ON(!valid_dma_direction(dir));
325 	if (dma_is_direct(ops))
326 		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
327 	else
328 		ents = ops->map_sg(dev, sg, nents, dir, attrs);
329 	BUG_ON(ents < 0);
330 	debug_dma_map_sg(dev, sg, nents, ents, dir);
331 
332 	return ents;
333 }
334 
dma_unmap_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)335 static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
336 				      int nents, enum dma_data_direction dir,
337 				      unsigned long attrs)
338 {
339 	const struct dma_map_ops *ops = get_dma_ops(dev);
340 
341 	BUG_ON(!valid_dma_direction(dir));
342 	debug_dma_unmap_sg(dev, sg, nents, dir);
343 	if (dma_is_direct(ops))
344 		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
345 	else if (ops->unmap_sg)
346 		ops->unmap_sg(dev, sg, nents, dir, attrs);
347 }
348 
dma_map_resource(struct device * dev,phys_addr_t phys_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)349 static inline dma_addr_t dma_map_resource(struct device *dev,
350 					  phys_addr_t phys_addr,
351 					  size_t size,
352 					  enum dma_data_direction dir,
353 					  unsigned long attrs)
354 {
355 	const struct dma_map_ops *ops = get_dma_ops(dev);
356 	dma_addr_t addr = DMA_MAPPING_ERROR;
357 
358 	BUG_ON(!valid_dma_direction(dir));
359 
360 	/* Don't allow RAM to be mapped */
361 	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
362 		return DMA_MAPPING_ERROR;
363 
364 	if (dma_is_direct(ops))
365 		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
366 	else if (ops->map_resource)
367 		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
368 
369 	debug_dma_map_resource(dev, phys_addr, size, dir, addr);
370 	return addr;
371 }
372 
dma_unmap_resource(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)373 static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
374 				      size_t size, enum dma_data_direction dir,
375 				      unsigned long attrs)
376 {
377 	const struct dma_map_ops *ops = get_dma_ops(dev);
378 
379 	BUG_ON(!valid_dma_direction(dir));
380 	if (!dma_is_direct(ops) && ops->unmap_resource)
381 		ops->unmap_resource(dev, addr, size, dir, attrs);
382 	debug_dma_unmap_resource(dev, addr, size, dir);
383 }
384 
dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)385 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
386 					   size_t size,
387 					   enum dma_data_direction dir)
388 {
389 	const struct dma_map_ops *ops = get_dma_ops(dev);
390 
391 	BUG_ON(!valid_dma_direction(dir));
392 	if (dma_is_direct(ops))
393 		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
394 	else if (ops->sync_single_for_cpu)
395 		ops->sync_single_for_cpu(dev, addr, size, dir);
396 	debug_dma_sync_single_for_cpu(dev, addr, size, dir);
397 }
398 
dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)399 static inline void dma_sync_single_for_device(struct device *dev,
400 					      dma_addr_t addr, size_t size,
401 					      enum dma_data_direction dir)
402 {
403 	const struct dma_map_ops *ops = get_dma_ops(dev);
404 
405 	BUG_ON(!valid_dma_direction(dir));
406 	if (dma_is_direct(ops))
407 		dma_direct_sync_single_for_device(dev, addr, size, dir);
408 	else if (ops->sync_single_for_device)
409 		ops->sync_single_for_device(dev, addr, size, dir);
410 	debug_dma_sync_single_for_device(dev, addr, size, dir);
411 }
412 
413 static inline void
dma_sync_sg_for_cpu(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)414 dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
415 		    int nelems, enum dma_data_direction dir)
416 {
417 	const struct dma_map_ops *ops = get_dma_ops(dev);
418 
419 	BUG_ON(!valid_dma_direction(dir));
420 	if (dma_is_direct(ops))
421 		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
422 	else if (ops->sync_sg_for_cpu)
423 		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
424 	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
425 }
426 
427 static inline void
dma_sync_sg_for_device(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)428 dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
429 		       int nelems, enum dma_data_direction dir)
430 {
431 	const struct dma_map_ops *ops = get_dma_ops(dev);
432 
433 	BUG_ON(!valid_dma_direction(dir));
434 	if (dma_is_direct(ops))
435 		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
436 	else if (ops->sync_sg_for_device)
437 		ops->sync_sg_for_device(dev, sg, nelems, dir);
438 	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
439 
440 }
441 
dma_mapping_error(struct device * dev,dma_addr_t dma_addr)442 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
443 {
444 	debug_dma_mapping_error(dev, dma_addr);
445 
446 	if (dma_addr == DMA_MAPPING_ERROR)
447 		return -ENOMEM;
448 	return 0;
449 }
450 
451 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
452 		gfp_t flag, unsigned long attrs);
453 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
454 		dma_addr_t dma_handle, unsigned long attrs);
455 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
456 		gfp_t gfp, unsigned long attrs);
457 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
458 		dma_addr_t dma_handle);
459 void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
460 		enum dma_data_direction dir);
461 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
462 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
463 		unsigned long attrs);
464 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
465 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
466 		unsigned long attrs);
467 bool dma_can_mmap(struct device *dev);
468 int dma_supported(struct device *dev, u64 mask);
469 int dma_set_mask(struct device *dev, u64 mask);
470 int dma_set_coherent_mask(struct device *dev, u64 mask);
471 u64 dma_get_required_mask(struct device *dev);
472 size_t dma_max_mapping_size(struct device *dev);
473 unsigned long dma_get_merge_boundary(struct device *dev);
474 #else /* CONFIG_HAS_DMA */
dma_map_page_attrs(struct device * dev,struct page * page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)475 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
476 		struct page *page, size_t offset, size_t size,
477 		enum dma_data_direction dir, unsigned long attrs)
478 {
479 	return DMA_MAPPING_ERROR;
480 }
dma_unmap_page_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)481 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
482 		size_t size, enum dma_data_direction dir, unsigned long attrs)
483 {
484 }
dma_map_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)485 static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
486 		int nents, enum dma_data_direction dir, unsigned long attrs)
487 {
488 	return 0;
489 }
dma_unmap_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)490 static inline void dma_unmap_sg_attrs(struct device *dev,
491 		struct scatterlist *sg, int nents, enum dma_data_direction dir,
492 		unsigned long attrs)
493 {
494 }
dma_map_resource(struct device * dev,phys_addr_t phys_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)495 static inline dma_addr_t dma_map_resource(struct device *dev,
496 		phys_addr_t phys_addr, size_t size, enum dma_data_direction dir,
497 		unsigned long attrs)
498 {
499 	return DMA_MAPPING_ERROR;
500 }
dma_unmap_resource(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)501 static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
502 		size_t size, enum dma_data_direction dir, unsigned long attrs)
503 {
504 }
dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)505 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
506 		size_t size, enum dma_data_direction dir)
507 {
508 }
dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)509 static inline void dma_sync_single_for_device(struct device *dev,
510 		dma_addr_t addr, size_t size, enum dma_data_direction dir)
511 {
512 }
dma_sync_sg_for_cpu(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)513 static inline void dma_sync_sg_for_cpu(struct device *dev,
514 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
515 {
516 }
dma_sync_sg_for_device(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)517 static inline void dma_sync_sg_for_device(struct device *dev,
518 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
519 {
520 }
dma_mapping_error(struct device * dev,dma_addr_t dma_addr)521 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
522 {
523 	return -ENOMEM;
524 }
dma_alloc_attrs(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flag,unsigned long attrs)525 static inline void *dma_alloc_attrs(struct device *dev, size_t size,
526 		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs)
527 {
528 	return NULL;
529 }
dma_free_attrs(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_handle,unsigned long attrs)530 static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
531 		dma_addr_t dma_handle, unsigned long attrs)
532 {
533 }
dmam_alloc_attrs(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp,unsigned long attrs)534 static inline void *dmam_alloc_attrs(struct device *dev, size_t size,
535 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
536 {
537 	return NULL;
538 }
dmam_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle)539 static inline void dmam_free_coherent(struct device *dev, size_t size,
540 		void *vaddr, dma_addr_t dma_handle)
541 {
542 }
dma_cache_sync(struct device * dev,void * vaddr,size_t size,enum dma_data_direction dir)543 static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
544 		enum dma_data_direction dir)
545 {
546 }
dma_get_sgtable_attrs(struct device * dev,struct sg_table * sgt,void * cpu_addr,dma_addr_t dma_addr,size_t size,unsigned long attrs)547 static inline int dma_get_sgtable_attrs(struct device *dev,
548 		struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr,
549 		size_t size, unsigned long attrs)
550 {
551 	return -ENXIO;
552 }
dma_mmap_attrs(struct device * dev,struct vm_area_struct * vma,void * cpu_addr,dma_addr_t dma_addr,size_t size,unsigned long attrs)553 static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
554 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
555 		unsigned long attrs)
556 {
557 	return -ENXIO;
558 }
dma_can_mmap(struct device * dev)559 static inline bool dma_can_mmap(struct device *dev)
560 {
561 	return false;
562 }
dma_supported(struct device * dev,u64 mask)563 static inline int dma_supported(struct device *dev, u64 mask)
564 {
565 	return 0;
566 }
dma_set_mask(struct device * dev,u64 mask)567 static inline int dma_set_mask(struct device *dev, u64 mask)
568 {
569 	return -EIO;
570 }
dma_set_coherent_mask(struct device * dev,u64 mask)571 static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
572 {
573 	return -EIO;
574 }
dma_get_required_mask(struct device * dev)575 static inline u64 dma_get_required_mask(struct device *dev)
576 {
577 	return 0;
578 }
dma_max_mapping_size(struct device * dev)579 static inline size_t dma_max_mapping_size(struct device *dev)
580 {
581 	return 0;
582 }
dma_get_merge_boundary(struct device * dev)583 static inline unsigned long dma_get_merge_boundary(struct device *dev)
584 {
585 	return 0;
586 }
587 #endif /* CONFIG_HAS_DMA */
588 
dma_map_single_attrs(struct device * dev,void * ptr,size_t size,enum dma_data_direction dir,unsigned long attrs)589 static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
590 		size_t size, enum dma_data_direction dir, unsigned long attrs)
591 {
592 	/* DMA must never operate on areas that might be remapped. */
593 	if (dev_WARN_ONCE(dev, is_vmalloc_addr(ptr),
594 			  "rejecting DMA map of vmalloc memory\n"))
595 		return DMA_MAPPING_ERROR;
596 	debug_dma_map_single(dev, ptr, size);
597 	return dma_map_page_attrs(dev, virt_to_page(ptr), offset_in_page(ptr),
598 			size, dir, attrs);
599 }
600 
dma_unmap_single_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)601 static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
602 		size_t size, enum dma_data_direction dir, unsigned long attrs)
603 {
604 	return dma_unmap_page_attrs(dev, addr, size, dir, attrs);
605 }
606 
dma_sync_single_range_for_cpu(struct device * dev,dma_addr_t addr,unsigned long offset,size_t size,enum dma_data_direction dir)607 static inline void dma_sync_single_range_for_cpu(struct device *dev,
608 		dma_addr_t addr, unsigned long offset, size_t size,
609 		enum dma_data_direction dir)
610 {
611 	return dma_sync_single_for_cpu(dev, addr + offset, size, dir);
612 }
613 
dma_sync_single_range_for_device(struct device * dev,dma_addr_t addr,unsigned long offset,size_t size,enum dma_data_direction dir)614 static inline void dma_sync_single_range_for_device(struct device *dev,
615 		dma_addr_t addr, unsigned long offset, size_t size,
616 		enum dma_data_direction dir)
617 {
618 	return dma_sync_single_for_device(dev, addr + offset, size, dir);
619 }
620 
621 /**
622  * dma_map_sgtable - Map the given buffer for DMA
623  * @dev:	The device for which to perform the DMA operation
624  * @sgt:	The sg_table object describing the buffer
625  * @dir:	DMA direction
626  * @attrs:	Optional DMA attributes for the map operation
627  *
628  * Maps a buffer described by a scatterlist stored in the given sg_table
629  * object for the @dir DMA operation by the @dev device. After success the
630  * ownership for the buffer is transferred to the DMA domain.  One has to
631  * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the
632  * ownership of the buffer back to the CPU domain before touching the
633  * buffer by the CPU.
634  *
635  * Returns 0 on success or -EINVAL on error during mapping the buffer.
636  */
dma_map_sgtable(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir,unsigned long attrs)637 static inline int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
638 		enum dma_data_direction dir, unsigned long attrs)
639 {
640 	int nents;
641 
642 	nents = dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);
643 	if (nents <= 0)
644 		return -EINVAL;
645 	sgt->nents = nents;
646 	return 0;
647 }
648 
649 /**
650  * dma_unmap_sgtable - Unmap the given buffer for DMA
651  * @dev:	The device for which to perform the DMA operation
652  * @sgt:	The sg_table object describing the buffer
653  * @dir:	DMA direction
654  * @attrs:	Optional DMA attributes for the unmap operation
655  *
656  * Unmaps a buffer described by a scatterlist stored in the given sg_table
657  * object for the @dir DMA operation by the @dev device. After this function
658  * the ownership of the buffer is transferred back to the CPU domain.
659  */
dma_unmap_sgtable(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir,unsigned long attrs)660 static inline void dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
661 		enum dma_data_direction dir, unsigned long attrs)
662 {
663 	dma_unmap_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);
664 }
665 
666 /**
667  * dma_sync_sgtable_for_cpu - Synchronize the given buffer for CPU access
668  * @dev:	The device for which to perform the DMA operation
669  * @sgt:	The sg_table object describing the buffer
670  * @dir:	DMA direction
671  *
672  * Performs the needed cache synchronization and moves the ownership of the
673  * buffer back to the CPU domain, so it is safe to perform any access to it
674  * by the CPU. Before doing any further DMA operations, one has to transfer
675  * the ownership of the buffer back to the DMA domain by calling the
676  * dma_sync_sgtable_for_device().
677  */
dma_sync_sgtable_for_cpu(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir)678 static inline void dma_sync_sgtable_for_cpu(struct device *dev,
679 		struct sg_table *sgt, enum dma_data_direction dir)
680 {
681 	dma_sync_sg_for_cpu(dev, sgt->sgl, sgt->orig_nents, dir);
682 }
683 
684 /**
685  * dma_sync_sgtable_for_device - Synchronize the given buffer for DMA
686  * @dev:	The device for which to perform the DMA operation
687  * @sgt:	The sg_table object describing the buffer
688  * @dir:	DMA direction
689  *
690  * Performs the needed cache synchronization and moves the ownership of the
691  * buffer back to the DMA domain, so it is safe to perform the DMA operation.
692  * Once finished, one has to call dma_sync_sgtable_for_cpu() or
693  * dma_unmap_sgtable().
694  */
dma_sync_sgtable_for_device(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir)695 static inline void dma_sync_sgtable_for_device(struct device *dev,
696 		struct sg_table *sgt, enum dma_data_direction dir)
697 {
698 	dma_sync_sg_for_device(dev, sgt->sgl, sgt->orig_nents, dir);
699 }
700 
701 #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
702 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
703 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
704 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
705 #define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
706 #define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
707 #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
708 #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
709 
710 extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
711 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
712 		unsigned long attrs);
713 
714 struct page **dma_common_find_pages(void *cpu_addr);
715 void *dma_common_contiguous_remap(struct page *page, size_t size,
716 			pgprot_t prot, const void *caller);
717 
718 void *dma_common_pages_remap(struct page **pages, size_t size,
719 			pgprot_t prot, const void *caller);
720 void dma_common_free_remap(void *cpu_addr, size_t size);
721 
722 bool dma_in_atomic_pool(void *start, size_t size);
723 void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags);
724 bool dma_free_from_pool(void *start, size_t size);
725 
726 int
727 dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
728 		dma_addr_t dma_addr, size_t size, unsigned long attrs);
729 
dma_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp)730 static inline void *dma_alloc_coherent(struct device *dev, size_t size,
731 		dma_addr_t *dma_handle, gfp_t gfp)
732 {
733 
734 	return dma_alloc_attrs(dev, size, dma_handle, gfp,
735 			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
736 }
737 
dma_free_coherent(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_handle)738 static inline void dma_free_coherent(struct device *dev, size_t size,
739 		void *cpu_addr, dma_addr_t dma_handle)
740 {
741 	return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
742 }
743 
744 
dma_get_mask(struct device * dev)745 static inline u64 dma_get_mask(struct device *dev)
746 {
747 	if (dev->dma_mask && *dev->dma_mask)
748 		return *dev->dma_mask;
749 	return DMA_BIT_MASK(32);
750 }
751 
752 /*
753  * Set both the DMA mask and the coherent DMA mask to the same thing.
754  * Note that we don't check the return value from dma_set_coherent_mask()
755  * as the DMA API guarantees that the coherent DMA mask can be set to
756  * the same or smaller than the streaming DMA mask.
757  */
dma_set_mask_and_coherent(struct device * dev,u64 mask)758 static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
759 {
760 	int rc = dma_set_mask(dev, mask);
761 	if (rc == 0)
762 		dma_set_coherent_mask(dev, mask);
763 	return rc;
764 }
765 
766 /*
767  * Similar to the above, except it deals with the case where the device
768  * does not have dev->dma_mask appropriately setup.
769  */
dma_coerce_mask_and_coherent(struct device * dev,u64 mask)770 static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
771 {
772 	dev->dma_mask = &dev->coherent_dma_mask;
773 	return dma_set_mask_and_coherent(dev, mask);
774 }
775 
776 /**
777  * dma_addressing_limited - return if the device is addressing limited
778  * @dev:	device to check
779  *
780  * Return %true if the devices DMA mask is too small to address all memory in
781  * the system, else %false.  Lack of addressing bits is the prime reason for
782  * bounce buffering, but might not be the only one.
783  */
dma_addressing_limited(struct device * dev)784 static inline bool dma_addressing_limited(struct device *dev)
785 {
786 	return min_not_zero(dma_get_mask(dev), dev->bus_dma_mask) <
787 			    dma_get_required_mask(dev);
788 }
789 
790 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
791 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
792 		const struct iommu_ops *iommu, bool coherent);
793 #else
arch_setup_dma_ops(struct device * dev,u64 dma_base,u64 size,const struct iommu_ops * iommu,bool coherent)794 static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
795 		u64 size, const struct iommu_ops *iommu, bool coherent)
796 {
797 }
798 #endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */
799 
800 #ifdef CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS
801 void arch_teardown_dma_ops(struct device *dev);
802 #else
arch_teardown_dma_ops(struct device * dev)803 static inline void arch_teardown_dma_ops(struct device *dev)
804 {
805 }
806 #endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */
807 
dma_get_max_seg_size(struct device * dev)808 static inline unsigned int dma_get_max_seg_size(struct device *dev)
809 {
810 	if (dev->dma_parms && dev->dma_parms->max_segment_size)
811 		return dev->dma_parms->max_segment_size;
812 	return SZ_64K;
813 }
814 
dma_set_max_seg_size(struct device * dev,unsigned int size)815 static inline int dma_set_max_seg_size(struct device *dev, unsigned int size)
816 {
817 	if (dev->dma_parms) {
818 		dev->dma_parms->max_segment_size = size;
819 		return 0;
820 	}
821 	return -EIO;
822 }
823 
dma_get_seg_boundary(struct device * dev)824 static inline unsigned long dma_get_seg_boundary(struct device *dev)
825 {
826 	if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
827 		return dev->dma_parms->segment_boundary_mask;
828 	return DMA_BIT_MASK(32);
829 }
830 
dma_set_seg_boundary(struct device * dev,unsigned long mask)831 static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask)
832 {
833 	if (dev->dma_parms) {
834 		dev->dma_parms->segment_boundary_mask = mask;
835 		return 0;
836 	}
837 	return -EIO;
838 }
839 
dma_get_cache_alignment(void)840 static inline int dma_get_cache_alignment(void)
841 {
842 #ifdef ARCH_DMA_MINALIGN
843 	return ARCH_DMA_MINALIGN;
844 #endif
845 	return 1;
846 }
847 
848 #ifdef CONFIG_DMA_DECLARE_COHERENT
849 int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
850 				dma_addr_t device_addr, size_t size);
851 #else
852 static inline int
dma_declare_coherent_memory(struct device * dev,phys_addr_t phys_addr,dma_addr_t device_addr,size_t size)853 dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
854 			    dma_addr_t device_addr, size_t size)
855 {
856 	return -ENOSYS;
857 }
858 #endif /* CONFIG_DMA_DECLARE_COHERENT */
859 
dmam_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp)860 static inline void *dmam_alloc_coherent(struct device *dev, size_t size,
861 		dma_addr_t *dma_handle, gfp_t gfp)
862 {
863 	return dmam_alloc_attrs(dev, size, dma_handle, gfp,
864 			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
865 }
866 
dma_alloc_wc(struct device * dev,size_t size,dma_addr_t * dma_addr,gfp_t gfp)867 static inline void *dma_alloc_wc(struct device *dev, size_t size,
868 				 dma_addr_t *dma_addr, gfp_t gfp)
869 {
870 	unsigned long attrs = DMA_ATTR_WRITE_COMBINE;
871 
872 	if (gfp & __GFP_NOWARN)
873 		attrs |= DMA_ATTR_NO_WARN;
874 
875 	return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);
876 }
877 
dma_free_wc(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_addr)878 static inline void dma_free_wc(struct device *dev, size_t size,
879 			       void *cpu_addr, dma_addr_t dma_addr)
880 {
881 	return dma_free_attrs(dev, size, cpu_addr, dma_addr,
882 			      DMA_ATTR_WRITE_COMBINE);
883 }
884 
dma_mmap_wc(struct device * dev,struct vm_area_struct * vma,void * cpu_addr,dma_addr_t dma_addr,size_t size)885 static inline int dma_mmap_wc(struct device *dev,
886 			      struct vm_area_struct *vma,
887 			      void *cpu_addr, dma_addr_t dma_addr,
888 			      size_t size)
889 {
890 	return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
891 			      DMA_ATTR_WRITE_COMBINE);
892 }
893 
894 #ifdef CONFIG_NEED_DMA_MAP_STATE
895 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)        dma_addr_t ADDR_NAME
896 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME)          __u32 LEN_NAME
897 #define dma_unmap_addr(PTR, ADDR_NAME)           ((PTR)->ADDR_NAME)
898 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  (((PTR)->ADDR_NAME) = (VAL))
899 #define dma_unmap_len(PTR, LEN_NAME)             ((PTR)->LEN_NAME)
900 #define dma_unmap_len_set(PTR, LEN_NAME, VAL)    (((PTR)->LEN_NAME) = (VAL))
901 #else
902 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
903 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
904 #define dma_unmap_addr(PTR, ADDR_NAME)           (0)
905 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  do { } while (0)
906 #define dma_unmap_len(PTR, LEN_NAME)             (0)
907 #define dma_unmap_len_set(PTR, LEN_NAME, VAL)    do { } while (0)
908 #endif
909 
910 #endif
911