115 static inline void dma_chan_get(struct dma_chan *chan)  in dma_chan_get()  argument
117 	if (unlikely(chan->slow_ref))  in dma_chan_get()
118 		kref_get(&chan->refcount);  in dma_chan_get()
120 		local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount));  in dma_chan_get()
125 static inline void dma_chan_put(struct dma_chan *chan)  in dma_chan_put()  argument
127 	if (unlikely(chan->slow_ref))  in dma_chan_put()
128 		kref_put(&chan->refcount, dma_chan_cleanup);  in dma_chan_put()
130 		local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount));  in dma_chan_put()
139 		struct dma_chan *chan, enum dma_event event);
188 	int (*device_alloc_chan_resources)(struct dma_chan *chan);
189 	void (*device_free_chan_resources)(struct dma_chan *chan);
190 	dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan,
192 	dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan,
195 	dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan,
198 	enum dma_status (*device_memcpy_complete)(struct dma_chan *chan,
201 	void (*device_memcpy_issue_pending)(struct dma_chan *chan);
223 static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,  in dma_async_memcpy_buf_to_buf()  argument
227 	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;  in dma_async_memcpy_buf_to_buf()
228 	per_cpu_ptr(chan->local, cpu)->memcpy_count++;  in dma_async_memcpy_buf_to_buf()
231 	return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len);  in dma_async_memcpy_buf_to_buf()
247 static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,  in dma_async_memcpy_buf_to_pg()  argument
251 	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;  in dma_async_memcpy_buf_to_pg()
252 	per_cpu_ptr(chan->local, cpu)->memcpy_count++;  in dma_async_memcpy_buf_to_pg()
255 	return chan->device->device_memcpy_buf_to_pg(chan, page, offset,  in dma_async_memcpy_buf_to_pg()
273 static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,  in dma_async_memcpy_pg_to_pg()  argument
278 	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;  in dma_async_memcpy_pg_to_pg()
279 	per_cpu_ptr(chan->local, cpu)->memcpy_count++;  in dma_async_memcpy_pg_to_pg()
282 	return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off,  in dma_async_memcpy_pg_to_pg()
293 static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan)  in dma_async_memcpy_issue_pending()  argument
295 	return chan->device->device_memcpy_issue_pending(chan);  in dma_async_memcpy_issue_pending()
309 static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan,  in dma_async_memcpy_complete()  argument
312 	return chan->device->device_memcpy_complete(chan, cookie, last, used);  in dma_async_memcpy_complete()
359 dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
361 dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,