• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Framework for buffer objects that can be shared across devices/subsystems.
4  *
5  * Copyright(C) 2011 Linaro Limited. All rights reserved.
6  * Author: Sumit Semwal <sumit.semwal@ti.com>
7  *
8  * Many thanks to linaro-mm-sig list, and specially
9  * Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and
10  * Daniel Vetter <daniel@ffwll.ch> for their support in creation and
11  * refining of this idea.
12  */
13 
14 #include <linux/fs.h>
15 #include <linux/slab.h>
16 #include <linux/dma-buf.h>
17 #include <linux/dma-fence.h>
18 #include <linux/anon_inodes.h>
19 #include <linux/export.h>
20 #include <linux/debugfs.h>
21 #include <linux/module.h>
22 #include <linux/seq_file.h>
23 #include <linux/poll.h>
24 #include <linux/dma-resv.h>
25 #include <linux/mm.h>
26 #include <linux/mount.h>
27 #include <linux/pseudo_fs.h>
28 
29 #include <uapi/linux/dma-buf.h>
30 #include <uapi/linux/magic.h>
31 
32 #include "dma-buf-sysfs-stats.h"
33 #include "dma-buf-process-info.h"
34 
35 static inline int is_dma_buf_file(struct file *);
36 
37 struct dma_buf_list {
38 	struct list_head head;
39 	struct mutex lock;
40 };
41 
42 static struct dma_buf_list db_list;
43 
dmabuffs_dname(struct dentry * dentry,char * buffer,int buflen)44 static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
45 {
46 	struct dma_buf *dmabuf;
47 	char name[DMA_BUF_NAME_LEN];
48 	size_t ret = 0;
49 
50 	dmabuf = dentry->d_fsdata;
51 	spin_lock(&dmabuf->name_lock);
52 	if (dmabuf->name)
53 		ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);
54 	spin_unlock(&dmabuf->name_lock);
55 
56 	return dynamic_dname(dentry, buffer, buflen, "/%s:%s",
57 			     dentry->d_name.name, ret > 0 ? name : "");
58 }
59 
dma_buf_release(struct dentry * dentry)60 static void dma_buf_release(struct dentry *dentry)
61 {
62 	struct dma_buf *dmabuf;
63 
64 	dmabuf = dentry->d_fsdata;
65 	if (unlikely(!dmabuf))
66 		return;
67 
68 	BUG_ON(dmabuf->vmapping_counter);
69 
70 	/*
71 	 * Any fences that a dma-buf poll can wait on should be signaled
72 	 * before releasing dma-buf. This is the responsibility of each
73 	 * driver that uses the reservation objects.
74 	 *
75 	 * If you hit this BUG() it means someone dropped their ref to the
76 	 * dma-buf while still having pending operation to the buffer.
77 	 */
78 	BUG_ON(dmabuf->cb_shared.active || dmabuf->cb_excl.active);
79 
80 	dmabuf->ops->release(dmabuf);
81 
82 	if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])
83 		dma_resv_fini(dmabuf->resv);
84 
85 	WARN_ON(!list_empty(&dmabuf->attachments));
86 	dma_buf_stats_teardown(dmabuf);
87 	module_put(dmabuf->owner);
88 	kfree(dmabuf->name);
89 	kfree(dmabuf);
90 }
91 
dma_buf_file_release(struct inode * inode,struct file * file)92 static int dma_buf_file_release(struct inode *inode, struct file *file)
93 {
94 	struct dma_buf *dmabuf;
95 
96 	if (!is_dma_buf_file(file))
97 		return -EINVAL;
98 
99 	dmabuf = file->private_data;
100 
101 	mutex_lock(&db_list.lock);
102 	list_del(&dmabuf->list_node);
103 	mutex_unlock(&db_list.lock);
104 
105 	return 0;
106 }
107 
108 static const struct dentry_operations dma_buf_dentry_ops = {
109 	.d_dname = dmabuffs_dname,
110 	.d_release = dma_buf_release,
111 };
112 
113 static struct vfsmount *dma_buf_mnt;
114 
dma_buf_fs_init_context(struct fs_context * fc)115 static int dma_buf_fs_init_context(struct fs_context *fc)
116 {
117 	struct pseudo_fs_context *ctx;
118 
119 	ctx = init_pseudo(fc, DMA_BUF_MAGIC);
120 	if (!ctx)
121 		return -ENOMEM;
122 	ctx->dops = &dma_buf_dentry_ops;
123 	return 0;
124 }
125 
126 static struct file_system_type dma_buf_fs_type = {
127 	.name = "dmabuf",
128 	.init_fs_context = dma_buf_fs_init_context,
129 	.kill_sb = kill_anon_super,
130 };
131 
dma_buf_mmap_internal(struct file * file,struct vm_area_struct * vma)132 static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
133 {
134 	struct dma_buf *dmabuf;
135 
136 	if (!is_dma_buf_file(file))
137 		return -EINVAL;
138 
139 	dmabuf = file->private_data;
140 
141 	/* check if buffer supports mmap */
142 	if (!dmabuf->ops->mmap)
143 		return -EINVAL;
144 
145 	/* check for overflowing the buffer's size */
146 	if (vma->vm_pgoff + vma_pages(vma) >
147 	    dmabuf->size >> PAGE_SHIFT)
148 		return -EINVAL;
149 
150 	return dmabuf->ops->mmap(dmabuf, vma);
151 }
152 
dma_buf_llseek(struct file * file,loff_t offset,int whence)153 static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
154 {
155 	struct dma_buf *dmabuf;
156 	loff_t base;
157 
158 	if (!is_dma_buf_file(file))
159 		return -EBADF;
160 
161 	dmabuf = file->private_data;
162 
163 	/* only support discovering the end of the buffer,
164 	   but also allow SEEK_SET to maintain the idiomatic
165 	   SEEK_END(0), SEEK_CUR(0) pattern */
166 	if (whence == SEEK_END)
167 		base = dmabuf->size;
168 	else if (whence == SEEK_SET)
169 		base = 0;
170 	else
171 		return -EINVAL;
172 
173 	if (offset != 0)
174 		return -EINVAL;
175 
176 	return base + offset;
177 }
178 
179 /**
180  * DOC: implicit fence polling
181  *
182  * To support cross-device and cross-driver synchronization of buffer access
183  * implicit fences (represented internally in the kernel with &struct dma_fence)
184  * can be attached to a &dma_buf. The glue for that and a few related things are
185  * provided in the &dma_resv structure.
186  *
187  * Userspace can query the state of these implicitly tracked fences using poll()
188  * and related system calls:
189  *
190  * - Checking for EPOLLIN, i.e. read access, can be use to query the state of the
191  *   most recent write or exclusive fence.
192  *
193  * - Checking for EPOLLOUT, i.e. write access, can be used to query the state of
194  *   all attached fences, shared and exclusive ones.
195  *
196  * Note that this only signals the completion of the respective fences, i.e. the
197  * DMA transfers are complete. Cache flushing and any other necessary
198  * preparations before CPU access can begin still need to happen.
199  */
200 
dma_buf_poll_cb(struct dma_fence * fence,struct dma_fence_cb * cb)201 static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
202 {
203 	struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb;
204 	unsigned long flags;
205 
206 	spin_lock_irqsave(&dcb->poll->lock, flags);
207 	wake_up_locked_poll(dcb->poll, dcb->active);
208 	dcb->active = 0;
209 	spin_unlock_irqrestore(&dcb->poll->lock, flags);
210 }
211 
dma_buf_poll(struct file * file,poll_table * poll)212 static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
213 {
214 	struct dma_buf *dmabuf;
215 	struct dma_resv *resv;
216 	struct dma_resv_list *fobj;
217 	struct dma_fence *fence_excl;
218 	__poll_t events;
219 	unsigned shared_count, seq;
220 
221 	dmabuf = file->private_data;
222 	if (!dmabuf || !dmabuf->resv)
223 		return EPOLLERR;
224 
225 	resv = dmabuf->resv;
226 
227 	poll_wait(file, &dmabuf->poll, poll);
228 
229 	events = poll_requested_events(poll) & (EPOLLIN | EPOLLOUT);
230 	if (!events)
231 		return 0;
232 
233 retry:
234 	seq = read_seqcount_begin(&resv->seq);
235 	rcu_read_lock();
236 
237 	fobj = rcu_dereference(resv->fence);
238 	if (fobj)
239 		shared_count = fobj->shared_count;
240 	else
241 		shared_count = 0;
242 	fence_excl = rcu_dereference(resv->fence_excl);
243 	if (read_seqcount_retry(&resv->seq, seq)) {
244 		rcu_read_unlock();
245 		goto retry;
246 	}
247 
248 	if (fence_excl && (!(events & EPOLLOUT) || shared_count == 0)) {
249 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;
250 		__poll_t pevents = EPOLLIN;
251 
252 		if (shared_count == 0)
253 			pevents |= EPOLLOUT;
254 
255 		spin_lock_irq(&dmabuf->poll.lock);
256 		if (dcb->active) {
257 			dcb->active |= pevents;
258 			events &= ~pevents;
259 		} else
260 			dcb->active = pevents;
261 		spin_unlock_irq(&dmabuf->poll.lock);
262 
263 		if (events & pevents) {
264 			if (!dma_fence_get_rcu(fence_excl)) {
265 				/* force a recheck */
266 				events &= ~pevents;
267 				dma_buf_poll_cb(NULL, &dcb->cb);
268 			} else if (!dma_fence_add_callback(fence_excl, &dcb->cb,
269 							   dma_buf_poll_cb)) {
270 				events &= ~pevents;
271 				dma_fence_put(fence_excl);
272 			} else {
273 				/*
274 				 * No callback queued, wake up any additional
275 				 * waiters.
276 				 */
277 				dma_fence_put(fence_excl);
278 				dma_buf_poll_cb(NULL, &dcb->cb);
279 			}
280 		}
281 	}
282 
283 	if ((events & EPOLLOUT) && shared_count > 0) {
284 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared;
285 		int i;
286 
287 		/* Only queue a new callback if no event has fired yet */
288 		spin_lock_irq(&dmabuf->poll.lock);
289 		if (dcb->active)
290 			events &= ~EPOLLOUT;
291 		else
292 			dcb->active = EPOLLOUT;
293 		spin_unlock_irq(&dmabuf->poll.lock);
294 
295 		if (!(events & EPOLLOUT))
296 			goto out;
297 
298 		for (i = 0; i < shared_count; ++i) {
299 			struct dma_fence *fence = rcu_dereference(fobj->shared[i]);
300 
301 			if (!dma_fence_get_rcu(fence)) {
302 				/*
303 				 * fence refcount dropped to zero, this means
304 				 * that fobj has been freed
305 				 *
306 				 * call dma_buf_poll_cb and force a recheck!
307 				 */
308 				events &= ~EPOLLOUT;
309 				dma_buf_poll_cb(NULL, &dcb->cb);
310 				break;
311 			}
312 			if (!dma_fence_add_callback(fence, &dcb->cb,
313 						    dma_buf_poll_cb)) {
314 				dma_fence_put(fence);
315 				events &= ~EPOLLOUT;
316 				break;
317 			}
318 			dma_fence_put(fence);
319 		}
320 
321 		/* No callback queued, wake up any additional waiters. */
322 		if (i == shared_count)
323 			dma_buf_poll_cb(NULL, &dcb->cb);
324 	}
325 
326 out:
327 	rcu_read_unlock();
328 	return events;
329 }
330 
331 /**
332  * dma_buf_set_name - Set a name to a specific dma_buf to track the usage.
333  * The name of the dma-buf buffer can only be set when the dma-buf is not
334  * attached to any devices. It could theoritically support changing the
335  * name of the dma-buf if the same piece of memory is used for multiple
336  * purpose between different devices.
337  *
338  * @dmabuf: [in]     dmabuf buffer that will be renamed.
339  * @buf:    [in]     A piece of userspace memory that contains the name of
340  *                   the dma-buf.
341  *
342  * Returns 0 on success. If the dma-buf buffer is already attached to
343  * devices, return -EBUSY.
344  *
345  */
dma_buf_set_name(struct dma_buf * dmabuf,const char __user * buf)346 static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)
347 {
348 	char *name = strndup_user(buf, DMA_BUF_NAME_LEN);
349 	long ret = 0;
350 
351 	if (IS_ERR(name))
352 		return PTR_ERR(name);
353 
354 	dma_resv_lock(dmabuf->resv, NULL);
355 	if (!list_empty(&dmabuf->attachments)) {
356 		ret = -EBUSY;
357 		kfree(name);
358 		goto out_unlock;
359 	}
360 	spin_lock(&dmabuf->name_lock);
361 	kfree(dmabuf->name);
362 	dmabuf->name = name;
363 	spin_unlock(&dmabuf->name_lock);
364 
365 out_unlock:
366 	dma_resv_unlock(dmabuf->resv);
367 	return ret;
368 }
369 
dma_buf_ioctl(struct file * file,unsigned int cmd,unsigned long arg)370 static long dma_buf_ioctl(struct file *file,
371 			  unsigned int cmd, unsigned long arg)
372 {
373 	struct dma_buf *dmabuf;
374 	struct dma_buf_sync sync;
375 	enum dma_data_direction direction;
376 	int ret;
377 
378 	dmabuf = file->private_data;
379 
380 	switch (cmd) {
381 	case DMA_BUF_IOCTL_SYNC:
382 		if (copy_from_user(&sync, (void __user *) arg, sizeof(sync)))
383 			return -EFAULT;
384 
385 		if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK)
386 			return -EINVAL;
387 
388 		switch (sync.flags & DMA_BUF_SYNC_RW) {
389 		case DMA_BUF_SYNC_READ:
390 			direction = DMA_FROM_DEVICE;
391 			break;
392 		case DMA_BUF_SYNC_WRITE:
393 			direction = DMA_TO_DEVICE;
394 			break;
395 		case DMA_BUF_SYNC_RW:
396 			direction = DMA_BIDIRECTIONAL;
397 			break;
398 		default:
399 			return -EINVAL;
400 		}
401 
402 		if (sync.flags & DMA_BUF_SYNC_END)
403 			ret = dma_buf_end_cpu_access(dmabuf, direction);
404 		else
405 			ret = dma_buf_begin_cpu_access(dmabuf, direction);
406 
407 		return ret;
408 
409 	case DMA_BUF_SET_NAME_A:
410 	case DMA_BUF_SET_NAME_B:
411 		return dma_buf_set_name(dmabuf, (const char __user *)arg);
412 
413 	default:
414 		return -ENOTTY;
415 	}
416 }
417 
dma_buf_show_fdinfo(struct seq_file * m,struct file * file)418 static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file)
419 {
420 	struct dma_buf *dmabuf = file->private_data;
421 
422 	seq_printf(m, "size:\t%zu\n", dmabuf->size);
423 	/* Don't count the temporary reference taken inside procfs seq_show */
424 	seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);
425 	seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);
426 	spin_lock(&dmabuf->name_lock);
427 	if (dmabuf->name)
428 		seq_printf(m, "name:\t%s\n", dmabuf->name);
429 	spin_unlock(&dmabuf->name_lock);
430 }
431 
432 static const struct file_operations dma_buf_fops = {
433 	.release	= dma_buf_file_release,
434 	.mmap		= dma_buf_mmap_internal,
435 	.llseek		= dma_buf_llseek,
436 	.poll		= dma_buf_poll,
437 	.unlocked_ioctl	= dma_buf_ioctl,
438 	.compat_ioctl	= compat_ptr_ioctl,
439 	.show_fdinfo	= dma_buf_show_fdinfo,
440 };
441 
442 /*
443  * is_dma_buf_file - Check if struct file* is associated with dma_buf
444  */
is_dma_buf_file(struct file * file)445 static inline int is_dma_buf_file(struct file *file)
446 {
447 	return file->f_op == &dma_buf_fops;
448 }
449 
dma_buf_getfile(struct dma_buf * dmabuf,int flags)450 static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
451 {
452 	struct file *file;
453 	struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
454 
455 	if (IS_ERR(inode))
456 		return ERR_CAST(inode);
457 
458 	inode->i_size = dmabuf->size;
459 	inode_set_bytes(inode, dmabuf->size);
460 
461 	file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
462 				 flags, &dma_buf_fops);
463 	if (IS_ERR(file))
464 		goto err_alloc_file;
465 	file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
466 	file->private_data = dmabuf;
467 	file->f_path.dentry->d_fsdata = dmabuf;
468 
469 	return file;
470 
471 err_alloc_file:
472 	iput(inode);
473 	return file;
474 }
475 
476 /**
477  * DOC: dma buf device access
478  *
479  * For device DMA access to a shared DMA buffer the usual sequence of operations
480  * is fairly simple:
481  *
482  * 1. The exporter defines his exporter instance using
483  *    DEFINE_DMA_BUF_EXPORT_INFO() and calls dma_buf_export() to wrap a private
484  *    buffer object into a &dma_buf. It then exports that &dma_buf to userspace
485  *    as a file descriptor by calling dma_buf_fd().
486  *
487  * 2. Userspace passes this file-descriptors to all drivers it wants this buffer
488  *    to share with: First the filedescriptor is converted to a &dma_buf using
489  *    dma_buf_get(). Then the buffer is attached to the device using
490  *    dma_buf_attach().
491  *
492  *    Up to this stage the exporter is still free to migrate or reallocate the
493  *    backing storage.
494  *
495  * 3. Once the buffer is attached to all devices userspace can initiate DMA
496  *    access to the shared buffer. In the kernel this is done by calling
497  *    dma_buf_map_attachment() and dma_buf_unmap_attachment().
498  *
499  * 4. Once a driver is done with a shared buffer it needs to call
500  *    dma_buf_detach() (after cleaning up any mappings) and then release the
501  *    reference acquired with dma_buf_get by calling dma_buf_put().
502  *
503  * For the detailed semantics exporters are expected to implement see
504  * &dma_buf_ops.
505  */
506 
507 /**
508  * dma_buf_export - Creates a new dma_buf, and associates an anon file
509  * with this buffer, so it can be exported.
510  * Also connect the allocator specific data and ops to the buffer.
511  * Additionally, provide a name string for exporter; useful in debugging.
512  *
513  * @exp_info:	[in]	holds all the export related information provided
514  *			by the exporter. see &struct dma_buf_export_info
515  *			for further details.
516  *
517  * Returns, on success, a newly created dma_buf object, which wraps the
518  * supplied private data and operations for dma_buf_ops. On either missing
519  * ops, or error in allocating struct dma_buf, will return negative error.
520  *
521  * For most cases the easiest way to create @exp_info is through the
522  * %DEFINE_DMA_BUF_EXPORT_INFO macro.
523  */
dma_buf_export(const struct dma_buf_export_info * exp_info)524 struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
525 {
526 	struct dma_buf *dmabuf;
527 	struct dma_resv *resv = exp_info->resv;
528 	struct file *file;
529 	size_t alloc_size = sizeof(struct dma_buf);
530 	int ret;
531 
532 	if (!exp_info->resv)
533 		alloc_size += sizeof(struct dma_resv);
534 	else
535 		/* prevent &dma_buf[1] == dma_buf->resv */
536 		alloc_size += 1;
537 
538 	if (WARN_ON(!exp_info->priv
539 			  || !exp_info->ops
540 			  || !exp_info->ops->map_dma_buf
541 			  || !exp_info->ops->unmap_dma_buf
542 			  || !exp_info->ops->release)) {
543 		return ERR_PTR(-EINVAL);
544 	}
545 
546 	if (WARN_ON(exp_info->ops->cache_sgt_mapping &&
547 		    (exp_info->ops->pin || exp_info->ops->unpin)))
548 		return ERR_PTR(-EINVAL);
549 
550 	if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin))
551 		return ERR_PTR(-EINVAL);
552 
553 	if (!try_module_get(exp_info->owner))
554 		return ERR_PTR(-ENOENT);
555 
556 	dmabuf = kzalloc(alloc_size, GFP_KERNEL);
557 	if (!dmabuf) {
558 		ret = -ENOMEM;
559 		goto err_module;
560 	}
561 
562 	dmabuf->priv = exp_info->priv;
563 	dmabuf->ops = exp_info->ops;
564 	dmabuf->size = exp_info->size;
565 	dmabuf->exp_name = exp_info->exp_name;
566 	dmabuf->owner = exp_info->owner;
567 	spin_lock_init(&dmabuf->name_lock);
568 	init_waitqueue_head(&dmabuf->poll);
569 	dmabuf->cb_excl.poll = dmabuf->cb_shared.poll = &dmabuf->poll;
570 	dmabuf->cb_excl.active = dmabuf->cb_shared.active = 0;
571 
572 	if (!resv) {
573 		resv = (struct dma_resv *)&dmabuf[1];
574 		dma_resv_init(resv);
575 	}
576 	dmabuf->resv = resv;
577 
578 	file = dma_buf_getfile(dmabuf, exp_info->flags);
579 	if (IS_ERR(file)) {
580 		ret = PTR_ERR(file);
581 		goto err_dmabuf;
582 	}
583 
584 	file->f_mode |= FMODE_LSEEK;
585 	dmabuf->file = file;
586 
587 	ret = dma_buf_stats_setup(dmabuf);
588 	if (ret)
589 		goto err_sysfs;
590 
591 	mutex_init(&dmabuf->lock);
592 	INIT_LIST_HEAD(&dmabuf->attachments);
593 
594 	mutex_lock(&db_list.lock);
595 	list_add(&dmabuf->list_node, &db_list.head);
596 	mutex_unlock(&db_list.lock);
597 
598 	init_dma_buf_task_info(dmabuf);
599 	return dmabuf;
600 
601 err_sysfs:
602 	/*
603 	 * Set file->f_path.dentry->d_fsdata to NULL so that when
604 	 * dma_buf_release() gets invoked by dentry_ops, it exits
605 	 * early before calling the release() dma_buf op.
606 	 */
607 	file->f_path.dentry->d_fsdata = NULL;
608 	fput(file);
609 err_dmabuf:
610 	kfree(dmabuf);
611 err_module:
612 	module_put(exp_info->owner);
613 	return ERR_PTR(ret);
614 }
615 EXPORT_SYMBOL_GPL(dma_buf_export);
616 
617 /**
618  * dma_buf_fd - returns a file descriptor for the given dma_buf
619  * @dmabuf:	[in]	pointer to dma_buf for which fd is required.
620  * @flags:      [in]    flags to give to fd
621  *
622  * On success, returns an associated 'fd'. Else, returns error.
623  */
dma_buf_fd(struct dma_buf * dmabuf,int flags)624 int dma_buf_fd(struct dma_buf *dmabuf, int flags)
625 {
626 	int fd;
627 
628 	if (!dmabuf || !dmabuf->file)
629 		return -EINVAL;
630 
631 	fd = get_unused_fd_flags(flags);
632 	if (fd < 0)
633 		return fd;
634 
635 	fd_install(fd, dmabuf->file);
636 
637 	return fd;
638 }
639 EXPORT_SYMBOL_GPL(dma_buf_fd);
640 
641 /**
642  * dma_buf_get - returns the dma_buf structure related to an fd
643  * @fd:	[in]	fd associated with the dma_buf to be returned
644  *
645  * On success, returns the dma_buf structure associated with an fd; uses
646  * file's refcounting done by fget to increase refcount. returns ERR_PTR
647  * otherwise.
648  */
dma_buf_get(int fd)649 struct dma_buf *dma_buf_get(int fd)
650 {
651 	struct file *file;
652 
653 	file = fget(fd);
654 
655 	if (!file)
656 		return ERR_PTR(-EBADF);
657 
658 	if (!is_dma_buf_file(file)) {
659 		fput(file);
660 		return ERR_PTR(-EINVAL);
661 	}
662 
663 	return file->private_data;
664 }
665 EXPORT_SYMBOL_GPL(dma_buf_get);
666 
667 /**
668  * dma_buf_put - decreases refcount of the buffer
669  * @dmabuf:	[in]	buffer to reduce refcount of
670  *
671  * Uses file's refcounting done implicitly by fput().
672  *
673  * If, as a result of this call, the refcount becomes 0, the 'release' file
674  * operation related to this fd is called. It calls &dma_buf_ops.release vfunc
675  * in turn, and frees the memory allocated for dmabuf when exported.
676  */
dma_buf_put(struct dma_buf * dmabuf)677 void dma_buf_put(struct dma_buf *dmabuf)
678 {
679 	if (WARN_ON(!dmabuf || !dmabuf->file))
680 		return;
681 
682 	fput(dmabuf->file);
683 }
684 EXPORT_SYMBOL_GPL(dma_buf_put);
685 
686 /**
687  * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list; optionally,
688  * calls attach() of dma_buf_ops to allow device-specific attach functionality
689  * @dmabuf:		[in]	buffer to attach device to.
690  * @dev:		[in]	device to be attached.
691  * @importer_ops:	[in]	importer operations for the attachment
692  * @importer_priv:	[in]	importer private pointer for the attachment
693  *
694  * Returns struct dma_buf_attachment pointer for this attachment. Attachments
695  * must be cleaned up by calling dma_buf_detach().
696  *
697  * Returns:
698  *
699  * A pointer to newly created &dma_buf_attachment on success, or a negative
700  * error code wrapped into a pointer on failure.
701  *
702  * Note that this can fail if the backing storage of @dmabuf is in a place not
703  * accessible to @dev, and cannot be moved to a more suitable place. This is
704  * indicated with the error code -EBUSY.
705  */
706 struct dma_buf_attachment *
dma_buf_dynamic_attach(struct dma_buf * dmabuf,struct device * dev,const struct dma_buf_attach_ops * importer_ops,void * importer_priv)707 dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
708 		       const struct dma_buf_attach_ops *importer_ops,
709 		       void *importer_priv)
710 {
711 	struct dma_buf_attachment *attach;
712 	int ret;
713 
714 	if (WARN_ON(!dmabuf || !dev))
715 		return ERR_PTR(-EINVAL);
716 
717 	if (WARN_ON(importer_ops && !importer_ops->move_notify))
718 		return ERR_PTR(-EINVAL);
719 
720 	attach = kzalloc(sizeof(*attach), GFP_KERNEL);
721 	if (!attach)
722 		return ERR_PTR(-ENOMEM);
723 
724 	attach->dev = dev;
725 	attach->dmabuf = dmabuf;
726 	if (importer_ops)
727 		attach->peer2peer = importer_ops->allow_peer2peer;
728 	attach->importer_ops = importer_ops;
729 	attach->importer_priv = importer_priv;
730 
731 	if (dmabuf->ops->attach) {
732 		ret = dmabuf->ops->attach(dmabuf, attach);
733 		if (ret)
734 			goto err_attach;
735 	}
736 	dma_resv_lock(dmabuf->resv, NULL);
737 	list_add(&attach->node, &dmabuf->attachments);
738 	dma_resv_unlock(dmabuf->resv);
739 
740 	/* When either the importer or the exporter can't handle dynamic
741 	 * mappings we cache the mapping here to avoid issues with the
742 	 * reservation object lock.
743 	 */
744 	if (dma_buf_attachment_is_dynamic(attach) !=
745 	    dma_buf_is_dynamic(dmabuf)) {
746 		struct sg_table *sgt;
747 
748 		if (dma_buf_is_dynamic(attach->dmabuf)) {
749 			dma_resv_lock(attach->dmabuf->resv, NULL);
750 			ret = dma_buf_pin(attach);
751 			if (ret)
752 				goto err_unlock;
753 		}
754 
755 		sgt = dmabuf->ops->map_dma_buf(attach, DMA_BIDIRECTIONAL);
756 		if (!sgt)
757 			sgt = ERR_PTR(-ENOMEM);
758 		if (IS_ERR(sgt)) {
759 			ret = PTR_ERR(sgt);
760 			goto err_unpin;
761 		}
762 		if (dma_buf_is_dynamic(attach->dmabuf))
763 			dma_resv_unlock(attach->dmabuf->resv);
764 		attach->sgt = sgt;
765 		attach->dir = DMA_BIDIRECTIONAL;
766 	}
767 
768 	return attach;
769 
770 err_attach:
771 	kfree(attach);
772 	return ERR_PTR(ret);
773 
774 err_unpin:
775 	if (dma_buf_is_dynamic(attach->dmabuf))
776 		dma_buf_unpin(attach);
777 
778 err_unlock:
779 	if (dma_buf_is_dynamic(attach->dmabuf))
780 		dma_resv_unlock(attach->dmabuf->resv);
781 
782 	dma_buf_detach(dmabuf, attach);
783 	return ERR_PTR(ret);
784 }
785 EXPORT_SYMBOL_GPL(dma_buf_dynamic_attach);
786 
787 /**
788  * dma_buf_attach - Wrapper for dma_buf_dynamic_attach
789  * @dmabuf:	[in]	buffer to attach device to.
790  * @dev:	[in]	device to be attached.
791  *
792  * Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static
793  * mapping.
794  */
dma_buf_attach(struct dma_buf * dmabuf,struct device * dev)795 struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
796 					  struct device *dev)
797 {
798 	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);
799 }
800 EXPORT_SYMBOL_GPL(dma_buf_attach);
801 
802 /**
803  * dma_buf_detach - Remove the given attachment from dmabuf's attachments list;
804  * optionally calls detach() of dma_buf_ops for device-specific detach
805  * @dmabuf:	[in]	buffer to detach from.
806  * @attach:	[in]	attachment to be detached; is free'd after this call.
807  *
808  * Clean up a device attachment obtained by calling dma_buf_attach().
809  */
dma_buf_detach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)810 void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
811 {
812 	if (WARN_ON(!dmabuf || !attach))
813 		return;
814 
815 	if (attach->sgt) {
816 		if (dma_buf_is_dynamic(attach->dmabuf))
817 			dma_resv_lock(attach->dmabuf->resv, NULL);
818 
819 		dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir);
820 
821 		if (dma_buf_is_dynamic(attach->dmabuf)) {
822 			dma_buf_unpin(attach);
823 			dma_resv_unlock(attach->dmabuf->resv);
824 		}
825 	}
826 
827 	dma_resv_lock(dmabuf->resv, NULL);
828 	list_del(&attach->node);
829 	dma_resv_unlock(dmabuf->resv);
830 	if (dmabuf->ops->detach)
831 		dmabuf->ops->detach(dmabuf, attach);
832 
833 	kfree(attach);
834 }
835 EXPORT_SYMBOL_GPL(dma_buf_detach);
836 
837 /**
838  * dma_buf_pin - Lock down the DMA-buf
839  *
840  * @attach:	[in]	attachment which should be pinned
841  *
842  * Returns:
843  * 0 on success, negative error code on failure.
844  */
dma_buf_pin(struct dma_buf_attachment * attach)845 int dma_buf_pin(struct dma_buf_attachment *attach)
846 {
847 	struct dma_buf *dmabuf = attach->dmabuf;
848 	int ret = 0;
849 
850 	dma_resv_assert_held(dmabuf->resv);
851 
852 	if (dmabuf->ops->pin)
853 		ret = dmabuf->ops->pin(attach);
854 
855 	return ret;
856 }
857 EXPORT_SYMBOL_GPL(dma_buf_pin);
858 
859 /**
860  * dma_buf_unpin - Remove lock from DMA-buf
861  *
862  * @attach:	[in]	attachment which should be unpinned
863  */
dma_buf_unpin(struct dma_buf_attachment * attach)864 void dma_buf_unpin(struct dma_buf_attachment *attach)
865 {
866 	struct dma_buf *dmabuf = attach->dmabuf;
867 
868 	dma_resv_assert_held(dmabuf->resv);
869 
870 	if (dmabuf->ops->unpin)
871 		dmabuf->ops->unpin(attach);
872 }
873 EXPORT_SYMBOL_GPL(dma_buf_unpin);
874 
875 /**
876  * dma_buf_map_attachment - Returns the scatterlist table of the attachment;
877  * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
878  * dma_buf_ops.
879  * @attach:	[in]	attachment whose scatterlist is to be returned
880  * @direction:	[in]	direction of DMA transfer
881  *
882  * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
883  * on error. May return -EINTR if it is interrupted by a signal.
884  *
885  * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
886  * the underlying backing storage is pinned for as long as a mapping exists,
887  * therefore users/importers should not hold onto a mapping for undue amounts of
888  * time.
889  */
dma_buf_map_attachment(struct dma_buf_attachment * attach,enum dma_data_direction direction)890 struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
891 					enum dma_data_direction direction)
892 {
893 	struct sg_table *sg_table;
894 	int r;
895 
896 	might_sleep();
897 
898 	if (WARN_ON(!attach || !attach->dmabuf))
899 		return ERR_PTR(-EINVAL);
900 
901 	if (dma_buf_attachment_is_dynamic(attach))
902 		dma_resv_assert_held(attach->dmabuf->resv);
903 
904 	if (attach->sgt) {
905 		/*
906 		 * Two mappings with different directions for the same
907 		 * attachment are not allowed.
908 		 */
909 		if (attach->dir != direction &&
910 		    attach->dir != DMA_BIDIRECTIONAL)
911 			return ERR_PTR(-EBUSY);
912 
913 		return attach->sgt;
914 	}
915 
916 	if (dma_buf_is_dynamic(attach->dmabuf)) {
917 		dma_resv_assert_held(attach->dmabuf->resv);
918 		if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) {
919 			r = dma_buf_pin(attach);
920 			if (r)
921 				return ERR_PTR(r);
922 		}
923 	}
924 
925 	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
926 	if (!sg_table)
927 		sg_table = ERR_PTR(-ENOMEM);
928 
929 	if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) &&
930 	     !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY))
931 		dma_buf_unpin(attach);
932 
933 	if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {
934 		attach->sgt = sg_table;
935 		attach->dir = direction;
936 	}
937 
938 	return sg_table;
939 }
940 EXPORT_SYMBOL_GPL(dma_buf_map_attachment);
941 
942 /**
943  * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might
944  * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
945  * dma_buf_ops.
946  * @attach:	[in]	attachment to unmap buffer from
947  * @sg_table:	[in]	scatterlist info of the buffer to unmap
948  * @direction:  [in]    direction of DMA transfer
949  *
950  * This unmaps a DMA mapping for @attached obtained by dma_buf_map_attachment().
951  */
dma_buf_unmap_attachment(struct dma_buf_attachment * attach,struct sg_table * sg_table,enum dma_data_direction direction)952 void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
953 				struct sg_table *sg_table,
954 				enum dma_data_direction direction)
955 {
956 	might_sleep();
957 
958 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
959 		return;
960 
961 	if (dma_buf_attachment_is_dynamic(attach))
962 		dma_resv_assert_held(attach->dmabuf->resv);
963 
964 	if (attach->sgt == sg_table)
965 		return;
966 
967 	if (dma_buf_is_dynamic(attach->dmabuf))
968 		dma_resv_assert_held(attach->dmabuf->resv);
969 
970 	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
971 
972 	if (dma_buf_is_dynamic(attach->dmabuf) &&
973 	    !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY))
974 		dma_buf_unpin(attach);
975 }
976 EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment);
977 
978 /**
979  * dma_buf_move_notify - notify attachments that DMA-buf is moving
980  *
981  * @dmabuf:	[in]	buffer which is moving
982  *
983  * Informs all attachmenst that they need to destroy and recreated all their
984  * mappings.
985  */
dma_buf_move_notify(struct dma_buf * dmabuf)986 void dma_buf_move_notify(struct dma_buf *dmabuf)
987 {
988 	struct dma_buf_attachment *attach;
989 
990 	dma_resv_assert_held(dmabuf->resv);
991 
992 	list_for_each_entry(attach, &dmabuf->attachments, node)
993 		if (attach->importer_ops)
994 			attach->importer_ops->move_notify(attach);
995 }
996 EXPORT_SYMBOL_GPL(dma_buf_move_notify);
997 
998 /**
999  * DOC: cpu access
1000  *
1001  * There are mutliple reasons for supporting CPU access to a dma buffer object:
1002  *
1003  * - Fallback operations in the kernel, for example when a device is connected
1004  *   over USB and the kernel needs to shuffle the data around first before
1005  *   sending it away. Cache coherency is handled by braketing any transactions
1006  *   with calls to dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
1007  *   access.
1008  *
1009  *   Since for most kernel internal dma-buf accesses need the entire buffer, a
1010  *   vmap interface is introduced. Note that on very old 32-bit architectures
1011  *   vmalloc space might be limited and result in vmap calls failing.
1012  *
1013  *   Interfaces::
1014  *      void \*dma_buf_vmap(struct dma_buf \*dmabuf)
1015  *      void dma_buf_vunmap(struct dma_buf \*dmabuf, void \*vaddr)
1016  *
1017  *   The vmap call can fail if there is no vmap support in the exporter, or if
1018  *   it runs out of vmalloc space. Fallback to kmap should be implemented. Note
1019  *   that the dma-buf layer keeps a reference count for all vmap access and
1020  *   calls down into the exporter's vmap function only when no vmapping exists,
1021  *   and only unmaps it once. Protection against concurrent vmap/vunmap calls is
1022  *   provided by taking the dma_buf->lock mutex.
1023  *
1024  * - For full compatibility on the importer side with existing userspace
1025  *   interfaces, which might already support mmap'ing buffers. This is needed in
1026  *   many processing pipelines (e.g. feeding a software rendered image into a
1027  *   hardware pipeline, thumbnail creation, snapshots, ...). Also, Android's ION
1028  *   framework already supported this and for DMA buffer file descriptors to
1029  *   replace ION buffers mmap support was needed.
1030  *
1031  *   There is no special interfaces, userspace simply calls mmap on the dma-buf
1032  *   fd. But like for CPU access there's a need to braket the actual access,
1033  *   which is handled by the ioctl (DMA_BUF_IOCTL_SYNC). Note that
1034  *   DMA_BUF_IOCTL_SYNC can fail with -EAGAIN or -EINTR, in which case it must
1035  *   be restarted.
1036  *
1037  *   Some systems might need some sort of cache coherency management e.g. when
1038  *   CPU and GPU domains are being accessed through dma-buf at the same time.
1039  *   To circumvent this problem there are begin/end coherency markers, that
1040  *   forward directly to existing dma-buf device drivers vfunc hooks. Userspace
1041  *   can make use of those markers through the DMA_BUF_IOCTL_SYNC ioctl. The
1042  *   sequence would be used like following:
1043  *
1044  *     - mmap dma-buf fd
1045  *     - for each drawing/upload cycle in CPU 1. SYNC_START ioctl, 2. read/write
1046  *       to mmap area 3. SYNC_END ioctl. This can be repeated as often as you
1047  *       want (with the new data being consumed by say the GPU or the scanout
1048  *       device)
1049  *     - munmap once you don't need the buffer any more
1050  *
1051  *    For correctness and optimal performance, it is always required to use
1052  *    SYNC_START and SYNC_END before and after, respectively, when accessing the
1053  *    mapped address. Userspace cannot rely on coherent access, even when there
1054  *    are systems where it just works without calling these ioctls.
1055  *
1056  * - And as a CPU fallback in userspace processing pipelines.
1057  *
1058  *   Similar to the motivation for kernel cpu access it is again important that
1059  *   the userspace code of a given importing subsystem can use the same
1060  *   interfaces with a imported dma-buf buffer object as with a native buffer
1061  *   object. This is especially important for drm where the userspace part of
1062  *   contemporary OpenGL, X, and other drivers is huge, and reworking them to
1063  *   use a different way to mmap a buffer rather invasive.
1064  *
1065  *   The assumption in the current dma-buf interfaces is that redirecting the
1066  *   initial mmap is all that's needed. A survey of some of the existing
1067  *   subsystems shows that no driver seems to do any nefarious thing like
1068  *   syncing up with outstanding asynchronous processing on the device or
1069  *   allocating special resources at fault time. So hopefully this is good
1070  *   enough, since adding interfaces to intercept pagefaults and allow pte
1071  *   shootdowns would increase the complexity quite a bit.
1072  *
1073  *   Interface::
1074  *      int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*,
1075  *		       unsigned long);
1076  *
1077  *   If the importing subsystem simply provides a special-purpose mmap call to
1078  *   set up a mapping in userspace, calling do_mmap with dma_buf->file will
1079  *   equally achieve that for a dma-buf object.
1080  */
1081 
__dma_buf_begin_cpu_access(struct dma_buf * dmabuf,enum dma_data_direction direction)1082 static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1083 				      enum dma_data_direction direction)
1084 {
1085 	bool write = (direction == DMA_BIDIRECTIONAL ||
1086 		      direction == DMA_TO_DEVICE);
1087 	struct dma_resv *resv = dmabuf->resv;
1088 	long ret;
1089 
1090 	/* Wait on any implicit rendering fences */
1091 	ret = dma_resv_wait_timeout_rcu(resv, write, true,
1092 						  MAX_SCHEDULE_TIMEOUT);
1093 	if (ret < 0)
1094 		return ret;
1095 
1096 	return 0;
1097 }
1098 
1099 /**
1100  * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the
1101  * cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific
1102  * preparations. Coherency is only guaranteed in the specified range for the
1103  * specified access direction.
1104  * @dmabuf:	[in]	buffer to prepare cpu access for.
1105  * @direction:	[in]	length of range for cpu access.
1106  *
1107  * After the cpu access is complete the caller should call
1108  * dma_buf_end_cpu_access(). Only when cpu access is braketed by both calls is
1109  * it guaranteed to be coherent with other DMA access.
1110  *
1111  * Can return negative error values, returns 0 on success.
1112  */
dma_buf_begin_cpu_access(struct dma_buf * dmabuf,enum dma_data_direction direction)1113 int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1114 			     enum dma_data_direction direction)
1115 {
1116 	int ret = 0;
1117 
1118 	if (WARN_ON(!dmabuf))
1119 		return -EINVAL;
1120 
1121 	if (dmabuf->ops->begin_cpu_access)
1122 		ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);
1123 
1124 	/* Ensure that all fences are waited upon - but we first allow
1125 	 * the native handler the chance to do so more efficiently if it
1126 	 * chooses. A double invocation here will be reasonably cheap no-op.
1127 	 */
1128 	if (ret == 0)
1129 		ret = __dma_buf_begin_cpu_access(dmabuf, direction);
1130 
1131 	return ret;
1132 }
1133 EXPORT_SYMBOL_GPL(dma_buf_begin_cpu_access);
1134 
1135 /**
1136  * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the
1137  * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific
1138  * actions. Coherency is only guaranteed in the specified range for the
1139  * specified access direction.
1140  * @dmabuf:	[in]	buffer to complete cpu access for.
1141  * @direction:	[in]	length of range for cpu access.
1142  *
1143  * This terminates CPU access started with dma_buf_begin_cpu_access().
1144  *
1145  * Can return negative error values, returns 0 on success.
1146  */
dma_buf_end_cpu_access(struct dma_buf * dmabuf,enum dma_data_direction direction)1147 int dma_buf_end_cpu_access(struct dma_buf *dmabuf,
1148 			   enum dma_data_direction direction)
1149 {
1150 	int ret = 0;
1151 
1152 	WARN_ON(!dmabuf);
1153 
1154 	if (dmabuf->ops->end_cpu_access)
1155 		ret = dmabuf->ops->end_cpu_access(dmabuf, direction);
1156 
1157 	return ret;
1158 }
1159 EXPORT_SYMBOL_GPL(dma_buf_end_cpu_access);
1160 
1161 
1162 /**
1163  * dma_buf_mmap - Setup up a userspace mmap with the given vma
1164  * @dmabuf:	[in]	buffer that should back the vma
1165  * @vma:	[in]	vma for the mmap
1166  * @pgoff:	[in]	offset in pages where this mmap should start within the
1167  *			dma-buf buffer.
1168  *
1169  * This function adjusts the passed in vma so that it points at the file of the
1170  * dma_buf operation. It also adjusts the starting pgoff and does bounds
1171  * checking on the size of the vma. Then it calls the exporters mmap function to
1172  * set up the mapping.
1173  *
1174  * Can return negative error values, returns 0 on success.
1175  */
dma_buf_mmap(struct dma_buf * dmabuf,struct vm_area_struct * vma,unsigned long pgoff)1176 int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
1177 		 unsigned long pgoff)
1178 {
1179 	struct file *oldfile;
1180 	int ret;
1181 
1182 	if (WARN_ON(!dmabuf || !vma))
1183 		return -EINVAL;
1184 
1185 	/* check if buffer supports mmap */
1186 	if (!dmabuf->ops->mmap)
1187 		return -EINVAL;
1188 
1189 	/* check for offset overflow */
1190 	if (pgoff + vma_pages(vma) < pgoff)
1191 		return -EOVERFLOW;
1192 
1193 	/* check for overflowing the buffer's size */
1194 	if (pgoff + vma_pages(vma) >
1195 	    dmabuf->size >> PAGE_SHIFT)
1196 		return -EINVAL;
1197 
1198 	/* readjust the vma */
1199 	get_file(dmabuf->file);
1200 	oldfile = vma->vm_file;
1201 	vma->vm_file = dmabuf->file;
1202 	vma->vm_pgoff = pgoff;
1203 
1204 	ret = dmabuf->ops->mmap(dmabuf, vma);
1205 	if (ret) {
1206 		/* restore old parameters on failure */
1207 		vma->vm_file = oldfile;
1208 		fput(dmabuf->file);
1209 	} else {
1210 		if (oldfile)
1211 			fput(oldfile);
1212 	}
1213 	return ret;
1214 
1215 }
1216 EXPORT_SYMBOL_GPL(dma_buf_mmap);
1217 
1218 /**
1219  * dma_buf_vmap - Create virtual mapping for the buffer object into kernel
1220  * address space. Same restrictions as for vmap and friends apply.
1221  * @dmabuf:	[in]	buffer to vmap
1222  *
1223  * This call may fail due to lack of virtual mapping address space.
1224  * These calls are optional in drivers. The intended use for them
1225  * is for mapping objects linear in kernel space for high use objects.
1226  * Please attempt to use kmap/kunmap before thinking about these interfaces.
1227  *
1228  * Returns NULL on error.
1229  */
dma_buf_vmap(struct dma_buf * dmabuf)1230 void *dma_buf_vmap(struct dma_buf *dmabuf)
1231 {
1232 	void *ptr;
1233 
1234 	if (WARN_ON(!dmabuf))
1235 		return NULL;
1236 
1237 	if (!dmabuf->ops->vmap)
1238 		return NULL;
1239 
1240 	mutex_lock(&dmabuf->lock);
1241 	if (dmabuf->vmapping_counter) {
1242 		dmabuf->vmapping_counter++;
1243 		BUG_ON(!dmabuf->vmap_ptr);
1244 		ptr = dmabuf->vmap_ptr;
1245 		goto out_unlock;
1246 	}
1247 
1248 	BUG_ON(dmabuf->vmap_ptr);
1249 
1250 	ptr = dmabuf->ops->vmap(dmabuf);
1251 	if (WARN_ON_ONCE(IS_ERR(ptr)))
1252 		ptr = NULL;
1253 	if (!ptr)
1254 		goto out_unlock;
1255 
1256 	dmabuf->vmap_ptr = ptr;
1257 	dmabuf->vmapping_counter = 1;
1258 
1259 out_unlock:
1260 	mutex_unlock(&dmabuf->lock);
1261 	return ptr;
1262 }
1263 EXPORT_SYMBOL_GPL(dma_buf_vmap);
1264 
1265 /**
1266  * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap.
1267  * @dmabuf:	[in]	buffer to vunmap
1268  * @vaddr:	[in]	vmap to vunmap
1269  */
dma_buf_vunmap(struct dma_buf * dmabuf,void * vaddr)1270 void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr)
1271 {
1272 	if (WARN_ON(!dmabuf))
1273 		return;
1274 
1275 	BUG_ON(!dmabuf->vmap_ptr);
1276 	BUG_ON(dmabuf->vmapping_counter == 0);
1277 	BUG_ON(dmabuf->vmap_ptr != vaddr);
1278 
1279 	mutex_lock(&dmabuf->lock);
1280 	if (--dmabuf->vmapping_counter == 0) {
1281 		if (dmabuf->ops->vunmap)
1282 			dmabuf->ops->vunmap(dmabuf, vaddr);
1283 		dmabuf->vmap_ptr = NULL;
1284 	}
1285 	mutex_unlock(&dmabuf->lock);
1286 }
1287 EXPORT_SYMBOL_GPL(dma_buf_vunmap);
1288 
1289 #ifdef CONFIG_DEBUG_FS
dma_buf_debug_show(struct seq_file * s,void * unused)1290 static int dma_buf_debug_show(struct seq_file *s, void *unused)
1291 {
1292 	int ret;
1293 	struct dma_buf *buf_obj;
1294 	struct dma_buf_attachment *attach_obj;
1295 	struct dma_resv *robj;
1296 	struct dma_resv_list *fobj;
1297 	struct dma_fence *fence;
1298 	unsigned seq;
1299 	int count = 0, attach_count, shared_count, i;
1300 	size_t size = 0;
1301 
1302 	ret = mutex_lock_interruptible(&db_list.lock);
1303 
1304 	if (ret)
1305 		return ret;
1306 
1307 	seq_puts(s, "\nDma-buf Objects:\n");
1308 	seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\t"
1309 		   "%-16s\t%-16s\t%-16s\n",
1310 		   "size", "flags", "mode", "count", "ino",
1311 		   "buf_name", "exp_pid",  "exp_task_comm");
1312 
1313 	list_for_each_entry(buf_obj, &db_list.head, list_node) {
1314 
1315 		ret = dma_resv_lock_interruptible(buf_obj->resv, NULL);
1316 		if (ret)
1317 			goto error_unlock;
1318 
1319 		seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\t"
1320 			   "%-16d\t%-16s\n",
1321 				buf_obj->size,
1322 				buf_obj->file->f_flags, buf_obj->file->f_mode,
1323 				file_count(buf_obj->file),
1324 				buf_obj->exp_name,
1325 				file_inode(buf_obj->file)->i_ino,
1326 				buf_obj->name ?: "NULL",
1327 				dma_buf_exp_pid(buf_obj),
1328 				dma_buf_exp_task_comm(buf_obj) ?: "NULL");
1329 
1330 		robj = buf_obj->resv;
1331 		while (true) {
1332 			seq = read_seqcount_begin(&robj->seq);
1333 			rcu_read_lock();
1334 			fobj = rcu_dereference(robj->fence);
1335 			shared_count = fobj ? fobj->shared_count : 0;
1336 			fence = rcu_dereference(robj->fence_excl);
1337 			if (!read_seqcount_retry(&robj->seq, seq))
1338 				break;
1339 			rcu_read_unlock();
1340 		}
1341 
1342 		if (fence)
1343 			seq_printf(s, "\tExclusive fence: %s %s %ssignalled\n",
1344 				   fence->ops->get_driver_name(fence),
1345 				   fence->ops->get_timeline_name(fence),
1346 				   dma_fence_is_signaled(fence) ? "" : "un");
1347 		for (i = 0; i < shared_count; i++) {
1348 			fence = rcu_dereference(fobj->shared[i]);
1349 			if (!dma_fence_get_rcu(fence))
1350 				continue;
1351 			seq_printf(s, "\tShared fence: %s %s %ssignalled\n",
1352 				   fence->ops->get_driver_name(fence),
1353 				   fence->ops->get_timeline_name(fence),
1354 				   dma_fence_is_signaled(fence) ? "" : "un");
1355 			dma_fence_put(fence);
1356 		}
1357 		rcu_read_unlock();
1358 
1359 		seq_puts(s, "\tAttached Devices:\n");
1360 		attach_count = 0;
1361 
1362 		list_for_each_entry(attach_obj, &buf_obj->attachments, node) {
1363 			seq_printf(s, "\t%s\n", dev_name(attach_obj->dev));
1364 			attach_count++;
1365 		}
1366 		dma_resv_unlock(buf_obj->resv);
1367 
1368 		seq_printf(s, "Total %d devices attached\n\n",
1369 				attach_count);
1370 
1371 		count++;
1372 		size += buf_obj->size;
1373 	}
1374 
1375 	seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size);
1376 
1377 	mutex_unlock(&db_list.lock);
1378 	return 0;
1379 
1380 error_unlock:
1381 	mutex_unlock(&db_list.lock);
1382 	return ret;
1383 }
1384 
1385 DEFINE_SHOW_ATTRIBUTE(dma_buf_debug);
1386 
1387 static struct dentry *dma_buf_debugfs_dir;
1388 
dma_buf_init_debugfs(void)1389 static int dma_buf_init_debugfs(void)
1390 {
1391 	struct dentry *d;
1392 	int err = 0;
1393 
1394 	d = debugfs_create_dir("dma_buf", NULL);
1395 	if (IS_ERR(d))
1396 		return PTR_ERR(d);
1397 
1398 	dma_buf_debugfs_dir = d;
1399 
1400 	d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir,
1401 				NULL, &dma_buf_debug_fops);
1402 	if (IS_ERR(d)) {
1403 		pr_debug("dma_buf: debugfs: failed to create node bufinfo\n");
1404 		debugfs_remove_recursive(dma_buf_debugfs_dir);
1405 		dma_buf_debugfs_dir = NULL;
1406 		err = PTR_ERR(d);
1407 	}
1408 
1409 	dma_buf_process_info_init_debugfs(dma_buf_debugfs_dir);
1410 	return err;
1411 }
1412 
dma_buf_uninit_debugfs(void)1413 static void dma_buf_uninit_debugfs(void)
1414 {
1415 	debugfs_remove_recursive(dma_buf_debugfs_dir);
1416 }
1417 #else
dma_buf_init_debugfs(void)1418 static inline int dma_buf_init_debugfs(void)
1419 {
1420 	return 0;
1421 }
dma_buf_uninit_debugfs(void)1422 static inline void dma_buf_uninit_debugfs(void)
1423 {
1424 }
1425 #endif
1426 
1427 #ifdef CONFIG_DMABUF_PROCESS_INFO
get_dma_buf_from_file(struct file * f)1428 struct dma_buf *get_dma_buf_from_file(struct file *f)
1429 {
1430 	if (IS_ERR_OR_NULL(f))
1431 		return NULL;
1432 
1433 	if (!is_dma_buf_file(f))
1434 		return NULL;
1435 
1436 	return f->private_data;
1437 }
1438 #endif /* CONFIG_DMABUF_PROCESS_INFO */
1439 
dma_buf_init(void)1440 static int __init dma_buf_init(void)
1441 {
1442 	int ret;
1443 
1444 	ret = dma_buf_init_sysfs_statistics();
1445 	if (ret)
1446 		return ret;
1447 
1448 	dma_buf_mnt = kern_mount(&dma_buf_fs_type);
1449 	if (IS_ERR(dma_buf_mnt))
1450 		return PTR_ERR(dma_buf_mnt);
1451 
1452 	mutex_init(&db_list.lock);
1453 	INIT_LIST_HEAD(&db_list.head);
1454 	dma_buf_init_debugfs();
1455 	dma_buf_process_info_init_procfs();
1456 	return 0;
1457 }
1458 subsys_initcall(dma_buf_init);
1459 
dma_buf_deinit(void)1460 static void __exit dma_buf_deinit(void)
1461 {
1462 	dma_buf_uninit_debugfs();
1463 	kern_unmount(dma_buf_mnt);
1464 	dma_buf_uninit_sysfs_statistics();
1465 	dma_buf_process_info_uninit_procfs();
1466 }
1467 __exitcall(dma_buf_deinit);
1468