Lines Matching +full:grant +full:- +full:dma
1 // SPDX-License-Identifier: GPL-2.0
4 * Xen dma-buf functionality for gntdev.
6 * DMA buffer implementation is based on drivers/gpu/drm/drm_prime.c.
13 #include <linux/dma-buf.h>
21 #include "gntdev-common.h"
22 #include "gntdev-dmabuf.h"
26 * Note on usage of grant reference 0 as invalid grant reference:
27 * grant reference 0 is valid, but never exposed to a driver,
50 /* Scatter-gather table of the imported buffer. */
52 /* dma-buf attachment of the imported buffer. */
75 /* List of exported DMA buffers. */
79 /* List of imported DMA buffers. */
84 * We reference this file while exporting dma-bufs, so
85 * the grant device context is not destroyed while there are
91 /* DMA buffer export support. */
93 /* Implementation of wait for exported DMA buffer to be released. */
105 return ERR_PTR(-ENOMEM); in dmabuf_exp_wait_obj_new()
107 init_completion(&obj->completion); in dmabuf_exp_wait_obj_new()
108 obj->gntdev_dmabuf = gntdev_dmabuf; in dmabuf_exp_wait_obj_new()
110 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_new()
111 list_add(&obj->next, &priv->exp_wait_list); in dmabuf_exp_wait_obj_new()
113 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); in dmabuf_exp_wait_obj_new()
114 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_new()
121 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_free()
122 list_del(&obj->next); in dmabuf_exp_wait_obj_free()
123 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_free()
130 if (wait_for_completion_timeout(&obj->completion, in dmabuf_exp_wait_obj_wait()
132 return -ETIMEDOUT; in dmabuf_exp_wait_obj_wait()
142 list_for_each_entry(obj, &priv->exp_wait_list, next) in dmabuf_exp_wait_obj_signal()
143 if (obj->gntdev_dmabuf == gntdev_dmabuf) { in dmabuf_exp_wait_obj_signal()
145 complete_all(&obj->completion); in dmabuf_exp_wait_obj_signal()
153 struct gntdev_dmabuf *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); in dmabuf_exp_wait_obj_get_dmabuf()
155 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_get_dmabuf()
156 list_for_each_entry(gntdev_dmabuf, &priv->exp_list, next) in dmabuf_exp_wait_obj_get_dmabuf()
157 if (gntdev_dmabuf->fd == fd) { in dmabuf_exp_wait_obj_get_dmabuf()
159 kref_get(&gntdev_dmabuf->u.exp.refcount); in dmabuf_exp_wait_obj_get_dmabuf()
163 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_get_dmabuf()
174 pr_debug("Will wait for dma-buf with fd %d\n", fd); in dmabuf_exp_wait_released()
176 * Try to find the DMA buffer: if not found means that in dmabuf_exp_wait_released()
198 /* DMA buffer export support. */
208 ret = -ENOMEM; in dmabuf_pages_to_sgt()
233 return -ENOMEM; in dmabuf_exp_ops_attach()
235 gntdev_dmabuf_attach->dir = DMA_NONE; in dmabuf_exp_ops_attach()
236 attach->priv = gntdev_dmabuf_attach; in dmabuf_exp_ops_attach()
243 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; in dmabuf_exp_ops_detach()
246 struct sg_table *sgt = gntdev_dmabuf_attach->sgt; in dmabuf_exp_ops_detach()
249 if (gntdev_dmabuf_attach->dir != DMA_NONE) in dmabuf_exp_ops_detach()
250 dma_unmap_sgtable(attach->dev, sgt, in dmabuf_exp_ops_detach()
251 gntdev_dmabuf_attach->dir, in dmabuf_exp_ops_detach()
258 attach->priv = NULL; in dmabuf_exp_ops_detach()
266 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; in dmabuf_exp_ops_map_dma_buf()
267 struct gntdev_dmabuf *gntdev_dmabuf = attach->dmabuf->priv; in dmabuf_exp_ops_map_dma_buf()
270 pr_debug("Mapping %d pages for dev %p\n", gntdev_dmabuf->nr_pages, in dmabuf_exp_ops_map_dma_buf()
271 attach->dev); in dmabuf_exp_ops_map_dma_buf()
274 return ERR_PTR(-EINVAL); in dmabuf_exp_ops_map_dma_buf()
277 if (gntdev_dmabuf_attach->dir == dir) in dmabuf_exp_ops_map_dma_buf()
278 return gntdev_dmabuf_attach->sgt; in dmabuf_exp_ops_map_dma_buf()
284 if (gntdev_dmabuf_attach->dir != DMA_NONE) in dmabuf_exp_ops_map_dma_buf()
285 return ERR_PTR(-EBUSY); in dmabuf_exp_ops_map_dma_buf()
287 sgt = dmabuf_pages_to_sgt(gntdev_dmabuf->pages, in dmabuf_exp_ops_map_dma_buf()
288 gntdev_dmabuf->nr_pages); in dmabuf_exp_ops_map_dma_buf()
290 if (dma_map_sgtable(attach->dev, sgt, dir, in dmabuf_exp_ops_map_dma_buf()
294 sgt = ERR_PTR(-ENOMEM); in dmabuf_exp_ops_map_dma_buf()
296 gntdev_dmabuf_attach->sgt = sgt; in dmabuf_exp_ops_map_dma_buf()
297 gntdev_dmabuf_attach->dir = dir; in dmabuf_exp_ops_map_dma_buf()
301 pr_debug("Failed to map sg table for dev %p\n", attach->dev); in dmabuf_exp_ops_map_dma_buf()
317 dmabuf_exp_wait_obj_signal(gntdev_dmabuf->priv, gntdev_dmabuf); in dmabuf_exp_release()
318 list_del(&gntdev_dmabuf->next); in dmabuf_exp_release()
319 fput(gntdev_dmabuf->priv->filp); in dmabuf_exp_release()
326 mutex_lock(&priv->lock); in dmabuf_exp_remove_map()
327 list_del(&map->next); in dmabuf_exp_remove_map()
329 mutex_unlock(&priv->lock); in dmabuf_exp_remove_map()
334 struct gntdev_dmabuf *gntdev_dmabuf = dma_buf->priv; in dmabuf_exp_ops_release()
335 struct gntdev_dmabuf_priv *priv = gntdev_dmabuf->priv; in dmabuf_exp_ops_release()
337 dmabuf_exp_remove_map(gntdev_dmabuf->u.exp.priv, in dmabuf_exp_ops_release()
338 gntdev_dmabuf->u.exp.map); in dmabuf_exp_ops_release()
339 mutex_lock(&priv->lock); in dmabuf_exp_ops_release()
340 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); in dmabuf_exp_ops_release()
341 mutex_unlock(&priv->lock); in dmabuf_exp_ops_release()
370 return -ENOMEM; in dmabuf_exp_from_pages()
372 kref_init(&gntdev_dmabuf->u.exp.refcount); in dmabuf_exp_from_pages()
374 gntdev_dmabuf->priv = args->dmabuf_priv; in dmabuf_exp_from_pages()
375 gntdev_dmabuf->nr_pages = args->count; in dmabuf_exp_from_pages()
376 gntdev_dmabuf->pages = args->pages; in dmabuf_exp_from_pages()
377 gntdev_dmabuf->u.exp.priv = args->priv; in dmabuf_exp_from_pages()
378 gntdev_dmabuf->u.exp.map = args->map; in dmabuf_exp_from_pages()
381 if (args->dev->driver && args->dev->driver->owner) in dmabuf_exp_from_pages()
382 exp_info.owner = args->dev->driver->owner; in dmabuf_exp_from_pages()
386 exp_info.size = args->count << PAGE_SHIFT; in dmabuf_exp_from_pages()
390 gntdev_dmabuf->dmabuf = dma_buf_export(&exp_info); in dmabuf_exp_from_pages()
391 if (IS_ERR(gntdev_dmabuf->dmabuf)) { in dmabuf_exp_from_pages()
392 ret = PTR_ERR(gntdev_dmabuf->dmabuf); in dmabuf_exp_from_pages()
393 gntdev_dmabuf->dmabuf = NULL; in dmabuf_exp_from_pages()
397 ret = dma_buf_fd(gntdev_dmabuf->dmabuf, O_CLOEXEC); in dmabuf_exp_from_pages()
401 gntdev_dmabuf->fd = ret; in dmabuf_exp_from_pages()
402 args->fd = ret; in dmabuf_exp_from_pages()
404 pr_debug("Exporting DMA buffer with fd %d\n", ret); in dmabuf_exp_from_pages()
406 mutex_lock(&args->dmabuf_priv->lock); in dmabuf_exp_from_pages()
407 list_add(&gntdev_dmabuf->next, &args->dmabuf_priv->exp_list); in dmabuf_exp_from_pages()
408 mutex_unlock(&args->dmabuf_priv->lock); in dmabuf_exp_from_pages()
409 get_file(gntdev_dmabuf->priv->filp); in dmabuf_exp_from_pages()
413 if (gntdev_dmabuf->dmabuf) in dmabuf_exp_from_pages()
414 dma_buf_put(gntdev_dmabuf->dmabuf); in dmabuf_exp_from_pages()
426 return ERR_PTR(-EINVAL); in dmabuf_exp_alloc_backing_storage()
430 pr_debug("Wrong dma-buf flags: 0x%x\n", dmabuf_flags); in dmabuf_exp_alloc_backing_storage()
431 return ERR_PTR(-EINVAL); in dmabuf_exp_alloc_backing_storage()
436 return ERR_PTR(-ENOMEM); in dmabuf_exp_alloc_backing_storage()
453 map->grants[i].domid = domid; in dmabuf_exp_from_refs()
454 map->grants[i].ref = refs[i]; in dmabuf_exp_from_refs()
457 mutex_lock(&priv->lock); in dmabuf_exp_from_refs()
459 mutex_unlock(&priv->lock); in dmabuf_exp_from_refs()
461 map->flags |= GNTMAP_host_map; in dmabuf_exp_from_refs()
463 map->flags |= GNTMAP_device_map; in dmabuf_exp_from_refs()
472 args.dev = priv->dma_dev; in dmabuf_exp_from_refs()
473 args.dmabuf_priv = priv->dmabuf_priv; in dmabuf_exp_from_refs()
474 args.count = map->count; in dmabuf_exp_from_refs()
475 args.pages = map->pages; in dmabuf_exp_from_refs()
476 args.fd = -1; /* Shut up unnecessary gcc warning for i386 */ in dmabuf_exp_from_refs()
490 /* DMA buffer import support. */
501 pr_debug("Cannot allocate grant references, ret %d\n", ret); in dmabuf_imp_grant_foreign_access()
511 pr_debug("Cannot claim grant reference, ret %d\n", ret); in dmabuf_imp_grant_foreign_access()
538 kfree(gntdev_dmabuf->pages); in dmabuf_imp_free_storage()
539 kfree(gntdev_dmabuf->u.imp.refs); in dmabuf_imp_free_storage()
552 gntdev_dmabuf->u.imp.refs = kcalloc(count, in dmabuf_imp_alloc_storage()
553 sizeof(gntdev_dmabuf->u.imp.refs[0]), in dmabuf_imp_alloc_storage()
555 if (!gntdev_dmabuf->u.imp.refs) in dmabuf_imp_alloc_storage()
558 gntdev_dmabuf->pages = kcalloc(count, in dmabuf_imp_alloc_storage()
559 sizeof(gntdev_dmabuf->pages[0]), in dmabuf_imp_alloc_storage()
561 if (!gntdev_dmabuf->pages) in dmabuf_imp_alloc_storage()
564 gntdev_dmabuf->nr_pages = count; in dmabuf_imp_alloc_storage()
567 gntdev_dmabuf->u.imp.refs[i] = GRANT_INVALID_REF; in dmabuf_imp_alloc_storage()
574 return ERR_PTR(-ENOMEM); in dmabuf_imp_alloc_storage()
598 gntdev_dmabuf->priv = priv; in dmabuf_imp_to_refs()
599 gntdev_dmabuf->fd = fd; in dmabuf_imp_to_refs()
607 gntdev_dmabuf->u.imp.attach = attach; in dmabuf_imp_to_refs()
616 if (sgt->sgl->offset) { in dmabuf_imp_to_refs()
617 ret = ERR_PTR(-EINVAL); in dmabuf_imp_to_refs()
618 pr_debug("DMA buffer has %d bytes offset, user-space expects 0\n", in dmabuf_imp_to_refs()
619 sgt->sgl->offset); in dmabuf_imp_to_refs()
624 if (attach->dmabuf->size != gntdev_dmabuf->nr_pages << PAGE_SHIFT) { in dmabuf_imp_to_refs()
625 ret = ERR_PTR(-EINVAL); in dmabuf_imp_to_refs()
626 pr_debug("DMA buffer has %zu pages, user-space expects %d\n", in dmabuf_imp_to_refs()
627 attach->dmabuf->size, gntdev_dmabuf->nr_pages); in dmabuf_imp_to_refs()
631 gntdev_dmabuf->u.imp.sgt = sgt; in dmabuf_imp_to_refs()
643 ret = ERR_PTR(-EINVAL); in dmabuf_imp_to_refs()
647 gntdev_dmabuf->pages[i++] = page; in dmabuf_imp_to_refs()
650 ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gntdev_dmabuf->pages, in dmabuf_imp_to_refs()
651 gntdev_dmabuf->u.imp.refs, in dmabuf_imp_to_refs()
656 pr_debug("Imported DMA buffer with fd %d\n", fd); in dmabuf_imp_to_refs()
658 mutex_lock(&priv->lock); in dmabuf_imp_to_refs()
659 list_add(&gntdev_dmabuf->next, &priv->imp_list); in dmabuf_imp_to_refs()
660 mutex_unlock(&priv->lock); in dmabuf_imp_to_refs()
665 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, count); in dmabuf_imp_to_refs()
678 * Find the hyper dma-buf by its file descriptor and remove
684 struct gntdev_dmabuf *q, *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); in dmabuf_imp_find_unlink()
686 mutex_lock(&priv->lock); in dmabuf_imp_find_unlink()
687 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) { in dmabuf_imp_find_unlink()
688 if (gntdev_dmabuf->fd == fd) { in dmabuf_imp_find_unlink()
691 list_del(&gntdev_dmabuf->next); in dmabuf_imp_find_unlink()
695 mutex_unlock(&priv->lock); in dmabuf_imp_find_unlink()
709 pr_debug("Releasing DMA buffer with fd %d\n", fd); in dmabuf_imp_release()
711 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, in dmabuf_imp_release()
712 gntdev_dmabuf->nr_pages); in dmabuf_imp_release()
714 attach = gntdev_dmabuf->u.imp.attach; in dmabuf_imp_release()
716 if (gntdev_dmabuf->u.imp.sgt) in dmabuf_imp_release()
717 dma_buf_unmap_attachment(attach, gntdev_dmabuf->u.imp.sgt, in dmabuf_imp_release()
719 dma_buf = attach->dmabuf; in dmabuf_imp_release()
720 dma_buf_detach(attach->dmabuf, attach); in dmabuf_imp_release()
731 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) in dmabuf_imp_release_all()
732 dmabuf_imp_release(priv, gntdev_dmabuf->fd); in dmabuf_imp_release_all()
735 /* DMA buffer IOCTL support. */
745 pr_debug("Cannot provide dma-buf: use_ptemode %d\n", in gntdev_ioctl_dmabuf_exp_from_refs()
747 return -EINVAL; in gntdev_ioctl_dmabuf_exp_from_refs()
751 return -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
754 return -EINVAL; in gntdev_ioctl_dmabuf_exp_from_refs()
758 return -ENOMEM; in gntdev_ioctl_dmabuf_exp_from_refs()
760 if (copy_from_user(refs, u->refs, sizeof(*refs) * op.count) != 0) { in gntdev_ioctl_dmabuf_exp_from_refs()
761 ret = -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
771 ret = -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
784 return -EFAULT; in gntdev_ioctl_dmabuf_exp_wait_released()
786 return dmabuf_exp_wait_released(priv->dmabuf_priv, op.fd, in gntdev_ioctl_dmabuf_exp_wait_released()
798 return -EFAULT; in gntdev_ioctl_dmabuf_imp_to_refs()
801 return -EINVAL; in gntdev_ioctl_dmabuf_imp_to_refs()
803 gntdev_dmabuf = dmabuf_imp_to_refs(priv->dmabuf_priv, in gntdev_ioctl_dmabuf_imp_to_refs()
804 priv->dma_dev, op.fd, in gntdev_ioctl_dmabuf_imp_to_refs()
809 if (copy_to_user(u->refs, gntdev_dmabuf->u.imp.refs, in gntdev_ioctl_dmabuf_imp_to_refs()
810 sizeof(*u->refs) * op.count) != 0) { in gntdev_ioctl_dmabuf_imp_to_refs()
811 ret = -EFAULT; in gntdev_ioctl_dmabuf_imp_to_refs()
817 dmabuf_imp_release(priv->dmabuf_priv, op.fd); in gntdev_ioctl_dmabuf_imp_to_refs()
827 return -EFAULT; in gntdev_ioctl_dmabuf_imp_release()
829 return dmabuf_imp_release(priv->dmabuf_priv, op.fd); in gntdev_ioctl_dmabuf_imp_release()
838 return ERR_PTR(-ENOMEM); in gntdev_dmabuf_init()
840 mutex_init(&priv->lock); in gntdev_dmabuf_init()
841 INIT_LIST_HEAD(&priv->exp_list); in gntdev_dmabuf_init()
842 INIT_LIST_HEAD(&priv->exp_wait_list); in gntdev_dmabuf_init()
843 INIT_LIST_HEAD(&priv->imp_list); in gntdev_dmabuf_init()
845 priv->filp = filp; in gntdev_dmabuf_init()