Lines Matching +full:fastrpc +full:- +full:compute +full:- +full:cb
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
20 #include <uapi/misc/fastrpc.h>
27 #define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/
36 #define FASTRPC_DEVICE_NAME "fastrpc"
243 if (map->table) { in fastrpc_free_map()
244 dma_buf_unmap_attachment(map->attach, map->table, in fastrpc_free_map()
246 dma_buf_detach(map->buf, map->attach); in fastrpc_free_map()
247 dma_buf_put(map->buf); in fastrpc_free_map()
250 if (map->fl) { in fastrpc_free_map()
251 spin_lock(&map->fl->lock); in fastrpc_free_map()
252 list_del(&map->node); in fastrpc_free_map()
253 spin_unlock(&map->fl->lock); in fastrpc_free_map()
254 map->fl = NULL; in fastrpc_free_map()
263 kref_put(&map->refcount, fastrpc_free_map); in fastrpc_map_put()
269 return -ENOENT; in fastrpc_map_get()
271 return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT; in fastrpc_map_get()
279 mutex_lock(&fl->mutex); in fastrpc_map_find()
280 list_for_each_entry(map, &fl->maps, node) { in fastrpc_map_find()
281 if (map->fd == fd) { in fastrpc_map_find()
284 mutex_unlock(&fl->mutex); in fastrpc_map_find()
288 mutex_unlock(&fl->mutex); in fastrpc_map_find()
290 return -ENOENT; in fastrpc_map_find()
295 dma_free_coherent(buf->dev, buf->size, buf->virt, in fastrpc_buf_free()
296 FASTRPC_PHYS(buf->phys)); in fastrpc_buf_free()
307 return -ENOMEM; in fastrpc_buf_alloc()
309 INIT_LIST_HEAD(&buf->attachments); in fastrpc_buf_alloc()
310 INIT_LIST_HEAD(&buf->node); in fastrpc_buf_alloc()
311 mutex_init(&buf->lock); in fastrpc_buf_alloc()
313 buf->fl = fl; in fastrpc_buf_alloc()
314 buf->virt = NULL; in fastrpc_buf_alloc()
315 buf->phys = 0; in fastrpc_buf_alloc()
316 buf->size = size; in fastrpc_buf_alloc()
317 buf->dev = dev; in fastrpc_buf_alloc()
318 buf->raddr = 0; in fastrpc_buf_alloc()
320 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, in fastrpc_buf_alloc()
322 if (!buf->virt) { in fastrpc_buf_alloc()
323 mutex_destroy(&buf->lock); in fastrpc_buf_alloc()
325 return -ENOMEM; in fastrpc_buf_alloc()
328 if (fl->sctx && fl->sctx->sid) in fastrpc_buf_alloc()
329 buf->phys += ((u64)fl->sctx->sid << 32); in fastrpc_buf_alloc()
347 kref_get(&cctx->refcount); in fastrpc_channel_ctx_get()
352 kref_put(&cctx->refcount, fastrpc_channel_ctx_free); in fastrpc_channel_ctx_put()
363 cctx = ctx->cctx; in fastrpc_context_free()
365 for (i = 0; i < ctx->nscalars; i++) in fastrpc_context_free()
366 fastrpc_map_put(ctx->maps[i]); in fastrpc_context_free()
368 if (ctx->buf) in fastrpc_context_free()
369 fastrpc_buf_free(ctx->buf); in fastrpc_context_free()
371 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_context_free()
372 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); in fastrpc_context_free()
373 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_free()
375 kfree(ctx->maps); in fastrpc_context_free()
376 kfree(ctx->olaps); in fastrpc_context_free()
384 kref_get(&ctx->refcount); in fastrpc_context_get()
389 kref_put(&ctx->refcount, fastrpc_context_free); in fastrpc_context_put()
400 #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
406 int st = CMP(pa->start, pb->start); in olaps_cmp()
408 int ed = CMP(pb->end, pa->end); in olaps_cmp()
418 for (i = 0; i < ctx->nbufs; ++i) { in fastrpc_get_buff_overlaps()
419 ctx->olaps[i].start = ctx->args[i].ptr; in fastrpc_get_buff_overlaps()
420 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length; in fastrpc_get_buff_overlaps()
421 ctx->olaps[i].raix = i; in fastrpc_get_buff_overlaps()
424 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL); in fastrpc_get_buff_overlaps()
426 for (i = 0; i < ctx->nbufs; ++i) { in fastrpc_get_buff_overlaps()
428 if (ctx->olaps[i].start < max_end) { in fastrpc_get_buff_overlaps()
429 ctx->olaps[i].mstart = max_end; in fastrpc_get_buff_overlaps()
430 ctx->olaps[i].mend = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
431 ctx->olaps[i].offset = max_end - ctx->olaps[i].start; in fastrpc_get_buff_overlaps()
433 if (ctx->olaps[i].end > max_end) { in fastrpc_get_buff_overlaps()
434 max_end = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
436 ctx->olaps[i].mend = 0; in fastrpc_get_buff_overlaps()
437 ctx->olaps[i].mstart = 0; in fastrpc_get_buff_overlaps()
441 ctx->olaps[i].mend = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
442 ctx->olaps[i].mstart = ctx->olaps[i].start; in fastrpc_get_buff_overlaps()
443 ctx->olaps[i].offset = 0; in fastrpc_get_buff_overlaps()
444 max_end = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
453 struct fastrpc_channel_ctx *cctx = user->cctx; in fastrpc_context_alloc()
460 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
462 INIT_LIST_HEAD(&ctx->node); in fastrpc_context_alloc()
463 ctx->fl = user; in fastrpc_context_alloc()
464 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc); in fastrpc_context_alloc()
465 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) + in fastrpc_context_alloc()
468 if (ctx->nscalars) { in fastrpc_context_alloc()
469 ctx->maps = kcalloc(ctx->nscalars, in fastrpc_context_alloc()
470 sizeof(*ctx->maps), GFP_KERNEL); in fastrpc_context_alloc()
471 if (!ctx->maps) { in fastrpc_context_alloc()
473 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
475 ctx->olaps = kcalloc(ctx->nscalars, in fastrpc_context_alloc()
476 sizeof(*ctx->olaps), GFP_KERNEL); in fastrpc_context_alloc()
477 if (!ctx->olaps) { in fastrpc_context_alloc()
478 kfree(ctx->maps); in fastrpc_context_alloc()
480 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
482 ctx->args = args; in fastrpc_context_alloc()
489 ctx->sc = sc; in fastrpc_context_alloc()
490 ctx->retval = -1; in fastrpc_context_alloc()
491 ctx->pid = current->pid; in fastrpc_context_alloc()
492 ctx->tgid = user->tgid; in fastrpc_context_alloc()
493 ctx->cctx = cctx; in fastrpc_context_alloc()
494 init_completion(&ctx->work); in fastrpc_context_alloc()
495 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq); in fastrpc_context_alloc()
497 spin_lock(&user->lock); in fastrpc_context_alloc()
498 list_add_tail(&ctx->node, &user->pending); in fastrpc_context_alloc()
499 spin_unlock(&user->lock); in fastrpc_context_alloc()
501 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_context_alloc()
502 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1, in fastrpc_context_alloc()
505 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_alloc()
508 ctx->ctxid = ret << 4; in fastrpc_context_alloc()
509 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_alloc()
511 kref_init(&ctx->refcount); in fastrpc_context_alloc()
515 spin_lock(&user->lock); in fastrpc_context_alloc()
516 list_del(&ctx->node); in fastrpc_context_alloc()
517 spin_unlock(&user->lock); in fastrpc_context_alloc()
519 kfree(ctx->maps); in fastrpc_context_alloc()
520 kfree(ctx->olaps); in fastrpc_context_alloc()
530 struct fastrpc_dma_buf_attachment *a = attachment->priv; in fastrpc_map_dma_buf()
534 table = &a->sgt; in fastrpc_map_dma_buf()
536 ret = dma_map_sgtable(attachment->dev, table, dir, 0); in fastrpc_map_dma_buf()
546 dma_unmap_sgtable(attach->dev, table, dir, 0); in fastrpc_unmap_dma_buf()
551 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_release()
560 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_dma_buf_attach()
565 return -ENOMEM; in fastrpc_dma_buf_attach()
567 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt, in fastrpc_dma_buf_attach()
568 FASTRPC_PHYS(buffer->phys), buffer->size); in fastrpc_dma_buf_attach()
570 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n"); in fastrpc_dma_buf_attach()
572 return -EINVAL; in fastrpc_dma_buf_attach()
575 a->dev = attachment->dev; in fastrpc_dma_buf_attach()
576 INIT_LIST_HEAD(&a->node); in fastrpc_dma_buf_attach()
577 attachment->priv = a; in fastrpc_dma_buf_attach()
579 mutex_lock(&buffer->lock); in fastrpc_dma_buf_attach()
580 list_add(&a->node, &buffer->attachments); in fastrpc_dma_buf_attach()
581 mutex_unlock(&buffer->lock); in fastrpc_dma_buf_attach()
589 struct fastrpc_dma_buf_attachment *a = attachment->priv; in fastrpc_dma_buf_detatch()
590 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_dma_buf_detatch()
592 mutex_lock(&buffer->lock); in fastrpc_dma_buf_detatch()
593 list_del(&a->node); in fastrpc_dma_buf_detatch()
594 mutex_unlock(&buffer->lock); in fastrpc_dma_buf_detatch()
595 sg_free_table(&a->sgt); in fastrpc_dma_buf_detatch()
601 struct fastrpc_buf *buf = dmabuf->priv; in fastrpc_vmap()
603 return buf->virt; in fastrpc_vmap()
609 struct fastrpc_buf *buf = dmabuf->priv; in fastrpc_mmap()
610 size_t size = vma->vm_end - vma->vm_start; in fastrpc_mmap()
612 return dma_mmap_coherent(buf->dev, vma, buf->virt, in fastrpc_mmap()
613 FASTRPC_PHYS(buf->phys), size); in fastrpc_mmap()
629 struct fastrpc_session_ctx *sess = fl->sctx; in fastrpc_map_create()
638 return -ENOMEM; in fastrpc_map_create()
640 INIT_LIST_HEAD(&map->node); in fastrpc_map_create()
641 map->fl = fl; in fastrpc_map_create()
642 map->fd = fd; in fastrpc_map_create()
643 map->buf = dma_buf_get(fd); in fastrpc_map_create()
644 if (IS_ERR(map->buf)) { in fastrpc_map_create()
645 err = PTR_ERR(map->buf); in fastrpc_map_create()
649 map->attach = dma_buf_attach(map->buf, sess->dev); in fastrpc_map_create()
650 if (IS_ERR(map->attach)) { in fastrpc_map_create()
651 dev_err(sess->dev, "Failed to attach dmabuf\n"); in fastrpc_map_create()
652 err = PTR_ERR(map->attach); in fastrpc_map_create()
656 map->table = dma_buf_map_attachment(map->attach, DMA_BIDIRECTIONAL); in fastrpc_map_create()
657 if (IS_ERR(map->table)) { in fastrpc_map_create()
658 err = PTR_ERR(map->table); in fastrpc_map_create()
662 map->phys = sg_dma_address(map->table->sgl); in fastrpc_map_create()
663 map->phys += ((u64)fl->sctx->sid << 32); in fastrpc_map_create()
664 map->size = len; in fastrpc_map_create()
665 map->va = sg_virt(map->table->sgl); in fastrpc_map_create()
666 map->len = len; in fastrpc_map_create()
667 kref_init(&map->refcount); in fastrpc_map_create()
669 spin_lock(&fl->lock); in fastrpc_map_create()
670 list_add_tail(&map->node, &fl->maps); in fastrpc_map_create()
671 spin_unlock(&fl->lock); in fastrpc_map_create()
677 dma_buf_detach(map->buf, map->attach); in fastrpc_map_create()
679 dma_buf_put(map->buf); in fastrpc_map_create()
687 * Fastrpc payload buffer with metadata looks like:
690 * +---------------------------------+
693 * | (0 - N) |
694 * +---------------------------------+
697 * | (0 - N) |
698 * +---------------------------------+
701 * | (0 - N) |
702 * +---------------------------------+
705 * +---------------------------------+
707 * +---------------------------------+
709 * | (0-N) |
710 * +---------------------------------+
719 sizeof(struct fastrpc_phy_page)) * ctx->nscalars + in fastrpc_get_meta_size()
732 for (oix = 0; oix < ctx->nbufs; oix++) { in fastrpc_get_payload_size()
733 int i = ctx->olaps[oix].raix; in fastrpc_get_payload_size()
735 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) { in fastrpc_get_payload_size()
737 if (ctx->olaps[oix].offset == 0) in fastrpc_get_payload_size()
740 size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart); in fastrpc_get_payload_size()
749 struct device *dev = ctx->fl->sctx->dev; in fastrpc_create_maps()
752 for (i = 0; i < ctx->nscalars; ++i) { in fastrpc_create_maps()
754 if (ctx->args[i].reserved) in fastrpc_create_maps()
755 return -EINVAL; in fastrpc_create_maps()
757 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 || in fastrpc_create_maps()
758 ctx->args[i].length == 0) in fastrpc_create_maps()
761 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, in fastrpc_create_maps()
762 ctx->args[i].length, &ctx->maps[i]); in fastrpc_create_maps()
765 return -EINVAL; in fastrpc_create_maps()
774 struct device *dev = ctx->fl->sctx->dev; in fastrpc_get_args()
784 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); in fastrpc_get_args()
792 ctx->msg_sz = pkt_size; in fastrpc_get_args()
794 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf); in fastrpc_get_args()
798 rpra = ctx->buf->virt; in fastrpc_get_args()
799 list = ctx->buf->virt + ctx->nscalars * sizeof(*rpra); in fastrpc_get_args()
800 pages = ctx->buf->virt + ctx->nscalars * (sizeof(*list) + in fastrpc_get_args()
802 args = (uintptr_t)ctx->buf->virt + metalen; in fastrpc_get_args()
803 rlen = pkt_size - metalen; in fastrpc_get_args()
804 ctx->rpra = rpra; in fastrpc_get_args()
806 for (oix = 0; oix < ctx->nbufs; ++oix) { in fastrpc_get_args()
809 i = ctx->olaps[oix].raix; in fastrpc_get_args()
810 len = ctx->args[i].length; in fastrpc_get_args()
820 if (ctx->maps[i]) { in fastrpc_get_args()
823 rpra[i].pv = (u64) ctx->args[i].ptr; in fastrpc_get_args()
824 pages[i].addr = ctx->maps[i]->phys; in fastrpc_get_args()
826 mmap_read_lock(current->mm); in fastrpc_get_args()
827 vma = find_vma(current->mm, ctx->args[i].ptr); in fastrpc_get_args()
829 pages[i].addr += ctx->args[i].ptr - in fastrpc_get_args()
830 vma->vm_start; in fastrpc_get_args()
831 mmap_read_unlock(current->mm); in fastrpc_get_args()
833 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT; in fastrpc_get_args()
834 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >> in fastrpc_get_args()
836 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE; in fastrpc_get_args()
840 if (ctx->olaps[oix].offset == 0) { in fastrpc_get_args()
841 rlen -= ALIGN(args, FASTRPC_ALIGN) - args; in fastrpc_get_args()
845 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart; in fastrpc_get_args()
850 rpra[i].pv = args - ctx->olaps[oix].offset; in fastrpc_get_args()
851 pages[i].addr = ctx->buf->phys - in fastrpc_get_args()
852 ctx->olaps[oix].offset + in fastrpc_get_args()
853 (pkt_size - rlen); in fastrpc_get_args()
857 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT; in fastrpc_get_args()
858 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE; in fastrpc_get_args()
860 rlen -= mlen; in fastrpc_get_args()
863 if (i < inbufs && !ctx->maps[i]) { in fastrpc_get_args()
865 void *src = (void *)(uintptr_t)ctx->args[i].ptr; in fastrpc_get_args()
870 err = -EFAULT; in fastrpc_get_args()
879 for (i = ctx->nbufs; i < ctx->nscalars; ++i) { in fastrpc_get_args()
880 rpra[i].pv = (u64) ctx->args[i].ptr; in fastrpc_get_args()
881 rpra[i].len = ctx->args[i].length; in fastrpc_get_args()
882 list[i].num = ctx->args[i].length ? 1 : 0; in fastrpc_get_args()
884 pages[i].addr = ctx->maps[i]->phys; in fastrpc_get_args()
885 pages[i].size = ctx->maps[i]->size; in fastrpc_get_args()
898 struct fastrpc_remote_arg *rpra = ctx->rpra; in fastrpc_put_args()
901 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); in fastrpc_put_args()
903 for (i = inbufs; i < ctx->nbufs; ++i) { in fastrpc_put_args()
905 void *dst = (void *)(uintptr_t)ctx->args[i].ptr; in fastrpc_put_args()
910 return -EFAULT; in fastrpc_put_args()
924 struct fastrpc_user *fl = ctx->fl; in fastrpc_invoke_send()
925 struct fastrpc_msg *msg = &ctx->msg; in fastrpc_invoke_send()
928 cctx = fl->cctx; in fastrpc_invoke_send()
929 msg->pid = fl->tgid; in fastrpc_invoke_send()
930 msg->tid = current->pid; in fastrpc_invoke_send()
933 msg->pid = 0; in fastrpc_invoke_send()
935 msg->ctx = ctx->ctxid | fl->pd; in fastrpc_invoke_send()
936 msg->handle = handle; in fastrpc_invoke_send()
937 msg->sc = ctx->sc; in fastrpc_invoke_send()
938 msg->addr = ctx->buf ? ctx->buf->phys : 0; in fastrpc_invoke_send()
939 msg->size = roundup(ctx->msg_sz, PAGE_SIZE); in fastrpc_invoke_send()
942 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg)); in fastrpc_invoke_send()
958 if (!fl->sctx) in fastrpc_internal_invoke()
959 return -EINVAL; in fastrpc_internal_invoke()
961 if (!fl->cctx->rpdev) in fastrpc_internal_invoke()
962 return -EPIPE; in fastrpc_internal_invoke()
965 …dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle… in fastrpc_internal_invoke()
966 return -EPERM; in fastrpc_internal_invoke()
973 if (ctx->nscalars) { in fastrpc_internal_invoke()
982 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle); in fastrpc_internal_invoke()
987 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) in fastrpc_internal_invoke()
988 err = -ETIMEDOUT; in fastrpc_internal_invoke()
990 err = wait_for_completion_interruptible(&ctx->work); in fastrpc_internal_invoke()
996 if (ctx->nscalars) { in fastrpc_internal_invoke()
1006 err = ctx->retval; in fastrpc_internal_invoke()
1011 if (err != -ERESTARTSYS && err != -ETIMEDOUT) { in fastrpc_internal_invoke()
1012 /* We are done with this compute context */ in fastrpc_internal_invoke()
1013 spin_lock(&fl->lock); in fastrpc_internal_invoke()
1014 list_del(&ctx->node); in fastrpc_internal_invoke()
1015 spin_unlock(&fl->lock); in fastrpc_internal_invoke()
1019 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err); in fastrpc_internal_invoke()
1046 return -ENOMEM; in fastrpc_init_create_process()
1049 err = -EFAULT; in fastrpc_init_create_process()
1054 err = -EINVAL; in fastrpc_init_create_process()
1058 inbuf.pgid = fl->tgid; in fastrpc_init_create_process()
1059 inbuf.namelen = strlen(current->comm) + 1; in fastrpc_init_create_process()
1064 fl->pd = USER_PD; in fastrpc_init_create_process()
1074 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen, in fastrpc_init_create_process()
1079 fl->init_mem = imem; in fastrpc_init_create_process()
1082 args[0].fd = -1; in fastrpc_init_create_process()
1084 args[1].ptr = (u64)(uintptr_t)current->comm; in fastrpc_init_create_process()
1086 args[1].fd = -1; in fastrpc_init_create_process()
1092 pages[0].addr = imem->phys; in fastrpc_init_create_process()
1093 pages[0].size = imem->size; in fastrpc_init_create_process()
1097 args[3].fd = -1; in fastrpc_init_create_process()
1101 args[4].fd = -1; in fastrpc_init_create_process()
1105 args[5].fd = -1; in fastrpc_init_create_process()
1121 fl->init_mem = NULL; in fastrpc_init_create_process()
1138 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_session_alloc()
1139 for (i = 0; i < cctx->sesscount; i++) { in fastrpc_session_alloc()
1140 if (!cctx->session[i].used && cctx->session[i].valid) { in fastrpc_session_alloc()
1141 cctx->session[i].used = true; in fastrpc_session_alloc()
1142 session = &cctx->session[i]; in fastrpc_session_alloc()
1146 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_session_alloc()
1156 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_session_free()
1157 session->used = false; in fastrpc_session_free()
1158 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_session_free()
1167 tgid = fl->tgid; in fastrpc_release_current_dsp_process()
1170 args[0].fd = -1; in fastrpc_release_current_dsp_process()
1180 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; in fastrpc_device_release()
1181 struct fastrpc_channel_ctx *cctx = fl->cctx; in fastrpc_device_release()
1189 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_device_release()
1190 list_del(&fl->user); in fastrpc_device_release()
1191 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_device_release()
1193 if (fl->init_mem) in fastrpc_device_release()
1194 fastrpc_buf_free(fl->init_mem); in fastrpc_device_release()
1196 list_for_each_entry_safe(ctx, n, &fl->pending, node) { in fastrpc_device_release()
1197 list_del(&ctx->node); in fastrpc_device_release()
1201 list_for_each_entry_safe(map, m, &fl->maps, node) in fastrpc_device_release()
1204 list_for_each_entry_safe(buf, b, &fl->mmaps, node) { in fastrpc_device_release()
1205 list_del(&buf->node); in fastrpc_device_release()
1209 fastrpc_session_free(cctx, fl->sctx); in fastrpc_device_release()
1212 mutex_destroy(&fl->mutex); in fastrpc_device_release()
1214 file->private_data = NULL; in fastrpc_device_release()
1221 struct fastrpc_channel_ctx *cctx = miscdev_to_cctx(filp->private_data); in fastrpc_device_open()
1227 return -ENOMEM; in fastrpc_device_open()
1232 filp->private_data = fl; in fastrpc_device_open()
1233 spin_lock_init(&fl->lock); in fastrpc_device_open()
1234 mutex_init(&fl->mutex); in fastrpc_device_open()
1235 INIT_LIST_HEAD(&fl->pending); in fastrpc_device_open()
1236 INIT_LIST_HEAD(&fl->maps); in fastrpc_device_open()
1237 INIT_LIST_HEAD(&fl->mmaps); in fastrpc_device_open()
1238 INIT_LIST_HEAD(&fl->user); in fastrpc_device_open()
1239 fl->tgid = current->tgid; in fastrpc_device_open()
1240 fl->cctx = cctx; in fastrpc_device_open()
1242 fl->sctx = fastrpc_session_alloc(cctx); in fastrpc_device_open()
1243 if (!fl->sctx) { in fastrpc_device_open()
1244 dev_err(&cctx->rpdev->dev, "No session available\n"); in fastrpc_device_open()
1245 mutex_destroy(&fl->mutex); in fastrpc_device_open()
1248 return -EBUSY; in fastrpc_device_open()
1251 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_device_open()
1252 list_add_tail(&fl->user, &cctx->users); in fastrpc_device_open()
1253 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_device_open()
1266 return -EFAULT; in fastrpc_dmabuf_alloc()
1268 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf); in fastrpc_dmabuf_alloc()
1275 buf->dmabuf = dma_buf_export(&exp_info); in fastrpc_dmabuf_alloc()
1276 if (IS_ERR(buf->dmabuf)) { in fastrpc_dmabuf_alloc()
1277 err = PTR_ERR(buf->dmabuf); in fastrpc_dmabuf_alloc()
1282 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE); in fastrpc_dmabuf_alloc()
1284 dma_buf_put(buf->dmabuf); in fastrpc_dmabuf_alloc()
1285 return -EINVAL; in fastrpc_dmabuf_alloc()
1297 return -EFAULT; in fastrpc_dmabuf_alloc()
1306 int tgid = fl->tgid; in fastrpc_init_attach()
1311 args[0].fd = -1; in fastrpc_init_attach()
1314 fl->pd = pd; in fastrpc_init_attach()
1328 return -EFAULT; in fastrpc_invoke()
1335 return -ENOMEM; in fastrpc_invoke()
1340 return -EFAULT; in fastrpc_invoke()
1356 struct device *dev = fl->sctx->dev; in fastrpc_req_munmap_impl()
1360 spin_lock(&fl->lock); in fastrpc_req_munmap_impl()
1361 list_for_each_entry_safe(iter, b, &fl->mmaps, node) { in fastrpc_req_munmap_impl()
1362 if ((iter->raddr == req->vaddrout) && (iter->size == req->size)) { in fastrpc_req_munmap_impl()
1367 spin_unlock(&fl->lock); in fastrpc_req_munmap_impl()
1371 return -EINVAL; in fastrpc_req_munmap_impl()
1374 req_msg.pgid = fl->tgid; in fastrpc_req_munmap_impl()
1375 req_msg.size = buf->size; in fastrpc_req_munmap_impl()
1376 req_msg.vaddr = buf->raddr; in fastrpc_req_munmap_impl()
1385 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); in fastrpc_req_munmap_impl()
1386 spin_lock(&fl->lock); in fastrpc_req_munmap_impl()
1387 list_del(&buf->node); in fastrpc_req_munmap_impl()
1388 spin_unlock(&fl->lock); in fastrpc_req_munmap_impl()
1391 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); in fastrpc_req_munmap_impl()
1402 return -EFAULT; in fastrpc_req_munmap()
1416 struct device *dev = fl->sctx->dev; in fastrpc_req_mmap()
1421 return -EFAULT; in fastrpc_req_mmap()
1425 return -EINVAL; in fastrpc_req_mmap()
1430 return -EINVAL; in fastrpc_req_mmap()
1433 err = fastrpc_buf_alloc(fl, fl->sctx->dev, req.size, &buf); in fastrpc_req_mmap()
1439 req_msg.pgid = fl->tgid; in fastrpc_req_mmap()
1447 pages.addr = buf->phys; in fastrpc_req_mmap()
1448 pages.size = buf->size; in fastrpc_req_mmap()
1460 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size); in fastrpc_req_mmap()
1465 buf->raddr = (uintptr_t) rsp_msg.vaddr; in fastrpc_req_mmap()
1470 spin_lock(&fl->lock); in fastrpc_req_mmap()
1471 list_add_tail(&buf->node, &fl->mmaps); in fastrpc_req_mmap()
1472 spin_unlock(&fl->lock); in fastrpc_req_mmap()
1476 req_unmap.vaddrout = buf->raddr; in fastrpc_req_mmap()
1477 req_unmap.size = buf->size; in fastrpc_req_mmap()
1479 return -EFAULT; in fastrpc_req_mmap()
1483 buf->raddr, buf->size); in fastrpc_req_mmap()
1496 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; in fastrpc_device_ioctl()
1523 err = -ENOTTY; in fastrpc_device_ioctl()
1541 struct device *dev = &pdev->dev; in fastrpc_cb_probe()
1546 cctx = dev_get_drvdata(dev->parent); in fastrpc_cb_probe()
1548 return -EINVAL; in fastrpc_cb_probe()
1550 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions); in fastrpc_cb_probe()
1552 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_cb_probe()
1553 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) { in fastrpc_cb_probe()
1554 dev_err(&pdev->dev, "too many sessions\n"); in fastrpc_cb_probe()
1555 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_probe()
1556 return -ENOSPC; in fastrpc_cb_probe()
1558 sess = &cctx->session[cctx->sesscount++]; in fastrpc_cb_probe()
1559 sess->used = false; in fastrpc_cb_probe()
1560 sess->valid = true; in fastrpc_cb_probe()
1561 sess->dev = dev; in fastrpc_cb_probe()
1564 if (of_property_read_u32(dev->of_node, "reg", &sess->sid)) in fastrpc_cb_probe()
1565 dev_info(dev, "FastRPC Session ID not specified in DT\n"); in fastrpc_cb_probe()
1571 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) in fastrpc_cb_probe()
1573 dup_sess = &cctx->session[cctx->sesscount++]; in fastrpc_cb_probe()
1577 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_probe()
1580 dev_err(dev, "32-bit DMA enable failed\n"); in fastrpc_cb_probe()
1589 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent); in fastrpc_cb_remove()
1590 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev); in fastrpc_cb_remove()
1594 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_cb_remove()
1596 if (cctx->session[i].sid == sess->sid) { in fastrpc_cb_remove()
1597 cctx->session[i].valid = false; in fastrpc_cb_remove()
1598 cctx->sesscount--; in fastrpc_cb_remove()
1601 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_remove()
1607 { .compatible = "qcom,fastrpc-compute-cb", },
1615 .name = "qcom,fastrpc-cb",
1623 struct device *rdev = &rpdev->dev; in fastrpc_rpmsg_probe()
1625 int i, err, domain_id = -1; in fastrpc_rpmsg_probe()
1628 err = of_property_read_string(rdev->of_node, "label", &domain); in fastrpc_rpmsg_probe()
1630 dev_info(rdev, "FastRPC Domain not specified in DT\n"); in fastrpc_rpmsg_probe()
1642 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id); in fastrpc_rpmsg_probe()
1643 return -EINVAL; in fastrpc_rpmsg_probe()
1648 return -ENOMEM; in fastrpc_rpmsg_probe()
1650 data->miscdev.minor = MISC_DYNAMIC_MINOR; in fastrpc_rpmsg_probe()
1651 data->miscdev.name = devm_kasprintf(rdev, GFP_KERNEL, "fastrpc-%s", in fastrpc_rpmsg_probe()
1653 data->miscdev.fops = &fastrpc_fops; in fastrpc_rpmsg_probe()
1654 err = misc_register(&data->miscdev); in fastrpc_rpmsg_probe()
1660 kref_init(&data->refcount); in fastrpc_rpmsg_probe()
1662 dev_set_drvdata(&rpdev->dev, data); in fastrpc_rpmsg_probe()
1664 INIT_LIST_HEAD(&data->users); in fastrpc_rpmsg_probe()
1665 spin_lock_init(&data->lock); in fastrpc_rpmsg_probe()
1666 idr_init(&data->ctx_idr); in fastrpc_rpmsg_probe()
1667 data->domain_id = domain_id; in fastrpc_rpmsg_probe()
1668 data->rpdev = rpdev; in fastrpc_rpmsg_probe()
1670 return of_platform_populate(rdev->of_node, NULL, NULL, rdev); in fastrpc_rpmsg_probe()
1677 spin_lock(&user->lock); in fastrpc_notify_users()
1678 list_for_each_entry(ctx, &user->pending, node) { in fastrpc_notify_users()
1679 ctx->retval = -EPIPE; in fastrpc_notify_users()
1680 complete(&ctx->work); in fastrpc_notify_users()
1682 spin_unlock(&user->lock); in fastrpc_notify_users()
1687 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); in fastrpc_rpmsg_remove()
1692 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_rpmsg_remove()
1693 cctx->rpdev = NULL; in fastrpc_rpmsg_remove()
1694 list_for_each_entry(user, &cctx->users, user) in fastrpc_rpmsg_remove()
1696 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_rpmsg_remove()
1698 misc_deregister(&cctx->miscdev); in fastrpc_rpmsg_remove()
1699 of_platform_depopulate(&rpdev->dev); in fastrpc_rpmsg_remove()
1707 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); in fastrpc_rpmsg_callback()
1714 return -EINVAL; in fastrpc_rpmsg_callback()
1716 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4); in fastrpc_rpmsg_callback()
1718 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_rpmsg_callback()
1719 ctx = idr_find(&cctx->ctx_idr, ctxid); in fastrpc_rpmsg_callback()
1720 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_rpmsg_callback()
1723 dev_err(&rpdev->dev, "No context ID matches response\n"); in fastrpc_rpmsg_callback()
1724 return -ENOENT; in fastrpc_rpmsg_callback()
1727 ctx->retval = rsp->retval; in fastrpc_rpmsg_callback()
1728 complete(&ctx->work); in fastrpc_rpmsg_callback()
1735 schedule_work(&ctx->put_work); in fastrpc_rpmsg_callback()
1741 { .compatible = "qcom,fastrpc" },
1751 .name = "qcom,fastrpc",
1762 pr_err("fastrpc: failed to register cb driver\n"); in fastrpc_init()
1768 pr_err("fastrpc: failed to register rpmsg driver\n"); in fastrpc_init()