1 // SPDX-License-Identifier: GPL-2.0
3  * Virtio-based remote processor messaging bus
8  * Ohad Ben-Cohen <ohad@wizery.com>
14 #include <linux/dma-mapping.h>
21 #include <linux/rpmsg.h>
34  * struct virtproc_info - virtual remote processor state
51  * @ns_ept:	the bus's name service endpoint
53  * This structure stores the rpmsg state of a given virtio remote processor
73 /* The feature bitmap for virtio rpmsg */
74 #define VIRTIO_RPMSG_F_NS	0 /* RP supports name service notifications */
77  * struct rpmsg_hdr - common header for all rpmsg messages
85  * Every message sent(/received) on the rpmsg bus begins with this header.
97  * struct rpmsg_ns_msg - dynamic name service announcement message
98  * @name: name of remote service that is published
104  * rpmsg channel (i.e device) is created/destroyed. In turn, the ->probe()
105  * or ->remove() handler of the appropriate rpmsg driver will be invoked
106  * (if/as-soon-as one is registered).
109 	char name[RPMSG_NAME_SIZE];  member
115  * enum rpmsg_ns_flags - dynamic name service announcement flags
126  * struct virtio_rpmsg_channel - rpmsg channel descriptor
127  * @rpdev: the rpmsg channel device
130  * This structure stores the channel that links the rpmsg device to the virtio
152  * We might also want to add support for user-provided buffers in time.
154  * to achieve zero-copy messaging.
156  * Note that these numbers are purely a decision of this driver - we
164  * Local addresses are dynamically allocated on-demand.
196  * rpmsg_sg_init - initialize scatterlist according to cpu address location
218  * __ept_release() - deallocate an rpmsg endpoint
245 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
251 	kref_init(&ept->refcount);  in __rpmsg_create_ept()
252 	mutex_init(&ept->cb_lock);  in __rpmsg_create_ept()
254 	ept->rpdev = rpdev;  in __rpmsg_create_ept()
255 	ept->cb = cb;  in __rpmsg_create_ept()
256 	ept->priv = priv;  in __rpmsg_create_ept()
257 	ept->ops = &virtio_endpoint_ops;  in __rpmsg_create_ept()
268 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
270 	/* bind the endpoint to an rpmsg address (and allocate one if needed) */  in __rpmsg_create_ept()
271 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
276 	ept->addr = id;  in __rpmsg_create_ept()
278 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
283 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
284 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_create_ept()
295 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
299  * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
304  * bound to an rpmsg channel. This is needed for handling the internal
305  * name service endpoint, which isn't bound to an rpmsg channel.
312 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
313 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
314 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
316 	/* make sure in-flight inbound messages won't invoke cb anymore */  in __rpmsg_destroy_ept()
317 	mutex_lock(&ept->cb_lock);  in __rpmsg_destroy_ept()
318 	ept->cb = NULL;  in __rpmsg_destroy_ept()
319 	mutex_unlock(&ept->cb_lock);  in __rpmsg_destroy_ept()
321 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_destroy_ept()
326 	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);  in virtio_rpmsg_destroy_ept()
328 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
334 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()
335 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_create()
338 	/* need to tell remote processor's name service about this channel ? */  in virtio_rpmsg_announce_create()
339 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_create()
340 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
343 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);  in virtio_rpmsg_announce_create()
344 		nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr);  in virtio_rpmsg_announce_create()
345 		nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_CREATE);  in virtio_rpmsg_announce_create()
347 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_create()
358 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()
359 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_destroy()
362 	/* tell remote processor's name service we're removing this channel */  in virtio_rpmsg_announce_destroy()
363 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_destroy()
364 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
367 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);  in virtio_rpmsg_announce_destroy()
368 		nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr);  in virtio_rpmsg_announce_destroy()
369 		nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_DESTROY);  in virtio_rpmsg_announce_destroy()
371 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_destroy()
394  * create an rpmsg channel using its name and address info.
403 	struct device *tmp, *dev = &vrp->vdev->dev;  in rpmsg_create_channel()
412 				chinfo->name, chinfo->src, chinfo->dst);  in rpmsg_create_channel()
421 	vch->vrp = vrp;  in rpmsg_create_channel()
424 	rpdev = &vch->rpdev;  in rpmsg_create_channel()
425 	rpdev->src = chinfo->src;  in rpmsg_create_channel()
426 	rpdev->dst = chinfo->dst;  in rpmsg_create_channel()
427 	rpdev->ops = &virtio_rpmsg_ops;  in rpmsg_create_channel()
430 	 * rpmsg server channels has predefined local address (for now),  in rpmsg_create_channel()
433 	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;  in rpmsg_create_channel()
435 	strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE);  in rpmsg_create_channel()
437 	rpdev->dev.parent = &vrp->vdev->dev;  in rpmsg_create_channel()
438 	rpdev->dev.release = virtio_rpmsg_release_device;  in rpmsg_create_channel()
453 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
459 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
460 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
463 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
465 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
471  * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed
489 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
492 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
493 		/* enable "tx-complete" interrupts before dozing off */  in rpmsg_upref_sleepers()
494 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
496 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
500  * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed
516 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
519 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
520 		/* disable "tx-complete" interrupts */  in rpmsg_downref_sleepers()
521 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
523 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
527  * rpmsg_send_offchannel_raw() - send a message across to the remote processor
528  * @rpdev: the rpmsg channel
535  * This function is the base implementation for all of the rpmsg sending API.
547  * case -ERESTARTSYS is returned. The number '15' itself was picked
552  * the function will immediately fail, and -ENOMEM will be returned.
556  * (see include/linux/rpmsg.h).
565 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()
566 	struct device *dev = &rpdev->dev;  in rpmsg_send_offchannel_raw()
574 		return -EINVAL;  in rpmsg_send_offchannel_raw()
578 	 * We currently use fixed-sized buffers, and therefore the payload  in rpmsg_send_offchannel_raw()
582 	 * user-provided buffers (and then we can also support zero-copy  in rpmsg_send_offchannel_raw()
584 	 * variable-length buffer sizes.  in rpmsg_send_offchannel_raw()
586 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
588 		return -EMSGSIZE;  in rpmsg_send_offchannel_raw()
594 		return -ENOMEM;  in rpmsg_send_offchannel_raw()
598 		/* enable "tx-complete" interrupts, if not already enabled */  in rpmsg_send_offchannel_raw()
607 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
611 		/* disable "tx-complete" interrupts if we're the last sleeper */  in rpmsg_send_offchannel_raw()
617 			return -ERESTARTSYS;  in rpmsg_send_offchannel_raw()
621 	msg->len = cpu_to_virtio16(vrp->vdev, len);  in rpmsg_send_offchannel_raw()
622 	msg->flags = 0;  in rpmsg_send_offchannel_raw()
623 	msg->src = cpu_to_virtio32(vrp->vdev, src);  in rpmsg_send_offchannel_raw()
624 	msg->dst = cpu_to_virtio32(vrp->vdev, dst);  in rpmsg_send_offchannel_raw()
625 	msg->reserved = 0;  in rpmsg_send_offchannel_raw()
626 	memcpy(msg->data, data, len);  in rpmsg_send_offchannel_raw()
629 		src, dst, len, msg->flags, msg->reserved);  in rpmsg_send_offchannel_raw()
637 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
640 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
644 		 * (memory won't leak, but rpmsg won't use it again for TX).  in rpmsg_send_offchannel_raw()
652 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
654 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
660 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send()
661 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_send()
669 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_sendto()
670 	u32 src = ept->addr;  in virtio_rpmsg_sendto()
678 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send_offchannel()
685 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend()
686 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_trysend()
694 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysendto()
695 	u32 src = ept->addr;  in virtio_rpmsg_trysendto()
703 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend_offchannel()
713 	unsigned int msg_len = virtio16_to_cpu(vrp->vdev, msg->len);  in rpmsg_recv_single()
717 		virtio32_to_cpu(vrp->vdev, msg->src),  in rpmsg_recv_single()
718 		virtio32_to_cpu(vrp->vdev, msg->dst), msg_len,  in rpmsg_recv_single()
719 		virtio16_to_cpu(vrp->vdev, msg->flags),  in rpmsg_recv_single()
720 		virtio32_to_cpu(vrp->vdev, msg->reserved));  in rpmsg_recv_single()
727 	 * We currently use fixed-sized buffers, so trivially sanitize  in rpmsg_recv_single()
730 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
731 	    msg_len > (len - sizeof(struct rpmsg_hdr))) {  in rpmsg_recv_single()
733 		return -EINVAL;  in rpmsg_recv_single()
737 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
739 	ept = idr_find(&vrp->endpoints, virtio32_to_cpu(vrp->vdev, msg->dst));  in rpmsg_recv_single()
743 		kref_get(&ept->refcount);  in rpmsg_recv_single()
745 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
748 		/* make sure ept->cb doesn't go away while we use it */  in rpmsg_recv_single()
749 		mutex_lock(&ept->cb_lock);  in rpmsg_recv_single()
751 		if (ept->cb)  in rpmsg_recv_single()
752 			ept->cb(ept->rpdev, msg->data, msg_len, ept->priv,  in rpmsg_recv_single()
753 				virtio32_to_cpu(vrp->vdev, msg->src));  in rpmsg_recv_single()
755 		mutex_unlock(&ept->cb_lock);  in rpmsg_recv_single()
758 		kref_put(&ept->refcount, __ept_release);  in rpmsg_recv_single()
763 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
766 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
778 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()
779 	struct device *dev = &rvq->vdev->dev;  in rpmsg_recv_done()
804 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
816 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()
818 	dev_dbg(&svq->vdev->dev, "%s\n", __func__);  in rpmsg_xmit_done()
821 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
824 /* invoked when a name service announcement arrives */
832 	struct device *dev = &vrp->vdev->dev;  in rpmsg_ns_cb()
842 		return -EINVAL;  in rpmsg_ns_cb()
846 	 * the name service ept does _not_ belong to a real rpmsg channel,  in rpmsg_ns_cb()
847 	 * and is handled by the rpmsg bus itself.  in rpmsg_ns_cb()
853 		return -EINVAL;  in rpmsg_ns_cb()
856 	/* don't trust the remote processor for null terminating the name */  in rpmsg_ns_cb()
857 	msg->name[RPMSG_NAME_SIZE - 1] = '\0';  in rpmsg_ns_cb()
859 	strncpy(chinfo.name, msg->name, sizeof(chinfo.name));  in rpmsg_ns_cb()
861 	chinfo.dst = virtio32_to_cpu(vrp->vdev, msg->addr);  in rpmsg_ns_cb()
864 		 virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY ?  in rpmsg_ns_cb()
865 		 "destroy" : "creat", msg->name, chinfo.dst);  in rpmsg_ns_cb()
867 	if (virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY) {  in rpmsg_ns_cb()
868 		ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo);  in rpmsg_ns_cb()
893 		return -ENOMEM;  in rpmsg_probe()
895 	vrp->vdev = vdev;  in rpmsg_probe()
897 	idr_init(&vrp->endpoints);  in rpmsg_probe()
898 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
899 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
900 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
907 	vrp->rvq = vqs[0];  in rpmsg_probe()
908 	vrp->svq = vqs[1];  in rpmsg_probe()
911 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
912 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
915 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
916 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
918 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
920 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
922 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
925 	bufs_va = dma_alloc_coherent(vdev->dev.parent,  in rpmsg_probe()
926 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
929 		err = -ENOMEM;  in rpmsg_probe()
933 	dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n",  in rpmsg_probe()
934 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
937 	vrp->rbufs = bufs_va;  in rpmsg_probe()
940 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
943 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
945 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
947 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
949 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
954 	/* suppress "tx-complete" interrupts */  in rpmsg_probe()
955 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
957 	vdev->priv = vrp;  in rpmsg_probe()
959 	/* if supported by the remote processor, enable the name service */  in rpmsg_probe()
961 		/* a dedicated endpoint handles the name service msgs */  in rpmsg_probe()
962 		vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,  in rpmsg_probe()
964 		if (!vrp->ns_ept) {  in rpmsg_probe()
965 			dev_err(&vdev->dev, "failed to create the ns ept\n");  in rpmsg_probe()
966 			err = -ENOMEM;  in rpmsg_probe()
972 	 * Prepare to kick but don't notify yet - we can't do this before  in rpmsg_probe()
975 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
986 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
988 	dev_info(&vdev->dev, "rpmsg host is online\n");  in rpmsg_probe()
993 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_probe()
994 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
996 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1011 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()
1012 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1015 	vdev->config->reset(vdev);  in rpmsg_remove()
1017 	ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device);  in rpmsg_remove()
1019 		dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret);  in rpmsg_remove()
1021 	if (vrp->ns_ept)  in rpmsg_remove()
1022 		__rpmsg_destroy_ept(vrp, vrp->ns_ept);  in rpmsg_remove()
1024 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1026 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1028 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_remove()
1029 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1046 	.driver.name	= KBUILD_MODNAME,
1072 MODULE_DESCRIPTION("Virtio-based remote processor messaging bus");