1 // SPDX-License-Identifier: GPL-2.0
3  * Virtio-based remote processor messaging bus
8  * Ohad Ben-Cohen <ohad@wizery.com>
14 #include <linux/dma-mapping.h>
21 #include <linux/rpmsg.h>
34  * struct virtproc_info - virtual remote processor state
53  * This structure stores the rpmsg state of a given virtio remote processor
73 /* The feature bitmap for virtio rpmsg */
77  * struct rpmsg_hdr - common header for all rpmsg messages
81  * @len: length of payload (in bytes)
85  * Every message sent(/received) on the rpmsg bus begins with this header.
97  * struct rpmsg_ns_msg - dynamic name service announcement message
104  * rpmsg channel (i.e device) is created/destroyed. In turn, the ->probe()
105  * or ->remove() handler of the appropriate rpmsg driver will be invoked
106  * (if/as-soon-as one is registered).
115  * enum rpmsg_ns_flags - dynamic name service announcement flags
126  * struct virtio_rpmsg_channel - rpmsg channel descriptor
127  * @rpdev: the rpmsg channel device
130  * This structure stores the channel that links the rpmsg device to the virtio
145  * by the vring, upto a maximum of 512 buffers (256 in each direction).
152  * We might also want to add support for user-provided buffers in time.
154  * to achieve zero-copy messaging.
156  * Note that these numbers are purely a decision of this driver - we
157  * can change this without changing anything in the firmware of the remote
164  * Local addresses are dynamically allocated on-demand.
166  * in order to reserve that address range for predefined services.
196  * rpmsg_sg_init - initialize scatterlist according to cpu address location
202  * location (in vmalloc or in kernel).
218  * __ept_release() - deallocate an rpmsg endpoint
245 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
251 	kref_init(&ept->refcount);  in __rpmsg_create_ept()
252 	mutex_init(&ept->cb_lock);  in __rpmsg_create_ept()
254 	ept->rpdev = rpdev;  in __rpmsg_create_ept()
255 	ept->cb = cb;  in __rpmsg_create_ept()
256 	ept->priv = priv;  in __rpmsg_create_ept()
257 	ept->ops = &virtio_endpoint_ops;  in __rpmsg_create_ept()
268 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
270 	/* bind the endpoint to an rpmsg address (and allocate one if needed) */  in __rpmsg_create_ept()
271 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
276 	ept->addr = id;  in __rpmsg_create_ept()
278 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
283 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
284 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_create_ept()
295 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
299  * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
304  * bound to an rpmsg channel. This is needed for handling the internal
305  * name service endpoint, which isn't bound to an rpmsg channel.
312 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
313 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
314 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
316 	/* make sure in-flight inbound messages won't invoke cb anymore */  in __rpmsg_destroy_ept()
317 	mutex_lock(&ept->cb_lock);  in __rpmsg_destroy_ept()
318 	ept->cb = NULL;  in __rpmsg_destroy_ept()
319 	mutex_unlock(&ept->cb_lock);  in __rpmsg_destroy_ept()
321 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_destroy_ept()
326 	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);  in virtio_rpmsg_destroy_ept()
328 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
334 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()
335 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_create()
339 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_create()
340 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
343 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);  in virtio_rpmsg_announce_create()
344 		nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr);  in virtio_rpmsg_announce_create()
345 		nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_CREATE);  in virtio_rpmsg_announce_create()
347 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_create()
358 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()
359 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_destroy()
363 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_destroy()
364 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
367 		strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE);  in virtio_rpmsg_announce_destroy()
368 		nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr);  in virtio_rpmsg_announce_destroy()
369 		nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_DESTROY);  in virtio_rpmsg_announce_destroy()
371 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_destroy()
390 	kfree(rpdev->driver_override);  in virtio_rpmsg_release_device()
395  * create an rpmsg channel using its name and address info.
404 	struct device *tmp, *dev = &vrp->vdev->dev;  in rpmsg_create_channel()
413 				chinfo->name, chinfo->src, chinfo->dst);  in rpmsg_create_channel()
422 	vch->vrp = vrp;  in rpmsg_create_channel()
425 	rpdev = &vch->rpdev;  in rpmsg_create_channel()
426 	rpdev->src = chinfo->src;  in rpmsg_create_channel()
427 	rpdev->dst = chinfo->dst;  in rpmsg_create_channel()
428 	rpdev->ops = &virtio_rpmsg_ops;  in rpmsg_create_channel()
431 	 * rpmsg server channels has predefined local address (for now),  in rpmsg_create_channel()
434 	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;  in rpmsg_create_channel()
436 	strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE);  in rpmsg_create_channel()
438 	rpdev->dev.parent = &vrp->vdev->dev;  in rpmsg_create_channel()
439 	rpdev->dev.release = virtio_rpmsg_release_device;  in rpmsg_create_channel()
454 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
460 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
461 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
464 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
466 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
472  * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed
483  * buffer is consumed (we rely on virtio's tx callback in order
490 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
493 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
494 		/* enable "tx-complete" interrupts before dozing off */  in rpmsg_upref_sleepers()
495 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
497 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
501  * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed
517 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
520 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
521 		/* disable "tx-complete" interrupts */  in rpmsg_downref_sleepers()
522 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
524 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
528  * rpmsg_send_offchannel_raw() - send a message across to the remote processor
529  * @rpdev: the rpmsg channel
534  * @wait: indicates whether caller should block in case no TX buffers available
536  * This function is the base implementation for all of the rpmsg sending API.
547  * sleep indefinitely due to misbehaving remote processors), and in that
548  * case -ERESTARTSYS is returned. The number '15' itself was picked
549  * arbitrarily; there's little point in asking drivers to provide a timeout
553  * the function will immediately fail, and -ENOMEM will be returned.
557  * (see include/linux/rpmsg.h).
566 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()
567 	struct device *dev = &rpdev->dev;  in rpmsg_send_offchannel_raw()
575 		return -EINVAL;  in rpmsg_send_offchannel_raw()
579 	 * We currently use fixed-sized buffers, and therefore the payload  in rpmsg_send_offchannel_raw()
583 	 * user-provided buffers (and then we can also support zero-copy  in rpmsg_send_offchannel_raw()
585 	 * variable-length buffer sizes.  in rpmsg_send_offchannel_raw()
587 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
589 		return -EMSGSIZE;  in rpmsg_send_offchannel_raw()
595 		return -ENOMEM;  in rpmsg_send_offchannel_raw()
599 		/* enable "tx-complete" interrupts, if not already enabled */  in rpmsg_send_offchannel_raw()
605 		 * little point in asking drivers to specify that.  in rpmsg_send_offchannel_raw()
608 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
612 		/* disable "tx-complete" interrupts if we're the last sleeper */  in rpmsg_send_offchannel_raw()
618 			return -ERESTARTSYS;  in rpmsg_send_offchannel_raw()
622 	msg->len = cpu_to_virtio16(vrp->vdev, len);  in rpmsg_send_offchannel_raw()
623 	msg->flags = 0;  in rpmsg_send_offchannel_raw()
624 	msg->src = cpu_to_virtio32(vrp->vdev, src);  in rpmsg_send_offchannel_raw()
625 	msg->dst = cpu_to_virtio32(vrp->vdev, dst);  in rpmsg_send_offchannel_raw()
626 	msg->reserved = 0;  in rpmsg_send_offchannel_raw()
627 	memcpy(msg->data, data, len);  in rpmsg_send_offchannel_raw()
630 		src, dst, len, msg->flags, msg->reserved);  in rpmsg_send_offchannel_raw()
638 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
641 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
645 		 * (memory won't leak, but rpmsg won't use it again for TX).  in rpmsg_send_offchannel_raw()
653 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
655 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
661 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send()
662 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_send()
670 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_sendto()
671 	u32 src = ept->addr;  in virtio_rpmsg_sendto()
679 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send_offchannel()
686 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend()
687 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_trysend()
695 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysendto()
696 	u32 src = ept->addr;  in virtio_rpmsg_trysendto()
704 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend_offchannel()
714 	unsigned int msg_len = virtio16_to_cpu(vrp->vdev, msg->len);  in rpmsg_recv_single()
718 		virtio32_to_cpu(vrp->vdev, msg->src),  in rpmsg_recv_single()
719 		virtio32_to_cpu(vrp->vdev, msg->dst), msg_len,  in rpmsg_recv_single()
720 		virtio16_to_cpu(vrp->vdev, msg->flags),  in rpmsg_recv_single()
721 		virtio32_to_cpu(vrp->vdev, msg->reserved));  in rpmsg_recv_single()
728 	 * We currently use fixed-sized buffers, so trivially sanitize  in rpmsg_recv_single()
731 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
732 	    msg_len > (len - sizeof(struct rpmsg_hdr))) {  in rpmsg_recv_single()
734 		return -EINVAL;  in rpmsg_recv_single()
738 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
740 	ept = idr_find(&vrp->endpoints, virtio32_to_cpu(vrp->vdev, msg->dst));  in rpmsg_recv_single()
744 		kref_get(&ept->refcount);  in rpmsg_recv_single()
746 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
749 		/* make sure ept->cb doesn't go away while we use it */  in rpmsg_recv_single()
750 		mutex_lock(&ept->cb_lock);  in rpmsg_recv_single()
752 		if (ept->cb)  in rpmsg_recv_single()
753 			ept->cb(ept->rpdev, msg->data, msg_len, ept->priv,  in rpmsg_recv_single()
754 				virtio32_to_cpu(vrp->vdev, msg->src));  in rpmsg_recv_single()
756 		mutex_unlock(&ept->cb_lock);  in rpmsg_recv_single()
759 		kref_put(&ept->refcount, __ept_release);  in rpmsg_recv_single()
764 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
767 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
779 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()
780 	struct device *dev = &rvq->vdev->dev;  in rpmsg_recv_done()
805 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
812  * Normally, though, we suppress this "tx complete" interrupt in order to
817 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()
819 	dev_dbg(&svq->vdev->dev, "%s\n", __func__);  in rpmsg_xmit_done()
822 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
833 	struct device *dev = &vrp->vdev->dev;  in rpmsg_ns_cb()
843 		return -EINVAL;  in rpmsg_ns_cb()
847 	 * the name service ept does _not_ belong to a real rpmsg channel,  in rpmsg_ns_cb()
848 	 * and is handled by the rpmsg bus itself.  in rpmsg_ns_cb()
850 	 * in somehow.  in rpmsg_ns_cb()
854 		return -EINVAL;  in rpmsg_ns_cb()
858 	msg->name[RPMSG_NAME_SIZE - 1] = '\0';  in rpmsg_ns_cb()
860 	strncpy(chinfo.name, msg->name, sizeof(chinfo.name));  in rpmsg_ns_cb()
862 	chinfo.dst = virtio32_to_cpu(vrp->vdev, msg->addr);  in rpmsg_ns_cb()
865 		 virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY ?  in rpmsg_ns_cb()
866 		 "destroy" : "creat", msg->name, chinfo.dst);  in rpmsg_ns_cb()
868 	if (virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY) {  in rpmsg_ns_cb()
869 		ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo);  in rpmsg_ns_cb()
894 		return -ENOMEM;  in rpmsg_probe()
896 	vrp->vdev = vdev;  in rpmsg_probe()
898 	idr_init(&vrp->endpoints);  in rpmsg_probe()
899 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
900 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
901 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
903 	/* We expect two virtqueues, rx and tx (and in this order) */  in rpmsg_probe()
908 	vrp->rvq = vqs[0];  in rpmsg_probe()
909 	vrp->svq = vqs[1];  in rpmsg_probe()
912 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
913 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
916 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
917 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
919 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
921 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
923 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
926 	bufs_va = dma_alloc_coherent(vdev->dev.parent,  in rpmsg_probe()
927 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
930 		err = -ENOMEM;  in rpmsg_probe()
934 	dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n",  in rpmsg_probe()
935 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
938 	vrp->rbufs = bufs_va;  in rpmsg_probe()
941 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
944 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
946 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
948 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
950 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
955 	/* suppress "tx-complete" interrupts */  in rpmsg_probe()
956 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
958 	vdev->priv = vrp;  in rpmsg_probe()
963 		vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,  in rpmsg_probe()
965 		if (!vrp->ns_ept) {  in rpmsg_probe()
966 			dev_err(&vdev->dev, "failed to create the ns ept\n");  in rpmsg_probe()
967 			err = -ENOMEM;  in rpmsg_probe()
973 	 * Prepare to kick but don't notify yet - we can't do this before  in rpmsg_probe()
976 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
987 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
989 	dev_info(&vdev->dev, "rpmsg host is online\n");  in rpmsg_probe()
994 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_probe()
995 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
997 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1012 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()
1013 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1016 	vdev->config->reset(vdev);  in rpmsg_remove()
1018 	ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device);  in rpmsg_remove()
1020 		dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret);  in rpmsg_remove()
1022 	if (vrp->ns_ept)  in rpmsg_remove()
1023 		__rpmsg_destroy_ept(vrp, vrp->ns_ept);  in rpmsg_remove()
1025 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1027 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1029 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_remove()
1030 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1073 MODULE_DESCRIPTION("Virtio-based remote processor messaging bus");