Lines Matching full:vring
26 /* Vring size. */
64 * @num: vring size (number of descriptors)
65 * @align: vring alignment size
66 * @index: vring index
67 * @vdev_id: vring virtio id (VIRTIO_ID_xxx)
88 /* Check whether vring is in drop mode. */
161 * @vring: Tx/Rx ring
176 struct mlxbf_tmfifo_vring *vring[2]; member
221 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_free_vrings() local
225 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_free_vrings()
226 if (vring->va) { in mlxbf_tmfifo_free_vrings()
227 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_free_vrings()
229 vring->va, vring->dma); in mlxbf_tmfifo_free_vrings()
230 vring->va = NULL; in mlxbf_tmfifo_free_vrings()
231 if (vring->vq) { in mlxbf_tmfifo_free_vrings()
232 vring_del_virtqueue(vring->vq); in mlxbf_tmfifo_free_vrings()
233 vring->vq = NULL; in mlxbf_tmfifo_free_vrings()
243 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_alloc_vrings() local
250 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_alloc_vrings()
251 vring->fifo = fifo; in mlxbf_tmfifo_alloc_vrings()
252 vring->num = MLXBF_TMFIFO_VRING_SIZE; in mlxbf_tmfifo_alloc_vrings()
253 vring->align = SMP_CACHE_BYTES; in mlxbf_tmfifo_alloc_vrings()
254 vring->index = i; in mlxbf_tmfifo_alloc_vrings()
255 vring->vdev_id = tm_vdev->vdev.id.device; in mlxbf_tmfifo_alloc_vrings()
256 vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN; in mlxbf_tmfifo_alloc_vrings()
259 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_alloc_vrings()
267 vring->va = va; in mlxbf_tmfifo_alloc_vrings()
268 vring->dma = dma; in mlxbf_tmfifo_alloc_vrings()
297 /* Get the next packet descriptor from the vring. */
299 mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_get_next_desc() argument
301 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_next_desc()
302 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_next_desc()
305 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx)) in mlxbf_tmfifo_get_next_desc()
311 idx = vring->next_avail % vr->num; in mlxbf_tmfifo_get_next_desc()
316 vring->next_avail++; in mlxbf_tmfifo_get_next_desc()
322 static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_release_desc() argument
325 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_release_desc()
326 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_release_desc()
344 static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_get_pkt_len() argument
347 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_pkt_len()
348 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_pkt_len()
362 static void mlxbf_tmfifo_release_pkt(struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_release_pkt() argument
367 if (vring->desc_head) { in mlxbf_tmfifo_release_pkt()
368 desc_head = vring->desc_head; in mlxbf_tmfifo_release_pkt()
369 len = vring->pkt_len; in mlxbf_tmfifo_release_pkt()
371 desc_head = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_release_pkt()
372 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head); in mlxbf_tmfifo_release_pkt()
376 mlxbf_tmfifo_release_desc(vring, desc_head, len); in mlxbf_tmfifo_release_pkt()
378 vring->pkt_len = 0; in mlxbf_tmfifo_release_pkt()
379 vring->desc = NULL; in mlxbf_tmfifo_release_pkt()
380 vring->desc_head = NULL; in mlxbf_tmfifo_release_pkt()
383 static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_init_net_desc() argument
386 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_init_net_desc()
395 mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx) in mlxbf_tmfifo_get_next_pkt() argument
399 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_get_next_pkt()
400 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) in mlxbf_tmfifo_get_next_pkt()
401 mlxbf_tmfifo_init_net_desc(vring, desc, is_rx); in mlxbf_tmfifo_get_next_pkt()
403 vring->desc_head = desc; in mlxbf_tmfifo_get_next_pkt()
404 vring->desc = desc; in mlxbf_tmfifo_get_next_pkt()
426 struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_console_output_one() argument
429 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_console_output_one()
459 struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_console_output() argument
464 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_console_output()
467 len = mlxbf_tmfifo_get_pkt_len(vring, desc); in mlxbf_tmfifo_console_output()
471 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
475 mlxbf_tmfifo_console_output_one(cons, vring, desc); in mlxbf_tmfifo_console_output()
476 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
477 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_console_output()
573 static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_word() argument
577 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_rxtx_word()
578 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_word()
589 if (vring->cur_len + sizeof(u64) <= len) { in mlxbf_tmfifo_rxtx_word()
592 if (!IS_VRING_DROP(vring)) in mlxbf_tmfifo_rxtx_word()
593 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
596 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
599 vring->cur_len += sizeof(u64); in mlxbf_tmfifo_rxtx_word()
603 if (!IS_VRING_DROP(vring)) in mlxbf_tmfifo_rxtx_word()
604 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
605 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
608 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
609 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
611 vring->cur_len = len; in mlxbf_tmfifo_rxtx_word()
622 * In Rx case, the packet might be found to belong to a different vring since
626 static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_header() argument
630 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_header()
662 * Check whether the new packet still belongs to this vring. in mlxbf_tmfifo_rxtx_header()
663 * If not, update the pkt_len of the new vring. in mlxbf_tmfifo_rxtx_header()
665 if (vdev_id != vring->vdev_id) { in mlxbf_tmfifo_rxtx_header()
670 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
671 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; in mlxbf_tmfifo_rxtx_header()
675 if (drop_rx && !IS_VRING_DROP(vring)) { in mlxbf_tmfifo_rxtx_header()
676 if (vring->desc_head) in mlxbf_tmfifo_rxtx_header()
677 mlxbf_tmfifo_release_pkt(vring); in mlxbf_tmfifo_rxtx_header()
678 *desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_header()
679 vring->desc_head = *desc; in mlxbf_tmfifo_rxtx_header()
680 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
683 vring->pkt_len = ntohs(hdr.len) + hdr_len; in mlxbf_tmfifo_rxtx_header()
686 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
688 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, *desc); in mlxbf_tmfifo_rxtx_header()
689 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
691 hdr.len = htons(vring->pkt_len - hdr_len); in mlxbf_tmfifo_rxtx_header()
695 vring->cur_len = hdr_len; in mlxbf_tmfifo_rxtx_header()
696 vring->rem_len = vring->pkt_len; in mlxbf_tmfifo_rxtx_header()
697 fifo->vring[is_rx] = vring; in mlxbf_tmfifo_rxtx_header()
705 static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_one_desc() argument
708 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
709 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_one_desc()
716 vdev = &fifo->vdev[vring->vdev_id]->vdev; in mlxbf_tmfifo_rxtx_one_desc()
719 if (!vring->desc) { in mlxbf_tmfifo_rxtx_one_desc()
720 desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx); in mlxbf_tmfifo_rxtx_one_desc()
724 desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_one_desc()
725 vring->desc_head = desc; in mlxbf_tmfifo_rxtx_one_desc()
726 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
732 desc = vring->desc; in mlxbf_tmfifo_rxtx_one_desc()
736 if (vring->pkt_len == 0) { in mlxbf_tmfifo_rxtx_one_desc()
737 mlxbf_tmfifo_rxtx_header(vring, &desc, is_rx, &vring_change); in mlxbf_tmfifo_rxtx_one_desc()
748 if (len > vring->rem_len) in mlxbf_tmfifo_rxtx_one_desc()
749 len = vring->rem_len; in mlxbf_tmfifo_rxtx_one_desc()
752 if (vring->cur_len < len) { in mlxbf_tmfifo_rxtx_one_desc()
753 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len); in mlxbf_tmfifo_rxtx_one_desc()
758 if (vring->cur_len == len) { in mlxbf_tmfifo_rxtx_one_desc()
759 vring->cur_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
760 vring->rem_len -= len; in mlxbf_tmfifo_rxtx_one_desc()
763 if (!IS_VRING_DROP(vring) && vring->rem_len > 0 && in mlxbf_tmfifo_rxtx_one_desc()
772 fifo->vring[is_rx] = NULL; in mlxbf_tmfifo_rxtx_one_desc()
773 if (!IS_VRING_DROP(vring)) { in mlxbf_tmfifo_rxtx_one_desc()
774 mlxbf_tmfifo_release_pkt(vring); in mlxbf_tmfifo_rxtx_one_desc()
776 vring->pkt_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
777 vring->desc_head = NULL; in mlxbf_tmfifo_rxtx_one_desc()
778 vring->desc = NULL; in mlxbf_tmfifo_rxtx_one_desc()
790 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
796 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
802 static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx) in mlxbf_tmfifo_rxtx() argument
804 int avail = 0, devid = vring->vdev_id; in mlxbf_tmfifo_rxtx()
808 fifo = vring->fifo; in mlxbf_tmfifo_rxtx()
814 /* Return if another vring is running. */ in mlxbf_tmfifo_rxtx()
815 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) in mlxbf_tmfifo_rxtx()
840 more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail); in mlxbf_tmfifo_rxtx()
849 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_work_rxtx() local
859 vring = &tm_vdev->vrings[queue_id]; in mlxbf_tmfifo_work_rxtx()
860 if (vring->vq) in mlxbf_tmfifo_work_rxtx()
861 mlxbf_tmfifo_rxtx(vring, is_rx); in mlxbf_tmfifo_work_rxtx()
891 struct mlxbf_tmfifo_vring *vring = vq->priv; in mlxbf_tmfifo_virtio_notify() local
896 fifo = vring->fifo; in mlxbf_tmfifo_virtio_notify()
902 if (vring->index & BIT(0)) { in mlxbf_tmfifo_virtio_notify()
905 * In such case, the vring needs to be served right away. For in mlxbf_tmfifo_virtio_notify()
909 if (vring->vdev_id == VIRTIO_ID_CONSOLE) { in mlxbf_tmfifo_virtio_notify()
912 mlxbf_tmfifo_console_output(tm_vdev, vring); in mlxbf_tmfifo_virtio_notify()
951 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_virtio_del_vqs() local
956 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_del_vqs()
959 if (vring->desc) in mlxbf_tmfifo_virtio_del_vqs()
960 mlxbf_tmfifo_release_pkt(vring); in mlxbf_tmfifo_virtio_del_vqs()
961 vq = vring->vq; in mlxbf_tmfifo_virtio_del_vqs()
963 vring->vq = NULL; in mlxbf_tmfifo_virtio_del_vqs()
979 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_virtio_find_vqs() local
991 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_find_vqs()
993 /* zero vring */ in mlxbf_tmfifo_virtio_find_vqs()
994 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_virtio_find_vqs()
995 memset(vring->va, 0, size); in mlxbf_tmfifo_virtio_find_vqs()
996 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, in mlxbf_tmfifo_virtio_find_vqs()
997 false, false, vring->va, in mlxbf_tmfifo_virtio_find_vqs()
1007 vring->vq = vq; in mlxbf_tmfifo_virtio_find_vqs()
1008 vq->priv = vring; in mlxbf_tmfifo_virtio_find_vqs()
1126 dev_err(dev, "unable to allocate vring\n"); in mlxbf_tmfifo_create_vdev()