Lines Matching full:vring
120 struct vring vring; member
147 } vring; member
316 * making all of the arch DMA ops work on the vring device itself
451 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
462 desc = vq->split.vring.desc; in virtqueue_add_split()
518 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
520 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, in virtqueue_add_split()
523 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev, in virtqueue_add_split()
533 vq->split.vring.desc[head].next); in virtqueue_add_split()
546 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
547 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
553 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
609 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
612 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
632 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
633 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
634 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next); in detach_buf_split()
638 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
639 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, in detach_buf_split()
656 vq->split.vring.desc[head].len); in detach_buf_split()
658 BUG_ON(!(vq->split.vring.desc[head].flags & in detach_buf_split()
675 vq->split.vring.used->idx); in more_used_split()
703 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
705 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
707 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
709 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
727 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
743 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
764 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
768 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
779 vq->split.vring.used->idx); in virtqueue_poll_split()
797 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
805 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
808 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
826 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
833 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
839 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
861 struct vring vring; in vring_create_virtqueue_split() local
892 vring_init(&vring, num, queue, vring_align); in vring_create_virtqueue_split()
894 vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, in vring_create_virtqueue_split()
1007 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_indirect_packed()
1031 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); in virtqueue_add_indirect_packed()
1032 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * in virtqueue_add_indirect_packed()
1034 vq->packed.vring.desc[head].id = cpu_to_le16(id); in virtqueue_add_indirect_packed()
1050 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1058 if (n >= vq->packed.vring.num) { in virtqueue_add_indirect_packed()
1138 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); in virtqueue_add_packed()
1140 desc = vq->packed.vring.desc; in virtqueue_add_packed()
1152 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_packed()
1184 if ((unlikely(++i >= vq->packed.vring.num))) { in virtqueue_add_packed()
1215 vq->packed.vring.desc[head].flags = head_flags; in virtqueue_add_packed()
1234 if (i >= vq->packed.vring.num) in virtqueue_add_packed()
1267 snapshot.u32 = *(u32 *)vq->packed.vring.device; in virtqueue_kick_prepare_packed()
1283 event_idx -= vq->packed.vring.num; in virtqueue_kick_prepare_packed()
1343 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); in is_used_desc_packed()
1381 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); in virtqueue_get_buf_ctx_packed()
1382 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); in virtqueue_get_buf_ctx_packed()
1384 if (unlikely(id >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1398 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1399 vq->last_used_idx -= vq->packed.vring.num; in virtqueue_get_buf_ctx_packed()
1410 &vq->packed.vring.driver->off_wrap, in virtqueue_get_buf_ctx_packed()
1427 vq->packed.vring.driver->flags = in virtqueue_disable_cb_packed()
1444 vq->packed.vring.driver->off_wrap = in virtqueue_enable_cb_prepare_packed()
1459 vq->packed.vring.driver->flags = in virtqueue_enable_cb_prepare_packed()
1495 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; in virtqueue_enable_cb_delayed_packed()
1499 if (used_idx >= vq->packed.vring.num) { in virtqueue_enable_cb_delayed_packed()
1500 used_idx -= vq->packed.vring.num; in virtqueue_enable_cb_delayed_packed()
1504 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | in virtqueue_enable_cb_delayed_packed()
1518 vq->packed.vring.driver->flags = in virtqueue_enable_cb_delayed_packed()
1547 for (i = 0; i < vq->packed.vring.num; i++) { in virtqueue_detach_unused_buf_packed()
1557 BUG_ON(vq->vq.num_free != vq->packed.vring.num); in virtqueue_detach_unused_buf_packed()
1640 vq->packed.vring.num = num; in vring_create_virtqueue_packed()
1641 vq->packed.vring.desc = ring; in vring_create_virtqueue_packed()
1642 vq->packed.vring.driver = driver; in vring_create_virtqueue_packed()
1643 vq->packed.vring.device = device; in vring_create_virtqueue_packed()
1677 vq->packed.vring.driver->flags = in vring_create_virtqueue_packed()
2069 struct vring vring, in __vring_new_virtqueue() argument
2091 vq->vq.num_free = vring.num; in __vring_new_virtqueue()
2115 vq->split.vring = vring; in __vring_new_virtqueue()
2123 vq->split.vring.avail->flags = cpu_to_virtio16(vdev, in __vring_new_virtqueue()
2127 vq->split.desc_state = kmalloc_array(vring.num, in __vring_new_virtqueue()
2136 for (i = 0; i < vring.num-1; i++) in __vring_new_virtqueue()
2137 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); in __vring_new_virtqueue()
2138 memset(vq->split.desc_state, 0, vring.num * in __vring_new_virtqueue()
2184 struct vring vring; in vring_new_virtqueue() local
2189 vring_init(&vring, num, pages, vring_align); in vring_new_virtqueue()
2190 return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, in vring_new_virtqueue()
2203 vq->packed.vring.desc, in vring_del_virtqueue()
2208 vq->packed.vring.driver, in vring_del_virtqueue()
2213 vq->packed.vring.device, in vring_del_virtqueue()
2221 vq->split.vring.desc, in vring_del_virtqueue()
2262 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2263 * @_vq: the struct virtqueue containing the vring of interest.
2265 * Returns the size of the vring. This is mainly used for boasting to
2273 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2327 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2341 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2346 const struct vring *virtqueue_get_vring(struct virtqueue *vq) in virtqueue_get_vring()
2348 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()