Lines Matching full:split
117 /* Available for split ring */
137 } split; member
364 * Split ring specific functions - *_split().
451 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
462 desc = vq->split.vring.desc; in virtqueue_add_split()
518 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
520 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, in virtqueue_add_split()
523 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev, in virtqueue_add_split()
533 vq->split.vring.desc[head].next); in virtqueue_add_split()
538 vq->split.desc_state[head].data = data; in virtqueue_add_split()
540 vq->split.desc_state[head].indir_desc = desc; in virtqueue_add_split()
542 vq->split.desc_state[head].indir_desc = ctx; in virtqueue_add_split()
546 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
547 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
552 vq->split.avail_idx_shadow++; in virtqueue_add_split()
553 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
554 vq->split.avail_idx_shadow); in virtqueue_add_split()
600 old = vq->split.avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare_split()
601 new = vq->split.avail_idx_shadow; in virtqueue_kick_prepare_split()
609 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
612 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
627 vq->split.desc_state[head].data = NULL; in detach_buf_split()
632 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
633 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
634 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next); in detach_buf_split()
638 vring_unmap_one_split(vq, &vq->split.vring.desc[i]); in detach_buf_split()
639 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, in detach_buf_split()
648 vq->split.desc_state[head].indir_desc; in detach_buf_split()
656 vq->split.vring.desc[head].len); in detach_buf_split()
658 BUG_ON(!(vq->split.vring.desc[head].flags & in detach_buf_split()
666 vq->split.desc_state[head].indir_desc = NULL; in detach_buf_split()
668 *ctx = vq->split.desc_state[head].indir_desc; in detach_buf_split()
675 vq->split.vring.used->idx); in more_used_split()
703 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
705 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
707 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
709 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
713 if (unlikely(!vq->split.desc_state[i].data)) { in virtqueue_get_buf_ctx_split()
719 ret = vq->split.desc_state[i].data; in virtqueue_get_buf_ctx_split()
725 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf_ctx_split()
727 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
740 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { in virtqueue_disable_cb_split()
741 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_disable_cb_split()
743 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
745 vq->split.avail_flags_shadow); in virtqueue_disable_cb_split()
761 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_prepare_split()
762 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_prepare_split()
764 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
766 vq->split.avail_flags_shadow); in virtqueue_enable_cb_prepare_split()
768 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
779 vq->split.vring.used->idx); in virtqueue_poll_split()
794 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_delayed_split()
795 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_delayed_split()
797 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
799 vq->split.avail_flags_shadow); in virtqueue_enable_cb_delayed_split()
802 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; in virtqueue_enable_cb_delayed_split()
805 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
808 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
826 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
827 if (!vq->split.desc_state[i].data) in virtqueue_detach_unused_buf_split()
830 buf = vq->split.desc_state[i].data; in virtqueue_detach_unused_buf_split()
832 vq->split.avail_idx_shadow--; in virtqueue_detach_unused_buf_split()
833 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
834 vq->split.avail_idx_shadow); in virtqueue_detach_unused_buf_split()
839 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
902 to_vvq(vq)->split.queue_dma_addr = dma_addr; in vring_create_virtqueue_split()
903 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes; in vring_create_virtqueue_split()
1826 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1846 * virtqueue_notify - second half of split virtqueue_kick call.
2067 /* Only available for split ring */
2112 vq->split.queue_dma_addr = 0; in __vring_new_virtqueue()
2113 vq->split.queue_size_in_bytes = 0; in __vring_new_virtqueue()
2115 vq->split.vring = vring; in __vring_new_virtqueue()
2116 vq->split.avail_flags_shadow = 0; in __vring_new_virtqueue()
2117 vq->split.avail_idx_shadow = 0; in __vring_new_virtqueue()
2121 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in __vring_new_virtqueue()
2123 vq->split.vring.avail->flags = cpu_to_virtio16(vdev, in __vring_new_virtqueue()
2124 vq->split.avail_flags_shadow); in __vring_new_virtqueue()
2127 vq->split.desc_state = kmalloc_array(vring.num, in __vring_new_virtqueue()
2129 if (!vq->split.desc_state) { in __vring_new_virtqueue()
2137 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); in __vring_new_virtqueue()
2138 memset(vq->split.desc_state, 0, vring.num * in __vring_new_virtqueue()
2172 /* Only available for split ring */
2220 vq->split.queue_size_in_bytes, in vring_del_virtqueue()
2221 vq->split.vring.desc, in vring_del_virtqueue()
2222 vq->split.queue_dma_addr); in vring_del_virtqueue()
2226 kfree(vq->split.desc_state); in vring_del_virtqueue()
2273 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2313 return vq->split.queue_dma_addr; in virtqueue_get_desc_addr()
2326 return vq->split.queue_dma_addr + in virtqueue_get_avail_addr()
2327 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2340 return vq->split.queue_dma_addr + in virtqueue_get_used_addr()
2341 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2345 /* Only available for split ring */
2348 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()