Lines Matching full:packed
90 /* Is this a packed ring? */
139 /* Available for packed ring */
177 } packed; member
911 * Packed ring specific functions - *_packed().
993 head = vq->packed.next_avail_idx; in virtqueue_add_indirect_packed()
1007 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_indirect_packed()
1031 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); in virtqueue_add_indirect_packed()
1032 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * in virtqueue_add_indirect_packed()
1034 vq->packed.vring.desc[head].id = cpu_to_le16(id); in virtqueue_add_indirect_packed()
1037 vq->packed.desc_extra[id].addr = addr; in virtqueue_add_indirect_packed()
1038 vq->packed.desc_extra[id].len = total_sg * in virtqueue_add_indirect_packed()
1040 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1041 vq->packed.avail_used_flags; in virtqueue_add_indirect_packed()
1050 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | in virtqueue_add_indirect_packed()
1051 vq->packed.avail_used_flags); in virtqueue_add_indirect_packed()
1058 if (n >= vq->packed.vring.num) { in virtqueue_add_indirect_packed()
1060 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_indirect_packed()
1061 vq->packed.avail_used_flags ^= in virtqueue_add_indirect_packed()
1065 vq->packed.next_avail_idx = n; in virtqueue_add_indirect_packed()
1066 vq->free_head = vq->packed.desc_state[id].next; in virtqueue_add_indirect_packed()
1069 vq->packed.desc_state[id].num = 1; in virtqueue_add_indirect_packed()
1070 vq->packed.desc_state[id].data = data; in virtqueue_add_indirect_packed()
1071 vq->packed.desc_state[id].indir_desc = desc; in virtqueue_add_indirect_packed()
1072 vq->packed.desc_state[id].last = id; in virtqueue_add_indirect_packed()
1135 head = vq->packed.next_avail_idx; in virtqueue_add_packed()
1136 avail_used_flags = vq->packed.avail_used_flags; in virtqueue_add_packed()
1138 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); in virtqueue_add_packed()
1140 desc = vq->packed.vring.desc; in virtqueue_add_packed()
1152 BUG_ON(id == vq->packed.vring.num); in virtqueue_add_packed()
1163 flags = cpu_to_le16(vq->packed.avail_used_flags | in virtqueue_add_packed()
1176 vq->packed.desc_extra[curr].addr = addr; in virtqueue_add_packed()
1177 vq->packed.desc_extra[curr].len = sg->length; in virtqueue_add_packed()
1178 vq->packed.desc_extra[curr].flags = in virtqueue_add_packed()
1182 curr = vq->packed.desc_state[curr].next; in virtqueue_add_packed()
1184 if ((unlikely(++i >= vq->packed.vring.num))) { in virtqueue_add_packed()
1186 vq->packed.avail_used_flags ^= in virtqueue_add_packed()
1194 vq->packed.avail_wrap_counter ^= 1; in virtqueue_add_packed()
1200 vq->packed.next_avail_idx = i; in virtqueue_add_packed()
1204 vq->packed.desc_state[id].num = descs_used; in virtqueue_add_packed()
1205 vq->packed.desc_state[id].data = data; in virtqueue_add_packed()
1206 vq->packed.desc_state[id].indir_desc = ctx; in virtqueue_add_packed()
1207 vq->packed.desc_state[id].last = prev; in virtqueue_add_packed()
1215 vq->packed.vring.desc[head].flags = head_flags; in virtqueue_add_packed()
1227 vq->packed.avail_used_flags = avail_used_flags; in virtqueue_add_packed()
1234 if (i >= vq->packed.vring.num) in virtqueue_add_packed()
1263 old = vq->packed.next_avail_idx - vq->num_added; in virtqueue_kick_prepare_packed()
1264 new = vq->packed.next_avail_idx; in virtqueue_kick_prepare_packed()
1267 snapshot.u32 = *(u32 *)vq->packed.vring.device; in virtqueue_kick_prepare_packed()
1282 if (wrap_counter != vq->packed.avail_wrap_counter) in virtqueue_kick_prepare_packed()
1283 event_idx -= vq->packed.vring.num; in virtqueue_kick_prepare_packed()
1298 state = &vq->packed.desc_state[id]; in detach_buf_packed()
1303 vq->packed.desc_state[state->last].next = vq->free_head; in detach_buf_packed()
1311 &vq->packed.desc_extra[curr]); in detach_buf_packed()
1312 curr = vq->packed.desc_state[curr].next; in detach_buf_packed()
1325 len = vq->packed.desc_extra[id].len; in detach_buf_packed()
1343 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); in is_used_desc_packed()
1353 vq->packed.used_wrap_counter); in more_used_packed()
1381 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); in virtqueue_get_buf_ctx_packed()
1382 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); in virtqueue_get_buf_ctx_packed()
1384 if (unlikely(id >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1388 if (unlikely(!vq->packed.desc_state[id].data)) { in virtqueue_get_buf_ctx_packed()
1394 ret = vq->packed.desc_state[id].data; in virtqueue_get_buf_ctx_packed()
1397 vq->last_used_idx += vq->packed.desc_state[id].num; in virtqueue_get_buf_ctx_packed()
1398 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) { in virtqueue_get_buf_ctx_packed()
1399 vq->last_used_idx -= vq->packed.vring.num; in virtqueue_get_buf_ctx_packed()
1400 vq->packed.used_wrap_counter ^= 1; in virtqueue_get_buf_ctx_packed()
1408 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) in virtqueue_get_buf_ctx_packed()
1410 &vq->packed.vring.driver->off_wrap, in virtqueue_get_buf_ctx_packed()
1412 (vq->packed.used_wrap_counter << in virtqueue_get_buf_ctx_packed()
1425 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_disable_cb_packed()
1426 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in virtqueue_disable_cb_packed()
1427 vq->packed.vring.driver->flags = in virtqueue_disable_cb_packed()
1428 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_disable_cb_packed()
1444 vq->packed.vring.driver->off_wrap = in virtqueue_enable_cb_prepare_packed()
1446 (vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1455 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_prepare_packed()
1456 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_prepare_packed()
1459 vq->packed.vring.driver->flags = in virtqueue_enable_cb_prepare_packed()
1460 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_prepare_packed()
1464 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter << in virtqueue_enable_cb_prepare_packed()
1495 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; in virtqueue_enable_cb_delayed_packed()
1496 wrap_counter = vq->packed.used_wrap_counter; in virtqueue_enable_cb_delayed_packed()
1499 if (used_idx >= vq->packed.vring.num) { in virtqueue_enable_cb_delayed_packed()
1500 used_idx -= vq->packed.vring.num; in virtqueue_enable_cb_delayed_packed()
1504 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | in virtqueue_enable_cb_delayed_packed()
1514 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { in virtqueue_enable_cb_delayed_packed()
1515 vq->packed.event_flags_shadow = vq->event ? in virtqueue_enable_cb_delayed_packed()
1518 vq->packed.vring.driver->flags = in virtqueue_enable_cb_delayed_packed()
1519 cpu_to_le16(vq->packed.event_flags_shadow); in virtqueue_enable_cb_delayed_packed()
1530 vq->packed.used_wrap_counter)) { in virtqueue_enable_cb_delayed_packed()
1547 for (i = 0; i < vq->packed.vring.num; i++) { in virtqueue_detach_unused_buf_packed()
1548 if (!vq->packed.desc_state[i].data) in virtqueue_detach_unused_buf_packed()
1551 buf = vq->packed.desc_state[i].data; in virtqueue_detach_unused_buf_packed()
1557 BUG_ON(vq->vq.num_free != vq->packed.vring.num); in virtqueue_detach_unused_buf_packed()
1633 vq->packed.ring_dma_addr = ring_dma_addr; in vring_create_virtqueue_packed()
1634 vq->packed.driver_event_dma_addr = driver_event_dma_addr; in vring_create_virtqueue_packed()
1635 vq->packed.device_event_dma_addr = device_event_dma_addr; in vring_create_virtqueue_packed()
1637 vq->packed.ring_size_in_bytes = ring_size_in_bytes; in vring_create_virtqueue_packed()
1638 vq->packed.event_size_in_bytes = event_size_in_bytes; in vring_create_virtqueue_packed()
1640 vq->packed.vring.num = num; in vring_create_virtqueue_packed()
1641 vq->packed.vring.desc = ring; in vring_create_virtqueue_packed()
1642 vq->packed.vring.driver = driver; in vring_create_virtqueue_packed()
1643 vq->packed.vring.device = device; in vring_create_virtqueue_packed()
1645 vq->packed.next_avail_idx = 0; in vring_create_virtqueue_packed()
1646 vq->packed.avail_wrap_counter = 1; in vring_create_virtqueue_packed()
1647 vq->packed.used_wrap_counter = 1; in vring_create_virtqueue_packed()
1648 vq->packed.event_flags_shadow = 0; in vring_create_virtqueue_packed()
1649 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; in vring_create_virtqueue_packed()
1651 vq->packed.desc_state = kmalloc_array(num, in vring_create_virtqueue_packed()
1654 if (!vq->packed.desc_state) in vring_create_virtqueue_packed()
1657 memset(vq->packed.desc_state, 0, in vring_create_virtqueue_packed()
1663 vq->packed.desc_state[i].next = i + 1; in vring_create_virtqueue_packed()
1665 vq->packed.desc_extra = kmalloc_array(num, in vring_create_virtqueue_packed()
1668 if (!vq->packed.desc_extra) in vring_create_virtqueue_packed()
1671 memset(vq->packed.desc_extra, 0, in vring_create_virtqueue_packed()
1676 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; in vring_create_virtqueue_packed()
1677 vq->packed.vring.driver->flags = in vring_create_virtqueue_packed()
1678 cpu_to_le16(vq->packed.event_flags_shadow); in vring_create_virtqueue_packed()
1687 kfree(vq->packed.desc_state); in vring_create_virtqueue_packed()
2202 vq->packed.ring_size_in_bytes, in vring_del_virtqueue()
2203 vq->packed.vring.desc, in vring_del_virtqueue()
2204 vq->packed.ring_dma_addr); in vring_del_virtqueue()
2207 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2208 vq->packed.vring.driver, in vring_del_virtqueue()
2209 vq->packed.driver_event_dma_addr); in vring_del_virtqueue()
2212 vq->packed.event_size_in_bytes, in vring_del_virtqueue()
2213 vq->packed.vring.device, in vring_del_virtqueue()
2214 vq->packed.device_event_dma_addr); in vring_del_virtqueue()
2216 kfree(vq->packed.desc_state); in vring_del_virtqueue()
2217 kfree(vq->packed.desc_extra); in vring_del_virtqueue()
2273 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2311 return vq->packed.ring_dma_addr; in virtqueue_get_desc_addr()
2324 return vq->packed.driver_event_dma_addr; in virtqueue_get_avail_addr()
2338 return vq->packed.device_event_dma_addr; in virtqueue_get_used_addr()