Lines Matching refs:vq
36 #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num]) argument
37 #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num]) argument
178 struct vhost_virtqueue *vq) in vhost_vq_reset() argument
180 vq->num = 1; in vhost_vq_reset()
181 vq->desc = NULL; in vhost_vq_reset()
182 vq->avail = NULL; in vhost_vq_reset()
183 vq->used = NULL; in vhost_vq_reset()
184 vq->last_avail_idx = 0; in vhost_vq_reset()
185 vq->avail_idx = 0; in vhost_vq_reset()
186 vq->last_used_idx = 0; in vhost_vq_reset()
187 vq->signalled_used = 0; in vhost_vq_reset()
188 vq->signalled_used_valid = false; in vhost_vq_reset()
189 vq->used_flags = 0; in vhost_vq_reset()
190 vq->log_used = false; in vhost_vq_reset()
191 vq->log_addr = -1ull; in vhost_vq_reset()
192 vq->private_data = NULL; in vhost_vq_reset()
193 vq->acked_features = 0; in vhost_vq_reset()
194 vq->log_base = NULL; in vhost_vq_reset()
195 vq->error_ctx = NULL; in vhost_vq_reset()
196 vq->error = NULL; in vhost_vq_reset()
197 vq->kick = NULL; in vhost_vq_reset()
198 vq->call_ctx = NULL; in vhost_vq_reset()
199 vq->call = NULL; in vhost_vq_reset()
200 vq->log_ctx = NULL; in vhost_vq_reset()
201 vq->memory = NULL; in vhost_vq_reset()
253 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) in vhost_vq_free_iovecs() argument
255 kfree(vq->indirect); in vhost_vq_free_iovecs()
256 vq->indirect = NULL; in vhost_vq_free_iovecs()
257 kfree(vq->log); in vhost_vq_free_iovecs()
258 vq->log = NULL; in vhost_vq_free_iovecs()
259 kfree(vq->heads); in vhost_vq_free_iovecs()
260 vq->heads = NULL; in vhost_vq_free_iovecs()
266 struct vhost_virtqueue *vq; in vhost_dev_alloc_iovecs() local
270 vq = dev->vqs[i]; in vhost_dev_alloc_iovecs()
271 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV, in vhost_dev_alloc_iovecs()
273 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL); in vhost_dev_alloc_iovecs()
274 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL); in vhost_dev_alloc_iovecs()
275 if (!vq->indirect || !vq->log || !vq->heads) in vhost_dev_alloc_iovecs()
297 struct vhost_virtqueue *vq; in vhost_dev_init() local
312 vq = dev->vqs[i]; in vhost_dev_init()
313 vq->log = NULL; in vhost_dev_init()
314 vq->indirect = NULL; in vhost_dev_init()
315 vq->heads = NULL; in vhost_dev_init()
316 vq->dev = dev; in vhost_dev_init()
317 mutex_init(&vq->mutex); in vhost_dev_init()
318 vhost_vq_reset(dev, vq); in vhost_dev_init()
319 if (vq->handle_kick) in vhost_dev_init()
320 vhost_poll_init(&vq->poll, vq->handle_kick, in vhost_dev_init()
549 static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, in vq_access_ok() argument
554 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vq_access_ok()
572 static int vq_log_access_ok(struct vhost_virtqueue *vq, in vq_log_access_ok() argument
575 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vq_log_access_ok()
577 return vq_memory_access_ok(log_base, vq->memory, in vq_log_access_ok()
578 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && in vq_log_access_ok()
579 (!vq->log_used || log_access_ok(log_base, vq->log_addr, in vq_log_access_ok()
580 sizeof *vq->used + in vq_log_access_ok()
581 vq->num * sizeof *vq->used->ring + s)); in vq_log_access_ok()
586 int vhost_vq_access_ok(struct vhost_virtqueue *vq) in vhost_vq_access_ok() argument
588 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) && in vhost_vq_access_ok()
589 vq_log_access_ok(vq, vq->log_base); in vhost_vq_access_ok()
639 struct vhost_virtqueue *vq; in vhost_vring_ioctl() local
652 vq = d->vqs[idx]; in vhost_vring_ioctl()
654 mutex_lock(&vq->mutex); in vhost_vring_ioctl()
660 if (vq->private_data) { in vhost_vring_ioctl()
672 vq->num = s.num; in vhost_vring_ioctl()
677 if (vq->private_data) { in vhost_vring_ioctl()
689 vq->last_avail_idx = s.num; in vhost_vring_ioctl()
691 vq->avail_idx = vq->last_avail_idx; in vhost_vring_ioctl()
695 s.num = vq->last_avail_idx; in vhost_vring_ioctl()
716 if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) || in vhost_vring_ioctl()
717 (a.used_user_addr & (sizeof *vq->used->ring - 1)) || in vhost_vring_ioctl()
718 (a.log_guest_addr & (sizeof *vq->used->ring - 1))) { in vhost_vring_ioctl()
726 if (vq->private_data) { in vhost_vring_ioctl()
727 if (!vq_access_ok(vq, vq->num, in vhost_vring_ioctl()
737 !log_access_ok(vq->log_base, a.log_guest_addr, in vhost_vring_ioctl()
738 sizeof *vq->used + in vhost_vring_ioctl()
739 vq->num * sizeof *vq->used->ring)) { in vhost_vring_ioctl()
745 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); in vhost_vring_ioctl()
746 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; in vhost_vring_ioctl()
747 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; in vhost_vring_ioctl()
748 vq->log_addr = a.log_guest_addr; in vhost_vring_ioctl()
749 vq->used = (void __user *)(unsigned long)a.used_user_addr; in vhost_vring_ioctl()
761 if (eventfp != vq->kick) { in vhost_vring_ioctl()
762 pollstop = (filep = vq->kick) != NULL; in vhost_vring_ioctl()
763 pollstart = (vq->kick = eventfp) != NULL; in vhost_vring_ioctl()
777 if (eventfp != vq->call) { in vhost_vring_ioctl()
778 filep = vq->call; in vhost_vring_ioctl()
779 ctx = vq->call_ctx; in vhost_vring_ioctl()
780 vq->call = eventfp; in vhost_vring_ioctl()
781 vq->call_ctx = eventfp ? in vhost_vring_ioctl()
796 if (eventfp != vq->error) { in vhost_vring_ioctl()
797 filep = vq->error; in vhost_vring_ioctl()
798 vq->error = eventfp; in vhost_vring_ioctl()
799 ctx = vq->error_ctx; in vhost_vring_ioctl()
800 vq->error_ctx = eventfp ? in vhost_vring_ioctl()
809 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
810 vhost_poll_stop(&vq->poll); in vhost_vring_ioctl()
817 if (pollstart && vq->handle_kick) in vhost_vring_ioctl()
818 r = vhost_poll_start(&vq->poll, vq->kick); in vhost_vring_ioctl()
820 mutex_unlock(&vq->mutex); in vhost_vring_ioctl()
822 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
823 vhost_poll_flush(&vq->poll); in vhost_vring_ioctl()
862 struct vhost_virtqueue *vq; in vhost_dev_ioctl() local
864 vq = d->vqs[i]; in vhost_dev_ioctl()
865 mutex_lock(&vq->mutex); in vhost_dev_ioctl()
867 if (vq->private_data && !vq_log_access_ok(vq, base)) in vhost_dev_ioctl()
870 vq->log_base = base; in vhost_dev_ioctl()
871 mutex_unlock(&vq->mutex); in vhost_dev_ioctl()
977 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, in vhost_log_write() argument
986 r = log_write(vq->log_base, log[i].addr, l); in vhost_log_write()
991 if (vq->log_ctx) in vhost_log_write()
992 eventfd_signal(vq->log_ctx, 1); in vhost_log_write()
1002 static int vhost_update_used_flags(struct vhost_virtqueue *vq) in vhost_update_used_flags() argument
1005 if (__put_user(vq->used_flags, &vq->used->flags) < 0) in vhost_update_used_flags()
1007 if (unlikely(vq->log_used)) { in vhost_update_used_flags()
1011 used = &vq->used->flags; in vhost_update_used_flags()
1012 log_write(vq->log_base, vq->log_addr + in vhost_update_used_flags()
1013 (used - (void __user *)vq->used), in vhost_update_used_flags()
1014 sizeof vq->used->flags); in vhost_update_used_flags()
1015 if (vq->log_ctx) in vhost_update_used_flags()
1016 eventfd_signal(vq->log_ctx, 1); in vhost_update_used_flags()
1021 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) in vhost_update_avail_event() argument
1023 if (__put_user(vq->avail_idx, vhost_avail_event(vq))) in vhost_update_avail_event()
1025 if (unlikely(vq->log_used)) { in vhost_update_avail_event()
1030 used = vhost_avail_event(vq); in vhost_update_avail_event()
1031 log_write(vq->log_base, vq->log_addr + in vhost_update_avail_event()
1032 (used - (void __user *)vq->used), in vhost_update_avail_event()
1033 sizeof *vhost_avail_event(vq)); in vhost_update_avail_event()
1034 if (vq->log_ctx) in vhost_update_avail_event()
1035 eventfd_signal(vq->log_ctx, 1); in vhost_update_avail_event()
1040 int vhost_init_used(struct vhost_virtqueue *vq) in vhost_init_used() argument
1043 if (!vq->private_data) in vhost_init_used()
1046 r = vhost_update_used_flags(vq); in vhost_init_used()
1049 vq->signalled_used_valid = false; in vhost_init_used()
1050 return get_user(vq->last_used_idx, &vq->used->idx); in vhost_init_used()
1054 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, in translate_desc() argument
1063 mem = vq->memory; in translate_desc()
1109 static int get_indirect(struct vhost_virtqueue *vq, in get_indirect() argument
1121 vq_err(vq, "Invalid length in indirect descriptor: " in get_indirect()
1128 ret = translate_desc(vq, indirect->addr, indirect->len, vq->indirect, in get_indirect()
1131 vq_err(vq, "Translation failure %d in indirect.\n", ret); in get_indirect()
1143 vq_err(vq, "Indirect buffer length too big: %d\n", in get_indirect()
1151 vq_err(vq, "Loop detected: last one at %u " in get_indirect()
1157 vq->indirect, sizeof desc))) { in get_indirect()
1158 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", in get_indirect()
1163 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", in get_indirect()
1168 ret = translate_desc(vq, desc.addr, desc.len, iov + iov_count, in get_indirect()
1171 vq_err(vq, "Translation failure %d indirect idx %d\n", in get_indirect()
1187 vq_err(vq, "Indirect descriptor " in get_indirect()
1205 int vhost_get_vq_desc(struct vhost_virtqueue *vq, in vhost_get_vq_desc() argument
1216 last_avail_idx = vq->last_avail_idx; in vhost_get_vq_desc()
1217 if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) { in vhost_get_vq_desc()
1218 vq_err(vq, "Failed to access avail idx at %p\n", in vhost_get_vq_desc()
1219 &vq->avail->idx); in vhost_get_vq_desc()
1223 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { in vhost_get_vq_desc()
1224 vq_err(vq, "Guest moved used index from %u to %u", in vhost_get_vq_desc()
1225 last_avail_idx, vq->avail_idx); in vhost_get_vq_desc()
1230 if (vq->avail_idx == last_avail_idx) in vhost_get_vq_desc()
1231 return vq->num; in vhost_get_vq_desc()
1239 &vq->avail->ring[last_avail_idx % vq->num]))) { in vhost_get_vq_desc()
1240 vq_err(vq, "Failed to read head: idx %d address %p\n", in vhost_get_vq_desc()
1242 &vq->avail->ring[last_avail_idx % vq->num]); in vhost_get_vq_desc()
1247 if (unlikely(head >= vq->num)) { in vhost_get_vq_desc()
1248 vq_err(vq, "Guest says index %u > %u is available", in vhost_get_vq_desc()
1249 head, vq->num); in vhost_get_vq_desc()
1261 if (unlikely(i >= vq->num)) { in vhost_get_vq_desc()
1262 vq_err(vq, "Desc index is %u > %u, head = %u", in vhost_get_vq_desc()
1263 i, vq->num, head); in vhost_get_vq_desc()
1266 if (unlikely(++found > vq->num)) { in vhost_get_vq_desc()
1267 vq_err(vq, "Loop detected: last one at %u " in vhost_get_vq_desc()
1269 i, vq->num, head); in vhost_get_vq_desc()
1272 ret = __copy_from_user(&desc, vq->desc + i, sizeof desc); in vhost_get_vq_desc()
1274 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", in vhost_get_vq_desc()
1275 i, vq->desc + i); in vhost_get_vq_desc()
1279 ret = get_indirect(vq, iov, iov_size, in vhost_get_vq_desc()
1283 vq_err(vq, "Failure detected " in vhost_get_vq_desc()
1290 ret = translate_desc(vq, desc.addr, desc.len, iov + iov_count, in vhost_get_vq_desc()
1293 vq_err(vq, "Translation failure %d descriptor idx %d\n", in vhost_get_vq_desc()
1310 vq_err(vq, "Descriptor has out after in: " in vhost_get_vq_desc()
1319 vq->last_avail_idx++; in vhost_get_vq_desc()
1323 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); in vhost_get_vq_desc()
1329 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) in vhost_discard_vq_desc() argument
1331 vq->last_avail_idx -= n; in vhost_discard_vq_desc()
1337 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) in vhost_add_used() argument
1341 return vhost_add_used_n(vq, &heads, 1); in vhost_add_used()
1345 static int __vhost_add_used_n(struct vhost_virtqueue *vq, in __vhost_add_used_n() argument
1353 start = vq->last_used_idx % vq->num; in __vhost_add_used_n()
1354 used = vq->used->ring + start; in __vhost_add_used_n()
1357 vq_err(vq, "Failed to write used id"); in __vhost_add_used_n()
1361 vq_err(vq, "Failed to write used len"); in __vhost_add_used_n()
1365 vq_err(vq, "Failed to write used"); in __vhost_add_used_n()
1368 if (unlikely(vq->log_used)) { in __vhost_add_used_n()
1372 log_write(vq->log_base, in __vhost_add_used_n()
1373 vq->log_addr + in __vhost_add_used_n()
1374 ((void __user *)used - (void __user *)vq->used), in __vhost_add_used_n()
1377 old = vq->last_used_idx; in __vhost_add_used_n()
1378 new = (vq->last_used_idx += count); in __vhost_add_used_n()
1383 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) in __vhost_add_used_n()
1384 vq->signalled_used_valid = false; in __vhost_add_used_n()
1390 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, in vhost_add_used_n() argument
1395 start = vq->last_used_idx % vq->num; in vhost_add_used_n()
1396 n = vq->num - start; in vhost_add_used_n()
1398 r = __vhost_add_used_n(vq, heads, n); in vhost_add_used_n()
1404 r = __vhost_add_used_n(vq, heads, count); in vhost_add_used_n()
1408 if (put_user(vq->last_used_idx, &vq->used->idx)) { in vhost_add_used_n()
1409 vq_err(vq, "Failed to increment used idx"); in vhost_add_used_n()
1412 if (unlikely(vq->log_used)) { in vhost_add_used_n()
1414 log_write(vq->log_base, in vhost_add_used_n()
1415 vq->log_addr + offsetof(struct vring_used, idx), in vhost_add_used_n()
1416 sizeof vq->used->idx); in vhost_add_used_n()
1417 if (vq->log_ctx) in vhost_add_used_n()
1418 eventfd_signal(vq->log_ctx, 1); in vhost_add_used_n()
1424 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_notify() argument
1433 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && in vhost_notify()
1434 unlikely(vq->avail_idx == vq->last_avail_idx)) in vhost_notify()
1437 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_notify()
1439 if (__get_user(flags, &vq->avail->flags)) { in vhost_notify()
1440 vq_err(vq, "Failed to get flags"); in vhost_notify()
1445 old = vq->signalled_used; in vhost_notify()
1446 v = vq->signalled_used_valid; in vhost_notify()
1447 new = vq->signalled_used = vq->last_used_idx; in vhost_notify()
1448 vq->signalled_used_valid = true; in vhost_notify()
1453 if (get_user(event, vhost_used_event(vq))) { in vhost_notify()
1454 vq_err(vq, "Failed to get used event idx"); in vhost_notify()
1461 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_signal() argument
1464 if (vq->call_ctx && vhost_notify(dev, vq)) in vhost_signal()
1465 eventfd_signal(vq->call_ctx, 1); in vhost_signal()
1471 struct vhost_virtqueue *vq, in vhost_add_used_and_signal() argument
1474 vhost_add_used(vq, head, len); in vhost_add_used_and_signal()
1475 vhost_signal(dev, vq); in vhost_add_used_and_signal()
1481 struct vhost_virtqueue *vq, in vhost_add_used_and_signal_n() argument
1484 vhost_add_used_n(vq, heads, count); in vhost_add_used_and_signal_n()
1485 vhost_signal(dev, vq); in vhost_add_used_and_signal_n()
1490 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_enable_notify() argument
1495 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) in vhost_enable_notify()
1497 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; in vhost_enable_notify()
1498 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_enable_notify()
1499 r = vhost_update_used_flags(vq); in vhost_enable_notify()
1501 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_enable_notify()
1502 &vq->used->flags, r); in vhost_enable_notify()
1506 r = vhost_update_avail_event(vq, vq->avail_idx); in vhost_enable_notify()
1508 vq_err(vq, "Failed to update avail event index at %p: %d\n", in vhost_enable_notify()
1509 vhost_avail_event(vq), r); in vhost_enable_notify()
1516 r = __get_user(avail_idx, &vq->avail->idx); in vhost_enable_notify()
1518 vq_err(vq, "Failed to check avail idx at %p: %d\n", in vhost_enable_notify()
1519 &vq->avail->idx, r); in vhost_enable_notify()
1523 return avail_idx != vq->avail_idx; in vhost_enable_notify()
1528 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_disable_notify() argument
1532 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) in vhost_disable_notify()
1534 vq->used_flags |= VRING_USED_F_NO_NOTIFY; in vhost_disable_notify()
1535 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_disable_notify()
1536 r = vhost_update_used_flags(vq); in vhost_disable_notify()
1538 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_disable_notify()
1539 &vq->used->flags, r); in vhost_disable_notify()