49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])  argument
50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])  argument
53 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)  in vhost_disable_cross_endian()
58 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)  in vhost_enable_cross_endian_big()
63 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)  in vhost_enable_cross_endian_little()
68 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)  in vhost_set_vring_endian()
90 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,  in vhost_get_vring_endian()
104 static void vhost_init_is_le(struct vhost_virtqueue *vq)  in vhost_init_is_le()
114 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)  in vhost_disable_cross_endian()
118 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)  in vhost_set_vring_endian()
123 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,  in vhost_get_vring_endian()
129 static void vhost_init_is_le(struct vhost_virtqueue *vq)  in vhost_init_is_le()
136 static void vhost_reset_is_le(struct vhost_virtqueue *vq)  in vhost_reset_is_le()
191 		     struct vhost_virtqueue *vq)  in vhost_poll_init()
249 bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work)  in vhost_vq_work_queue()
266 void vhost_vq_flush(struct vhost_virtqueue *vq)  in vhost_vq_flush()
322 bool vhost_vq_has_work(struct vhost_virtqueue *vq)  in vhost_vq_has_work()
343 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)  in __vhost_vq_meta_reset()
365 bool vhost_vq_is_setup(struct vhost_virtqueue *vq)  in vhost_vq_is_setup()
372 			   struct vhost_virtqueue *vq)  in vhost_vq_reset()
432 	struct vhost_virtqueue *vq;  in vhost_worker_killed()  local
462 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)  in vhost_vq_free_iovecs()
475 	struct vhost_virtqueue *vq;  in vhost_dev_alloc_iovecs()  local
506 bool vhost_exceeds_weight(struct vhost_virtqueue *vq,  in vhost_exceeds_weight()
521 static size_t vhost_get_avail_size(struct vhost_virtqueue *vq,  in vhost_get_avail_size()
530 static size_t vhost_get_used_size(struct vhost_virtqueue *vq,  in vhost_get_used_size()
539 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq,  in vhost_get_desc_size()
552 	struct vhost_virtqueue *vq;  in vhost_dev_init()  local
706 static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq,  in __vhost_vq_attach_worker()
775 static int vhost_vq_attach_worker(struct vhost_virtqueue *vq,  in vhost_vq_attach_worker()
836 				  struct vhost_virtqueue **vq, u32 *id)  in vhost_get_vq_from_user()
863 	struct vhost_virtqueue *vq;  in vhost_worker_ioctl()  local
1119 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,  in vhost_vq_meta_fetch()
1160 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,  in vhost_copy_to_user()
1195 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,  in vhost_copy_from_user()
1235 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,  in __vhost_get_user_slow()
1266 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,  in __vhost_get_user()
1278 #define vhost_put_user(vq, x, ptr)		\  argument
1295 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq)  in vhost_put_avail_event()
1301 static inline int vhost_put_used(struct vhost_virtqueue *vq,  in vhost_put_used()
1309 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq)  in vhost_put_used_flags()
1316 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq)  in vhost_put_used_idx()
1323 #define vhost_get_user(vq, x, ptr, type)		\  argument
1341 #define vhost_get_avail(vq, x, ptr) \  argument
1344 #define vhost_get_used(vq, x, ptr) \  argument
1361 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq,  in vhost_get_avail_idx()
1367 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,  in vhost_get_avail_head()
1374 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq,  in vhost_get_avail_flags()
1380 static inline int vhost_get_used_event(struct vhost_virtqueue *vq,  in vhost_get_used_event()
1386 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq,  in vhost_get_used_idx()
1392 static inline int vhost_get_desc(struct vhost_virtqueue *vq,  in vhost_get_desc()
1632 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)  in vhost_iotlb_miss()
1659 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,  in vq_access_ok()
1675 static void vhost_vq_meta_update(struct vhost_virtqueue *vq,  in vhost_vq_meta_update()
1686 static bool iotlb_access_ok(struct vhost_virtqueue *vq,  in iotlb_access_ok()
1720 int vq_meta_prefetch(struct vhost_virtqueue *vq)  in vq_meta_prefetch()
1745 static bool vq_log_used_access_ok(struct vhost_virtqueue *vq,  in vq_log_used_access_ok()
1761 static bool vq_log_access_ok(struct vhost_virtqueue *vq,  in vq_log_access_ok()
1771 bool vhost_vq_access_ok(struct vhost_virtqueue *vq)  in vhost_vq_access_ok()
1848 				struct vhost_virtqueue *vq,  in vhost_vring_set_num()
1869 				 struct vhost_virtqueue *vq,  in vhost_vring_set_addr()
1921 				     struct vhost_virtqueue *vq,  in vhost_vring_set_num_addr()
1949 	struct vhost_virtqueue *vq;  in vhost_vring_ioctl()  local
2096 		struct vhost_virtqueue *vq = d->vqs[i];  in vhost_init_device_iotlb()  local
2143 			struct vhost_virtqueue *vq;  in vhost_dev_ioctl()  local
2231 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len)  in log_write_hva()
2270 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len)  in log_used()
2293 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,  in vhost_log_write()
2329 static int vhost_update_used_flags(struct vhost_virtqueue *vq)  in vhost_update_used_flags()
2347 static int vhost_update_avail_event(struct vhost_virtqueue *vq)  in vhost_update_avail_event()
2365 int vhost_vq_init_access(struct vhost_virtqueue *vq)  in vhost_vq_init_access()
2400 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,  in translate_desc()
2448 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)  in next_desc()
2461 static int get_indirect(struct vhost_virtqueue *vq,  in get_indirect()
2562 int vhost_get_vq_desc(struct vhost_virtqueue *vq,  in vhost_get_vq_desc()
2704 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)  in vhost_discard_vq_desc()
2712 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)  in vhost_add_used()
2723 static int __vhost_add_used_n(struct vhost_virtqueue *vq,  in __vhost_add_used_n()
2757 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,  in vhost_add_used_n()
2792 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_notify()
2830 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_signal()
2840 			       struct vhost_virtqueue *vq,  in vhost_add_used_and_signal()
2850 				 struct vhost_virtqueue *vq,  in vhost_add_used_and_signal_n()
2859 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_vq_avail_empty()
2887 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_enable_notify()
2936 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_disable_notify()
2953 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type)  in vhost_new_msg()
2996 	struct vhost_virtqueue *vq;  in vhost_set_backend_features()  local