229 					 struct vduse_virtqueue *vq,  in vduse_dev_get_vq_state_packed()  argument
236 	msg.req.vq_state.index = vq->index;  in vduse_dev_get_vq_state_packed()
255 					struct vduse_virtqueue *vq,  in vduse_dev_get_vq_state_split()  argument
262 	msg.req.vq_state.index = vq->index;  in vduse_dev_get_vq_state_split()
426 		struct vduse_virtqueue *vq = &dev->vqs[i];  in vduse_dev_reset()  local
428 		vq->ready = false;  in vduse_dev_reset()
429 		vq->desc_addr = 0;  in vduse_dev_reset()
430 		vq->driver_addr = 0;  in vduse_dev_reset()
431 		vq->device_addr = 0;  in vduse_dev_reset()
432 		vq->num = 0;  in vduse_dev_reset()
433 		memset(&vq->state, 0, sizeof(vq->state));  in vduse_dev_reset()
435 		spin_lock(&vq->kick_lock);  in vduse_dev_reset()
436 		vq->kicked = false;  in vduse_dev_reset()
437 		if (vq->kickfd)  in vduse_dev_reset()
438 			eventfd_ctx_put(vq->kickfd);  in vduse_dev_reset()
439 		vq->kickfd = NULL;  in vduse_dev_reset()
440 		spin_unlock(&vq->kick_lock);  in vduse_dev_reset()
442 		spin_lock(&vq->irq_lock);  in vduse_dev_reset()
443 		vq->cb.callback = NULL;  in vduse_dev_reset()
444 		vq->cb.private = NULL;  in vduse_dev_reset()
445 		spin_unlock(&vq->irq_lock);  in vduse_dev_reset()
446 		flush_work(&vq->inject);  in vduse_dev_reset()
447 		flush_work(&vq->kick);  in vduse_dev_reset()
458 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_set_vq_address()  local
460 	vq->desc_addr = desc_area;  in vduse_vdpa_set_vq_address()
461 	vq->driver_addr = driver_area;  in vduse_vdpa_set_vq_address()
462 	vq->device_addr = device_area;  in vduse_vdpa_set_vq_address()
467 static void vduse_vq_kick(struct vduse_virtqueue *vq)  in vduse_vq_kick()  argument
469 	spin_lock(&vq->kick_lock);  in vduse_vq_kick()
470 	if (!vq->ready)  in vduse_vq_kick()
473 	if (vq->kickfd)  in vduse_vq_kick()
474 		eventfd_signal(vq->kickfd, 1);  in vduse_vq_kick()
476 		vq->kicked = true;  in vduse_vq_kick()
478 	spin_unlock(&vq->kick_lock);  in vduse_vq_kick()
483 	struct vduse_virtqueue *vq = container_of(work,  in vduse_vq_kick_work()  local
486 	vduse_vq_kick(vq);  in vduse_vq_kick_work()
492 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_kick_vq()  local
495 		schedule_work(&vq->kick);  in vduse_vdpa_kick_vq()
498 	vduse_vq_kick(vq);  in vduse_vdpa_kick_vq()
505 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_set_vq_cb()  local
507 	spin_lock(&vq->irq_lock);  in vduse_vdpa_set_vq_cb()
508 	vq->cb.callback = cb->callback;  in vduse_vdpa_set_vq_cb()
509 	vq->cb.private = cb->private;  in vduse_vdpa_set_vq_cb()
510 	spin_unlock(&vq->irq_lock);  in vduse_vdpa_set_vq_cb()
516 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_set_vq_num()  local
518 	vq->num = num;  in vduse_vdpa_set_vq_num()
525 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_set_vq_ready()  local
527 	vq->ready = ready;  in vduse_vdpa_set_vq_ready()
533 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_get_vq_ready()  local
535 	return vq->ready;  in vduse_vdpa_get_vq_ready()
542 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_set_vq_state()  local
545 		vq->state.packed.last_avail_counter =  in vduse_vdpa_set_vq_state()
547 		vq->state.packed.last_avail_idx = state->packed.last_avail_idx;  in vduse_vdpa_set_vq_state()
548 		vq->state.packed.last_used_counter =  in vduse_vdpa_set_vq_state()
550 		vq->state.packed.last_used_idx = state->packed.last_used_idx;  in vduse_vdpa_set_vq_state()
552 		vq->state.split.avail_index = state->split.avail_index;  in vduse_vdpa_set_vq_state()
561 	struct vduse_virtqueue *vq = &dev->vqs[idx];  in vduse_vdpa_get_vq_state()  local
564 		return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);  in vduse_vdpa_get_vq_state()
566 	return vduse_dev_get_vq_state_split(dev, vq, &state->split);  in vduse_vdpa_get_vq_state()
839 	struct vduse_virtqueue *vq;  in vduse_kickfd_setup()  local
846 	vq = &dev->vqs[index];  in vduse_kickfd_setup()
854 	spin_lock(&vq->kick_lock);  in vduse_kickfd_setup()
855 	if (vq->kickfd)  in vduse_kickfd_setup()
856 		eventfd_ctx_put(vq->kickfd);  in vduse_kickfd_setup()
857 	vq->kickfd = ctx;  in vduse_kickfd_setup()
858 	if (vq->ready && vq->kicked && vq->kickfd) {  in vduse_kickfd_setup()
859 		eventfd_signal(vq->kickfd, 1);  in vduse_kickfd_setup()
860 		vq->kicked = false;  in vduse_kickfd_setup()
862 	spin_unlock(&vq->kick_lock);  in vduse_kickfd_setup()
890 	struct vduse_virtqueue *vq = container_of(work,  in vduse_vq_irq_inject()  local
893 	spin_lock_bh(&vq->irq_lock);  in vduse_vq_irq_inject()
894 	if (vq->ready && vq->cb.callback)  in vduse_vq_irq_inject()
895 		vq->cb.callback(vq->cb.private);  in vduse_vq_irq_inject()
896 	spin_unlock_bh(&vq->irq_lock);  in vduse_vq_irq_inject()
1023 		struct vduse_virtqueue *vq;  in vduse_dev_ioctl()  local
1035 		vq = &dev->vqs[index];  in vduse_dev_ioctl()
1036 		vq_info.desc_addr = vq->desc_addr;  in vduse_dev_ioctl()
1037 		vq_info.driver_addr = vq->driver_addr;  in vduse_dev_ioctl()
1038 		vq_info.device_addr = vq->device_addr;  in vduse_dev_ioctl()
1039 		vq_info.num = vq->num;  in vduse_dev_ioctl()
1043 				vq->state.packed.last_avail_counter;  in vduse_dev_ioctl()
1045 				vq->state.packed.last_avail_idx;  in vduse_dev_ioctl()
1047 				vq->state.packed.last_used_counter;  in vduse_dev_ioctl()
1049 				vq->state.packed.last_used_idx;  in vduse_dev_ioctl()
1052 				vq->state.split.avail_index;  in vduse_dev_ioctl()
1054 		vq_info.ready = vq->ready;  in vduse_dev_ioctl()