37 /* Returns vring->num if empty, -ve on error. */
46 	err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);  in __vringh_get_head()
49 			   &vrh->vring.avail->idx);  in __vringh_get_head()
54 		return vrh->vring.num;  in __vringh_get_head()
59 	i = *last_avail_idx & (vrh->vring.num - 1);  in __vringh_get_head()
61 	err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
64 			   *last_avail_idx, &vrh->vring.avail->ring[i]);  in __vringh_get_head()
68 	if (head >= vrh->vring.num) {  in __vringh_get_head()
70 			   head, vrh->vring.num);  in __vringh_get_head()
223 	*descs = vrh->vring.desc;  in return_from_indirect()
224 	*desc_max = vrh->vring.num;  in return_from_indirect()
282 	/* We start traversing vring's descriptor table. */  in __vringh_iov()
283 	descs = vrh->vring.desc;  in __vringh_iov()
284 	desc_max = vrh->vring.num;  in __vringh_iov()
339 		if (count > vrh->vring.num || indirect_count > desc_max) {  in __vringh_iov()
433 	used_ring = vrh->vring.used;  in __vringh_complete()
436 	off = used_idx % vrh->vring.num;  in __vringh_complete()
439 	if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {  in __vringh_complete()
440 		u16 part = vrh->vring.num - off;  in __vringh_complete()
457 	err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);  in __vringh_complete()
460 			   &vrh->vring.used->idx);  in __vringh_complete()
486 		err = getu16(vrh, &flags, &vrh->vring.avail->flags);  in __vringh_need_notify()
489 				   &vrh->vring.avail->flags);  in __vringh_need_notify()
496 	err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));  in __vringh_need_notify()
499 			   &vring_used_event(&vrh->vring));  in __vringh_need_notify()
526 		if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {  in __vringh_notify_enable()
528 				   &vrh->vring.used->flags);  in __vringh_notify_enable()
532 		if (putu16(vrh, &vring_avail_event(&vrh->vring),  in __vringh_notify_enable()
535 				   &vring_avail_event(&vrh->vring));  in __vringh_notify_enable()
544 	if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {  in __vringh_notify_enable()
546 			   &vrh->vring.avail->idx);  in __vringh_notify_enable()
562 		if (putu16(vrh, &vrh->vring.used->flags,  in __vringh_notify_disable()
565 				   &vrh->vring.used->flags);  in __vringh_notify_disable()
616  * vringh_init_user - initialize a vringh for a userspace vring.
646 	vrh->vring.num = num;  in vringh_init_user()
647 	/* vring expects kernel addresses, but only used via accessors. */  in vringh_init_user()
648 	vrh->vring.desc = (__force struct vring_desc *)desc;  in vringh_init_user()
649 	vrh->vring.avail = (__force struct vring_avail *)avail;  in vringh_init_user()
650 	vrh->vring.used = (__force struct vring_used *)used;  in vringh_init_user()
657  * @vrh: the userspace vring.
667  * *head will be vrh->vring.num.  You may be able to ignore an invalid
681 	*head = vrh->vring.num;  in vringh_getdesc_user()
687 	if (err == vrh->vring.num)  in vringh_getdesc_user()
754  * @vrh: the vring.
770  * @vrh: the vring.
789  * @vrh: the vring.
807  * @vrh: the vring.
810  * now more buffers available in the vring.
820  * @vrh: the vring.
833  * @vrh: the vring we've called vringh_complete_user() on.
888  * vringh_init_kern - initialize a vringh for a kernelspace vring.
917 	vrh->vring.num = num;  in vringh_init_kern()
918 	vrh->vring.desc = desc;  in vringh_init_kern()
919 	vrh->vring.avail = avail;  in vringh_init_kern()
920 	vrh->vring.used = used;  in vringh_init_kern()
927  * @vrh: the kernelspace vring.
937  * *head will be vrh->vring.num.  You may be able to ignore an invalid
955 	if (err == vrh->vring.num)  in vringh_getdesc_kern()
999  * @vrh: the vring.
1015  * @vrh: the vring.
1035  * @vrh: the vring.
1038  * now more buffers available in the vring.
1048  * @vrh: the vring.
1061  * @vrh: the vring we've called vringh_complete_kern() on.
1267  * @vrh: the vring
1268  * @iotlb: iotlb associated with this vring
1279  * @vrh: the kernelspace vring.
1289  * *head will be vrh->vring.num.  You may be able to ignore an invalid
1307 	if (err == vrh->vring.num)  in vringh_getdesc_iotlb()
1322  * @vrh: the vring.
1339  * @vrh: the vring.
1356  * @vrh: the vring.
1373  * @vrh: the vring.
1393  * @vrh: the vring.
1396  * now more buffers available in the vring.
1406  * @vrh: the vring.
1419  * @vrh: the vring we've called vringh_complete_iotlb() on.