112 static inline bool xskq_cons_read_addr_unchecked(struct xsk_queue *q, u64 *addr)  in xskq_cons_read_addr_unchecked()  argument
114 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_cons_read_addr_unchecked()
116 	if (q->cached_cons != q->cached_prod) {  in xskq_cons_read_addr_unchecked()
117 		u32 idx = q->cached_cons & q->ring_mask;  in xskq_cons_read_addr_unchecked()
174 static inline bool xskq_cons_is_valid_desc(struct xsk_queue *q,  in xskq_cons_is_valid_desc()  argument
179 		q->invalid_descs++;  in xskq_cons_is_valid_desc()
185 static inline bool xskq_cons_read_desc(struct xsk_queue *q,  in xskq_cons_read_desc()  argument
189 	while (q->cached_cons != q->cached_prod) {  in xskq_cons_read_desc()
190 		struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;  in xskq_cons_read_desc()
191 		u32 idx = q->cached_cons & q->ring_mask;  in xskq_cons_read_desc()
194 		if (xskq_cons_is_valid_desc(q, desc, pool))  in xskq_cons_read_desc()
197 		q->cached_cons++;  in xskq_cons_read_desc()
205 static inline void __xskq_cons_release(struct xsk_queue *q)  in __xskq_cons_release()  argument
208 	WRITE_ONCE(q->ring->consumer, q->cached_cons);  in __xskq_cons_release()
211 static inline void __xskq_cons_peek(struct xsk_queue *q)  in __xskq_cons_peek()  argument
214 	q->cached_prod = READ_ONCE(q->ring->producer);  in __xskq_cons_peek()
218 static inline void xskq_cons_get_entries(struct xsk_queue *q)  in xskq_cons_get_entries()  argument
220 	__xskq_cons_release(q);  in xskq_cons_get_entries()
221 	__xskq_cons_peek(q);  in xskq_cons_get_entries()
224 static inline bool xskq_cons_has_entries(struct xsk_queue *q, u32 cnt)  in xskq_cons_has_entries()  argument
226 	u32 entries = q->cached_prod - q->cached_cons;  in xskq_cons_has_entries()
231 	__xskq_cons_peek(q);  in xskq_cons_has_entries()
232 	entries = q->cached_prod - q->cached_cons;  in xskq_cons_has_entries()
237 static inline bool xskq_cons_peek_addr_unchecked(struct xsk_queue *q, u64 *addr)  in xskq_cons_peek_addr_unchecked()  argument
239 	if (q->cached_prod == q->cached_cons)  in xskq_cons_peek_addr_unchecked()
240 		xskq_cons_get_entries(q);  in xskq_cons_peek_addr_unchecked()
241 	return xskq_cons_read_addr_unchecked(q, addr);  in xskq_cons_peek_addr_unchecked()
244 static inline bool xskq_cons_peek_desc(struct xsk_queue *q,  in xskq_cons_peek_desc()  argument
248 	if (q->cached_prod == q->cached_cons)  in xskq_cons_peek_desc()
249 		xskq_cons_get_entries(q);  in xskq_cons_peek_desc()
250 	return xskq_cons_read_desc(q, desc, pool);  in xskq_cons_peek_desc()
253 static inline void xskq_cons_release(struct xsk_queue *q)  in xskq_cons_release()  argument
260 	q->cached_cons++;  in xskq_cons_release()
263 static inline bool xskq_cons_is_full(struct xsk_queue *q)  in xskq_cons_is_full()  argument
266 	return READ_ONCE(q->ring->producer) - READ_ONCE(q->ring->consumer) ==  in xskq_cons_is_full()
267 		q->nentries;  in xskq_cons_is_full()
270 static inline u32 xskq_cons_present_entries(struct xsk_queue *q)  in xskq_cons_present_entries()  argument
273 	return READ_ONCE(q->ring->producer) - READ_ONCE(q->ring->consumer);  in xskq_cons_present_entries()
278 static inline bool xskq_prod_is_full(struct xsk_queue *q)  in xskq_prod_is_full()  argument
280 	u32 free_entries = q->nentries - (q->cached_prod - q->cached_cons);  in xskq_prod_is_full()
286 	q->cached_cons = READ_ONCE(q->ring->consumer);  in xskq_prod_is_full()
287 	free_entries = q->nentries - (q->cached_prod - q->cached_cons);  in xskq_prod_is_full()
292 static inline void xskq_prod_cancel(struct xsk_queue *q)  in xskq_prod_cancel()  argument
294 	q->cached_prod--;  in xskq_prod_cancel()
297 static inline int xskq_prod_reserve(struct xsk_queue *q)  in xskq_prod_reserve()  argument
299 	if (xskq_prod_is_full(q))  in xskq_prod_reserve()
303 	q->cached_prod++;  in xskq_prod_reserve()
307 static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr)  in xskq_prod_reserve_addr()  argument
309 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_prod_reserve_addr()
311 	if (xskq_prod_is_full(q))  in xskq_prod_reserve_addr()
315 	ring->desc[q->cached_prod++ & q->ring_mask] = addr;  in xskq_prod_reserve_addr()
319 static inline int xskq_prod_reserve_desc(struct xsk_queue *q,  in xskq_prod_reserve_desc()  argument
322 	struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;  in xskq_prod_reserve_desc()
325 	if (xskq_prod_is_full(q))  in xskq_prod_reserve_desc()
329 	idx = q->cached_prod++ & q->ring_mask;  in xskq_prod_reserve_desc()
336 static inline void __xskq_prod_submit(struct xsk_queue *q, u32 idx)  in __xskq_prod_submit()  argument
340 	WRITE_ONCE(q->ring->producer, idx);  in __xskq_prod_submit()
343 static inline void xskq_prod_submit(struct xsk_queue *q)  in xskq_prod_submit()  argument
345 	__xskq_prod_submit(q, q->cached_prod);  in xskq_prod_submit()
348 static inline void xskq_prod_submit_addr(struct xsk_queue *q, u64 addr)  in xskq_prod_submit_addr()  argument
350 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;  in xskq_prod_submit_addr()
351 	u32 idx = q->ring->producer;  in xskq_prod_submit_addr()
353 	ring->desc[idx++ & q->ring_mask] = addr;  in xskq_prod_submit_addr()
355 	__xskq_prod_submit(q, idx);  in xskq_prod_submit_addr()
358 static inline void xskq_prod_submit_n(struct xsk_queue *q, u32 nb_entries)  in xskq_prod_submit_n()  argument
360 	__xskq_prod_submit(q, q->ring->producer + nb_entries);  in xskq_prod_submit_n()
363 static inline bool xskq_prod_is_empty(struct xsk_queue *q)  in xskq_prod_is_empty()  argument
366 	return READ_ONCE(q->ring->consumer) == READ_ONCE(q->ring->producer);  in xskq_prod_is_empty()
371 static inline u64 xskq_nb_invalid_descs(struct xsk_queue *q)  in xskq_nb_invalid_descs()  argument
373 	return q ? q->invalid_descs : 0;  in xskq_nb_invalid_descs()
376 static inline u64 xskq_nb_queue_empty_descs(struct xsk_queue *q)  in xskq_nb_queue_empty_descs()  argument
378 	return q ? q->queue_empty_descs : 0;  in xskq_nb_queue_empty_descs()