4  * Copyright(c) 2015 - 2018 Intel Corporation.
26  *  - Redistributions of source code must retain the above copyright
28  *  - Redistributions in binary form must reproduce the above copyright
32  *  - Neither the name of Intel Corporation nor the names of its
52 #include <linux/wait.h>
58  * typedef (*restart_t)() - restart callback
98  * @list: used to add/insert into QP/PQ wait lists
104  * @lock: lock protected head of wait queue
106  * @wait_dma: wait for sdma_busy == 0
107  * @wait_pio: wait for pio_busy == 0
112  * @flags: wait flags (one per QP)
113  * @wait: SE array for multiple legs
143 		struct iowait_work *wait,
148 	void (*wakeup)(struct iowait *wait, int reason);
149 	void (*sdma_drained)(struct iowait *wait);
150 	void (*init_priority)(struct iowait *wait);
162 	struct iowait_work wait[IOWAIT_SES];  member
167 void iowait_set_flag(struct iowait *wait, u32 flag);
168 bool iowait_flag_set(struct iowait *wait, u32 flag);
169 void iowait_clear_flag(struct iowait *wait, u32 flag);
171 void iowait_init(struct iowait *wait, u32 tx_limit,
175 			      struct iowait_work *wait,
179 		 void (*wakeup)(struct iowait *wait, int reason),
180 		 void (*sdma_drained)(struct iowait *wait),
181 		 void (*init_priority)(struct iowait *wait));
184  * iowait_schedule() - schedule the default send engine work
185  * @wait: wait struct to schedule
189 static inline bool iowait_schedule(struct iowait *wait,  in iowait_schedule()  argument
192 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);  in iowait_schedule()
196  * iowait_tid_schedule - schedule the tid SE
197  * @wait: the iowait structure
198  * @wq: the work queue
201 static inline bool iowait_tid_schedule(struct iowait *wait,  in iowait_tid_schedule()  argument
204 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);  in iowait_tid_schedule()
208  * iowait_sdma_drain() - wait for DMAs to drain
210  * @wait: iowait structure
215 static inline void iowait_sdma_drain(struct iowait *wait)  in iowait_sdma_drain()  argument
217 	wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));  in iowait_sdma_drain()
221  * iowait_sdma_pending() - return sdma pending count
223  * @wait: iowait structure
226 static inline int iowait_sdma_pending(struct iowait *wait)  in iowait_sdma_pending()  argument
228 	return atomic_read(&wait->sdma_busy);  in iowait_sdma_pending()
232  * iowait_sdma_inc - note sdma io pending
233  * @wait: iowait structure
235 static inline void iowait_sdma_inc(struct iowait *wait)  in iowait_sdma_inc()  argument
237 	atomic_inc(&wait->sdma_busy);  in iowait_sdma_inc()
241  * iowait_sdma_add - add count to pending
242  * @wait: iowait structure
244 static inline void iowait_sdma_add(struct iowait *wait, int count)  in iowait_sdma_add()  argument
246 	atomic_add(count, &wait->sdma_busy);  in iowait_sdma_add()
250  * iowait_sdma_dec - note sdma complete
251  * @wait: iowait structure
253 static inline int iowait_sdma_dec(struct iowait *wait)  in iowait_sdma_dec()  argument
255 	if (!wait)  in iowait_sdma_dec()
257 	return atomic_dec_and_test(&wait->sdma_busy);  in iowait_sdma_dec()
261  * iowait_pio_drain() - wait for pios to drain
263  * @wait: iowait structure
268 static inline void iowait_pio_drain(struct iowait *wait)  in iowait_pio_drain()  argument
270 	wait_event_timeout(wait->wait_pio,  in iowait_pio_drain()
271 			   !atomic_read(&wait->pio_busy),  in iowait_pio_drain()
276  * iowait_pio_pending() - return pio pending count
278  * @wait: iowait structure
281 static inline int iowait_pio_pending(struct iowait *wait)  in iowait_pio_pending()  argument
283 	return atomic_read(&wait->pio_busy);  in iowait_pio_pending()
287  * iowait_pio_inc - note pio pending
288  * @wait: iowait structure
290 static inline void iowait_pio_inc(struct iowait *wait)  in iowait_pio_inc()  argument
292 	atomic_inc(&wait->pio_busy);  in iowait_pio_inc()
296  * iowait_pio_dec - note pio complete
297  * @wait: iowait structure
299 static inline int iowait_pio_dec(struct iowait *wait)  in iowait_pio_dec()  argument
301 	if (!wait)  in iowait_pio_dec()
303 	return atomic_dec_and_test(&wait->pio_busy);  in iowait_pio_dec()
307  * iowait_drain_wakeup() - trigger iowait_drain() waiter
309  * @wait: iowait structure
313 static inline void iowait_drain_wakeup(struct iowait *wait)  in iowait_drain_wakeup()  argument
315 	wake_up(&wait->wait_dma);  in iowait_drain_wakeup()
316 	wake_up(&wait->wait_pio);  in iowait_drain_wakeup()
317 	if (wait->sdma_drained)  in iowait_drain_wakeup()
318 		wait->sdma_drained(wait);  in iowait_drain_wakeup()
322  * iowait_get_txhead() - get packet off of iowait list
324  * @wait iowait_work struture
326 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)  in iowait_get_txhead()  argument
330 	if (!list_empty(&wait->tx_head)) {  in iowait_get_txhead()
332 			&wait->tx_head,  in iowait_get_txhead()
335 		list_del_init(&tx->list);  in iowait_get_txhead()
345 	if (!list_empty(&w->tx_head)) {  in iowait_get_desc()
346 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,  in iowait_get_desc()
348 		num_desc = tx->num_desc;  in iowait_get_desc()
349 		if (tx->flags & SDMA_TXREQ_F_VIP)  in iowait_get_desc()
350 			w->iow->priority++;  in iowait_get_desc()
359 	num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);  in iowait_get_all_desc()
360 	num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);  in iowait_get_all_desc()
368 	if (!list_empty(&w->tx_head)) {  in iowait_update_priority()
369 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,  in iowait_update_priority()
371 		if (tx->flags & SDMA_TXREQ_F_VIP)  in iowait_update_priority()
372 			w->iow->priority++;  in iowait_update_priority()
378 	iowait_update_priority(&w->wait[IOWAIT_IB_SE]);  in iowait_update_all_priority()
379 	iowait_update_priority(&w->wait[IOWAIT_TID_SE]);  in iowait_update_all_priority()
384 	w->priority = 0;  in iowait_init_priority()
385 	if (w->init_priority)  in iowait_init_priority()
386 		w->init_priority(w);  in iowait_init_priority()
396  * iowait_queue - Put the iowait on a wait queue
399  * @wait_head: the wait queue
402  * wait queue after a resource (eg, sdma descriptor or pio
409 	 * To play fair, insert the iowait at the tail of the wait queue if it  in iowait_queue()
415 		w->starved_cnt = 0;  in iowait_queue()
417 		w->starved_cnt++;  in iowait_queue()
419 	if (w->priority > 0 || !pkts_sent)  in iowait_queue()
420 		list_add(&w->list, wait_head);  in iowait_queue()
422 		list_add_tail(&w->list, wait_head);  in iowait_queue()
426  * iowait_starve_clear - clear the wait queue's starve count
436 		w->starved_cnt = 0;  in iowait_starve_clear()
445  * iowait_packet_queued() - determine if a packet is queued
446  * @wait: the iowait_work structure
448 static inline bool iowait_packet_queued(struct iowait_work *wait)  in iowait_packet_queued()  argument
450 	return !list_empty(&wait->tx_head);  in iowait_packet_queued()
454  * inc_wait_count - increment wait counts
462 	w->iow->tx_count++;  in iowait_inc_wait_count()
463 	w->iow->count += n;  in iowait_inc_wait_count()
467  * iowait_get_tid_work - return iowait_work for tid SE
472 	return &w->wait[IOWAIT_TID_SE];  in iowait_get_tid_work()
476  * iowait_get_ib_work - return iowait_work for ib SE
481 	return &w->wait[IOWAIT_IB_SE];  in iowait_get_ib_work()
485  * iowait_ioww_to_iow - return iowait given iowait_work
491 		return w->iow;  in iowait_ioww_to_iow()