38 	struct intel_wait *wait;  in __intel_breadcrumbs_wakeup()  local
43 	wait = b->irq_wait;  in __intel_breadcrumbs_wakeup()
44 	if (wait) {  in __intel_breadcrumbs_wakeup()
55 		bool was_asleep = task_asleep(wait->tsk);  in __intel_breadcrumbs_wakeup()
58 		if (wake_up_process(wait->tsk) && was_asleep)  in __intel_breadcrumbs_wakeup()
118 	 * system) and wait until it sleeps before declaring a missed interrupt.  in intel_breadcrumbs_hangcheck()
237 	struct intel_wait *wait, *n;  in intel_engine_disarm_breadcrumbs()  local
258 	rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {  in intel_engine_disarm_breadcrumbs()
260 					      wait->seqno));  in intel_engine_disarm_breadcrumbs()
261 		RB_CLEAR_NODE(&wait->node);  in intel_engine_disarm_breadcrumbs()
262 		wake_up_process(wait->tsk);  in intel_engine_disarm_breadcrumbs()
350 					      struct intel_wait *wait)  in __intel_breadcrumbs_finish()  argument
353 	GEM_BUG_ON(b->irq_wait == wait);  in __intel_breadcrumbs_finish()
364 	rb_erase(&wait->node, &b->waiters);  in __intel_breadcrumbs_finish()
365 	RB_CLEAR_NODE(&wait->node);  in __intel_breadcrumbs_finish()
367 	if (wait->tsk->state != TASK_RUNNING)  in __intel_breadcrumbs_finish()
368 		wake_up_process(wait->tsk); /* implicit smp_wmb() */  in __intel_breadcrumbs_finish()
391 				    struct intel_wait *wait)  in __intel_engine_add_wait()  argument
398 	GEM_BUG_ON(!wait->seqno);  in __intel_engine_add_wait()
423 	if (i915_seqno_passed(seqno, wait->seqno)) {  in __intel_engine_add_wait()
424 		RB_CLEAR_NODE(&wait->node);  in __intel_engine_add_wait()
431 		if (wait->seqno == to_wait(parent)->seqno) {  in __intel_engine_add_wait()
437 			if (wait->tsk->prio > to_wait(parent)->tsk->prio) {  in __intel_engine_add_wait()
443 		} else if (i915_seqno_passed(wait->seqno,  in __intel_engine_add_wait()
454 	rb_link_node(&wait->node, parent, p);  in __intel_engine_add_wait()
455 	rb_insert_color(&wait->node, &b->waiters);  in __intel_engine_add_wait()
459 		b->irq_wait = wait;  in __intel_engine_add_wait()
480 			GEM_BUG_ON(next == &wait->node);  in __intel_engine_add_wait()
499 			   struct intel_wait *wait)  in intel_engine_add_wait()  argument
505 	armed = __intel_engine_add_wait(engine, wait);  in intel_engine_add_wait()
512 				 wait->seqno - 1);  in intel_engine_add_wait()
530 				       struct intel_wait *wait)  in __intel_engine_remove_wait()  argument
536 	if (RB_EMPTY_NODE(&wait->node))  in __intel_engine_remove_wait()
539 	if (b->irq_wait == wait) {  in __intel_engine_remove_wait()
540 		const int priority = wakeup_priority(b, wait->tsk);  in __intel_engine_remove_wait()
550 		next = rb_next(&wait->node);  in __intel_engine_remove_wait()
577 		GEM_BUG_ON(rb_first(&b->waiters) == &wait->node);  in __intel_engine_remove_wait()
580 	GEM_BUG_ON(RB_EMPTY_NODE(&wait->node));  in __intel_engine_remove_wait()
581 	rb_erase(&wait->node, &b->waiters);  in __intel_engine_remove_wait()
582 	RB_CLEAR_NODE(&wait->node);  in __intel_engine_remove_wait()
585 	GEM_BUG_ON(b->irq_wait == wait);  in __intel_engine_remove_wait()
591 			      struct intel_wait *wait)  in intel_engine_remove_wait()  argument
599 	if (RB_EMPTY_NODE(&wait->node)) {  in intel_engine_remove_wait()
600 		GEM_BUG_ON(READ_ONCE(b->irq_wait) == wait);  in intel_engine_remove_wait()
605 	__intel_engine_remove_wait(engine, wait);  in intel_engine_remove_wait()
640 		 * need to wait for a new interrupt from the GPU or for  in intel_breadcrumbs_signaler()
647 			u32 this = rq->signaling.wait.seqno;  in intel_breadcrumbs_signaler()
649 			GEM_BUG_ON(!rq->signaling.wait.seqno);  in intel_breadcrumbs_signaler()
656 							   &rq->signaling.wait);  in intel_breadcrumbs_signaler()
658 				rq->signaling.wait.seqno = 0;  in intel_breadcrumbs_signaler()
737 		if (i915_seqno_passed(seqno, iter->signaling.wait.seqno))  in insert_signal()
747 	struct intel_wait *wait = &request->signaling.wait;  in intel_engine_enable_signaling()  local
766 	GEM_BUG_ON(wait->seqno);  in intel_engine_enable_signaling()
767 	wait->tsk = b->signaler;  in intel_engine_enable_signaling()
768 	wait->request = request;  in intel_engine_enable_signaling()
769 	wait->seqno = seqno;  in intel_engine_enable_signaling()
782 	wakeup &= __intel_engine_add_wait(engine, wait);  in intel_engine_enable_signaling()
787 		return !intel_wait_complete(wait);  in intel_engine_enable_signaling()
801 	if (!READ_ONCE(request->signaling.wait.seqno))  in intel_engine_cancel_signaling()
805 	__intel_engine_remove_wait(engine, &request->signaling.wait);  in intel_engine_cancel_signaling()
806 	if (fetch_and_zero(&request->signaling.wait.seqno))  in intel_engine_cancel_signaling()