47 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
49 	debug_object_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_init()
52 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
54 	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);  in debug_fence_init_onstack()
57 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
59 	debug_object_activate(fence, &i915_sw_fence_debug_descr);  in debug_fence_activate()
62 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
65 	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);  in debug_fence_set_state()
68 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
70 	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);  in debug_fence_deactivate()
73 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
75 	debug_object_destroy(fence, &i915_sw_fence_debug_descr);  in debug_fence_destroy()
78 static inline void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
80 	debug_object_free(fence, &i915_sw_fence_debug_descr);  in debug_fence_free()
84 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
86 	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);  in debug_fence_assert()
91 static inline void debug_fence_init(struct i915_sw_fence *fence)  in debug_fence_init()  argument
95 static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)  in debug_fence_init_onstack()  argument
99 static inline void debug_fence_activate(struct i915_sw_fence *fence)  in debug_fence_activate()  argument
103 static inline void debug_fence_set_state(struct i915_sw_fence *fence,  in debug_fence_set_state()  argument
108 static inline void debug_fence_deactivate(struct i915_sw_fence *fence)  in debug_fence_deactivate()  argument
112 static inline void debug_fence_destroy(struct i915_sw_fence *fence)  in debug_fence_destroy()  argument
116 static inline void debug_fence_free(struct i915_sw_fence *fence)  in debug_fence_free()  argument
120 static inline void debug_fence_assert(struct i915_sw_fence *fence)  in debug_fence_assert()  argument
126 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,  in __i915_sw_fence_notify()  argument
131 	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);  in __i915_sw_fence_notify()
132 	return fn(fence, state);  in __i915_sw_fence_notify()
136 void i915_sw_fence_fini(struct i915_sw_fence *fence)  in i915_sw_fence_fini()  argument
138 	debug_fence_free(fence);  in i915_sw_fence_fini()
142 static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,  in __i915_sw_fence_wake_up_all()  argument
145 	wait_queue_head_t *x = &fence->wait;  in __i915_sw_fence_wake_up_all()
149 	debug_fence_deactivate(fence);  in __i915_sw_fence_wake_up_all()
150 	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */  in __i915_sw_fence_wake_up_all()
176 					wake_flags = fence->error;  in __i915_sw_fence_wake_up_all()
189 	debug_fence_assert(fence);  in __i915_sw_fence_wake_up_all()
192 static void __i915_sw_fence_complete(struct i915_sw_fence *fence,  in __i915_sw_fence_complete()  argument
195 	debug_fence_assert(fence);  in __i915_sw_fence_complete()
197 	if (!atomic_dec_and_test(&fence->pending))  in __i915_sw_fence_complete()
200 	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);  in __i915_sw_fence_complete()
202 	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)  in __i915_sw_fence_complete()
205 	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);  in __i915_sw_fence_complete()
207 	__i915_sw_fence_wake_up_all(fence, continuation);  in __i915_sw_fence_complete()
209 	debug_fence_destroy(fence);  in __i915_sw_fence_complete()
210 	__i915_sw_fence_notify(fence, FENCE_FREE);  in __i915_sw_fence_complete()
213 void i915_sw_fence_complete(struct i915_sw_fence *fence)  in i915_sw_fence_complete()  argument
215 	debug_fence_assert(fence);  in i915_sw_fence_complete()
217 	if (WARN_ON(i915_sw_fence_done(fence)))  in i915_sw_fence_complete()
220 	__i915_sw_fence_complete(fence, NULL);  in i915_sw_fence_complete()
223 bool i915_sw_fence_await(struct i915_sw_fence *fence)  in i915_sw_fence_await()  argument
231 	pending = atomic_read(&fence->pending);  in i915_sw_fence_await()
235 	} while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1));  in i915_sw_fence_await()
240 void __i915_sw_fence_init(struct i915_sw_fence *fence,  in __i915_sw_fence_init()  argument
247 	__init_waitqueue_head(&fence->wait, name, key);  in __i915_sw_fence_init()
248 	fence->flags = (unsigned long)fn;  in __i915_sw_fence_init()
250 	i915_sw_fence_reinit(fence);  in __i915_sw_fence_init()
253 void i915_sw_fence_reinit(struct i915_sw_fence *fence)  in i915_sw_fence_reinit()  argument
255 	debug_fence_init(fence);  in i915_sw_fence_reinit()
257 	atomic_set(&fence->pending, 1);  in i915_sw_fence_reinit()
258 	fence->error = 0;  in i915_sw_fence_reinit()
260 	I915_SW_FENCE_BUG_ON(!fence->flags);  in i915_sw_fence_reinit()
261 	I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head));  in i915_sw_fence_reinit()
264 void i915_sw_fence_commit(struct i915_sw_fence *fence)  in i915_sw_fence_commit()  argument
266 	debug_fence_activate(fence);  in i915_sw_fence_commit()
267 	i915_sw_fence_complete(fence);  in i915_sw_fence_commit()
282 static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in __i915_sw_fence_check_if_after()  argument
287 	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_check_if_after()
290 	if (fence == signaler)  in __i915_sw_fence_check_if_after()
293 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_check_if_after()
304 static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)  in __i915_sw_fence_clear_checked_bit()  argument
308 	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))  in __i915_sw_fence_clear_checked_bit()
311 	list_for_each_entry(wq, &fence->wait.head, entry) {  in __i915_sw_fence_clear_checked_bit()
319 static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,  in i915_sw_fence_check_if_after()  argument
329 	err = __i915_sw_fence_check_if_after(fence, signaler);  in i915_sw_fence_check_if_after()
330 	__i915_sw_fence_clear_checked_bit(fence);  in i915_sw_fence_check_if_after()
336 static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in __i915_sw_fence_await_sw_fence()  argument
343 	debug_fence_assert(fence);  in __i915_sw_fence_await_sw_fence()
347 		i915_sw_fence_set_error_once(fence, signaler->error);  in __i915_sw_fence_await_sw_fence()
354 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))  in __i915_sw_fence_await_sw_fence()
365 			i915_sw_fence_set_error_once(fence, signaler->error);  in __i915_sw_fence_await_sw_fence()
375 	wq->private = fence;  in __i915_sw_fence_await_sw_fence()
377 	i915_sw_fence_await(fence);  in __i915_sw_fence_await_sw_fence()
392 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence()  argument
396 	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);  in i915_sw_fence_await_sw_fence()
399 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,  in i915_sw_fence_await_sw_fence_gfp()  argument
403 	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);  in i915_sw_fence_await_sw_fence_gfp()
419 	i915_sw_fence_set_error_once(cb->fence, dma->error);  in dma_i915_sw_fence_wake()
420 	i915_sw_fence_complete(cb->fence);  in dma_i915_sw_fence_wake()
427 	struct i915_sw_fence *fence;  in timer_i915_sw_fence_wake()  local
429 	fence = xchg(&cb->base.fence, NULL);  in timer_i915_sw_fence_wake()
430 	if (!fence)  in timer_i915_sw_fence_wake()
437 		  i915_sw_fence_debug_hint(fence));  in timer_i915_sw_fence_wake()
439 	i915_sw_fence_set_error_once(fence, -ETIMEDOUT);  in timer_i915_sw_fence_wake()
440 	i915_sw_fence_complete(fence);  in timer_i915_sw_fence_wake()
448 	struct i915_sw_fence *fence;  in dma_i915_sw_fence_wake_timer()  local
450 	fence = xchg(&cb->base.fence, NULL);  in dma_i915_sw_fence_wake_timer()
451 	if (fence) {  in dma_i915_sw_fence_wake_timer()
452 		i915_sw_fence_set_error_once(fence, dma->error);  in dma_i915_sw_fence_wake_timer()
453 		i915_sw_fence_complete(fence);  in dma_i915_sw_fence_wake_timer()
470 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,  in i915_sw_fence_await_dma_fence()  argument
479 	debug_fence_assert(fence);  in i915_sw_fence_await_dma_fence()
483 		i915_sw_fence_set_error_once(fence, dma->error);  in i915_sw_fence_await_dma_fence()
499 		i915_sw_fence_set_error_once(fence, dma->error);  in i915_sw_fence_await_dma_fence()
503 	cb->fence = fence;  in i915_sw_fence_await_dma_fence()
504 	i915_sw_fence_await(fence);  in i915_sw_fence_await_dma_fence()
538 	i915_sw_fence_set_error_once(cb->fence, dma->error);  in __dma_i915_sw_fence_wake()
539 	i915_sw_fence_complete(cb->fence);  in __dma_i915_sw_fence_wake()
542 int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,  in __i915_sw_fence_await_dma_fence()  argument
548 	debug_fence_assert(fence);  in __i915_sw_fence_await_dma_fence()
551 		i915_sw_fence_set_error_once(fence, dma->error);  in __i915_sw_fence_await_dma_fence()
555 	cb->fence = fence;  in __i915_sw_fence_await_dma_fence()
556 	i915_sw_fence_await(fence);  in __i915_sw_fence_await_dma_fence()
568 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,  in i915_sw_fence_await_reservation()  argument
578 	debug_fence_assert(fence);  in i915_sw_fence_await_reservation()
593 			pending = i915_sw_fence_await_dma_fence(fence,  in i915_sw_fence_await_reservation()
613 		pending = i915_sw_fence_await_dma_fence(fence,  in i915_sw_fence_await_reservation()