68 int dma_fence_signal_locked(struct dma_fence *fence)  in dma_fence_signal_locked()  argument
73 	lockdep_assert_held(fence->lock);  in dma_fence_signal_locked()
75 	if (WARN_ON(!fence))  in dma_fence_signal_locked()
78 	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_signal_locked()
86 		fence->timestamp = ktime_get();  in dma_fence_signal_locked()
87 		set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_locked()
88 		trace_dma_fence_signaled(fence);  in dma_fence_signal_locked()
91 	list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in dma_fence_signal_locked()
93 		cur->func(fence, cur);  in dma_fence_signal_locked()
109 int dma_fence_signal(struct dma_fence *fence)  in dma_fence_signal()  argument
113 	if (!fence)  in dma_fence_signal()
116 	if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_signal()
119 	fence->timestamp = ktime_get();  in dma_fence_signal()
120 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal()
121 	trace_dma_fence_signaled(fence);  in dma_fence_signal()
123 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {  in dma_fence_signal()
126 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
127 		list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {  in dma_fence_signal()
129 			cur->func(fence, cur);  in dma_fence_signal()
131 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
154 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_wait_timeout()  argument
161 	trace_dma_fence_wait_start(fence);  in dma_fence_wait_timeout()
162 	ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
163 	trace_dma_fence_wait_end(fence);  in dma_fence_wait_timeout()
170 	struct dma_fence *fence =  in dma_fence_release()  local
173 	trace_dma_fence_destroy(fence);  in dma_fence_release()
175 	WARN_ON(!list_empty(&fence->cb_list));  in dma_fence_release()
177 	if (fence->ops->release)  in dma_fence_release()
178 		fence->ops->release(fence);  in dma_fence_release()
180 		dma_fence_free(fence);  in dma_fence_release()
184 void dma_fence_free(struct dma_fence *fence)  in dma_fence_free()  argument
186 	kfree_rcu(fence, rcu);  in dma_fence_free()
197 void dma_fence_enable_sw_signaling(struct dma_fence *fence)  in dma_fence_enable_sw_signaling()  argument
202 			      &fence->flags) &&  in dma_fence_enable_sw_signaling()
203 	    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_enable_sw_signaling()
204 		trace_dma_fence_enable_signal(fence);  in dma_fence_enable_sw_signaling()
206 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
208 		if (!fence->ops->enable_signaling(fence))  in dma_fence_enable_sw_signaling()
209 			dma_fence_signal_locked(fence);  in dma_fence_enable_sw_signaling()
211 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
241 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
248 	if (WARN_ON(!fence || !func))  in dma_fence_add_callback()
251 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
256 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
259 				   &fence->flags);  in dma_fence_add_callback()
261 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_add_callback()
264 		trace_dma_fence_enable_signal(fence);  in dma_fence_add_callback()
266 		if (!fence->ops->enable_signaling(fence)) {  in dma_fence_add_callback()
267 			dma_fence_signal_locked(fence);  in dma_fence_add_callback()
274 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
277 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
295 int dma_fence_get_status(struct dma_fence *fence)  in dma_fence_get_status()  argument
300 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
301 	status = dma_fence_get_status_locked(fence);  in dma_fence_get_status()
302 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
324 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
329 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
334 		if (list_empty(&fence->cb_list))  in dma_fence_remove_callback()
335 			if (fence->ops->disable_signaling)  in dma_fence_remove_callback()
336 				fence->ops->disable_signaling(fence);  in dma_fence_remove_callback()
339 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
351 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
372 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_default_wait()  argument
379 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
382 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
390 				   &fence->flags);  in dma_fence_default_wait()
392 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
396 		trace_dma_fence_enable_signal(fence);  in dma_fence_default_wait()
398 		if (!fence->ops->enable_signaling(fence)) {  in dma_fence_default_wait()
399 			dma_fence_signal_locked(fence);  in dma_fence_default_wait()
411 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
413 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
418 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
422 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
432 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
444 		struct dma_fence *fence = fences[i];  in dma_fence_test_signaled_any()  local
445 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
501 		struct dma_fence *fence = fences[i];  in dma_fence_wait_any_timeout()  local
503 		if (fence->ops->wait != dma_fence_default_wait) {  in dma_fence_wait_any_timeout()
509 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
563 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init()  argument
570 	kref_init(&fence->refcount);  in dma_fence_init()
571 	fence->ops = ops;  in dma_fence_init()
572 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
573 	fence->lock = lock;  in dma_fence_init()
574 	fence->context = context;  in dma_fence_init()
575 	fence->seqno = seqno;  in dma_fence_init()
576 	fence->flags = 0UL;  in dma_fence_init()
577 	fence->error = 0;  in dma_fence_init()
579 	trace_dma_fence_init(fence);  in dma_fence_init()