3  * Fence mechanism for dma-buf and to allow for asynchronous dma access
16 #include <linux/dma-fence.h>
30  * fence context counter: each execution context should have its own
31  * fence context, this allows checking if fences belong to the same
44  * A fence is initialized using dma_fence_init() and completed using
59  *   fence to be updated.
68  * DOC: fence cross-driver contract
94  *   This means any code required for fence completion cannot acquire a
99  *   callbacks. This means any code required for fence completion cannot
104  *   for fence completeion cannot allocate memory with GFP_NOFS or GFP_NOIO.
113 static const char *dma_fence_stub_get_name(struct dma_fence *fence)  in dma_fence_stub_get_name()  argument
124  * dma_fence_get_stub - return a signaled fence
126  * Return a stub fence which is already signaled.
145  * dma_fence_context_alloc - allocate an array of fence contexts
148  * This function will return the first index of the number of fence contexts
149  * allocated.  The fence context is used for setting &dma_fence.context to a
160  * DOC: fence signalling annotation
200  * prevents it from signalling the fence the previous thread is stuck waiting
221  *   point where a fence is accessible to other threads, to the point where
254  * dma_fence_begin_signalling - begin a critical DMA fence signalling section
285  * dma_fence_end_signalling - end a critical DMA fence signalling section
315  * dma_fence_signal_locked - signal completion of a fence
316  * @fence: the fence to signal
318  * Signal completion for software callbacks on a fence, this will unblock
320  * dma_fence_add_callback(). Can be called multiple times, but since a fence
327  * Returns 0 on success and a negative error value when @fence has been
330 int dma_fence_signal_locked(struct dma_fence *fence)  in dma_fence_signal_locked()  argument
335 	lockdep_assert_held(fence->lock);  in dma_fence_signal_locked()
338 				      &fence->flags)))  in dma_fence_signal_locked()
342 	list_replace(&fence->cb_list, &cb_list);  in dma_fence_signal_locked()
344 	fence->timestamp = ktime_get();  in dma_fence_signal_locked()
345 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);  in dma_fence_signal_locked()
346 	trace_dma_fence_signaled(fence);  in dma_fence_signal_locked()
350 		cur->func(fence, cur);  in dma_fence_signal_locked()
358  * dma_fence_signal - signal completion of a fence
359  * @fence: the fence to signal
361  * Signal completion for software callbacks on a fence, this will unblock
363  * dma_fence_add_callback(). Can be called multiple times, but since a fence
367  * Returns 0 on success and a negative error value when @fence has been
370 int dma_fence_signal(struct dma_fence *fence)  in dma_fence_signal()  argument
376 	if (!fence)  in dma_fence_signal()
381 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_signal()
382 	ret = dma_fence_signal_locked(fence);  in dma_fence_signal()
383 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_signal()
392  * dma_fence_wait_timeout - sleep until the fence gets signaled
394  * @fence: the fence to wait on
402  * Performs a synchronous wait on this fence. It is assumed the caller
404  * holds a reference to the fence, otherwise the fence might be
410 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_wait_timeout()  argument
421 	trace_dma_fence_wait_start(fence);  in dma_fence_wait_timeout()
422 	if (fence->ops->wait)  in dma_fence_wait_timeout()
423 		ret = fence->ops->wait(fence, intr, timeout);  in dma_fence_wait_timeout()
425 		ret = dma_fence_default_wait(fence, intr, timeout);  in dma_fence_wait_timeout()
426 	trace_dma_fence_wait_end(fence);  in dma_fence_wait_timeout()
440 	struct dma_fence *fence =  in dma_fence_release()  local
443 	trace_dma_fence_destroy(fence);  in dma_fence_release()
445 	if (WARN(!list_empty(&fence->cb_list) &&  in dma_fence_release()
446 		 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),  in dma_fence_release()
447 		 "Fence %s:%s:%llx:%llx released with pending signals!\n",  in dma_fence_release()
448 		 fence->ops->get_driver_name(fence),  in dma_fence_release()
449 		 fence->ops->get_timeline_name(fence),  in dma_fence_release()
450 		 fence->context, fence->seqno)) {  in dma_fence_release()
460 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_release()
461 		fence->error = -EDEADLK;  in dma_fence_release()
462 		dma_fence_signal_locked(fence);  in dma_fence_release()
463 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_release()
466 	if (fence->ops->release)  in dma_fence_release()
467 		fence->ops->release(fence);  in dma_fence_release()
469 		dma_fence_free(fence);  in dma_fence_release()
475  * @fence: fence to release
478  * kfree_rcu() on @fence.
480 void dma_fence_free(struct dma_fence *fence)  in dma_fence_free()  argument
482 	kfree_rcu(fence, rcu);  in dma_fence_free()
486 static bool __dma_fence_enable_signaling(struct dma_fence *fence)  in __dma_fence_enable_signaling()  argument
490 	lockdep_assert_held(fence->lock);  in __dma_fence_enable_signaling()
493 				   &fence->flags);  in __dma_fence_enable_signaling()
495 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in __dma_fence_enable_signaling()
498 	if (!was_set && fence->ops->enable_signaling) {  in __dma_fence_enable_signaling()
499 		trace_dma_fence_enable_signal(fence);  in __dma_fence_enable_signaling()
501 		if (!fence->ops->enable_signaling(fence)) {  in __dma_fence_enable_signaling()
502 			dma_fence_signal_locked(fence);  in __dma_fence_enable_signaling()
511  * dma_fence_enable_sw_signaling - enable signaling on fence
512  * @fence: the fence to enable
514  * This will request for sw signaling to be enabled, to make the fence
518 void dma_fence_enable_sw_signaling(struct dma_fence *fence)  in dma_fence_enable_sw_signaling()  argument
522 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_enable_sw_signaling()
525 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_enable_sw_signaling()
526 	__dma_fence_enable_signaling(fence);  in dma_fence_enable_sw_signaling()
527 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_enable_sw_signaling()
532  * dma_fence_add_callback - add a callback to be called when the fence
534  * @fence: the fence to wait on
540  * to a fence, but a callback can only be registered to one fence at a time.
543  * fence is already signaled, this function will return -ENOENT (and
546  * Add a software callback to the fence. Same restrictions apply to
548  * keep a refcount to fence afterward dma_fence_add_callback() has returned:
549  * when software access is enabled, the creator of the fence is required to keep
550  * the fence alive until after it signals with dma_fence_signal(). The callback
553  * Returns 0 in case of success, -ENOENT if the fence is already signaled
556 int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,  in dma_fence_add_callback()  argument
562 	if (WARN_ON(!fence || !func))  in dma_fence_add_callback()
565 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_add_callback()
570 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_add_callback()
572 	if (__dma_fence_enable_signaling(fence)) {  in dma_fence_add_callback()
574 		list_add_tail(&cb->node, &fence->cb_list);  in dma_fence_add_callback()
580 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_add_callback()
588  * @fence: the dma_fence to query
591  * condition on a signaled fence. See dma_fence_get_status_locked() for more
594  * Returns 0 if the fence has not yet been signaled, 1 if the fence has
596  * if the fence has been completed in err.
598 int dma_fence_get_status(struct dma_fence *fence)  in dma_fence_get_status()  argument
603 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_get_status()
604 	status = dma_fence_get_status_locked(fence);  in dma_fence_get_status()
605 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_get_status()
613  * @fence: the fence to wait on
616  * Remove a previously queued callback from the fence. This function returns
617  * true if the callback is successfully removed, or false if the fence has
624  * with a reference held to the fence.
626  * Behaviour is undefined if @cb has not been added to @fence using
630 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_remove_callback()  argument
635 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_remove_callback()
641 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_remove_callback()
653 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)  in dma_fence_default_wait_cb()  argument
662  * dma_fence_default_wait - default sleep until the fence gets signaled
664  * @fence: the fence to wait on
670  * returned if the fence is already signaled for consistency with other
674 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)  in dma_fence_default_wait()  argument
680 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))  in dma_fence_default_wait()
683 	spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
690 	if (!__dma_fence_enable_signaling(fence))  in dma_fence_default_wait()
700 	list_add(&cb.base.node, &fence->cb_list);  in dma_fence_default_wait()
702 	while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {  in dma_fence_default_wait()
707 		spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
711 		spin_lock_irqsave(fence->lock, flags);  in dma_fence_default_wait()
721 	spin_unlock_irqrestore(fence->lock, flags);  in dma_fence_default_wait()
733 		struct dma_fence *fence = fences[i];  in dma_fence_test_signaled_any()  local
734 		if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {  in dma_fence_test_signaled_any()
744  * dma_fence_wait_any_timeout - sleep until any fence gets signaled
750  * @idx: used to store the first signaled fence index, meaningful only on
753  * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
757  * Synchronous waits for the first fence in the array to be signaled. The
759  * fence might be freed before return, resulting in undefined behavior.
792 		struct dma_fence *fence = fences[i];  in dma_fence_wait_any_timeout()  local
795 		if (dma_fence_add_callback(fence, &cb[i].base,  in dma_fence_wait_any_timeout()
797 			/* This fence is already signaled */  in dma_fence_wait_any_timeout()
833  * dma_fence_init - Initialize a custom fence.
834  * @fence: the fence to initialize
835  * @ops: the dma_fence_ops for operations on this fence
836  * @lock: the irqsafe spinlock to use for locking this fence
837  * @context: the execution context this fence is run on
840  * Initializes an allocated fence, the caller doesn't have to keep its
841  * refcount after committing with this fence, but it will need to hold a
845  * to check which fence is later by simply using dma_fence_later().
848 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,  in dma_fence_init()  argument
854 	kref_init(&fence->refcount);  in dma_fence_init()
855 	fence->ops = ops;  in dma_fence_init()
856 	INIT_LIST_HEAD(&fence->cb_list);  in dma_fence_init()
857 	fence->lock = lock;  in dma_fence_init()
858 	fence->context = context;  in dma_fence_init()
859 	fence->seqno = seqno;  in dma_fence_init()
860 	fence->flags = 0UL;  in dma_fence_init()
861 	fence->error = 0;  in dma_fence_init()
863 	trace_dma_fence_init(fence);  in dma_fence_init()