1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (c) 2010-2013, NVIDIA Corporation.
40 	u32 thresh = waiter->thresh;  in add_waiter_to_queue()
43 		if ((s32)(pos->thresh - thresh) <= 0) {  in add_waiter_to_queue()
44 			list_add(&waiter->list, &pos->list);  in add_waiter_to_queue()
48 	list_add(&waiter->list, queue);  in add_waiter_to_queue()
53  * run through a waiter queue for a single sync point ID
63 		if ((s32)(waiter->thresh - sync) > 0)  in remove_completed_waiters()
66 		dest = completed + waiter->action;  in remove_completed_waiters()
69 		if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&  in remove_completed_waiters()
71 			prev = list_entry(dest->prev,  in remove_completed_waiters()
73 			if (prev->data == waiter->data) {  in remove_completed_waiters()
74 				prev->count++;  in remove_completed_waiters()
79 		/* PENDING->REMOVED or CANCELLED->HANDLED */  in remove_completed_waiters()
80 		if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {  in remove_completed_waiters()
81 			list_del(&waiter->list);  in remove_completed_waiters()
82 			kref_put(&waiter->refcount, waiter_release);  in remove_completed_waiters()
84 			list_move_tail(&waiter->list, dest);  in remove_completed_waiters()
88 static void reset_threshold_interrupt(struct host1x *host,  in reset_threshold_interrupt()  argument
90 				      unsigned int id)  in reset_threshold_interrupt()  argument
93 		list_first_entry(head, struct host1x_waitlist, list)->thresh;  in reset_threshold_interrupt()
95 	host1x_hw_intr_set_syncpt_threshold(host, id, thresh);  in reset_threshold_interrupt()
96 	host1x_hw_intr_enable_syncpt_intr(host, id);  in reset_threshold_interrupt()
101 	struct host1x_channel *channel = waiter->data;  in action_submit_complete()
103 	host1x_cdma_update(&channel->cdma);  in action_submit_complete()
106 	trace_host1x_channel_submit_complete(dev_name(channel->dev),  in action_submit_complete()
107 					     waiter->count, waiter->thresh);  in action_submit_complete()
112 	wait_queue_head_t *wq = waiter->data;  in action_wakeup()
119 	wait_queue_head_t *wq = waiter->data;  in action_wakeup_interruptible()
142 			list_del(&waiter->list);  in run_handlers()
144 			WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=  in run_handlers()
146 			kref_put(&waiter->refcount, waiter_release);  in run_handlers()
154 static int process_wait_list(struct host1x *host,  in process_wait_list()  argument
165 	spin_lock(&syncpt->intr.lock);  in process_wait_list()
167 	remove_completed_waiters(&syncpt->intr.wait_head, threshold,  in process_wait_list()
170 	empty = list_empty(&syncpt->intr.wait_head);  in process_wait_list()
172 		host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);  in process_wait_list()
174 		reset_threshold_interrupt(host, &syncpt->intr.wait_head,  in process_wait_list()
175 					  syncpt->id);  in process_wait_list()
177 	spin_unlock(&syncpt->intr.lock);  in process_wait_list()
195 	unsigned int id = syncpt->id;  in syncpt_thresh_work()  local
196 	struct host1x *host = syncpt->host;  in syncpt_thresh_work()  local
198 	(void)process_wait_list(host, syncpt,  in syncpt_thresh_work()
199 				host1x_syncpt_load(host->syncpt + id));  in syncpt_thresh_work()
202 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,  in host1x_intr_add_action()  argument
211 		return -EINVAL;  in host1x_intr_add_action()
215 	INIT_LIST_HEAD(&waiter->list);  in host1x_intr_add_action()
216 	kref_init(&waiter->refcount);  in host1x_intr_add_action()
218 		kref_get(&waiter->refcount);  in host1x_intr_add_action()
219 	waiter->thresh = thresh;  in host1x_intr_add_action()
220 	waiter->action = action;  in host1x_intr_add_action()
221 	atomic_set(&waiter->state, WLS_PENDING);  in host1x_intr_add_action()
222 	waiter->data = data;  in host1x_intr_add_action()
223 	waiter->count = 1;  in host1x_intr_add_action()
225 	spin_lock(&syncpt->intr.lock);  in host1x_intr_add_action()
227 	queue_was_empty = list_empty(&syncpt->intr.wait_head);  in host1x_intr_add_action()
229 	if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {  in host1x_intr_add_action()
230 		/* added at head of list - new threshold value */  in host1x_intr_add_action()
231 		host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);  in host1x_intr_add_action()
233 		/* added as first waiter - enable interrupt */  in host1x_intr_add_action()
235 			host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);  in host1x_intr_add_action()
238 	spin_unlock(&syncpt->intr.lock);  in host1x_intr_add_action()
245 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref)  in host1x_intr_put_ref()  argument
250 	while (atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED) ==  in host1x_intr_put_ref()
254 	syncpt = host->syncpt + id;  in host1x_intr_put_ref()
255 	(void)process_wait_list(host, syncpt,  in host1x_intr_put_ref()
256 				host1x_syncpt_load(host->syncpt + id));  in host1x_intr_put_ref()
258 	kref_put(&waiter->refcount, waiter_release);  in host1x_intr_put_ref()
261 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)  in host1x_intr_init()  argument
263 	unsigned int id;  in host1x_intr_init()  local
264 	u32 nb_pts = host1x_syncpt_nb_pts(host);  in host1x_intr_init()
266 	mutex_init(&host->intr_mutex);  in host1x_intr_init()
267 	host->intr_syncpt_irq = irq_sync;  in host1x_intr_init()
269 	for (id = 0; id < nb_pts; ++id) {  in host1x_intr_init()
270 		struct host1x_syncpt *syncpt = host->syncpt + id;  in host1x_intr_init()
272 		spin_lock_init(&syncpt->intr.lock);  in host1x_intr_init()
273 		INIT_LIST_HEAD(&syncpt->intr.wait_head);  in host1x_intr_init()
274 		snprintf(syncpt->intr.thresh_irq_name,  in host1x_intr_init()
275 			 sizeof(syncpt->intr.thresh_irq_name),  in host1x_intr_init()
276 			 "host1x_sp_%02u", id);  in host1x_intr_init()
279 	host1x_intr_start(host);  in host1x_intr_init()
284 void host1x_intr_deinit(struct host1x *host)  in host1x_intr_deinit()  argument
286 	host1x_intr_stop(host);  in host1x_intr_deinit()
289 void host1x_intr_start(struct host1x *host)  in host1x_intr_start()  argument
291 	u32 hz = clk_get_rate(host->clk);  in host1x_intr_start()
294 	mutex_lock(&host->intr_mutex);  in host1x_intr_start()
295 	err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),  in host1x_intr_start()
298 		mutex_unlock(&host->intr_mutex);  in host1x_intr_start()
301 	mutex_unlock(&host->intr_mutex);  in host1x_intr_start()
304 void host1x_intr_stop(struct host1x *host)  in host1x_intr_stop()  argument
306 	unsigned int id;  in host1x_intr_stop()  local
307 	struct host1x_syncpt *syncpt = host->syncpt;  in host1x_intr_stop()
308 	u32 nb_pts = host1x_syncpt_nb_pts(host);  in host1x_intr_stop()
310 	mutex_lock(&host->intr_mutex);  in host1x_intr_stop()
312 	host1x_hw_intr_disable_all_syncpt_intrs(host);  in host1x_intr_stop()
314 	for (id = 0; id < nb_pts; ++id) {  in host1x_intr_stop()
318 			&syncpt[id].intr.wait_head, list) {  in host1x_intr_stop()
319 			if (atomic_cmpxchg(&waiter->state,  in host1x_intr_stop()
321 				list_del(&waiter->list);  in host1x_intr_stop()
322 				kref_put(&waiter->refcount, waiter_release);  in host1x_intr_stop()
326 		if (!list_empty(&syncpt[id].intr.wait_head)) {  in host1x_intr_stop()
328 			mutex_unlock(&host->intr_mutex);  in host1x_intr_stop()
329 			pr_warn("%s cannot stop syncpt intr id=%u\n",  in host1x_intr_stop()
330 				__func__, id);  in host1x_intr_stop()
335 	host1x_hw_intr_free_syncpt_irq(host);  in host1x_intr_stop()
337 	mutex_unlock(&host->intr_mutex);  in host1x_intr_stop()