Lines Matching refs:task
45 static void rpc_release_task(struct rpc_task *task);
61 rpc_task_timeout(const struct rpc_task *task) in rpc_task_timeout() argument
63 unsigned long timeout = READ_ONCE(task->tk_timeout); in rpc_task_timeout()
80 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) in __rpc_disable_timer() argument
82 if (list_empty(&task->u.tk_wait.timer_list)) in __rpc_disable_timer()
84 task->tk_timeout = 0; in __rpc_disable_timer()
85 list_del(&task->u.tk_wait.timer_list); in __rpc_disable_timer()
106 __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task, in __rpc_add_timer() argument
109 task->tk_timeout = timeout; in __rpc_add_timer()
112 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); in __rpc_add_timer()
132 __rpc_list_enqueue_task(struct list_head *q, struct rpc_task *task) in __rpc_list_enqueue_task() argument
137 if (t->tk_owner == task->tk_owner) { in __rpc_list_enqueue_task()
138 list_add_tail(&task->u.tk_wait.links, in __rpc_list_enqueue_task()
141 task->u.tk_wait.list.next = q; in __rpc_list_enqueue_task()
142 task->u.tk_wait.list.prev = NULL; in __rpc_list_enqueue_task()
146 INIT_LIST_HEAD(&task->u.tk_wait.links); in __rpc_list_enqueue_task()
147 list_add_tail(&task->u.tk_wait.list, q); in __rpc_list_enqueue_task()
154 __rpc_list_dequeue_task(struct rpc_task *task) in __rpc_list_dequeue_task() argument
159 if (task->u.tk_wait.list.prev == NULL) { in __rpc_list_dequeue_task()
160 list_del(&task->u.tk_wait.links); in __rpc_list_dequeue_task()
163 if (!list_empty(&task->u.tk_wait.links)) { in __rpc_list_dequeue_task()
164 t = list_first_entry(&task->u.tk_wait.links, in __rpc_list_dequeue_task()
170 list_del(&task->u.tk_wait.links); in __rpc_list_dequeue_task()
172 list_del(&task->u.tk_wait.list); in __rpc_list_dequeue_task()
179 struct rpc_task *task, in __rpc_add_wait_queue_priority() argument
184 __rpc_list_enqueue_task(&queue->tasks[queue_priority], task); in __rpc_add_wait_queue_priority()
191 struct rpc_task *task, in __rpc_add_wait_queue() argument
194 INIT_LIST_HEAD(&task->u.tk_wait.timer_list); in __rpc_add_wait_queue()
196 __rpc_add_wait_queue_priority(queue, task, queue_priority); in __rpc_add_wait_queue()
198 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); in __rpc_add_wait_queue()
199 task->tk_waitqueue = queue; in __rpc_add_wait_queue()
203 rpc_set_queued(task); in __rpc_add_wait_queue()
209 static void __rpc_remove_wait_queue_priority(struct rpc_task *task) in __rpc_remove_wait_queue_priority() argument
211 __rpc_list_dequeue_task(task); in __rpc_remove_wait_queue_priority()
218 static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) in __rpc_remove_wait_queue() argument
220 __rpc_disable_timer(queue, task); in __rpc_remove_wait_queue()
222 __rpc_remove_wait_queue_priority(task); in __rpc_remove_wait_queue()
224 list_del(&task->u.tk_wait.list); in __rpc_remove_wait_queue()
271 static void rpc_task_set_debuginfo(struct rpc_task *task) in rpc_task_set_debuginfo() argument
275 task->tk_pid = atomic_inc_return(&rpc_pid); in rpc_task_set_debuginfo()
278 static inline void rpc_task_set_debuginfo(struct rpc_task *task) in rpc_task_set_debuginfo() argument
283 static void rpc_set_active(struct rpc_task *task) in rpc_set_active() argument
285 rpc_task_set_debuginfo(task); in rpc_set_active()
286 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); in rpc_set_active()
287 trace_rpc_task_begin(task, NULL); in rpc_set_active()
294 static int rpc_complete_task(struct rpc_task *task) in rpc_complete_task() argument
296 void *m = &task->tk_runstate; in rpc_complete_task()
302 trace_rpc_task_complete(task, NULL); in rpc_complete_task()
305 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); in rpc_complete_task()
306 ret = atomic_dec_and_test(&task->tk_count); in rpc_complete_task()
320 int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) in __rpc_wait_for_completion_task() argument
324 return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, in __rpc_wait_for_completion_task()
341 struct rpc_task *task) in rpc_make_runnable() argument
343 bool need_wakeup = !rpc_test_and_set_running(task); in rpc_make_runnable()
345 rpc_clear_queued(task); in rpc_make_runnable()
348 if (RPC_IS_ASYNC(task)) { in rpc_make_runnable()
349 INIT_WORK(&task->u.tk_work, rpc_async_schedule); in rpc_make_runnable()
350 queue_work(wq, &task->u.tk_work); in rpc_make_runnable()
352 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); in rpc_make_runnable()
362 struct rpc_task *task, in __rpc_do_sleep_on_priority() argument
365 trace_rpc_task_sleep(task, q); in __rpc_do_sleep_on_priority()
367 __rpc_add_wait_queue(q, task, queue_priority); in __rpc_do_sleep_on_priority()
371 struct rpc_task *task, in __rpc_sleep_on_priority() argument
374 if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) in __rpc_sleep_on_priority()
376 __rpc_do_sleep_on_priority(q, task, queue_priority); in __rpc_sleep_on_priority()
380 struct rpc_task *task, unsigned long timeout, in __rpc_sleep_on_priority_timeout() argument
383 if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) in __rpc_sleep_on_priority_timeout()
386 __rpc_do_sleep_on_priority(q, task, queue_priority); in __rpc_sleep_on_priority_timeout()
387 __rpc_add_timer(q, task, timeout); in __rpc_sleep_on_priority_timeout()
389 task->tk_status = -ETIMEDOUT; in __rpc_sleep_on_priority_timeout()
392 static void rpc_set_tk_callback(struct rpc_task *task, rpc_action action) in rpc_set_tk_callback() argument
394 if (action && !WARN_ON_ONCE(task->tk_callback != NULL)) in rpc_set_tk_callback()
395 task->tk_callback = action; in rpc_set_tk_callback()
398 static bool rpc_sleep_check_activated(struct rpc_task *task) in rpc_sleep_check_activated() argument
401 if (WARN_ON_ONCE(!RPC_IS_ACTIVATED(task))) { in rpc_sleep_check_activated()
402 task->tk_status = -EIO; in rpc_sleep_check_activated()
403 rpc_put_task_async(task); in rpc_sleep_check_activated()
409 void rpc_sleep_on_timeout(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on_timeout() argument
412 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_timeout()
415 rpc_set_tk_callback(task, action); in rpc_sleep_on_timeout()
421 __rpc_sleep_on_priority_timeout(q, task, timeout, task->tk_priority); in rpc_sleep_on_timeout()
426 void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on() argument
429 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on()
432 rpc_set_tk_callback(task, action); in rpc_sleep_on()
434 WARN_ON_ONCE(task->tk_timeout != 0); in rpc_sleep_on()
439 __rpc_sleep_on_priority(q, task, task->tk_priority); in rpc_sleep_on()
445 struct rpc_task *task, unsigned long timeout, int priority) in rpc_sleep_on_priority_timeout() argument
447 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_priority_timeout()
455 __rpc_sleep_on_priority_timeout(q, task, timeout, priority); in rpc_sleep_on_priority_timeout()
460 void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, in rpc_sleep_on_priority() argument
463 if (!rpc_sleep_check_activated(task)) in rpc_sleep_on_priority()
466 WARN_ON_ONCE(task->tk_timeout != 0); in rpc_sleep_on_priority()
472 __rpc_sleep_on_priority(q, task, priority); in rpc_sleep_on_priority()
487 struct rpc_task *task) in __rpc_do_wake_up_task_on_wq() argument
490 if (!RPC_IS_ACTIVATED(task)) { in __rpc_do_wake_up_task_on_wq()
491 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); in __rpc_do_wake_up_task_on_wq()
495 trace_rpc_task_wakeup(task, queue); in __rpc_do_wake_up_task_on_wq()
497 __rpc_remove_wait_queue(queue, task); in __rpc_do_wake_up_task_on_wq()
499 rpc_make_runnable(wq, task); in __rpc_do_wake_up_task_on_wq()
507 struct rpc_wait_queue *queue, struct rpc_task *task, in rpc_wake_up_task_on_wq_queue_action_locked() argument
510 if (RPC_IS_QUEUED(task)) { in rpc_wake_up_task_on_wq_queue_action_locked()
512 if (task->tk_waitqueue == queue) { in rpc_wake_up_task_on_wq_queue_action_locked()
513 if (action == NULL || action(task, data)) { in rpc_wake_up_task_on_wq_queue_action_locked()
514 __rpc_do_wake_up_task_on_wq(wq, queue, task); in rpc_wake_up_task_on_wq_queue_action_locked()
515 return task; in rpc_wake_up_task_on_wq_queue_action_locked()
526 struct rpc_task *task) in rpc_wake_up_task_queue_locked() argument
529 task, NULL, NULL); in rpc_wake_up_task_queue_locked()
535 void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) in rpc_wake_up_queued_task() argument
537 if (!RPC_IS_QUEUED(task)) in rpc_wake_up_queued_task()
540 rpc_wake_up_task_queue_locked(queue, task); in rpc_wake_up_queued_task()
545 static bool rpc_task_action_set_status(struct rpc_task *task, void *status) in rpc_task_action_set_status() argument
547 task->tk_status = *(int *)status; in rpc_task_action_set_status()
553 struct rpc_task *task, int status) in rpc_wake_up_task_queue_set_status_locked() argument
556 task, rpc_task_action_set_status, &status); in rpc_wake_up_task_queue_set_status_locked()
570 struct rpc_task *task, int status) in rpc_wake_up_queued_task_set_status() argument
572 if (!RPC_IS_QUEUED(task)) in rpc_wake_up_queued_task_set_status()
575 rpc_wake_up_task_queue_set_status_locked(queue, task, status); in rpc_wake_up_queued_task_set_status()
585 struct rpc_task *task; in __rpc_find_next_queued_priority() local
592 task = list_first_entry(q, struct rpc_task, u.tk_wait.list); in __rpc_find_next_queued_priority()
602 task = list_first_entry(q, struct rpc_task, u.tk_wait.list); in __rpc_find_next_queued_priority()
615 task = list_first_entry(q, struct rpc_task, u.tk_wait.list); in __rpc_find_next_queued_priority()
626 return task; in __rpc_find_next_queued_priority()
645 struct rpc_task *task = NULL; in rpc_wake_up_first_on_wq() local
648 task = __rpc_find_next_queued(queue); in rpc_wake_up_first_on_wq()
649 if (task != NULL) in rpc_wake_up_first_on_wq()
650 task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue, in rpc_wake_up_first_on_wq()
651 task, func, data); in rpc_wake_up_first_on_wq()
654 return task; in rpc_wake_up_first_on_wq()
667 static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) in rpc_wake_up_next_func() argument
688 struct rpc_task *task; in rpc_wake_up_locked() local
691 task = __rpc_find_next_queued(queue); in rpc_wake_up_locked()
692 if (task == NULL) in rpc_wake_up_locked()
694 rpc_wake_up_task_queue_locked(queue, task); in rpc_wake_up_locked()
719 struct rpc_task *task; in rpc_wake_up_status_locked() local
722 task = __rpc_find_next_queued(queue); in rpc_wake_up_status_locked()
723 if (task == NULL) in rpc_wake_up_status_locked()
725 rpc_wake_up_task_queue_set_status_locked(queue, task, status); in rpc_wake_up_status_locked()
749 struct rpc_task *task, *n; in __rpc_queue_timer_fn() local
754 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { in __rpc_queue_timer_fn()
755 timeo = task->tk_timeout; in __rpc_queue_timer_fn()
757 trace_rpc_task_timeout(task, task->tk_action); in __rpc_queue_timer_fn()
758 task->tk_status = -ETIMEDOUT; in __rpc_queue_timer_fn()
759 rpc_wake_up_task_queue_locked(queue, task); in __rpc_queue_timer_fn()
770 static void __rpc_atrun(struct rpc_task *task) in __rpc_atrun() argument
772 if (task->tk_status == -ETIMEDOUT) in __rpc_atrun()
773 task->tk_status = 0; in __rpc_atrun()
779 void rpc_delay(struct rpc_task *task, unsigned long delay) in rpc_delay() argument
781 rpc_sleep_on_timeout(&delay_queue, task, __rpc_atrun, jiffies + delay); in rpc_delay()
788 void rpc_prepare_task(struct rpc_task *task) in rpc_prepare_task() argument
790 task->tk_ops->rpc_call_prepare(task, task->tk_calldata); in rpc_prepare_task()
794 rpc_init_task_statistics(struct rpc_task *task) in rpc_init_task_statistics() argument
797 task->tk_garb_retry = 2; in rpc_init_task_statistics()
798 task->tk_cred_retry = 2; in rpc_init_task_statistics()
801 task->tk_start = ktime_get(); in rpc_init_task_statistics()
805 rpc_reset_task_statistics(struct rpc_task *task) in rpc_reset_task_statistics() argument
807 task->tk_timeouts = 0; in rpc_reset_task_statistics()
808 task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_SENT); in rpc_reset_task_statistics()
809 rpc_init_task_statistics(task); in rpc_reset_task_statistics()
815 void rpc_exit_task(struct rpc_task *task) in rpc_exit_task() argument
817 trace_rpc_task_end(task, task->tk_action); in rpc_exit_task()
818 task->tk_action = NULL; in rpc_exit_task()
819 if (task->tk_ops->rpc_count_stats) in rpc_exit_task()
820 task->tk_ops->rpc_count_stats(task, task->tk_calldata); in rpc_exit_task()
821 else if (task->tk_client) in rpc_exit_task()
822 rpc_count_iostats(task, task->tk_client->cl_metrics); in rpc_exit_task()
823 if (task->tk_ops->rpc_call_done != NULL) { in rpc_exit_task()
824 task->tk_ops->rpc_call_done(task, task->tk_calldata); in rpc_exit_task()
825 if (task->tk_action != NULL) { in rpc_exit_task()
827 xprt_release(task); in rpc_exit_task()
828 rpc_reset_task_statistics(task); in rpc_exit_task()
833 void rpc_signal_task(struct rpc_task *task) in rpc_signal_task() argument
837 if (!RPC_IS_ACTIVATED(task)) in rpc_signal_task()
840 trace_rpc_task_signalled(task, task->tk_action); in rpc_signal_task()
841 set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); in rpc_signal_task()
843 queue = READ_ONCE(task->tk_waitqueue); in rpc_signal_task()
845 rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS); in rpc_signal_task()
848 void rpc_exit(struct rpc_task *task, int status) in rpc_exit() argument
850 task->tk_status = status; in rpc_exit()
851 task->tk_action = rpc_exit_task; in rpc_exit()
852 rpc_wake_up_queued_task(task->tk_waitqueue, task); in rpc_exit()
865 static void __rpc_execute(struct rpc_task *task) in __rpc_execute() argument
868 int task_is_async = RPC_IS_ASYNC(task); in __rpc_execute()
871 WARN_ON_ONCE(RPC_IS_QUEUED(task)); in __rpc_execute()
872 if (RPC_IS_QUEUED(task)) in __rpc_execute()
885 do_action = task->tk_action; in __rpc_execute()
886 if (task->tk_callback) { in __rpc_execute()
887 do_action = task->tk_callback; in __rpc_execute()
888 task->tk_callback = NULL; in __rpc_execute()
892 trace_rpc_task_run_action(task, do_action); in __rpc_execute()
893 do_action(task); in __rpc_execute()
898 if (!RPC_IS_QUEUED(task)) in __rpc_execute()
904 if (RPC_SIGNALLED(task)) { in __rpc_execute()
905 task->tk_rpc_status = -ERESTARTSYS; in __rpc_execute()
906 rpc_exit(task, -ERESTARTSYS); in __rpc_execute()
918 queue = task->tk_waitqueue; in __rpc_execute()
920 if (!RPC_IS_QUEUED(task)) { in __rpc_execute()
924 rpc_clear_running(task); in __rpc_execute()
930 trace_rpc_task_sync_sleep(task, task->tk_action); in __rpc_execute()
931 status = out_of_line_wait_on_bit(&task->tk_runstate, in __rpc_execute()
941 trace_rpc_task_signalled(task, task->tk_action); in __rpc_execute()
942 set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); in __rpc_execute()
943 task->tk_rpc_status = -ERESTARTSYS; in __rpc_execute()
944 rpc_exit(task, -ERESTARTSYS); in __rpc_execute()
946 trace_rpc_task_sync_wake(task, task->tk_action); in __rpc_execute()
950 rpc_release_task(task); in __rpc_execute()
962 void rpc_execute(struct rpc_task *task) in rpc_execute() argument
964 bool is_async = RPC_IS_ASYNC(task); in rpc_execute()
966 rpc_set_active(task); in rpc_execute()
967 rpc_make_runnable(rpciod_workqueue, task); in rpc_execute()
970 __rpc_execute(task); in rpc_execute()
1000 int rpc_malloc(struct rpc_task *task) in rpc_malloc() argument
1002 struct rpc_rqst *rqst = task->tk_rqstp; in rpc_malloc()
1007 if (RPC_IS_ASYNC(task)) in rpc_malloc()
1009 if (RPC_IS_SWAPPER(task)) in rpc_malloc()
1033 void rpc_free(struct rpc_task *task) in rpc_free() argument
1035 void *buffer = task->tk_rqstp->rq_buffer; in rpc_free()
1052 static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) in rpc_init_task() argument
1054 memset(task, 0, sizeof(*task)); in rpc_init_task()
1055 atomic_set(&task->tk_count, 1); in rpc_init_task()
1056 task->tk_flags = task_setup_data->flags; in rpc_init_task()
1057 task->tk_ops = task_setup_data->callback_ops; in rpc_init_task()
1058 task->tk_calldata = task_setup_data->callback_data; in rpc_init_task()
1059 INIT_LIST_HEAD(&task->tk_task); in rpc_init_task()
1061 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; in rpc_init_task()
1062 task->tk_owner = current->tgid; in rpc_init_task()
1065 task->tk_workqueue = task_setup_data->workqueue; in rpc_init_task()
1067 task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client, in rpc_init_task()
1070 task->tk_op_cred = get_rpccred(task_setup_data->rpc_op_cred); in rpc_init_task()
1072 if (task->tk_ops->rpc_call_prepare != NULL) in rpc_init_task()
1073 task->tk_action = rpc_prepare_task; in rpc_init_task()
1075 rpc_init_task_statistics(task); in rpc_init_task()
1089 struct rpc_task *task = setup_data->task; in rpc_new_task() local
1092 if (task == NULL) { in rpc_new_task()
1093 task = rpc_alloc_task(); in rpc_new_task()
1097 rpc_init_task(task, setup_data); in rpc_new_task()
1098 task->tk_flags |= flags; in rpc_new_task()
1099 return task; in rpc_new_task()
1121 static void rpc_free_task(struct rpc_task *task) in rpc_free_task() argument
1123 unsigned short tk_flags = task->tk_flags; in rpc_free_task()
1125 put_rpccred(task->tk_op_cred); in rpc_free_task()
1126 rpc_release_calldata(task->tk_ops, task->tk_calldata); in rpc_free_task()
1129 mempool_free(task, rpc_task_mempool); in rpc_free_task()
1140 static void rpc_release_resources_task(struct rpc_task *task) in rpc_release_resources_task() argument
1142 xprt_release(task); in rpc_release_resources_task()
1143 if (task->tk_msg.rpc_cred) { in rpc_release_resources_task()
1144 if (!(task->tk_flags & RPC_TASK_CRED_NOREF)) in rpc_release_resources_task()
1145 put_cred(task->tk_msg.rpc_cred); in rpc_release_resources_task()
1146 task->tk_msg.rpc_cred = NULL; in rpc_release_resources_task()
1148 rpc_task_release_client(task); in rpc_release_resources_task()
1151 static void rpc_final_put_task(struct rpc_task *task, in rpc_final_put_task() argument
1155 INIT_WORK(&task->u.tk_work, rpc_async_release); in rpc_final_put_task()
1156 queue_work(q, &task->u.tk_work); in rpc_final_put_task()
1158 rpc_free_task(task); in rpc_final_put_task()
1161 static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) in rpc_do_put_task() argument
1163 if (atomic_dec_and_test(&task->tk_count)) { in rpc_do_put_task()
1164 rpc_release_resources_task(task); in rpc_do_put_task()
1165 rpc_final_put_task(task, q); in rpc_do_put_task()
1169 void rpc_put_task(struct rpc_task *task) in rpc_put_task() argument
1171 rpc_do_put_task(task, NULL); in rpc_put_task()
1175 void rpc_put_task_async(struct rpc_task *task) in rpc_put_task_async() argument
1177 rpc_do_put_task(task, task->tk_workqueue); in rpc_put_task_async()
1181 static void rpc_release_task(struct rpc_task *task) in rpc_release_task() argument
1183 WARN_ON_ONCE(RPC_IS_QUEUED(task)); in rpc_release_task()
1185 rpc_release_resources_task(task); in rpc_release_task()
1192 if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { in rpc_release_task()
1194 if (!rpc_complete_task(task)) in rpc_release_task()
1197 if (!atomic_dec_and_test(&task->tk_count)) in rpc_release_task()
1200 rpc_final_put_task(task, task->tk_workqueue); in rpc_release_task()