Lines Matching full:work
27 IO_WORKER_F_BOUND = 8, /* is doing bounded work */
60 struct work_struct work; member
163 struct io_wq_work *work) in io_work_get_acct() argument
165 return io_get_acct(wqe, !(work->flags & IO_WQ_WORK_UNBOUND)); in io_work_get_acct()
286 * Most likely an attempt to queue unbounded work on an io_wq that in io_wqe_create_worker()
369 * work item after we canceled in io_wq_exit_workers(). in io_queue_worker_create()
405 * Worker will start processing some work. Move it to the busy list, if
409 struct io_wq_work *work) in __io_worker_busy() argument
419 * No work, worker going to sleep. Move to freelist, and unuse mm if we
434 static inline unsigned int io_get_work_hash(struct io_wq_work *work) in io_get_work_hash() argument
436 return work->flags >> IO_WQ_HASH_SHIFT; in io_get_work_hash()
462 struct io_wq_work *work, *tail; in io_get_next_work() local
469 work = container_of(node, struct io_wq_work, list); in io_get_next_work()
472 if (!io_wq_is_hashed(work)) { in io_get_next_work()
474 return work; in io_get_next_work()
477 hash = io_get_work_hash(work); in io_get_next_work()
478 /* all items with this hash lie in [work, tail] */ in io_get_next_work()
485 return work; in io_get_next_work()
498 * work being added and clearing the stalled bit. in io_get_next_work()
525 struct io_wq_work *work) in io_assign_current_work() argument
527 if (work) { in io_assign_current_work()
533 worker->cur_work = work; in io_assign_current_work()
537 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
548 struct io_wq_work *work; in io_worker_handle_work() local
551 * If we got some work, mark us as busy. If we didn't, but in io_worker_handle_work()
552 * the list isn't empty, it means we stalled on hashed work. in io_worker_handle_work()
553 * Mark us stalled so we don't keep looking for work when we in io_worker_handle_work()
554 * can't make progress, any work completion or insertion will in io_worker_handle_work()
557 work = io_get_next_work(acct, worker); in io_worker_handle_work()
558 if (work) in io_worker_handle_work()
559 __io_worker_busy(wqe, worker, work); in io_worker_handle_work()
562 if (!work) in io_worker_handle_work()
564 io_assign_current_work(worker, work); in io_worker_handle_work()
570 unsigned int hash = io_get_work_hash(work); in io_worker_handle_work()
572 next_hashed = wq_next_work(work); in io_worker_handle_work()
574 if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND)) in io_worker_handle_work()
575 work->flags |= IO_WQ_WORK_CANCEL; in io_worker_handle_work()
576 wq->do_work(work); in io_worker_handle_work()
579 linked = wq->free_work(work); in io_worker_handle_work()
580 work = next_hashed; in io_worker_handle_work()
581 if (!work && linked && !io_wq_is_hashed(linked)) { in io_worker_handle_work()
582 work = linked; in io_worker_handle_work()
585 io_assign_current_work(worker, work); in io_worker_handle_work()
599 if (!work) in io_worker_handle_work()
603 } while (work); in io_worker_handle_work()
684 * running and we have work pending, wake up a free one or create a new one.
720 static bool io_wq_work_match_all(struct io_wq_work *work, void *data) in io_wq_work_match_all() argument
782 schedule_work(&worker->work); in create_worker_cont()
785 static void io_workqueue_create(struct work_struct *work) in io_workqueue_create() argument
787 struct io_worker *worker = container_of(work, struct io_worker, work); in io_workqueue_create()
828 INIT_WORK(&worker->work, io_workqueue_create); in create_io_worker()
829 schedule_work(&worker->work); in create_io_worker()
867 static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) in io_run_cancel() argument
872 work->flags |= IO_WQ_WORK_CANCEL; in io_run_cancel()
873 wq->do_work(work); in io_run_cancel()
874 work = wq->free_work(work); in io_run_cancel()
875 } while (work); in io_run_cancel()
878 static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) in io_wqe_insert_work() argument
880 struct io_wqe_acct *acct = io_work_get_acct(wqe, work); in io_wqe_insert_work()
884 if (!io_wq_is_hashed(work)) { in io_wqe_insert_work()
886 wq_list_add_tail(&work->list, &acct->work_list); in io_wqe_insert_work()
890 hash = io_get_work_hash(work); in io_wqe_insert_work()
892 wqe->hash_tail[hash] = work; in io_wqe_insert_work()
896 wq_list_add_after(&work->list, &tail->list, &acct->work_list); in io_wqe_insert_work()
899 static bool io_wq_work_match_item(struct io_wq_work *work, void *data) in io_wq_work_match_item() argument
901 return work == data; in io_wq_work_match_item()
904 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) in io_wqe_enqueue() argument
906 struct io_wqe_acct *acct = io_work_get_acct(wqe, work); in io_wqe_enqueue()
907 unsigned work_flags = work->flags; in io_wqe_enqueue()
915 (work->flags & IO_WQ_WORK_CANCEL)) { in io_wqe_enqueue()
916 io_run_cancel(work, wqe); in io_wqe_enqueue()
921 io_wqe_insert_work(wqe, work); in io_wqe_enqueue()
943 .data = work, in io_wqe_enqueue()
954 void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) in io_wq_enqueue() argument
958 io_wqe_enqueue(wqe, work); in io_wq_enqueue()
962 * Work items that hash to the same value will not be done in parallel.
965 void io_wq_hash_work(struct io_wq_work *work, void *val) in io_wq_hash_work() argument
970 work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); in io_wq_hash_work()
979 * may dereference the passed in work. in io_wq_worker_cancel()
993 struct io_wq_work *work, in io_wqe_remove_pending() argument
996 struct io_wqe_acct *acct = io_work_get_acct(wqe, work); in io_wqe_remove_pending()
997 unsigned int hash = io_get_work_hash(work); in io_wqe_remove_pending()
1000 if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { in io_wqe_remove_pending()
1008 wq_list_del(&acct->work_list, &work->list, prev); in io_wqe_remove_pending()
1017 struct io_wq_work *work; in io_acct_cancel_pending_work() local
1020 work = container_of(node, struct io_wq_work, list); in io_acct_cancel_pending_work()
1021 if (!match->fn(work, match->data)) in io_acct_cancel_pending_work()
1023 io_wqe_remove_pending(wqe, work, prev); in io_acct_cancel_pending_work()
1025 io_run_cancel(work, wqe); in io_acct_cancel_pending_work()
1072 * from there. CANCEL_OK means that the work is returned as-new, in io_wq_cancel_cb()
1084 * Now check if a free (going busy) or busy worker has the work in io_wq_cancel_cb()