Lines Matching full:work
34 IO_WORKER_F_BOUND = 16, /* is doing bounded work */
39 IO_WQ_BIT_CANCEL = 1, /* cancel work on list */
202 struct io_wq_work *work) in io_work_get_acct() argument
204 if (work->flags & IO_WQ_WORK_UNBOUND) in io_work_get_acct()
299 * Most likely an attempt to queue unbounded work on an io_wq that in io_wqe_wake_worker()
343 * Worker will start processing some work. Move it to the busy list, if
347 struct io_wq_work *work) in __io_worker_busy() argument
362 work_bound = (work->flags & IO_WQ_WORK_UNBOUND) == 0; in __io_worker_busy()
381 * No work, worker going to sleep. Move to freelist, and unuse mm if we
398 static inline unsigned int io_get_work_hash(struct io_wq_work *work) in io_get_work_hash() argument
400 return work->flags >> IO_WQ_HASH_SHIFT; in io_get_work_hash()
407 struct io_wq_work *work, *tail; in io_get_next_work() local
411 work = container_of(node, struct io_wq_work, list); in io_get_next_work()
414 if (!io_wq_is_hashed(work)) { in io_get_next_work()
416 return work; in io_get_next_work()
420 hash = io_get_work_hash(work); in io_get_next_work()
423 /* all items with this hash lie in [work, tail] */ in io_get_next_work()
427 return work; in io_get_next_work()
434 static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work) in io_wq_switch_mm() argument
442 if (mmget_not_zero(work->identity->mm)) { in io_wq_switch_mm()
443 kthread_use_mm(work->identity->mm); in io_wq_switch_mm()
444 worker->mm = work->identity->mm; in io_wq_switch_mm()
448 /* failed grabbing mm, ensure work gets cancelled */ in io_wq_switch_mm()
449 work->flags |= IO_WQ_WORK_CANCEL; in io_wq_switch_mm()
453 struct io_wq_work *work) in io_wq_switch_blkcg() argument
456 if (!(work->flags & IO_WQ_WORK_BLKCG)) in io_wq_switch_blkcg()
458 if (work->identity->blkcg_css != worker->blkcg_css) { in io_wq_switch_blkcg()
459 kthread_associate_blkcg(work->identity->blkcg_css); in io_wq_switch_blkcg()
460 worker->blkcg_css = work->identity->blkcg_css; in io_wq_switch_blkcg()
466 struct io_wq_work *work) in io_wq_switch_creds() argument
468 const struct cred *old_creds = override_creds(work->identity->creds); in io_wq_switch_creds()
470 worker->cur_creds = work->identity->creds; in io_wq_switch_creds()
478 struct io_wq_work *work) in io_impersonate_work() argument
480 if ((work->flags & IO_WQ_WORK_FILES) && in io_impersonate_work()
481 current->files != work->identity->files) { in io_impersonate_work()
483 current->files = work->identity->files; in io_impersonate_work()
484 current->nsproxy = work->identity->nsproxy; in io_impersonate_work()
486 if (!work->identity->files) { in io_impersonate_work()
487 /* failed grabbing files, ensure work gets cancelled */ in io_impersonate_work()
488 work->flags |= IO_WQ_WORK_CANCEL; in io_impersonate_work()
491 if ((work->flags & IO_WQ_WORK_FS) && current->fs != work->identity->fs) in io_impersonate_work()
492 current->fs = work->identity->fs; in io_impersonate_work()
493 if ((work->flags & IO_WQ_WORK_MM) && work->identity->mm != worker->mm) in io_impersonate_work()
494 io_wq_switch_mm(worker, work); in io_impersonate_work()
495 if ((work->flags & IO_WQ_WORK_CREDS) && in io_impersonate_work()
496 worker->cur_creds != work->identity->creds) in io_impersonate_work()
497 io_wq_switch_creds(worker, work); in io_impersonate_work()
498 if (work->flags & IO_WQ_WORK_FSIZE) in io_impersonate_work()
499 current->signal->rlim[RLIMIT_FSIZE].rlim_cur = work->identity->fsize; in io_impersonate_work()
502 io_wq_switch_blkcg(worker, work); in io_impersonate_work()
504 current->loginuid = work->identity->loginuid; in io_impersonate_work()
505 current->sessionid = work->identity->sessionid; in io_impersonate_work()
510 struct io_wq_work *work) in io_assign_current_work() argument
512 if (work) { in io_assign_current_work()
513 /* flush pending signals before assigning new work */ in io_assign_current_work()
525 worker->cur_work = work; in io_assign_current_work()
529 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work);
538 struct io_wq_work *work; in io_worker_handle_work() local
541 * If we got some work, mark us as busy. If we didn't, but in io_worker_handle_work()
542 * the list isn't empty, it means we stalled on hashed work. in io_worker_handle_work()
543 * Mark us stalled so we don't keep looking for work when we in io_worker_handle_work()
544 * can't make progress, any work completion or insertion will in io_worker_handle_work()
547 work = io_get_next_work(wqe); in io_worker_handle_work()
548 if (work) in io_worker_handle_work()
549 __io_worker_busy(wqe, worker, work); in io_worker_handle_work()
554 if (!work) in io_worker_handle_work()
556 io_assign_current_work(worker, work); in io_worker_handle_work()
561 unsigned int hash = io_get_work_hash(work); in io_worker_handle_work()
563 next_hashed = wq_next_work(work); in io_worker_handle_work()
564 io_impersonate_work(worker, work); in io_worker_handle_work()
567 * work, the worker function will do the right thing. in io_worker_handle_work()
570 work->flags |= IO_WQ_WORK_CANCEL; in io_worker_handle_work()
572 old_work = work; in io_worker_handle_work()
573 linked = wq->do_work(work); in io_worker_handle_work()
575 work = next_hashed; in io_worker_handle_work()
576 if (!work && linked && !io_wq_is_hashed(linked)) { in io_worker_handle_work()
577 work = linked; in io_worker_handle_work()
580 io_assign_current_work(worker, work); in io_worker_handle_work()
591 if (!work) in io_worker_handle_work()
595 } while (work); in io_worker_handle_work()
664 * running and we have work pending, wake up a free one or have the manager
730 /* if we have available workers or no work, no need */ in io_wqe_need_worker()
840 struct io_wq_work *work) in io_wq_can_queue() argument
844 if (!(work->flags & IO_WQ_WORK_UNBOUND)) in io_wq_can_queue()
862 static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) in io_run_cancel() argument
867 struct io_wq_work *old_work = work; in io_run_cancel()
869 work->flags |= IO_WQ_WORK_CANCEL; in io_run_cancel()
870 work = wq->do_work(work); in io_run_cancel()
872 } while (work); in io_run_cancel()
875 static void io_wqe_insert_work(struct io_wqe *wqe, struct io_wq_work *work) in io_wqe_insert_work() argument
880 if (!io_wq_is_hashed(work)) { in io_wqe_insert_work()
882 wq_list_add_tail(&work->list, &wqe->work_list); in io_wqe_insert_work()
886 hash = io_get_work_hash(work); in io_wqe_insert_work()
888 wqe->hash_tail[hash] = work; in io_wqe_insert_work()
892 wq_list_add_after(&work->list, &tail->list, &wqe->work_list); in io_wqe_insert_work()
895 static void io_wqe_enqueue(struct io_wqe *wqe, struct io_wq_work *work) in io_wqe_enqueue() argument
897 struct io_wqe_acct *acct = io_work_get_acct(wqe, work); in io_wqe_enqueue()
907 if (unlikely(!io_wq_can_queue(wqe, acct, work))) { in io_wqe_enqueue()
908 io_run_cancel(work, wqe); in io_wqe_enqueue()
913 io_wqe_insert_work(wqe, work); in io_wqe_enqueue()
915 do_wake = (work->flags & IO_WQ_WORK_CONCURRENT) || in io_wqe_enqueue()
923 void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work) in io_wq_enqueue() argument
927 io_wqe_enqueue(wqe, work); in io_wq_enqueue()
931 * Work items that hash to the same value will not be done in parallel.
934 void io_wq_hash_work(struct io_wq_work *work, void *val) in io_wq_hash_work() argument
939 work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT)); in io_wq_hash_work()
972 * may dereference the passed in work. in io_wq_worker_cancel()
987 struct io_wq_work *work, in io_wqe_remove_pending() argument
990 unsigned int hash = io_get_work_hash(work); in io_wqe_remove_pending()
993 if (io_wq_is_hashed(work) && work == wqe->hash_tail[hash]) { in io_wqe_remove_pending()
1001 wq_list_del(&wqe->work_list, &work->list, prev); in io_wqe_remove_pending()
1008 struct io_wq_work *work; in io_wqe_cancel_pending_work() local
1014 work = container_of(node, struct io_wq_work, list); in io_wqe_cancel_pending_work()
1015 if (!match->fn(work, match->data)) in io_wqe_cancel_pending_work()
1017 io_wqe_remove_pending(wqe, work, prev); in io_wqe_cancel_pending_work()
1019 io_run_cancel(work, wqe); in io_wqe_cancel_pending_work()
1050 * from there. CANCEL_OK means that the work is returned as-new, in io_wq_cancel_cb()
1062 * Now check if a free (going busy) or busy worker has the work in io_wq_cancel_cb()