Lines Matching +full:full +full:- +full:pwr +full:- +full:cycle
1 // SPDX-License-Identifier: GPL-2.0-only
3 * kernel/workqueue.c - generic async execution with shared worker pool
10 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
22 * pools for workqueues which are not bound to any specific CPU - the
25 * Please read Documentation/core-api/workqueue.rst for details.
90 UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
115 * I: Modifiable by initialization/destruction paths and read-only for
121 * L: pool->lock protected. Access with pool->lock held.
123 * X: During normal operation, modification requires pool->lock and should
125 * cpu or grabbing pool->lock is enough for read access. If
134 * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
136 * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
139 * WQ: wq->mutex protected.
141 * WR: wq->mutex protected for writes. RCU protected for reads.
195 * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
196 * of work_struct->data are used for flags and the remaining high bits
211 struct list_head pwqs_node; /* WR: node on wq->pwqs */
212 struct list_head mayday_node; /* MD: node on wq->maydays */
218 * determined without grabbing wq->mutex.
279 struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
280 struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
304 static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
319 * following always forces round-robin CPU selection on unbound work items
329 /* the per-cpu worker pools */
334 /* PL: hash of all unbound pools keyed by pool->attrs */
372 !lockdep_is_held(&wq->mutex) && \
374 "RCU, wq->mutex or wq_pool_mutex should be held")
382 * for_each_pool - iterate through all worker_pools in the system
399 * for_each_pool_worker - iterate through all workers of a worker_pool
409 list_for_each_entry((worker), &(pool)->workers, node) \
414 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
418 * This must be called either with wq->mutex held or RCU read locked.
426 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
427 lockdep_is_held(&(wq->mutex)))
435 return ((struct work_struct *) addr)->func; in work_debug_hint()
447 * - an active object is initialized
465 * - an active object is freed
516 destroy_timer_on_stack(&work->timer); in destroy_delayed_work_on_stack()
517 debug_object_free(&work->work, &work_debug_descr); in destroy_delayed_work_on_stack()
527 * worker_pool_assign_id - allocate ID and assing it to @pool
531 * successfully, -errno on failure.
542 pool->id = ret; in worker_pool_assign_id()
549 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
553 * This must be called with any of wq_pool_mutex, wq->mutex or RCU
567 * delayed item is pending. The plan is to keep CPU -> NODE in unbound_pwq_by_node()
572 return wq->dfl_pwq; in unbound_pwq_by_node()
574 return rcu_dereference_raw(wq->numa_pwq_tbl[node]); in unbound_pwq_by_node()
585 ((1 << WORK_STRUCT_COLOR_BITS) - 1); in get_work_color()
600 * work->data. These functions should only be called while the work is
601 * owned - ie. while the PENDING bit is set.
617 atomic_long_set(&work->data, data | flags | work_static(work)); in set_work_data()
648 * work->current_func, which is executed afterwards. This possible in set_work_pool_and_clear_pending()
653 * ---------------------------- -------------------------------- in set_work_pool_and_clear_pending()
661 * 7 work->current_func() { in set_work_pool_and_clear_pending()
665 * Without an explicit full barrier speculative LOAD on line 8 can in set_work_pool_and_clear_pending()
684 unsigned long data = atomic_long_read(&work->data); in get_work_pwq()
693 * get_work_pool - return the worker_pool a given work was associated with
709 unsigned long data = atomic_long_read(&work->data); in get_work_pool()
716 (data & WORK_STRUCT_WQ_DATA_MASK))->pool; in get_work_pool()
726 * get_work_pool_id - return the worker pool ID a given work is associated with
734 unsigned long data = atomic_long_read(&work->data); in get_work_pool_id()
738 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; in get_work_pool_id()
753 unsigned long data = atomic_long_read(&work->data); in work_is_canceling()
761 * they're being called with pool->lock held.
766 return !atomic_read(&pool->nr_running); in __need_more_worker()
779 return !list_empty(&pool->worklist) && __need_more_worker(pool); in need_more_worker()
785 return pool->nr_idle; in may_start_working()
791 return !list_empty(&pool->worklist) && in keep_working()
792 atomic_read(&pool->nr_running) <= 1; in keep_working()
804 bool managing = pool->flags & POOL_MANAGER_ACTIVE; in too_many_workers()
805 int nr_idle = pool->nr_idle + managing; /* manager is considered idle */ in too_many_workers()
806 int nr_busy = pool->nr_workers - nr_idle; in too_many_workers()
808 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy; in too_many_workers()
818 if (unlikely(list_empty(&pool->idle_list))) in first_idle_worker()
821 return list_first_entry(&pool->idle_list, struct worker, entry); in first_idle_worker()
825 * wake_up_worker - wake up an idle worker
831 * raw_spin_lock_irq(pool->lock).
838 wake_up_process(worker->task); in wake_up_worker()
842 * wq_worker_running - a worker is running again
851 if (!worker->sleeping) in wq_worker_running()
857 * and leave with an unexpected pool->nr_running == 1 on the newly unbound in wq_worker_running()
861 if (!(worker->flags & WORKER_NOT_RUNNING)) in wq_worker_running()
862 atomic_inc(&worker->pool->nr_running); in wq_worker_running()
864 worker->sleeping = 0; in wq_worker_running()
868 * wq_worker_sleeping - a worker is going to sleep
872 * going to sleep. Preemption needs to be disabled to protect ->sleeping
885 if (worker->flags & WORKER_NOT_RUNNING) in wq_worker_sleeping()
888 pool = worker->pool; in wq_worker_sleeping()
891 if (worker->sleeping) in wq_worker_sleeping()
894 worker->sleeping = 1; in wq_worker_sleeping()
895 raw_spin_lock_irq(&pool->lock); in wq_worker_sleeping()
908 if (atomic_dec_and_test(&pool->nr_running) && in wq_worker_sleeping()
909 !list_empty(&pool->worklist)) { in wq_worker_sleeping()
912 wake_up_process(next->task); in wq_worker_sleeping()
914 raw_spin_unlock_irq(&pool->lock); in wq_worker_sleeping()
918 * wq_worker_last_func - retrieve worker's last work function
925 * raw_spin_lock_irq(rq->lock)
929 * dequeuing, to allow periodic aggregation to shut-off when that
932 * As this function doesn't involve any workqueue-related locking, it
945 return worker->last_func; in wq_worker_last_func()
949 * worker_set_flags - set worker flags and adjust nr_running accordingly
953 * Set @flags in @worker->flags and adjust nr_running accordingly.
956 * raw_spin_lock_irq(pool->lock)
960 struct worker_pool *pool = worker->pool; in worker_set_flags()
962 WARN_ON_ONCE(worker->task != current); in worker_set_flags()
966 !(worker->flags & WORKER_NOT_RUNNING)) { in worker_set_flags()
967 atomic_dec(&pool->nr_running); in worker_set_flags()
970 worker->flags |= flags; in worker_set_flags()
974 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
978 * Clear @flags in @worker->flags and adjust nr_running accordingly.
981 * raw_spin_lock_irq(pool->lock)
985 struct worker_pool *pool = worker->pool; in worker_clr_flags()
986 unsigned int oflags = worker->flags; in worker_clr_flags()
988 WARN_ON_ONCE(worker->task != current); in worker_clr_flags()
990 worker->flags &= ~flags; in worker_clr_flags()
998 if (!(worker->flags & WORKER_NOT_RUNNING)) in worker_clr_flags()
999 atomic_inc(&pool->nr_running); in worker_clr_flags()
1003 * find_worker_executing_work - find worker which is executing a work
1008 * @pool->busy_hash which is keyed by the address of @work. For a worker
1029 * raw_spin_lock_irq(pool->lock).
1040 hash_for_each_possible(pool->busy_hash, worker, hentry, in find_worker_executing_work()
1042 if (worker->current_work == work && in find_worker_executing_work()
1043 worker->current_func == work->func) in find_worker_executing_work()
1050 * move_linked_works - move linked works to a list
1064 * raw_spin_lock_irq(pool->lock).
1076 list_move_tail(&work->entry, head); in move_linked_works()
1091 * get_pwq - get an extra reference on the specified pool_workqueue
1095 * @pwq has positive refcnt and be holding the matching pool->lock.
1099 lockdep_assert_held(&pwq->pool->lock); in get_pwq()
1100 WARN_ON_ONCE(pwq->refcnt <= 0); in get_pwq()
1101 pwq->refcnt++; in get_pwq()
1105 * put_pwq - put a pool_workqueue reference
1109 * destruction. The caller should be holding the matching pool->lock.
1113 lockdep_assert_held(&pwq->pool->lock); in put_pwq()
1114 if (likely(--pwq->refcnt)) in put_pwq()
1116 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND))) in put_pwq()
1119 * @pwq can't be released under pool->lock, bounce to in put_pwq()
1121 * pool->lock as this path is taken only for unbound workqueues and in put_pwq()
1122 * the release work item is scheduled on a per-cpu workqueue. To in put_pwq()
1123 * avoid lockdep warning, unbound pool->locks are given lockdep in put_pwq()
1126 schedule_work(&pwq->unbound_release_work); in put_pwq()
1130 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1142 raw_spin_lock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1144 raw_spin_unlock_irq(&pwq->pool->lock); in put_pwq_unlocked()
1153 if (list_empty(&pwq->pool->worklist)) in pwq_activate_delayed_work()
1154 pwq->pool->watchdog_ts = jiffies; in pwq_activate_delayed_work()
1155 move_linked_works(work, &pwq->pool->worklist, NULL); in pwq_activate_delayed_work()
1157 pwq->nr_active++; in pwq_activate_delayed_work()
1162 struct work_struct *work = list_first_entry(&pwq->delayed_works, in pwq_activate_first_delayed()
1169 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1177 * raw_spin_lock_irq(pool->lock).
1185 pwq->nr_in_flight[color]--; in pwq_dec_nr_in_flight()
1187 pwq->nr_active--; in pwq_dec_nr_in_flight()
1188 if (!list_empty(&pwq->delayed_works)) { in pwq_dec_nr_in_flight()
1190 if (pwq->nr_active < pwq->max_active) in pwq_dec_nr_in_flight()
1195 if (likely(pwq->flush_color != color)) in pwq_dec_nr_in_flight()
1198 /* are there still in-flight works? */ in pwq_dec_nr_in_flight()
1199 if (pwq->nr_in_flight[color]) in pwq_dec_nr_in_flight()
1203 pwq->flush_color = -1; in pwq_dec_nr_in_flight()
1209 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush)) in pwq_dec_nr_in_flight()
1210 complete(&pwq->wq->first_flusher->done); in pwq_dec_nr_in_flight()
1216 * try_to_grab_pending - steal work item from worklist and disable irq
1222 * stable state - idle, on timer or on worklist.
1229 * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
1230 * -ENOENT if someone else is canceling @work, this state may persist
1237 * disabled on entry. This, combined with delayed_work->timer being
1238 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1258 * dwork->timer is irqsafe. If del_timer() fails, it's in try_to_grab_pending()
1262 if (likely(del_timer(&dwork->timer))) in try_to_grab_pending()
1273 * steal it from ->worklist without clearing WORK_STRUCT_PENDING. in try_to_grab_pending()
1279 raw_spin_lock(&pool->lock); in try_to_grab_pending()
1281 * work->data is guaranteed to point to pwq only while the work in try_to_grab_pending()
1282 * item is queued on pwq->wq, and both updating work->data to point in try_to_grab_pending()
1284 * pwq->pool->lock. This in turn guarantees that, if work->data in try_to_grab_pending()
1289 if (pwq && pwq->pool == pool) { in try_to_grab_pending()
1295 * on the delayed_list, will confuse pwq->nr_active in try_to_grab_pending()
1302 list_del_init(&work->entry); in try_to_grab_pending()
1305 /* work->data points to pwq iff queued, point to pool */ in try_to_grab_pending()
1306 set_work_pool_and_keep_pending(work, pool->id); in try_to_grab_pending()
1308 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1312 raw_spin_unlock(&pool->lock); in try_to_grab_pending()
1317 return -ENOENT; in try_to_grab_pending()
1319 return -EAGAIN; in try_to_grab_pending()
1323 * insert_work - insert a work into a pool
1333 * raw_spin_lock_irq(pool->lock).
1338 struct worker_pool *pool = pwq->pool; in insert_work()
1342 list_add_tail(&work->entry, head); in insert_work()
1369 return worker && worker->current_pwq->wq == wq; in is_chained_work()
1386 pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n"); in wq_select_unbound_cpu()
1416 * steal the PENDING will busy-loop waiting for it to either get in __queue_work()
1424 if (unlikely(wq->flags & __WQ_DRAINING) && in __queue_work()
1430 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1437 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in __queue_work()
1443 * pool to guarantee non-reentrancy. in __queue_work()
1446 if (last_pool && last_pool != pwq->pool) { in __queue_work()
1449 raw_spin_lock(&last_pool->lock); in __queue_work()
1453 if (worker && worker->current_pwq->wq == wq) { in __queue_work()
1454 pwq = worker->current_pwq; in __queue_work()
1457 raw_spin_unlock(&last_pool->lock); in __queue_work()
1458 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1461 raw_spin_lock(&pwq->pool->lock); in __queue_work()
1470 * make forward-progress. in __queue_work()
1472 if (unlikely(!pwq->refcnt)) { in __queue_work()
1473 if (wq->flags & WQ_UNBOUND) { in __queue_work()
1474 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1479 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt", in __queue_work()
1480 wq->name, cpu); in __queue_work()
1486 if (WARN_ON(!list_empty(&work->entry))) in __queue_work()
1489 pwq->nr_in_flight[pwq->work_color]++; in __queue_work()
1490 work_flags = work_color_to_flags(pwq->work_color); in __queue_work()
1492 if (likely(pwq->nr_active < pwq->max_active)) { in __queue_work()
1494 pwq->nr_active++; in __queue_work()
1495 worklist = &pwq->pool->worklist; in __queue_work()
1497 pwq->pool->watchdog_ts = jiffies; in __queue_work()
1500 worklist = &pwq->delayed_works; in __queue_work()
1507 raw_spin_unlock(&pwq->pool->lock); in __queue_work()
1512 * queue_work_on - queue work on specific cpu
1541 * workqueue_select_cpu_near - Select a CPU based on NUMA node
1574 * queue_work_node - queue work on a "random" cpu for a given NUMA node
1604 * If this is used with a per-cpu workqueue then the logic in in queue_work_node()
1608 WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); in queue_work_node()
1629 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in delayed_work_timer_fn()
1636 struct timer_list *timer = &dwork->timer; in __queue_delayed_work()
1637 struct work_struct *work = &dwork->work; in __queue_delayed_work()
1640 WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn); in __queue_delayed_work()
1642 WARN_ON_ONCE(!list_empty(&work->entry)); in __queue_delayed_work()
1645 * If @delay is 0, queue @dwork->work immediately. This is for in __queue_delayed_work()
1651 __queue_work(cpu, wq, &dwork->work); in __queue_delayed_work()
1655 dwork->wq = wq; in __queue_delayed_work()
1656 dwork->cpu = cpu; in __queue_delayed_work()
1657 timer->expires = jiffies + delay; in __queue_delayed_work()
1666 * queue_delayed_work_on - queue work on specific CPU after delay
1679 struct work_struct *work = &dwork->work; in queue_delayed_work_on()
1697 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1721 ret = try_to_grab_pending(&dwork->work, true, &flags); in mod_delayed_work_on()
1722 } while (unlikely(ret == -EAGAIN)); in mod_delayed_work_on()
1729 /* -ENOENT from try_to_grab_pending() becomes %true */ in mod_delayed_work_on()
1740 __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work); in rcu_work_rcufn()
1745 * queue_rcu_work - queue work after a RCU grace period
1750 * that a full RCU grace period is guaranteed only after a %true return.
1752 * execution may happen before a full RCU grace period has passed.
1756 struct work_struct *work = &rwork->work; in queue_rcu_work()
1759 rwork->wq = wq; in queue_rcu_work()
1760 call_rcu(&rwork->rcu, rcu_work_rcufn); in queue_rcu_work()
1769 * worker_enter_idle - enter idle state
1776 * raw_spin_lock_irq(pool->lock).
1780 struct worker_pool *pool = worker->pool; in worker_enter_idle()
1782 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || in worker_enter_idle()
1783 WARN_ON_ONCE(!list_empty(&worker->entry) && in worker_enter_idle()
1784 (worker->hentry.next || worker->hentry.pprev))) in worker_enter_idle()
1788 worker->flags |= WORKER_IDLE; in worker_enter_idle()
1789 pool->nr_idle++; in worker_enter_idle()
1790 worker->last_active = jiffies; in worker_enter_idle()
1793 list_add(&worker->entry, &pool->idle_list); in worker_enter_idle()
1795 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) in worker_enter_idle()
1796 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); in worker_enter_idle()
1800 * pool->lock between setting %WORKER_UNBOUND and zapping in worker_enter_idle()
1804 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in worker_enter_idle()
1805 pool->nr_workers == pool->nr_idle && in worker_enter_idle()
1806 atomic_read(&pool->nr_running)); in worker_enter_idle()
1810 * worker_leave_idle - leave idle state
1816 * raw_spin_lock_irq(pool->lock).
1820 struct worker_pool *pool = worker->pool; in worker_leave_idle()
1822 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) in worker_leave_idle()
1825 pool->nr_idle--; in worker_leave_idle()
1826 list_del_init(&worker->entry); in worker_leave_idle()
1835 INIT_LIST_HEAD(&worker->entry); in alloc_worker()
1836 INIT_LIST_HEAD(&worker->scheduled); in alloc_worker()
1837 INIT_LIST_HEAD(&worker->node); in alloc_worker()
1839 worker->flags = WORKER_PREP; in alloc_worker()
1845 * worker_attach_to_pool() - attach a worker to a pool
1850 * cpu-binding of @worker are kept coordinated with the pool across
1851 * cpu-[un]hotplugs.
1863 if (pool->flags & POOL_DISASSOCIATED) in worker_attach_to_pool()
1864 worker->flags |= WORKER_UNBOUND; in worker_attach_to_pool()
1866 if (worker->rescue_wq) in worker_attach_to_pool()
1867 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); in worker_attach_to_pool()
1869 list_add_tail(&worker->node, &pool->workers); in worker_attach_to_pool()
1870 worker->pool = pool; in worker_attach_to_pool()
1876 * worker_detach_from_pool() - detach a worker from its pool
1885 struct worker_pool *pool = worker->pool; in worker_detach_from_pool()
1890 list_del(&worker->node); in worker_detach_from_pool()
1891 worker->pool = NULL; in worker_detach_from_pool()
1893 if (list_empty(&pool->workers)) in worker_detach_from_pool()
1894 detach_completion = pool->detach_completion; in worker_detach_from_pool()
1897 /* clear leftover flags without pool->lock after it is detached */ in worker_detach_from_pool()
1898 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); in worker_detach_from_pool()
1905 * create_worker - create a new workqueue worker
1919 int id = -1; in create_worker()
1923 id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL); in create_worker()
1927 worker = alloc_worker(pool->node); in create_worker()
1931 worker->id = id; in create_worker()
1933 if (pool->cpu >= 0) in create_worker()
1934 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id, in create_worker()
1935 pool->attrs->nice < 0 ? "H" : ""); in create_worker()
1937 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id); in create_worker()
1939 worker->task = kthread_create_on_node(worker_thread, worker, pool->node, in create_worker()
1941 if (IS_ERR(worker->task)) in create_worker()
1944 set_user_nice(worker->task, pool->attrs->nice); in create_worker()
1945 kthread_bind_mask(worker->task, pool->attrs->cpumask); in create_worker()
1951 raw_spin_lock_irq(&pool->lock); in create_worker()
1952 worker->pool->nr_workers++; in create_worker()
1954 wake_up_process(worker->task); in create_worker()
1955 raw_spin_unlock_irq(&pool->lock); in create_worker()
1961 ida_simple_remove(&pool->worker_ida, id); in create_worker()
1967 * destroy_worker - destroy a workqueue worker
1974 * raw_spin_lock_irq(pool->lock).
1978 struct worker_pool *pool = worker->pool; in destroy_worker()
1980 lockdep_assert_held(&pool->lock); in destroy_worker()
1983 if (WARN_ON(worker->current_work) || in destroy_worker()
1984 WARN_ON(!list_empty(&worker->scheduled)) || in destroy_worker()
1985 WARN_ON(!(worker->flags & WORKER_IDLE))) in destroy_worker()
1988 pool->nr_workers--; in destroy_worker()
1989 pool->nr_idle--; in destroy_worker()
1991 list_del_init(&worker->entry); in destroy_worker()
1992 worker->flags |= WORKER_DIE; in destroy_worker()
1993 wake_up_process(worker->task); in destroy_worker()
2000 raw_spin_lock_irq(&pool->lock); in idle_worker_timeout()
2007 worker = list_entry(pool->idle_list.prev, struct worker, entry); in idle_worker_timeout()
2008 expires = worker->last_active + IDLE_WORKER_TIMEOUT; in idle_worker_timeout()
2011 mod_timer(&pool->idle_timer, expires); in idle_worker_timeout()
2018 raw_spin_unlock_irq(&pool->lock); in idle_worker_timeout()
2024 struct workqueue_struct *wq = pwq->wq; in send_mayday()
2028 if (!wq->rescuer) in send_mayday()
2032 if (list_empty(&pwq->mayday_node)) { in send_mayday()
2039 list_add_tail(&pwq->mayday_node, &wq->maydays); in send_mayday()
2040 wake_up_process(wq->rescuer->task); in send_mayday()
2049 raw_spin_lock_irq(&pool->lock); in pool_mayday_timeout()
2050 raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */ in pool_mayday_timeout()
2059 list_for_each_entry(work, &pool->worklist, entry) in pool_mayday_timeout()
2064 raw_spin_unlock_irq(&pool->lock); in pool_mayday_timeout()
2066 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL); in pool_mayday_timeout()
2070 * maybe_create_worker - create a new worker if necessary
2083 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2088 __releases(&pool->lock) in maybe_create_worker()
2089 __acquires(&pool->lock) in maybe_create_worker()
2092 raw_spin_unlock_irq(&pool->lock); in maybe_create_worker()
2095 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); in maybe_create_worker()
2107 del_timer_sync(&pool->mayday_timer); in maybe_create_worker()
2108 raw_spin_lock_irq(&pool->lock); in maybe_create_worker()
2111 * created as @pool->lock was dropped and the new worker might have in maybe_create_worker()
2119 * manage_workers - manage worker pool
2131 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2142 struct worker_pool *pool = worker->pool; in manage_workers()
2144 if (pool->flags & POOL_MANAGER_ACTIVE) in manage_workers()
2147 pool->flags |= POOL_MANAGER_ACTIVE; in manage_workers()
2148 pool->manager = worker; in manage_workers()
2152 pool->manager = NULL; in manage_workers()
2153 pool->flags &= ~POOL_MANAGER_ACTIVE; in manage_workers()
2159 * process_one_work - process single work
2170 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2173 __releases(&pool->lock) in process_one_work()
2174 __acquires(&pool->lock) in process_one_work()
2177 struct worker_pool *pool = worker->pool; in process_one_work()
2178 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE; in process_one_work()
2187 * work->lockdep_map, make a copy and use that here. in process_one_work()
2191 lockdep_copy_map(&lockdep_map, &work->lockdep_map); in process_one_work()
2194 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && in process_one_work()
2195 raw_smp_processor_id() != pool->cpu); in process_one_work()
2205 move_linked_works(work, &collision->scheduled, NULL); in process_one_work()
2211 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); in process_one_work()
2212 worker->current_work = work; in process_one_work()
2213 worker->current_func = work->func; in process_one_work()
2214 worker->current_pwq = pwq; in process_one_work()
2221 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
2223 list_del_init(&work->entry); in process_one_work()
2236 * false for normal per-cpu workers since nr_running would always in process_one_work()
2246 * update to @work. Also, do this inside @pool->lock so that in process_one_work()
2250 set_work_pool_and_clear_pending(work, pool->id); in process_one_work()
2252 raw_spin_unlock_irq(&pool->lock); in process_one_work()
2254 lock_map_acquire(&pwq->wq->lockdep_map); in process_one_work()
2267 * Which would create W1->C->W1 dependencies, even though there is no in process_one_work()
2269 * read-recursive acquire on the work(queue) 'locks', but this will then in process_one_work()
2274 * flush_work() and complete() primitives (except for single-threaded in process_one_work()
2279 worker->current_func(work); in process_one_work()
2284 trace_workqueue_execute_end(work, worker->current_func); in process_one_work()
2286 lock_map_release(&pwq->wq->lockdep_map); in process_one_work()
2291 current->comm, preempt_count(), task_pid_nr(current), in process_one_work()
2292 worker->current_func); in process_one_work()
2307 raw_spin_lock_irq(&pool->lock); in process_one_work()
2314 worker->last_func = worker->current_func; in process_one_work()
2317 hash_del(&worker->hentry); in process_one_work()
2318 worker->current_work = NULL; in process_one_work()
2319 worker->current_func = NULL; in process_one_work()
2320 worker->current_pwq = NULL; in process_one_work()
2325 * process_scheduled_works - process scheduled works
2333 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2338 while (!list_empty(&worker->scheduled)) { in process_scheduled_works()
2339 struct work_struct *work = list_first_entry(&worker->scheduled, in process_scheduled_works()
2349 current->flags |= PF_WQ_WORKER; in set_pf_worker()
2351 current->flags &= ~PF_WQ_WORKER; in set_pf_worker()
2356 * worker_thread - the worker thread function
2359 * The worker thread function. All workers belong to a worker_pool -
2360 * either a per-cpu one or dynamic unbound one. These workers process all
2370 struct worker_pool *pool = worker->pool; in worker_thread()
2375 raw_spin_lock_irq(&pool->lock); in worker_thread()
2378 if (unlikely(worker->flags & WORKER_DIE)) { in worker_thread()
2379 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2380 WARN_ON_ONCE(!list_empty(&worker->entry)); in worker_thread()
2383 set_task_comm(worker->task, "kworker/dying"); in worker_thread()
2384 ida_simple_remove(&pool->worker_ida, worker->id); in worker_thread()
2401 * ->scheduled list can only be filled while a worker is in worker_thread()
2405 WARN_ON_ONCE(!list_empty(&worker->scheduled)); in worker_thread()
2418 list_first_entry(&pool->worklist, in worker_thread()
2421 pool->watchdog_ts = jiffies; in worker_thread()
2426 if (unlikely(!list_empty(&worker->scheduled))) in worker_thread()
2429 move_linked_works(work, &worker->scheduled, NULL); in worker_thread()
2437 * pool->lock is held and there's no work to process and no need to in worker_thread()
2439 * pool->lock or from local cpu, so setting the current state in worker_thread()
2440 * before releasing pool->lock is enough to prevent losing any in worker_thread()
2445 raw_spin_unlock_irq(&pool->lock); in worker_thread()
2451 * rescuer_thread - the rescuer thread function
2474 struct workqueue_struct *wq = rescuer->rescue_wq; in rescuer_thread()
2475 struct list_head *scheduled = &rescuer->scheduled; in rescuer_thread()
2490 * shouldn't have any work pending, but @wq->maydays may still have in rescuer_thread()
2491 * pwq(s) queued. This can happen by non-rescuer workers consuming in rescuer_thread()
2493 * @wq->maydays processing before acting on should_stop so that the in rescuer_thread()
2501 while (!list_empty(&wq->maydays)) { in rescuer_thread()
2502 struct pool_workqueue *pwq = list_first_entry(&wq->maydays, in rescuer_thread()
2504 struct worker_pool *pool = pwq->pool; in rescuer_thread()
2509 list_del_init(&pwq->mayday_node); in rescuer_thread()
2515 raw_spin_lock_irq(&pool->lock); in rescuer_thread()
2522 list_for_each_entry_safe(work, n, &pool->worklist, entry) { in rescuer_thread()
2525 pool->watchdog_ts = jiffies; in rescuer_thread()
2539 * that such back-to-back work items, which may be in rescuer_thread()
2543 if (pwq->nr_active && need_to_create_worker(pool)) { in rescuer_thread()
2549 if (wq->rescuer && list_empty(&pwq->mayday_node)) { in rescuer_thread()
2551 list_add_tail(&pwq->mayday_node, &wq->maydays); in rescuer_thread()
2571 raw_spin_unlock_irq(&pool->lock); in rescuer_thread()
2587 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING)); in rescuer_thread()
2593 * check_flush_dependency - check for flush dependency sanity
2600 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2606 work_func_t target_func = target_work ? target_work->func : NULL; in check_flush_dependency()
2609 if (target_wq->flags & WQ_MEM_RECLAIM) in check_flush_dependency()
2614 WARN_ONCE(current->flags & PF_MEMALLOC, in check_flush_dependency()
2616 current->pid, current->comm, target_wq->name, target_func); in check_flush_dependency()
2617 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & in check_flush_dependency()
2620 worker->current_pwq->wq->name, worker->current_func, in check_flush_dependency()
2621 target_wq->name, target_func); in check_flush_dependency()
2633 complete(&barr->done); in wq_barrier_func()
2637 * insert_wq_barrier - insert a barrier work
2654 * Note that when @worker is non-NULL, @target may be modified
2658 * raw_spin_lock_irq(pool->lock).
2668 * debugobject calls are safe here even with pool->lock locked in insert_wq_barrier()
2673 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func); in insert_wq_barrier()
2674 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work)); in insert_wq_barrier()
2676 init_completion_map(&barr->done, &target->lockdep_map); in insert_wq_barrier()
2678 barr->task = current; in insert_wq_barrier()
2685 head = worker->scheduled.next; in insert_wq_barrier()
2689 head = target->entry.next; in insert_wq_barrier()
2695 debug_work_activate(&barr->work); in insert_wq_barrier()
2696 insert_work(pwq, &barr->work, head, in insert_wq_barrier()
2701 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2703 * @flush_color: new flush color, < 0 for no-op
2704 * @work_color: new work color, < 0 for no-op
2708 * If @flush_color is non-negative, flush_color on all pwqs should be
2709 * -1. If no pwq has in-flight commands at the specified color, all
2710 * pwq->flush_color's stay at -1 and %false is returned. If any pwq
2711 * has in flight commands, its pwq->flush_color is set to
2712 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2715 * The caller should have initialized @wq->first_flusher prior to
2716 * calling this function with non-negative @flush_color. If
2720 * If @work_color is non-negative, all pwqs should have the same
2725 * mutex_lock(wq->mutex).
2738 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush)); in flush_workqueue_prep_pwqs()
2739 atomic_set(&wq->nr_pwqs_to_flush, 1); in flush_workqueue_prep_pwqs()
2743 struct worker_pool *pool = pwq->pool; in flush_workqueue_prep_pwqs()
2745 raw_spin_lock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2748 WARN_ON_ONCE(pwq->flush_color != -1); in flush_workqueue_prep_pwqs()
2750 if (pwq->nr_in_flight[flush_color]) { in flush_workqueue_prep_pwqs()
2751 pwq->flush_color = flush_color; in flush_workqueue_prep_pwqs()
2752 atomic_inc(&wq->nr_pwqs_to_flush); in flush_workqueue_prep_pwqs()
2758 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color)); in flush_workqueue_prep_pwqs()
2759 pwq->work_color = work_color; in flush_workqueue_prep_pwqs()
2762 raw_spin_unlock_irq(&pool->lock); in flush_workqueue_prep_pwqs()
2765 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush)) in flush_workqueue_prep_pwqs()
2766 complete(&wq->first_flusher->done); in flush_workqueue_prep_pwqs()
2772 * flush_workqueue - ensure that any scheduled work has run to completion.
2782 .flush_color = -1, in flush_workqueue()
2783 .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map), in flush_workqueue()
2790 lock_map_acquire(&wq->lockdep_map); in flush_workqueue()
2791 lock_map_release(&wq->lockdep_map); in flush_workqueue()
2793 mutex_lock(&wq->mutex); in flush_workqueue()
2796 * Start-to-wait phase in flush_workqueue()
2798 next_color = work_next_color(wq->work_color); in flush_workqueue()
2800 if (next_color != wq->flush_color) { in flush_workqueue()
2802 * Color space is not full. The current work_color in flush_workqueue()
2806 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow)); in flush_workqueue()
2807 this_flusher.flush_color = wq->work_color; in flush_workqueue()
2808 wq->work_color = next_color; in flush_workqueue()
2810 if (!wq->first_flusher) { in flush_workqueue()
2812 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2814 wq->first_flusher = &this_flusher; in flush_workqueue()
2816 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color, in flush_workqueue()
2817 wq->work_color)) { in flush_workqueue()
2819 wq->flush_color = next_color; in flush_workqueue()
2820 wq->first_flusher = NULL; in flush_workqueue()
2825 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color); in flush_workqueue()
2826 list_add_tail(&this_flusher.list, &wq->flusher_queue); in flush_workqueue()
2827 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2831 * Oops, color space is full, wait on overflow queue. in flush_workqueue()
2835 list_add_tail(&this_flusher.list, &wq->flusher_overflow); in flush_workqueue()
2840 mutex_unlock(&wq->mutex); in flush_workqueue()
2845 * Wake-up-and-cascade phase in flush_workqueue()
2848 * handling overflow. Non-first flushers can simply return. in flush_workqueue()
2850 if (READ_ONCE(wq->first_flusher) != &this_flusher) in flush_workqueue()
2853 mutex_lock(&wq->mutex); in flush_workqueue()
2856 if (wq->first_flusher != &this_flusher) in flush_workqueue()
2859 WRITE_ONCE(wq->first_flusher, NULL); in flush_workqueue()
2862 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color); in flush_workqueue()
2868 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) { in flush_workqueue()
2869 if (next->flush_color != wq->flush_color) in flush_workqueue()
2871 list_del_init(&next->list); in flush_workqueue()
2872 complete(&next->done); in flush_workqueue()
2875 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) && in flush_workqueue()
2876 wq->flush_color != work_next_color(wq->work_color)); in flush_workqueue()
2879 wq->flush_color = work_next_color(wq->flush_color); in flush_workqueue()
2882 if (!list_empty(&wq->flusher_overflow)) { in flush_workqueue()
2886 * flusher_queue. This is the start-to-wait in flush_workqueue()
2889 list_for_each_entry(tmp, &wq->flusher_overflow, list) in flush_workqueue()
2890 tmp->flush_color = wq->work_color; in flush_workqueue()
2892 wq->work_color = work_next_color(wq->work_color); in flush_workqueue()
2894 list_splice_tail_init(&wq->flusher_overflow, in flush_workqueue()
2895 &wq->flusher_queue); in flush_workqueue()
2896 flush_workqueue_prep_pwqs(wq, -1, wq->work_color); in flush_workqueue()
2899 if (list_empty(&wq->flusher_queue)) { in flush_workqueue()
2900 WARN_ON_ONCE(wq->flush_color != wq->work_color); in flush_workqueue()
2908 WARN_ON_ONCE(wq->flush_color == wq->work_color); in flush_workqueue()
2909 WARN_ON_ONCE(wq->flush_color != next->flush_color); in flush_workqueue()
2911 list_del_init(&next->list); in flush_workqueue()
2912 wq->first_flusher = next; in flush_workqueue()
2914 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1)) in flush_workqueue()
2921 wq->first_flusher = NULL; in flush_workqueue()
2925 mutex_unlock(&wq->mutex); in flush_workqueue()
2930 * drain_workqueue - drain a workqueue
2947 * hotter than drain_workqueue() and already looks at @wq->flags. in drain_workqueue()
2950 mutex_lock(&wq->mutex); in drain_workqueue()
2951 if (!wq->nr_drainers++) in drain_workqueue()
2952 wq->flags |= __WQ_DRAINING; in drain_workqueue()
2953 mutex_unlock(&wq->mutex); in drain_workqueue()
2957 mutex_lock(&wq->mutex); in drain_workqueue()
2962 raw_spin_lock_irq(&pwq->pool->lock); in drain_workqueue()
2963 drained = !pwq->nr_active && list_empty(&pwq->delayed_works); in drain_workqueue()
2964 raw_spin_unlock_irq(&pwq->pool->lock); in drain_workqueue()
2972 wq->name, flush_cnt); in drain_workqueue()
2974 mutex_unlock(&wq->mutex); in drain_workqueue()
2978 if (!--wq->nr_drainers) in drain_workqueue()
2979 wq->flags &= ~__WQ_DRAINING; in drain_workqueue()
2980 mutex_unlock(&wq->mutex); in drain_workqueue()
3000 raw_spin_lock_irq(&pool->lock); in start_flush_work()
3004 if (unlikely(pwq->pool != pool)) in start_flush_work()
3010 pwq = worker->current_pwq; in start_flush_work()
3013 check_flush_dependency(pwq->wq, work); in start_flush_work()
3016 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3020 * single-threaded or rescuer equipped workqueue. in start_flush_work()
3028 (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) { in start_flush_work()
3029 lock_map_acquire(&pwq->wq->lockdep_map); in start_flush_work()
3030 lock_map_release(&pwq->wq->lockdep_map); in start_flush_work()
3035 raw_spin_unlock_irq(&pool->lock); in start_flush_work()
3047 if (WARN_ON(!work->func)) in __flush_work()
3051 lock_map_acquire(&work->lockdep_map); in __flush_work()
3052 lock_map_release(&work->lockdep_map); in __flush_work()
3065 * flush_work - wait for a work to finish executing the last queueing instance
3090 if (cwait->work != key) in cwt_wakefn()
3108 * CANCELING - flush_work() will return false immediately in __cancel_work_timer()
3110 * return -ENOENT as @work is still being canceled and the in __cancel_work_timer()
3119 if (unlikely(ret == -ENOENT)) { in __cancel_work_timer()
3160 * cancel_work_sync - cancel a work and wait for it to finish
3164 * can be used even if the work re-queues itself or migrates to
3168 * cancel_work_sync(&delayed_work->work) must not be used for
3184 * flush_delayed_work - wait for a dwork to finish executing the last queueing
3198 if (del_timer_sync(&dwork->timer)) in flush_delayed_work()
3199 __queue_work(dwork->cpu, dwork->wq, &dwork->work); in flush_delayed_work()
3201 return flush_work(&dwork->work); in flush_delayed_work()
3206 * flush_rcu_work - wait for a rwork to finish executing the last queueing
3215 if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) { in flush_rcu_work()
3217 flush_work(&rwork->work); in flush_rcu_work()
3220 return flush_work(&rwork->work); in flush_rcu_work()
3232 } while (unlikely(ret == -EAGAIN)); in __cancel_work()
3243 * cancel_delayed_work - cancel a delayed work
3253 * it returns %true and the work doesn't re-arm itself. Explicitly flush or
3260 return __cancel_work(&dwork->work, true); in cancel_delayed_work()
3265 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3275 return __cancel_work_timer(&dwork->work, true); in cancel_delayed_work_sync()
3280 * schedule_on_each_cpu - execute a function synchronously on each online CPU
3288 * 0 on success, -errno on failure.
3297 return -ENOMEM; in schedule_on_each_cpu()
3317 * execute_in_process_context - reliably execute the routine with user context
3325 * Return: 0 - function was executed
3326 * 1 - function was scheduled for execution
3331 fn(&ew->work); in execute_in_process_context()
3335 INIT_WORK(&ew->work, fn); in execute_in_process_context()
3336 schedule_work(&ew->work); in execute_in_process_context()
3343 * free_workqueue_attrs - free a workqueue_attrs
3351 free_cpumask_var(attrs->cpumask); in free_workqueue_attrs()
3357 * alloc_workqueue_attrs - allocate a workqueue_attrs
3371 if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL)) in alloc_workqueue_attrs()
3374 cpumask_copy(attrs->cpumask, cpu_possible_mask); in alloc_workqueue_attrs()
3384 to->nice = from->nice; in copy_workqueue_attrs()
3385 cpumask_copy(to->cpumask, from->cpumask); in copy_workqueue_attrs()
3388 * ->no_numa as it is used for both pool and wq attrs. Instead, in copy_workqueue_attrs()
3389 * get_unbound_pool() explicitly clears ->no_numa after copying. in copy_workqueue_attrs()
3391 to->no_numa = from->no_numa; in copy_workqueue_attrs()
3399 hash = jhash_1word(attrs->nice, hash); in wqattrs_hash()
3400 hash = jhash(cpumask_bits(attrs->cpumask), in wqattrs_hash()
3409 if (a->nice != b->nice) in wqattrs_equal()
3411 if (!cpumask_equal(a->cpumask, b->cpumask)) in wqattrs_equal()
3417 * init_worker_pool - initialize a newly zalloc'd worker_pool
3420 * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
3422 * Return: 0 on success, -errno on failure. Even on failure, all fields
3428 raw_spin_lock_init(&pool->lock); in init_worker_pool()
3429 pool->id = -1; in init_worker_pool()
3430 pool->cpu = -1; in init_worker_pool()
3431 pool->node = NUMA_NO_NODE; in init_worker_pool()
3432 pool->flags |= POOL_DISASSOCIATED; in init_worker_pool()
3433 pool->watchdog_ts = jiffies; in init_worker_pool()
3434 INIT_LIST_HEAD(&pool->worklist); in init_worker_pool()
3435 INIT_LIST_HEAD(&pool->idle_list); in init_worker_pool()
3436 hash_init(pool->busy_hash); in init_worker_pool()
3438 timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); in init_worker_pool()
3440 timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); in init_worker_pool()
3442 INIT_LIST_HEAD(&pool->workers); in init_worker_pool()
3444 ida_init(&pool->worker_ida); in init_worker_pool()
3445 INIT_HLIST_NODE(&pool->hash_node); in init_worker_pool()
3446 pool->refcnt = 1; in init_worker_pool()
3449 pool->attrs = alloc_workqueue_attrs(); in init_worker_pool()
3450 if (!pool->attrs) in init_worker_pool()
3451 return -ENOMEM; in init_worker_pool()
3460 lockdep_register_key(&wq->key); in wq_init_lockdep()
3461 lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name); in wq_init_lockdep()
3463 lock_name = wq->name; in wq_init_lockdep()
3465 wq->lock_name = lock_name; in wq_init_lockdep()
3466 lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0); in wq_init_lockdep()
3471 lockdep_unregister_key(&wq->key); in wq_unregister_lockdep()
3476 if (wq->lock_name != wq->name) in wq_free_lockdep()
3477 kfree(wq->lock_name); in wq_free_lockdep()
3500 if (!(wq->flags & WQ_UNBOUND)) in rcu_free_wq()
3501 free_percpu(wq->cpu_pwqs); in rcu_free_wq()
3503 free_workqueue_attrs(wq->unbound_attrs); in rcu_free_wq()
3512 ida_destroy(&pool->worker_ida); in rcu_free_pool()
3513 free_workqueue_attrs(pool->attrs); in rcu_free_pool()
3520 raw_spin_lock_irq(&pool->lock); in wq_manager_inactive()
3522 if (pool->flags & POOL_MANAGER_ACTIVE) { in wq_manager_inactive()
3523 raw_spin_unlock_irq(&pool->lock); in wq_manager_inactive()
3530 * put_unbound_pool - put a worker_pool
3547 if (--pool->refcnt) in put_unbound_pool()
3551 if (WARN_ON(!(pool->cpu < 0)) || in put_unbound_pool()
3552 WARN_ON(!list_empty(&pool->worklist))) in put_unbound_pool()
3556 if (pool->id >= 0) in put_unbound_pool()
3557 idr_remove(&worker_pool_idr, pool->id); in put_unbound_pool()
3558 hash_del(&pool->hash_node); in put_unbound_pool()
3569 pool->flags |= POOL_MANAGER_ACTIVE; in put_unbound_pool()
3573 WARN_ON(pool->nr_workers || pool->nr_idle); in put_unbound_pool()
3574 raw_spin_unlock_irq(&pool->lock); in put_unbound_pool()
3577 if (!list_empty(&pool->workers)) in put_unbound_pool()
3578 pool->detach_completion = &detach_completion; in put_unbound_pool()
3581 if (pool->detach_completion) in put_unbound_pool()
3582 wait_for_completion(pool->detach_completion); in put_unbound_pool()
3585 del_timer_sync(&pool->idle_timer); in put_unbound_pool()
3586 del_timer_sync(&pool->mayday_timer); in put_unbound_pool()
3589 call_rcu(&pool->rcu, rcu_free_pool); in put_unbound_pool()
3593 * get_unbound_pool - get a worker_pool with the specified attributes
3617 if (wqattrs_equal(pool->attrs, attrs)) { in get_unbound_pool()
3618 pool->refcnt++; in get_unbound_pool()
3626 if (cpumask_subset(attrs->cpumask, in get_unbound_pool()
3639 lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */ in get_unbound_pool()
3640 copy_workqueue_attrs(pool->attrs, attrs); in get_unbound_pool()
3641 pool->node = target_node; in get_unbound_pool()
3647 pool->attrs->no_numa = false; in get_unbound_pool()
3657 hash_add(unbound_pool_hash, &pool->hash_node, hash); in get_unbound_pool()
3680 struct workqueue_struct *wq = pwq->wq; in pwq_unbound_release_workfn()
3681 struct worker_pool *pool = pwq->pool; in pwq_unbound_release_workfn()
3688 if (!list_empty(&pwq->pwqs_node)) { in pwq_unbound_release_workfn()
3689 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND))) in pwq_unbound_release_workfn()
3692 mutex_lock(&wq->mutex); in pwq_unbound_release_workfn()
3693 list_del_rcu(&pwq->pwqs_node); in pwq_unbound_release_workfn()
3694 is_last = list_empty(&wq->pwqs); in pwq_unbound_release_workfn()
3695 mutex_unlock(&wq->mutex); in pwq_unbound_release_workfn()
3702 call_rcu(&pwq->rcu, rcu_free_pwq); in pwq_unbound_release_workfn()
3710 call_rcu(&wq->rcu, rcu_free_wq); in pwq_unbound_release_workfn()
3715 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3718 * If @pwq isn't freezing, set @pwq->max_active to the associated
3720 * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
3724 struct workqueue_struct *wq = pwq->wq; in pwq_adjust_max_active()
3725 bool freezable = wq->flags & WQ_FREEZABLE; in pwq_adjust_max_active()
3728 /* for @wq->saved_max_active */ in pwq_adjust_max_active()
3729 lockdep_assert_held(&wq->mutex); in pwq_adjust_max_active()
3731 /* fast exit for non-freezable wqs */ in pwq_adjust_max_active()
3732 if (!freezable && pwq->max_active == wq->saved_max_active) in pwq_adjust_max_active()
3736 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3746 pwq->max_active = wq->saved_max_active; in pwq_adjust_max_active()
3748 while (!list_empty(&pwq->delayed_works) && in pwq_adjust_max_active()
3749 pwq->nr_active < pwq->max_active) { in pwq_adjust_max_active()
3761 wake_up_worker(pwq->pool); in pwq_adjust_max_active()
3763 pwq->max_active = 0; in pwq_adjust_max_active()
3766 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in pwq_adjust_max_active()
3777 pwq->pool = pool; in init_pwq()
3778 pwq->wq = wq; in init_pwq()
3779 pwq->flush_color = -1; in init_pwq()
3780 pwq->refcnt = 1; in init_pwq()
3781 INIT_LIST_HEAD(&pwq->delayed_works); in init_pwq()
3782 INIT_LIST_HEAD(&pwq->pwqs_node); in init_pwq()
3783 INIT_LIST_HEAD(&pwq->mayday_node); in init_pwq()
3784 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn); in init_pwq()
3790 struct workqueue_struct *wq = pwq->wq; in link_pwq()
3792 lockdep_assert_held(&wq->mutex); in link_pwq()
3795 if (!list_empty(&pwq->pwqs_node)) in link_pwq()
3799 pwq->work_color = wq->work_color; in link_pwq()
3805 list_add_rcu(&pwq->pwqs_node, &wq->pwqs); in link_pwq()
3821 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node); in alloc_unbound_pwq()
3832 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3842 * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
3845 * @attrs->cpumask.
3850 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3856 if (!wq_numa_enabled || attrs->no_numa) in wq_calc_node_cpumask()
3860 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask); in wq_calc_node_cpumask()
3868 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]); in wq_calc_node_cpumask()
3876 return !cpumask_equal(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3879 cpumask_copy(cpumask, attrs->cpumask); in wq_calc_node_cpumask()
3891 lockdep_assert_held(&wq->mutex); in numa_pwq_tbl_install()
3896 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in numa_pwq_tbl_install()
3897 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq); in numa_pwq_tbl_install()
3917 put_pwq_unlocked(ctx->pwq_tbl[node]); in apply_wqattrs_cleanup()
3918 put_pwq_unlocked(ctx->dfl_pwq); in apply_wqattrs_cleanup()
3920 free_workqueue_attrs(ctx->attrs); in apply_wqattrs_cleanup()
3950 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3951 if (unlikely(cpumask_empty(new_attrs->cpumask))) in apply_wqattrs_prepare()
3952 cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); in apply_wqattrs_prepare()
3963 * the default pwq covering whole @attrs->cpumask. Always create in apply_wqattrs_prepare()
3966 ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs); in apply_wqattrs_prepare()
3967 if (!ctx->dfl_pwq) in apply_wqattrs_prepare()
3971 if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) { in apply_wqattrs_prepare()
3972 ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs); in apply_wqattrs_prepare()
3973 if (!ctx->pwq_tbl[node]) in apply_wqattrs_prepare()
3976 ctx->dfl_pwq->refcnt++; in apply_wqattrs_prepare()
3977 ctx->pwq_tbl[node] = ctx->dfl_pwq; in apply_wqattrs_prepare()
3983 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask); in apply_wqattrs_prepare()
3984 ctx->attrs = new_attrs; in apply_wqattrs_prepare()
3986 ctx->wq = wq; in apply_wqattrs_prepare()
4003 mutex_lock(&ctx->wq->mutex); in apply_wqattrs_commit()
4005 copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); in apply_wqattrs_commit()
4009 ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node, in apply_wqattrs_commit()
4010 ctx->pwq_tbl[node]); in apply_wqattrs_commit()
4013 link_pwq(ctx->dfl_pwq); in apply_wqattrs_commit()
4014 swap(ctx->wq->dfl_pwq, ctx->dfl_pwq); in apply_wqattrs_commit()
4016 mutex_unlock(&ctx->wq->mutex); in apply_wqattrs_commit()
4038 if (WARN_ON(!(wq->flags & WQ_UNBOUND))) in apply_workqueue_attrs_locked()
4039 return -EINVAL; in apply_workqueue_attrs_locked()
4042 if (!list_empty(&wq->pwqs)) { in apply_workqueue_attrs_locked()
4043 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in apply_workqueue_attrs_locked()
4044 return -EINVAL; in apply_workqueue_attrs_locked()
4046 wq->flags &= ~__WQ_ORDERED; in apply_workqueue_attrs_locked()
4051 return -ENOMEM; in apply_workqueue_attrs_locked()
4061 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4067 * possibles CPUs in @attrs->cpumask so that work items are affine to the
4068 * NUMA node it was issued on. Older pwqs are released as in-flight work
4070 * back-to-back will stay on its current pwq.
4076 * Return: 0 on success and -errno on failure.
4093 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4103 * falls back to @wq->dfl_pwq which may not be optimal but is always
4109 * affinity and may execute on any CPU. This is similar to how per-cpu
4118 int cpu_off = online ? -1 : cpu; in wq_update_unbound_numa()
4125 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) || in wq_update_unbound_numa()
4126 wq->unbound_attrs->no_numa) in wq_update_unbound_numa()
4135 cpumask = target_attrs->cpumask; in wq_update_unbound_numa()
4137 copy_workqueue_attrs(target_attrs, wq->unbound_attrs); in wq_update_unbound_numa()
4146 if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) { in wq_update_unbound_numa()
4147 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask)) in wq_update_unbound_numa()
4157 wq->name); in wq_update_unbound_numa()
4162 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4167 mutex_lock(&wq->mutex); in wq_update_unbound_numa()
4168 raw_spin_lock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4169 get_pwq(wq->dfl_pwq); in wq_update_unbound_numa()
4170 raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock); in wq_update_unbound_numa()
4171 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq); in wq_update_unbound_numa()
4173 mutex_unlock(&wq->mutex); in wq_update_unbound_numa()
4179 bool highpri = wq->flags & WQ_HIGHPRI; in alloc_and_link_pwqs()
4182 if (!(wq->flags & WQ_UNBOUND)) { in alloc_and_link_pwqs()
4183 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue); in alloc_and_link_pwqs()
4184 if (!wq->cpu_pwqs) in alloc_and_link_pwqs()
4185 return -ENOMEM; in alloc_and_link_pwqs()
4189 per_cpu_ptr(wq->cpu_pwqs, cpu); in alloc_and_link_pwqs()
4195 mutex_lock(&wq->mutex); in alloc_and_link_pwqs()
4197 mutex_unlock(&wq->mutex); in alloc_and_link_pwqs()
4203 if (wq->flags & __WQ_ORDERED) { in alloc_and_link_pwqs()
4206 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node || in alloc_and_link_pwqs()
4207 wq->pwqs.prev != &wq->dfl_pwq->pwqs_node), in alloc_and_link_pwqs()
4208 "ordering guarantee broken for workqueue %s\n", wq->name); in alloc_and_link_pwqs()
4238 if (!(wq->flags & WQ_MEM_RECLAIM)) in init_rescuer()
4243 return -ENOMEM; in init_rescuer()
4245 rescuer->rescue_wq = wq; in init_rescuer()
4246 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name); in init_rescuer()
4247 if (IS_ERR(rescuer->task)) { in init_rescuer()
4248 ret = PTR_ERR(rescuer->task); in init_rescuer()
4253 wq->rescuer = rescuer; in init_rescuer()
4254 kthread_bind_mask(rescuer->task, cpu_possible_mask); in init_rescuer()
4255 wake_up_process(rescuer->task); in init_rescuer()
4272 * longer the case on NUMA machines due to per-node pools. While in alloc_workqueue()
4286 tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]); in alloc_workqueue()
4293 wq->unbound_attrs = alloc_workqueue_attrs(); in alloc_workqueue()
4294 if (!wq->unbound_attrs) in alloc_workqueue()
4299 vsnprintf(wq->name, sizeof(wq->name), fmt, args); in alloc_workqueue()
4303 max_active = wq_clamp_max_active(max_active, flags, wq->name); in alloc_workqueue()
4306 wq->flags = flags; in alloc_workqueue()
4307 wq->saved_max_active = max_active; in alloc_workqueue()
4308 mutex_init(&wq->mutex); in alloc_workqueue()
4309 atomic_set(&wq->nr_pwqs_to_flush, 0); in alloc_workqueue()
4310 INIT_LIST_HEAD(&wq->pwqs); in alloc_workqueue()
4311 INIT_LIST_HEAD(&wq->flusher_queue); in alloc_workqueue()
4312 INIT_LIST_HEAD(&wq->flusher_overflow); in alloc_workqueue()
4313 INIT_LIST_HEAD(&wq->maydays); in alloc_workqueue()
4316 INIT_LIST_HEAD(&wq->list); in alloc_workqueue()
4324 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq)) in alloc_workqueue()
4334 mutex_lock(&wq->mutex); in alloc_workqueue()
4337 mutex_unlock(&wq->mutex); in alloc_workqueue()
4339 list_add_tail_rcu(&wq->list, &workqueues); in alloc_workqueue()
4349 free_workqueue_attrs(wq->unbound_attrs); in alloc_workqueue()
4363 if (pwq->nr_in_flight[i]) in pwq_busy()
4366 if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1)) in pwq_busy()
4368 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) in pwq_busy()
4375 * destroy_workqueue - safely terminate a workqueue
4395 if (wq->rescuer) { in destroy_workqueue()
4396 struct worker *rescuer = wq->rescuer; in destroy_workqueue()
4400 wq->rescuer = NULL; in destroy_workqueue()
4404 kthread_stop(rescuer->task); in destroy_workqueue()
4409 * Sanity checks - grab all the locks so that we wait for all in destroy_workqueue()
4410 * in-flight operations which may do put_pwq(). in destroy_workqueue()
4413 mutex_lock(&wq->mutex); in destroy_workqueue()
4415 raw_spin_lock_irq(&pwq->pool->lock); in destroy_workqueue()
4418 __func__, wq->name); in destroy_workqueue()
4420 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4421 mutex_unlock(&wq->mutex); in destroy_workqueue()
4426 raw_spin_unlock_irq(&pwq->pool->lock); in destroy_workqueue()
4428 mutex_unlock(&wq->mutex); in destroy_workqueue()
4434 list_del_rcu(&wq->list); in destroy_workqueue()
4437 if (!(wq->flags & WQ_UNBOUND)) { in destroy_workqueue()
4440 * The base ref is never dropped on per-cpu pwqs. Directly in destroy_workqueue()
4443 call_rcu(&wq->rcu, rcu_free_wq); in destroy_workqueue()
4451 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]); in destroy_workqueue()
4452 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL); in destroy_workqueue()
4460 pwq = wq->dfl_pwq; in destroy_workqueue()
4461 wq->dfl_pwq = NULL; in destroy_workqueue()
4468 * workqueue_set_max_active - adjust max_active of a workqueue
4482 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_set_max_active()
4485 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); in workqueue_set_max_active()
4487 mutex_lock(&wq->mutex); in workqueue_set_max_active()
4489 wq->flags &= ~__WQ_ORDERED; in workqueue_set_max_active()
4490 wq->saved_max_active = max_active; in workqueue_set_max_active()
4495 mutex_unlock(&wq->mutex); in workqueue_set_max_active()
4500 * current_work - retrieve %current task's work struct
4511 return worker ? worker->current_work : NULL; in current_work()
4516 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4527 return worker && worker->rescue_wq; in current_is_workqueue_rescuer()
4531 * workqueue_congested - test whether a workqueue is congested
4540 * Note that both per-cpu and unbound workqueues may be associated with
4559 if (!(wq->flags & WQ_UNBOUND)) in workqueue_congested()
4560 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); in workqueue_congested()
4564 ret = !list_empty(&pwq->delayed_works); in workqueue_congested()
4573 * work_busy - test whether a work is currently pending or running
4595 raw_spin_lock_irqsave(&pool->lock, flags); in work_busy()
4598 raw_spin_unlock_irqrestore(&pool->lock, flags); in work_busy()
4607 * set_worker_desc - set description for the current work item
4608 * @fmt: printf-style format string
4623 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); in set_worker_desc()
4630 * print_worker_info - print out worker information and description
4651 if (!(task->flags & PF_WQ_WORKER)) in print_worker_info()
4664 copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn)); in print_worker_info()
4665 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq)); in print_worker_info()
4666 copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq)); in print_worker_info()
4667 copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1); in print_worker_info()
4668 copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1); in print_worker_info()
4680 pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask); in pr_cont_pool_info()
4681 if (pool->node != NUMA_NO_NODE) in pr_cont_pool_info()
4682 pr_cont(" node=%d", pool->node); in pr_cont_pool_info()
4683 pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice); in pr_cont_pool_info()
4688 if (work->func == wq_barrier_func) { in pr_cont_work()
4694 task_pid_nr(barr->task)); in pr_cont_work()
4696 pr_cont("%s %ps", comma ? "," : "", work->func); in pr_cont_work()
4702 struct worker_pool *pool = pwq->pool; in show_pwq()
4708 pr_info(" pwq %d:", pool->id); in show_pwq()
4712 pwq->nr_active, pwq->max_active, pwq->refcnt, in show_pwq()
4713 !list_empty(&pwq->mayday_node) ? " MAYDAY" : ""); in show_pwq()
4715 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4716 if (worker->current_pwq == pwq) { in show_pwq()
4724 pr_info(" in-flight:"); in show_pwq()
4725 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4726 if (worker->current_pwq != pwq) in show_pwq()
4730 task_pid_nr(worker->task), in show_pwq()
4731 worker->rescue_wq ? "(RESCUER)" : "", in show_pwq()
4732 worker->current_func); in show_pwq()
4733 list_for_each_entry(work, &worker->scheduled, entry) in show_pwq()
4740 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4750 list_for_each_entry(work, &pool->worklist, entry) { in show_pwq()
4760 if (!list_empty(&pwq->delayed_works)) { in show_pwq()
4764 list_for_each_entry(work, &pwq->delayed_works, entry) { in show_pwq()
4773 * show_workqueue_state - dump workqueue state
4794 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) { in show_workqueue_state()
4802 pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); in show_workqueue_state()
4805 raw_spin_lock_irqsave(&pwq->pool->lock, flags); in show_workqueue_state()
4806 if (pwq->nr_active || !list_empty(&pwq->delayed_works)) in show_workqueue_state()
4808 raw_spin_unlock_irqrestore(&pwq->pool->lock, flags); in show_workqueue_state()
4811 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4822 raw_spin_lock_irqsave(&pool->lock, flags); in show_workqueue_state()
4823 if (pool->nr_workers == pool->nr_idle) in show_workqueue_state()
4826 pr_info("pool %d:", pool->id); in show_workqueue_state()
4829 jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000, in show_workqueue_state()
4830 pool->nr_workers); in show_workqueue_state()
4831 if (pool->manager) in show_workqueue_state()
4833 task_pid_nr(pool->manager->task)); in show_workqueue_state()
4834 list_for_each_entry(worker, &pool->idle_list, entry) { in show_workqueue_state()
4836 task_pid_nr(worker->task)); in show_workqueue_state()
4841 raw_spin_unlock_irqrestore(&pool->lock, flags); in show_workqueue_state()
4844 * sysrq-t -> show_workqueue_state(). Avoid triggering in show_workqueue_state()
4859 off = strscpy(buf, task->comm, size); in wq_worker_comm()
4866 if (task->flags & PF_WQ_WORKER) { in wq_worker_comm()
4868 struct worker_pool *pool = worker->pool; in wq_worker_comm()
4871 raw_spin_lock_irq(&pool->lock); in wq_worker_comm()
4873 * ->desc tracks information (wq name or in wq_worker_comm()
4875 * current, prepend '+', otherwise '-'. in wq_worker_comm()
4877 if (worker->desc[0] != '\0') { in wq_worker_comm()
4878 if (worker->current_work) in wq_worker_comm()
4879 scnprintf(buf + off, size - off, "+%s", in wq_worker_comm()
4880 worker->desc); in wq_worker_comm()
4882 scnprintf(buf + off, size - off, "-%s", in wq_worker_comm()
4883 worker->desc); in wq_worker_comm()
4885 raw_spin_unlock_irq(&pool->lock); in wq_worker_comm()
4916 raw_spin_lock_irq(&pool->lock); in unbind_workers()
4926 worker->flags |= WORKER_UNBOUND; in unbind_workers()
4928 pool->flags |= POOL_DISASSOCIATED; in unbind_workers()
4930 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
4934 * Call schedule() so that we cross rq->lock and thus can in unbind_workers()
4949 atomic_set(&pool->nr_running, 0); in unbind_workers()
4956 raw_spin_lock_irq(&pool->lock); in unbind_workers()
4958 raw_spin_unlock_irq(&pool->lock); in unbind_workers()
4963 * rebind_workers - rebind all workers of a pool to the associated CPU
4966 * @pool->cpu is coming online. Rebind all workers to the CPU.
4976 * be on the run-queue of the associated CPU before any local in rebind_workers()
4977 * wake-ups for concurrency management happen, restore CPU affinity in rebind_workers()
4982 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, in rebind_workers()
4983 pool->attrs->cpumask) < 0); in rebind_workers()
4985 raw_spin_lock_irq(&pool->lock); in rebind_workers()
4987 pool->flags &= ~POOL_DISASSOCIATED; in rebind_workers()
4990 unsigned int worker_flags = worker->flags; in rebind_workers()
4994 * of the associated CPU for local wake-ups targeting it to in rebind_workers()
4998 * be bound before @pool->lock is released. in rebind_workers()
5001 wake_up_process(worker->task); in rebind_workers()
5008 * it initiates the next execution cycle thus restoring in rebind_workers()
5012 * WRITE_ONCE() is necessary because @worker->flags may be in rebind_workers()
5021 WRITE_ONCE(worker->flags, worker_flags); in rebind_workers()
5024 raw_spin_unlock_irq(&pool->lock); in rebind_workers()
5028 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
5045 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask)) in restore_unbound_workers_cpumask()
5048 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); in restore_unbound_workers_cpumask()
5052 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); in restore_unbound_workers_cpumask()
5060 if (pool->nr_workers) in workqueue_prepare_cpu()
5063 return -ENOMEM; in workqueue_prepare_cpu()
5079 if (pool->cpu == cpu) in workqueue_online_cpu()
5081 else if (pool->cpu < 0) in workqueue_online_cpu()
5099 /* unbinding per-cpu workers should happen on the local CPU */ in workqueue_offline_cpu()
5101 return -1; in workqueue_offline_cpu()
5125 wfc->ret = wfc->fn(wfc->arg); in work_for_cpu_fn()
5129 * work_on_cpu - run a function in thread context on a particular cpu
5152 * work_on_cpu_safe - run a function in thread context on a particular cpu
5164 long ret = -ENODEV; in work_on_cpu_safe()
5178 * freeze_workqueues_begin - begin freezing workqueues
5182 * pool->worklist.
5185 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5198 mutex_lock(&wq->mutex); in freeze_workqueues_begin()
5201 mutex_unlock(&wq->mutex); in freeze_workqueues_begin()
5208 * freeze_workqueues_busy - are freezable workqueues still busy?
5231 if (!(wq->flags & WQ_FREEZABLE)) in freeze_workqueues_busy()
5239 WARN_ON_ONCE(pwq->nr_active < 0); in freeze_workqueues_busy()
5240 if (pwq->nr_active) { in freeze_workqueues_busy()
5254 * thaw_workqueues - thaw workqueues
5260 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5276 mutex_lock(&wq->mutex); in thaw_workqueues()
5279 mutex_unlock(&wq->mutex); in thaw_workqueues()
5297 if (!(wq->flags & WQ_UNBOUND)) in workqueue_apply_unbound_cpumask()
5300 if (wq->flags & __WQ_ORDERED) in workqueue_apply_unbound_cpumask()
5303 ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs); in workqueue_apply_unbound_cpumask()
5305 ret = -ENOMEM; in workqueue_apply_unbound_cpumask()
5309 list_add_tail(&ctx->list, &ctxs); in workqueue_apply_unbound_cpumask()
5322 * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5325 * The low-level workqueues cpumask is a global cpumask that limits
5329 * Retun: 0 - Success
5330 * -EINVAL - Invalid @cpumask
5331 * -ENOMEM - Failed to allocate memory for attrs or pwqs.
5335 int ret = -EINVAL; in workqueue_set_unbound_cpumask()
5351 ret = -ENOMEM; in workqueue_set_unbound_cpumask()
5380 * per_cpu RO bool : whether the workqueue is per-cpu or unbound
5381 * max_active RW int : maximum number of in-flight work items
5399 return wq_dev->wq; in dev_to_wq()
5407 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND)); in per_cpu_show()
5416 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active); in max_active_show()
5427 return -EINVAL; in max_active_store()
5451 written += scnprintf(buf + written, PAGE_SIZE - written, in wq_pool_ids_show()
5453 unbound_pwq_by_node(wq, node)->pool->id); in wq_pool_ids_show()
5456 written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); in wq_pool_ids_show()
5469 mutex_lock(&wq->mutex); in wq_nice_show()
5470 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice); in wq_nice_show()
5471 mutex_unlock(&wq->mutex); in wq_nice_show()
5487 copy_workqueue_attrs(attrs, wq->unbound_attrs); in wq_sysfs_prep_attrs()
5496 int ret = -ENOMEM; in wq_nice_store()
5504 if (sscanf(buf, "%d", &attrs->nice) == 1 && in wq_nice_store()
5505 attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE) in wq_nice_store()
5508 ret = -EINVAL; in wq_nice_store()
5522 mutex_lock(&wq->mutex); in wq_cpumask_show()
5524 cpumask_pr_args(wq->unbound_attrs->cpumask)); in wq_cpumask_show()
5525 mutex_unlock(&wq->mutex); in wq_cpumask_show()
5535 int ret = -ENOMEM; in wq_cpumask_store()
5543 ret = cpumask_parse(buf, attrs->cpumask); in wq_cpumask_store()
5559 mutex_lock(&wq->mutex); in wq_numa_show()
5561 !wq->unbound_attrs->no_numa); in wq_numa_show()
5562 mutex_unlock(&wq->mutex); in wq_numa_show()
5572 int v, ret = -ENOMEM; in wq_numa_store()
5580 ret = -EINVAL; in wq_numa_store()
5582 attrs->no_numa = !v; in wq_numa_store()
5625 return -ENOMEM; in wq_unbound_cpumask_store()
5659 * workqueue_sysfs_register - make a workqueue visible in sysfs
5671 * Return: 0 on success, -errno on failure.
5683 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT)) in workqueue_sysfs_register()
5684 return -EINVAL; in workqueue_sysfs_register()
5686 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL); in workqueue_sysfs_register()
5688 return -ENOMEM; in workqueue_sysfs_register()
5690 wq_dev->wq = wq; in workqueue_sysfs_register()
5691 wq_dev->dev.bus = &wq_subsys; in workqueue_sysfs_register()
5692 wq_dev->dev.release = wq_device_release; in workqueue_sysfs_register()
5693 dev_set_name(&wq_dev->dev, "%s", wq->name); in workqueue_sysfs_register()
5699 dev_set_uevent_suppress(&wq_dev->dev, true); in workqueue_sysfs_register()
5701 ret = device_register(&wq_dev->dev); in workqueue_sysfs_register()
5703 put_device(&wq_dev->dev); in workqueue_sysfs_register()
5704 wq->wq_dev = NULL; in workqueue_sysfs_register()
5708 if (wq->flags & WQ_UNBOUND) { in workqueue_sysfs_register()
5711 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) { in workqueue_sysfs_register()
5712 ret = device_create_file(&wq_dev->dev, attr); in workqueue_sysfs_register()
5714 device_unregister(&wq_dev->dev); in workqueue_sysfs_register()
5715 wq->wq_dev = NULL; in workqueue_sysfs_register()
5721 dev_set_uevent_suppress(&wq_dev->dev, false); in workqueue_sysfs_register()
5722 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD); in workqueue_sysfs_register()
5727 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5734 struct wq_device *wq_dev = wq->wq_dev; in workqueue_sysfs_unregister()
5736 if (!wq->wq_dev) in workqueue_sysfs_unregister()
5739 wq->wq_dev = NULL; in workqueue_sysfs_unregister()
5740 device_unregister(&wq_dev->dev); in workqueue_sysfs_unregister()
5749 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5757 * forward progress is defined as the first item on ->worklist changing.
5796 if (list_empty(&pool->worklist)) in wq_watchdog_timer_fn()
5806 pool_ts = READ_ONCE(pool->watchdog_ts); in wq_watchdog_timer_fn()
5814 if (pool->cpu >= 0) { in wq_watchdog_timer_fn()
5817 pool->cpu)); in wq_watchdog_timer_fn()
5825 pr_emerg("BUG: workqueue lockup - pool"); in wq_watchdog_timer_fn()
5828 jiffies_to_msecs(now - pool_ts) / 1000); in wq_watchdog_timer_fn()
5944 * workqueue_init_early - early init for workqueue subsystem
5946 * This is the first half of two-staged workqueue subsystem initialization
5947 * and invoked as soon as the bare basics - memory allocation, cpumasks and
5973 pool->cpu = cpu; in workqueue_init_early()
5974 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu)); in workqueue_init_early()
5975 pool->attrs->nice = std_nice[i++]; in workqueue_init_early()
5976 pool->node = cpu_to_node(cpu); in workqueue_init_early()
5990 attrs->nice = std_nice[i]; in workqueue_init_early()
5999 attrs->nice = std_nice[i]; in workqueue_init_early()
6000 attrs->no_numa = true; in workqueue_init_early()
6023 * workqueue_init - bring workqueue subsystem fully online
6025 * This is the latter half of two-staged workqueue subsystem initialization
6040 * archs such as power and arm64. As per-cpu pools created in workqueue_init()
6052 pool->node = cpu_to_node(cpu); in workqueue_init()
6060 wq->name); in workqueue_init()
6068 pool->flags &= ~POOL_DISASSOCIATED; in workqueue_init()