Lines Matching +full:cs +full:- +full:0
7 * Copyright (C) 2004-2007 Silicon Graphics, Inc.
11 * sysfs is Copyright (c) 2001-3 Patrick Mochel
13 * 2003-10-10 Written by Simon Derr.
14 * 2003-10-22 Updates by Stephen Hemminger.
15 * 2004 May-July Rework by Paul Jackson.
24 #include "cgroup-internal.h"
61 #include <linux/backing-dev.h>
91 * The user-configured masks can only be changed by writing to
105 * The user-configured masks are always the same with effective masks.
108 /* user-configured CPUs and Memory Nodes allow to tasks */
118 * CPUs allocated to child sub-partitions (default hierarchy only)
119 * - CPUs granted by the parent = effective_cpus U subparts_cpus
120 * - effective_cpus and subparts_cpus are mutually exclusive.
130 * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
131 * - A new cpuset's old_mems_allowed is initialized when some
133 * - old_mems_allowed is used in cpuset_migrate_mm() when we change
143 * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
161 * use_parent_ecpus - set if using parent's effective_cpus
162 * child_ecpus_count - # of children with use_parent_ecpus set
179 * 0 - not a partition root
181 * 1 - partition root
183 * -1 - invalid partition root
190 #define PRS_DISABLED 0
192 #define PRS_ERROR -1
214 static inline struct cpuset *parent_cs(struct cpuset *cs) in parent_cs() argument
216 return css_cs(cs->css.parent); in parent_cs()
221 struct cpuset *cs = task_cs(p); in inc_dl_tasks_cs() local
223 cs->nr_deadline_tasks++; in inc_dl_tasks_cs()
228 struct cpuset *cs = task_cs(p); in dec_dl_tasks_cs() local
230 cs->nr_deadline_tasks--; in dec_dl_tasks_cs()
246 static inline bool is_cpuset_online(struct cpuset *cs) in is_cpuset_online() argument
248 return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css); in is_cpuset_online()
251 static inline int is_cpu_exclusive(const struct cpuset *cs) in is_cpu_exclusive() argument
253 return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); in is_cpu_exclusive()
256 static inline int is_mem_exclusive(const struct cpuset *cs) in is_mem_exclusive() argument
258 return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); in is_mem_exclusive()
261 static inline int is_mem_hardwall(const struct cpuset *cs) in is_mem_hardwall() argument
263 return test_bit(CS_MEM_HARDWALL, &cs->flags); in is_mem_hardwall()
266 static inline int is_sched_load_balance(const struct cpuset *cs) in is_sched_load_balance() argument
268 return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in is_sched_load_balance()
271 static inline int is_memory_migrate(const struct cpuset *cs) in is_memory_migrate() argument
273 return test_bit(CS_MEMORY_MIGRATE, &cs->flags); in is_memory_migrate()
276 static inline int is_spread_page(const struct cpuset *cs) in is_spread_page() argument
278 return test_bit(CS_SPREAD_PAGE, &cs->flags); in is_spread_page()
281 static inline int is_spread_slab(const struct cpuset *cs) in is_spread_slab() argument
283 return test_bit(CS_SPREAD_SLAB, &cs->flags); in is_spread_slab()
286 static inline int is_partition_root(const struct cpuset *cs) in is_partition_root() argument
288 return cs->partition_root_state > 0; in is_partition_root()
298 * cpuset_for_each_child - traverse online children of a cpuset
307 css_for_each_child((pos_css), &(parent_cs)->css) \
311 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
322 css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
326 * There are two global locks guarding cpuset structures - cpuset_mutex and
346 * If a task is only holding callback_lock, then it has read-only
354 * small pieces of code, such as when reading out possibly multi-word
396 (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); in is_in_v2_mode()
404 * One way or another, we guarantee to return some non-empty subset
409 static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask) in guarantee_online_cpus() argument
411 while (!cpumask_intersects(cs->effective_cpus, cpu_online_mask)) { in guarantee_online_cpus()
412 cs = parent_cs(cs); in guarantee_online_cpus()
413 if (unlikely(!cs)) { in guarantee_online_cpus()
425 cpumask_and(pmask, cs->effective_cpus, cpu_online_mask); in guarantee_online_cpus()
434 * One way or another, we guarantee to return some non-empty subset
439 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) in guarantee_online_mems() argument
441 while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) in guarantee_online_mems()
442 cs = parent_cs(cs); in guarantee_online_mems()
443 nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); in guarantee_online_mems()
451 static void cpuset_update_task_spread_flag(struct cpuset *cs, in cpuset_update_task_spread_flag() argument
454 if (is_spread_page(cs)) in cpuset_update_task_spread_flag()
459 if (is_spread_slab(cs)) in cpuset_update_task_spread_flag()
466 * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
475 return cpumask_subset(p->cpus_requested, q->cpus_requested) && in is_cpuset_subset()
476 nodes_subset(p->mems_allowed, q->mems_allowed) && in is_cpuset_subset()
482 * alloc_cpumasks - allocate three cpumasks for cpuset
483 * @cs: the cpuset that have cpumasks to be allocated.
485 * Return: 0 if successful, -ENOMEM otherwise.
487 * Only one of the two input arguments should be non-NULL.
489 static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) in alloc_cpumasks() argument
493 if (cs) { in alloc_cpumasks()
494 pmask1 = &cs->cpus_allowed; in alloc_cpumasks()
495 pmask2 = &cs->effective_cpus; in alloc_cpumasks()
496 pmask3 = &cs->subparts_cpus; in alloc_cpumasks()
497 pmask4 = &cs->cpus_requested; in alloc_cpumasks()
499 pmask1 = &tmp->new_cpus; in alloc_cpumasks()
500 pmask2 = &tmp->addmask; in alloc_cpumasks()
501 pmask3 = &tmp->delmask; in alloc_cpumasks()
505 return -ENOMEM; in alloc_cpumasks()
513 if (cs && !zalloc_cpumask_var(pmask4, GFP_KERNEL)) in alloc_cpumasks()
516 return 0; in alloc_cpumasks()
524 return -ENOMEM; in alloc_cpumasks()
528 * free_cpumasks - free cpumasks in a tmpmasks structure
529 * @cs: the cpuset that have cpumasks to be free.
532 static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp) in free_cpumasks() argument
534 if (cs) { in free_cpumasks()
535 free_cpumask_var(cs->cpus_allowed); in free_cpumasks()
536 free_cpumask_var(cs->cpus_requested); in free_cpumasks()
537 free_cpumask_var(cs->effective_cpus); in free_cpumasks()
538 free_cpumask_var(cs->subparts_cpus); in free_cpumasks()
541 free_cpumask_var(tmp->new_cpus); in free_cpumasks()
542 free_cpumask_var(tmp->addmask); in free_cpumasks()
543 free_cpumask_var(tmp->delmask); in free_cpumasks()
548 * alloc_trial_cpuset - allocate a trial cpuset
549 * @cs: the cpuset that the trial cpuset duplicates
551 static struct cpuset *alloc_trial_cpuset(struct cpuset *cs) in alloc_trial_cpuset() argument
555 trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL); in alloc_trial_cpuset()
564 cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); in alloc_trial_cpuset()
565 cpumask_copy(trial->cpus_requested, cs->cpus_requested); in alloc_trial_cpuset()
566 cpumask_copy(trial->effective_cpus, cs->effective_cpus); in alloc_trial_cpuset()
571 * free_cpuset - free the cpuset
572 * @cs: the cpuset to be freed
574 static inline void free_cpuset(struct cpuset *cs) in free_cpuset() argument
576 free_cpumasks(cs, NULL); in free_cpuset()
577 kfree(cs); in free_cpuset()
581 * validate_change() - Used to validate that any proposed cpuset change
589 * 'cur' is the address of an actual, in-use cpuset. Operations
597 * Return 0 if valid, -errno if not.
609 ret = -EBUSY; in validate_change()
615 ret = 0; in validate_change()
622 ret = -EACCES; in validate_change()
630 ret = -EINVAL; in validate_change()
634 cpumask_intersects(trial->cpus_requested, in validate_change()
635 c->cpus_requested)) in validate_change()
639 nodes_intersects(trial->mems_allowed, c->mems_allowed)) in validate_change()
644 * Cpusets with tasks - existing or newly being attached - can't in validate_change()
647 ret = -ENOSPC; in validate_change()
648 if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { in validate_change()
649 if (!cpumask_empty(cur->cpus_allowed) && in validate_change()
650 cpumask_empty(trial->cpus_allowed)) in validate_change()
652 if (!nodes_empty(cur->mems_allowed) && in validate_change()
653 nodes_empty(trial->mems_allowed)) in validate_change()
661 ret = -EBUSY; in validate_change()
663 !cpuset_cpumask_can_shrink(cur->cpus_allowed, in validate_change()
664 trial->cpus_allowed)) in validate_change()
667 ret = 0; in validate_change()
680 return cpumask_intersects(a->effective_cpus, b->effective_cpus); in cpusets_overlap()
686 if (dattr->relax_domain_level < c->relax_domain_level) in update_domain_attr()
687 dattr->relax_domain_level = c->relax_domain_level; in update_domain_attr()
700 if (cpumask_empty(cp->cpus_allowed)) { in update_domain_attr_tree()
714 /* jump label reference count + the top-level cpuset */ in nr_cpusets()
722 * A 'partial partition' is a set of non-overlapping subsets whose
729 * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
740 * cp - cpuset pointer, used (together with pos_css) to perform a
741 * top-down scan of all cpusets. For our purposes, rebuilding
744 * csa - (for CpuSet Array) Array of pointers to all the cpusets
751 * doms - Conversion of 'csa' to an array of cpumasks, for passing to
774 struct cpuset *cp; /* top-down scan of cpusets */ in generate_sched_domains()
780 int ndoms = 0; /* number of sched domains in result */ in generate_sched_domains()
801 cpumask_and(doms[0], top_cpuset.effective_cpus, in generate_sched_domains()
810 csn = 0; in generate_sched_domains()
826 * If root is load-balancing, we can skip @cp if it in generate_sched_domains()
829 if (!cpumask_empty(cp->cpus_allowed) && in generate_sched_domains()
831 cpumask_intersects(cp->cpus_allowed, in generate_sched_domains()
836 cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus)) in generate_sched_domains()
840 !cpumask_empty(cp->effective_cpus)) in generate_sched_domains()
849 for (i = 0; i < csn; i++) in generate_sched_domains()
850 csa[i]->pn = i; in generate_sched_domains()
855 for (i = 0; i < csn; i++) { in generate_sched_domains()
857 int apn = a->pn; in generate_sched_domains()
859 for (j = 0; j < csn; j++) { in generate_sched_domains()
861 int bpn = b->pn; in generate_sched_domains()
864 for (k = 0; k < csn; k++) { in generate_sched_domains()
867 if (c->pn == bpn) in generate_sched_domains()
868 c->pn = apn; in generate_sched_domains()
870 ndoms--; /* one less element */ in generate_sched_domains()
891 for (nslot = 0, i = 0; i < csn; i++) { in generate_sched_domains()
894 int apn = a->pn; in generate_sched_domains()
896 if (apn < 0) { in generate_sched_domains()
908 warnings--; in generate_sched_domains()
919 if (apn == b->pn) { in generate_sched_domains()
920 cpumask_or(dp, dp, b->effective_cpus); in generate_sched_domains()
926 b->pn = -1; in generate_sched_domains()
948 static void dl_update_tasks_root_domain(struct cpuset *cs) in dl_update_tasks_root_domain() argument
953 if (cs->nr_deadline_tasks == 0) in dl_update_tasks_root_domain()
956 css_task_iter_start(&cs->css, 0, &it); in dl_update_tasks_root_domain()
966 struct cpuset *cs = NULL; in dl_rebuild_rd_accounting() local
981 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in dl_rebuild_rd_accounting()
983 if (cpumask_empty(cs->effective_cpus)) { in dl_rebuild_rd_accounting()
988 css_get(&cs->css); in dl_rebuild_rd_accounting()
992 dl_update_tasks_root_domain(cs); in dl_rebuild_rd_accounting()
995 css_put(&cs->css); in dl_rebuild_rd_accounting()
1013 * If the flag 'sched_load_balance' of any cpuset with non-empty
1015 * which has that flag enabled, or if any cpuset with a non-empty
1026 struct cpuset *cs; in rebuild_sched_domains_locked() local
1052 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in rebuild_sched_domains_locked()
1053 if (!is_partition_root(cs)) { in rebuild_sched_domains_locked()
1057 if (!cpumask_subset(cs->effective_cpus, in rebuild_sched_domains_locked()
1088 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
1089 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
1091 * Iterate through each task of @cs updating its cpus_allowed to the
1095 static void update_tasks_cpumask(struct cpuset *cs) in update_tasks_cpumask() argument
1099 bool top_cs = cs == &top_cpuset; in update_tasks_cpumask()
1101 css_task_iter_start(&cs->css, 0, &it); in update_tasks_cpumask()
1106 if (top_cs && (task->flags & PF_KTHREAD) && in update_tasks_cpumask()
1109 set_cpus_allowed_ptr(task, cs->effective_cpus); in update_tasks_cpumask()
1115 * compute_effective_cpumask - Compute the effective cpumask of the cpuset
1117 * @cs: the cpuset the need to recompute the new effective_cpus mask
1126 struct cpuset *cs, struct cpuset *parent) in compute_effective_cpumask() argument
1128 if (parent->nr_subparts_cpus) { in compute_effective_cpumask()
1129 cpumask_or(new_cpus, parent->effective_cpus, in compute_effective_cpumask()
1130 parent->subparts_cpus); in compute_effective_cpumask()
1131 cpumask_and(new_cpus, new_cpus, cs->cpus_requested); in compute_effective_cpumask()
1134 cpumask_and(new_cpus, cs->cpus_requested, in compute_effective_cpumask()
1135 parent->effective_cpus); in compute_effective_cpumask()
1149 * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset
1154 * Return: 0, 1 or an error code
1156 * For partcmd_enable, the cpuset is being transformed from a non-partition
1159 * effective_cpus. The function will return 0 if all the CPUs listed in
1163 * root back to a non-partition root. Any CPUs in cpus_allowed that are in
1165 * into parent's effective_cpus. 0 should always be returned.
1173 * parent's subparts_cpus and effective_cpus happen or 0 otherwise.
1174 * Error code should only be returned when newmask is non-NULL.
1209 (!newmask && cpumask_empty(cpuset->cpus_allowed))) in update_parent_subparts_cpumask()
1210 return -EINVAL; in update_parent_subparts_cpumask()
1216 if ((cmd != partcmd_update) && css_has_online_children(&cpuset->css)) in update_parent_subparts_cpumask()
1217 return -EBUSY; in update_parent_subparts_cpumask()
1225 (!cpumask_subset(cpuset->cpus_allowed, parent->effective_cpus) || in update_parent_subparts_cpumask()
1226 cpumask_equal(cpuset->cpus_allowed, parent->effective_cpus))) in update_parent_subparts_cpumask()
1227 return -EINVAL; in update_parent_subparts_cpumask()
1233 new_prs = cpuset->partition_root_state; in update_parent_subparts_cpumask()
1235 cpumask_copy(tmp->addmask, cpuset->cpus_allowed); in update_parent_subparts_cpumask()
1238 deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1239 parent->subparts_cpus); in update_parent_subparts_cpumask()
1244 * delmask = cpus_allowed & ~newmask & parent->subparts_cpus in update_parent_subparts_cpumask()
1245 * addmask = newmask & parent->effective_cpus in update_parent_subparts_cpumask()
1246 * & ~parent->subparts_cpus in update_parent_subparts_cpumask()
1248 cpumask_andnot(tmp->delmask, cpuset->cpus_allowed, newmask); in update_parent_subparts_cpumask()
1249 deleting = cpumask_and(tmp->delmask, tmp->delmask, in update_parent_subparts_cpumask()
1250 parent->subparts_cpus); in update_parent_subparts_cpumask()
1252 cpumask_and(tmp->addmask, newmask, parent->effective_cpus); in update_parent_subparts_cpumask()
1253 adding = cpumask_andnot(tmp->addmask, tmp->addmask, in update_parent_subparts_cpumask()
1254 parent->subparts_cpus); in update_parent_subparts_cpumask()
1259 cpumask_equal(parent->effective_cpus, tmp->addmask)) { in update_parent_subparts_cpumask()
1261 return -EINVAL; in update_parent_subparts_cpumask()
1267 if (!cpumask_and(tmp->addmask, tmp->delmask, in update_parent_subparts_cpumask()
1269 return -EINVAL; in update_parent_subparts_cpumask()
1270 cpumask_copy(tmp->addmask, parent->effective_cpus); in update_parent_subparts_cpumask()
1276 * addmask = cpus_allowed & parent->effective_cpus in update_parent_subparts_cpumask()
1279 * pre-shrunk in case there is a change in the cpu list. in update_parent_subparts_cpumask()
1282 adding = cpumask_and(tmp->addmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1283 parent->effective_cpus); in update_parent_subparts_cpumask()
1284 part_error = cpumask_equal(tmp->addmask, in update_parent_subparts_cpumask()
1285 parent->effective_cpus); in update_parent_subparts_cpumask()
1289 int prev_prs = cpuset->partition_root_state; in update_parent_subparts_cpumask()
1295 switch (cpuset->partition_root_state) { in update_parent_subparts_cpumask()
1312 return 0; /* Nothing need to be done */ in update_parent_subparts_cpumask()
1319 deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, in update_parent_subparts_cpumask()
1320 parent->subparts_cpus); in update_parent_subparts_cpumask()
1323 if (!adding && !deleting && (new_prs == cpuset->partition_root_state)) in update_parent_subparts_cpumask()
1324 return 0; in update_parent_subparts_cpumask()
1333 cpumask_or(parent->subparts_cpus, in update_parent_subparts_cpumask()
1334 parent->subparts_cpus, tmp->addmask); in update_parent_subparts_cpumask()
1335 cpumask_andnot(parent->effective_cpus, in update_parent_subparts_cpumask()
1336 parent->effective_cpus, tmp->addmask); in update_parent_subparts_cpumask()
1339 cpumask_andnot(parent->subparts_cpus, in update_parent_subparts_cpumask()
1340 parent->subparts_cpus, tmp->delmask); in update_parent_subparts_cpumask()
1344 cpumask_and(tmp->delmask, tmp->delmask, cpu_active_mask); in update_parent_subparts_cpumask()
1345 cpumask_or(parent->effective_cpus, in update_parent_subparts_cpumask()
1346 parent->effective_cpus, tmp->delmask); in update_parent_subparts_cpumask()
1349 parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus); in update_parent_subparts_cpumask()
1351 if (cpuset->partition_root_state != new_prs) in update_parent_subparts_cpumask()
1352 cpuset->partition_root_state = new_prs; in update_parent_subparts_cpumask()
1359 * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
1360 * @cs: the cpuset to consider
1370 static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) in update_cpumasks_hier() argument
1378 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_cpumasks_hier()
1381 compute_effective_cpumask(tmp->new_cpus, cp, parent); in update_cpumasks_hier()
1387 if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) { in update_cpumasks_hier()
1388 cpumask_copy(tmp->new_cpus, parent->effective_cpus); in update_cpumasks_hier()
1389 if (!cp->use_parent_ecpus) { in update_cpumasks_hier()
1390 cp->use_parent_ecpus = true; in update_cpumasks_hier()
1391 parent->child_ecpus_count++; in update_cpumasks_hier()
1393 } else if (cp->use_parent_ecpus) { in update_cpumasks_hier()
1394 cp->use_parent_ecpus = false; in update_cpumasks_hier()
1395 WARN_ON_ONCE(!parent->child_ecpus_count); in update_cpumasks_hier()
1396 parent->child_ecpus_count--; in update_cpumasks_hier()
1403 if (!cp->partition_root_state && in update_cpumasks_hier()
1404 cpumask_equal(tmp->new_cpus, cp->effective_cpus)) { in update_cpumasks_hier()
1411 * for cs already in update_cpumask(). We should also call in update_cpumasks_hier()
1415 new_prs = cp->partition_root_state; in update_cpumasks_hier()
1416 if ((cp != cs) && new_prs) { in update_cpumasks_hier()
1417 switch (parent->partition_root_state) { in update_cpumasks_hier()
1424 WARN_ON_ONCE(cp->partition_root_state in update_cpumasks_hier()
1435 clear_bit(CS_CPU_EXCLUSIVE, &cp->flags); in update_cpumasks_hier()
1452 if (!css_tryget_online(&cp->css)) in update_cpumasks_hier()
1458 cpumask_copy(cp->effective_cpus, tmp->new_cpus); in update_cpumasks_hier()
1459 if (cp->nr_subparts_cpus && (new_prs != PRS_ENABLED)) { in update_cpumasks_hier()
1460 cp->nr_subparts_cpus = 0; in update_cpumasks_hier()
1461 cpumask_clear(cp->subparts_cpus); in update_cpumasks_hier()
1462 } else if (cp->nr_subparts_cpus) { in update_cpumasks_hier()
1468 * becomes empty. we clear cp->nr_subparts_cpus and in update_cpumasks_hier()
1472 cpumask_andnot(cp->effective_cpus, cp->effective_cpus, in update_cpumasks_hier()
1473 cp->subparts_cpus); in update_cpumasks_hier()
1474 if (cpumask_empty(cp->effective_cpus)) { in update_cpumasks_hier()
1475 cpumask_copy(cp->effective_cpus, tmp->new_cpus); in update_cpumasks_hier()
1476 cpumask_clear(cp->subparts_cpus); in update_cpumasks_hier()
1477 cp->nr_subparts_cpus = 0; in update_cpumasks_hier()
1478 } else if (!cpumask_subset(cp->subparts_cpus, in update_cpumasks_hier()
1479 tmp->new_cpus)) { in update_cpumasks_hier()
1480 cpumask_andnot(cp->subparts_cpus, in update_cpumasks_hier()
1481 cp->subparts_cpus, tmp->new_cpus); in update_cpumasks_hier()
1482 cp->nr_subparts_cpus in update_cpumasks_hier()
1483 = cpumask_weight(cp->subparts_cpus); in update_cpumasks_hier()
1487 if (new_prs != cp->partition_root_state) in update_cpumasks_hier()
1488 cp->partition_root_state = new_prs; in update_cpumasks_hier()
1493 !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); in update_cpumasks_hier()
1498 * On legacy hierarchy, if the effective cpumask of any non- in update_cpumasks_hier()
1503 if (!cpumask_empty(cp->cpus_allowed) && in update_cpumasks_hier()
1510 css_put(&cp->css); in update_cpumasks_hier()
1519 * update_sibling_cpumasks - Update siblings cpumasks
1521 * @cs: Current cpuset
1524 static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, in update_sibling_cpumasks() argument
1542 if (sibling == cs) in update_sibling_cpumasks()
1544 if (!sibling->use_parent_ecpus) in update_sibling_cpumasks()
1546 if (!css_tryget_online(&sibling->css)) in update_sibling_cpumasks()
1552 css_put(&sibling->css); in update_sibling_cpumasks()
1558 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
1559 * @cs: the cpuset to consider
1563 static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, in update_cpumask() argument
1569 /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ in update_cpumask()
1570 if (cs == &top_cpuset) in update_cpumask()
1571 return -EACCES; in update_cpumask()
1580 cpumask_clear(trialcs->cpus_requested); in update_cpumask()
1582 retval = cpulist_parse(buf, trialcs->cpus_requested); in update_cpumask()
1583 if (retval < 0) in update_cpumask()
1587 if (!cpumask_subset(trialcs->cpus_requested, cpu_present_mask)) in update_cpumask()
1588 return -EINVAL; in update_cpumask()
1590 cpumask_and(trialcs->cpus_allowed, trialcs->cpus_requested, in update_cpumask()
1594 if (cpumask_equal(cs->cpus_requested, trialcs->cpus_requested)) in update_cpumask()
1595 return 0; in update_cpumask()
1597 retval = validate_change(cs, trialcs); in update_cpumask()
1598 if (retval < 0) in update_cpumask()
1606 tmp.addmask = trialcs->subparts_cpus; in update_cpumask()
1607 tmp.delmask = trialcs->effective_cpus; in update_cpumask()
1608 tmp.new_cpus = trialcs->cpus_allowed; in update_cpumask()
1611 if (cs->partition_root_state) { in update_cpumask()
1613 if (cpumask_empty(trialcs->cpus_allowed)) in update_cpumask()
1614 return -EINVAL; in update_cpumask()
1615 if (update_parent_subparts_cpumask(cs, partcmd_update, in update_cpumask()
1616 trialcs->cpus_allowed, &tmp) < 0) in update_cpumask()
1617 return -EINVAL; in update_cpumask()
1621 cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); in update_cpumask()
1622 cpumask_copy(cs->cpus_requested, trialcs->cpus_requested); in update_cpumask()
1627 if (cs->nr_subparts_cpus) { in update_cpumask()
1628 cpumask_and(cs->subparts_cpus, cs->subparts_cpus, cs->cpus_allowed); in update_cpumask()
1629 cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus); in update_cpumask()
1633 update_cpumasks_hier(cs, &tmp); in update_cpumask()
1635 if (cs->partition_root_state) { in update_cpumask()
1636 struct cpuset *parent = parent_cs(cs); in update_cpumask()
1642 if (parent->child_ecpus_count) in update_cpumask()
1643 update_sibling_cpumasks(parent, cs, &tmp); in update_cpumask()
1645 return 0; in update_cpumask()
1669 do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); in cpuset_migrate_mm_workfn()
1670 mmput(mwork->mm); in cpuset_migrate_mm_workfn()
1681 mwork->mm = mm; in cpuset_migrate_mm()
1682 mwork->from = *from; in cpuset_migrate_mm()
1683 mwork->to = *to; in cpuset_migrate_mm()
1684 INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); in cpuset_migrate_mm()
1685 queue_work(cpuset_migrate_mm_wq, &mwork->work); in cpuset_migrate_mm()
1697 * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
1701 * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
1712 write_seqcount_begin(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
1714 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); in cpuset_change_task_nodemask()
1716 tsk->mems_allowed = *newmems; in cpuset_change_task_nodemask()
1718 write_seqcount_end(&tsk->mems_allowed_seq); in cpuset_change_task_nodemask()
1727 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
1728 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
1730 * Iterate through each task of @cs updating its mems_allowed to the
1734 static void update_tasks_nodemask(struct cpuset *cs) in update_tasks_nodemask() argument
1740 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ in update_tasks_nodemask()
1742 guarantee_online_mems(cs, &newmems); in update_tasks_nodemask()
1746 * take while holding tasklist_lock. Forks can happen - the in update_tasks_nodemask()
1754 css_task_iter_start(&cs->css, 0, &it); in update_tasks_nodemask()
1765 migrate = is_memory_migrate(cs); in update_tasks_nodemask()
1767 mpol_rebind_mm(mm, &cs->mems_allowed); in update_tasks_nodemask()
1769 cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); in update_tasks_nodemask()
1777 * cs->old_mems_allowed. in update_tasks_nodemask()
1779 cs->old_mems_allowed = newmems; in update_tasks_nodemask()
1786 * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
1787 * @cs: the cpuset to consider
1797 static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) in update_nodemasks_hier() argument
1803 cpuset_for_each_descendant_pre(cp, pos_css, cs) { in update_nodemasks_hier()
1806 nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); in update_nodemasks_hier()
1813 *new_mems = parent->effective_mems; in update_nodemasks_hier()
1816 if (nodes_equal(*new_mems, cp->effective_mems)) { in update_nodemasks_hier()
1821 if (!css_tryget_online(&cp->css)) in update_nodemasks_hier()
1826 cp->effective_mems = *new_mems; in update_nodemasks_hier()
1830 !nodes_equal(cp->mems_allowed, cp->effective_mems)); in update_nodemasks_hier()
1835 css_put(&cp->css); in update_nodemasks_hier()
1849 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
1850 * lock each such tasks mm->mmap_lock, scan its vma's and rebind
1853 static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, in update_nodemask() argument
1860 * it's read-only in update_nodemask()
1862 if (cs == &top_cpuset) { in update_nodemask()
1863 retval = -EACCES; in update_nodemask()
1874 nodes_clear(trialcs->mems_allowed); in update_nodemask()
1876 retval = nodelist_parse(buf, trialcs->mems_allowed); in update_nodemask()
1877 if (retval < 0) in update_nodemask()
1880 if (!nodes_subset(trialcs->mems_allowed, in update_nodemask()
1882 retval = -EINVAL; in update_nodemask()
1887 if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { in update_nodemask()
1888 retval = 0; /* Too easy - nothing to do */ in update_nodemask()
1891 retval = validate_change(cs, trialcs); in update_nodemask()
1892 if (retval < 0) in update_nodemask()
1896 cs->mems_allowed = trialcs->mems_allowed; in update_nodemask()
1899 /* use trialcs->mems_allowed as a temp variable */ in update_nodemask()
1900 update_nodemasks_hier(cs, &trialcs->mems_allowed); in update_nodemask()
1916 static int update_relax_domain_level(struct cpuset *cs, s64 val) in update_relax_domain_level() argument
1919 if (val < -1 || val >= sched_domain_level_max) in update_relax_domain_level()
1920 return -EINVAL; in update_relax_domain_level()
1923 if (val != cs->relax_domain_level) { in update_relax_domain_level()
1924 cs->relax_domain_level = val; in update_relax_domain_level()
1925 if (!cpumask_empty(cs->cpus_allowed) && in update_relax_domain_level()
1926 is_sched_load_balance(cs)) in update_relax_domain_level()
1930 return 0; in update_relax_domain_level()
1934 * update_tasks_flags - update the spread flags of tasks in the cpuset.
1935 * @cs: the cpuset in which each task's spread flags needs to be changed
1937 * Iterate through each task of @cs updating its spread flags. As this
1941 static void update_tasks_flags(struct cpuset *cs) in update_tasks_flags() argument
1946 css_task_iter_start(&cs->css, 0, &it); in update_tasks_flags()
1948 cpuset_update_task_spread_flag(cs, task); in update_tasks_flags()
1953 * update_flag - read a 0 or a 1 in a file and update associated flag
1955 * cs: the cpuset to update
1961 static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, in update_flag() argument
1969 trialcs = alloc_trial_cpuset(cs); in update_flag()
1971 return -ENOMEM; in update_flag()
1974 set_bit(bit, &trialcs->flags); in update_flag()
1976 clear_bit(bit, &trialcs->flags); in update_flag()
1978 err = validate_change(cs, trialcs); in update_flag()
1979 if (err < 0) in update_flag()
1982 balance_flag_changed = (is_sched_load_balance(cs) != in update_flag()
1985 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) in update_flag()
1986 || (is_spread_page(cs) != is_spread_page(trialcs))); in update_flag()
1989 cs->flags = trialcs->flags; in update_flag()
1992 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) in update_flag()
1996 update_tasks_flags(cs); in update_flag()
2003 * update_prstate - update partititon_root_state
2004 * cs: the cpuset to update
2009 static int update_prstate(struct cpuset *cs, int new_prs) in update_prstate() argument
2011 int err, old_prs = cs->partition_root_state; in update_prstate()
2012 struct cpuset *parent = parent_cs(cs); in update_prstate()
2016 return 0; in update_prstate()
2023 return -EINVAL; in update_prstate()
2026 return -ENOMEM; in update_prstate()
2028 err = -EINVAL; in update_prstate()
2035 if (cpumask_empty(cs->cpus_allowed)) in update_prstate()
2038 err = update_flag(CS_CPU_EXCLUSIVE, cs, 1); in update_prstate()
2042 err = update_parent_subparts_cpumask(cs, partcmd_enable, in update_prstate()
2045 update_flag(CS_CPU_EXCLUSIVE, cs, 0); in update_prstate()
2054 update_flag(CS_CPU_EXCLUSIVE, cs, 0); in update_prstate()
2055 err = 0; in update_prstate()
2059 err = update_parent_subparts_cpumask(cs, partcmd_disable, in update_prstate()
2065 update_flag(CS_CPU_EXCLUSIVE, cs, 0); in update_prstate()
2070 if (parent->child_ecpus_count) in update_prstate()
2071 update_sibling_cpumasks(parent, cs, &tmpmask); in update_prstate()
2077 cs->partition_root_state = new_prs; in update_prstate()
2086 * Frequency meter - How fast is some event occurring?
2090 * fmeter_init() - initialize a frequency meter.
2091 * fmeter_markevent() - called each time the event happens.
2092 * fmeter_getrate() - returns the recent rate of such events.
2093 * fmeter_update() - internal routine used to update fmeter.
2100 * The filter is single-pole low-pass recursive (IIR). The time unit
2101 * is 1 second. Arithmetic is done using 32-bit integers scaled to
2105 * has a half-life of 10 seconds, meaning that if the events quit
2130 #define FM_COEF 933 /* coefficient for half-life of 10 secs */
2138 fmp->cnt = 0; in fmeter_init()
2139 fmp->val = 0; in fmeter_init()
2140 fmp->time = 0; in fmeter_init()
2141 spin_lock_init(&fmp->lock); in fmeter_init()
2144 /* Internal meter update - process cnt events and update value */
2151 ticks = now - fmp->time; in fmeter_update()
2153 if (ticks == 0) in fmeter_update()
2157 while (ticks-- > 0) in fmeter_update()
2158 fmp->val = (FM_COEF * fmp->val) / FM_SCALE; in fmeter_update()
2159 fmp->time = now; in fmeter_update()
2161 fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; in fmeter_update()
2162 fmp->cnt = 0; in fmeter_update()
2168 spin_lock(&fmp->lock); in fmeter_markevent()
2170 fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); in fmeter_markevent()
2171 spin_unlock(&fmp->lock); in fmeter_markevent()
2179 spin_lock(&fmp->lock); in fmeter_getrate()
2181 val = fmp->val; in fmeter_getrate()
2182 spin_unlock(&fmp->lock); in fmeter_getrate()
2188 static void reset_migrate_dl_data(struct cpuset *cs) in reset_migrate_dl_data() argument
2190 cs->nr_migrate_dl_tasks = 0; in reset_migrate_dl_data()
2191 cs->sum_migrate_dl_bw = 0; in reset_migrate_dl_data()
2198 struct cpuset *cs, *oldcs; in cpuset_can_attach() local
2205 cs = css_cs(css); in cpuset_can_attach()
2210 ret = -ENOSPC; in cpuset_can_attach()
2212 (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) in cpuset_can_attach()
2224 cs->nr_migrate_dl_tasks++; in cpuset_can_attach()
2225 cs->sum_migrate_dl_bw += task->dl.dl_bw; in cpuset_can_attach()
2229 if (!cs->nr_migrate_dl_tasks) in cpuset_can_attach()
2232 if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) { in cpuset_can_attach()
2233 int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); in cpuset_can_attach()
2236 reset_migrate_dl_data(cs); in cpuset_can_attach()
2237 ret = -EINVAL; in cpuset_can_attach()
2241 ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); in cpuset_can_attach()
2243 reset_migrate_dl_data(cs); in cpuset_can_attach()
2253 cs->attach_in_progress++; in cpuset_can_attach()
2254 ret = 0; in cpuset_can_attach()
2263 struct cpuset *cs; in cpuset_cancel_attach() local
2266 cs = css_cs(css); in cpuset_cancel_attach()
2269 cs->attach_in_progress--; in cpuset_cancel_attach()
2270 if (!cs->attach_in_progress) in cpuset_cancel_attach()
2273 if (cs->nr_migrate_dl_tasks) { in cpuset_cancel_attach()
2274 int cpu = cpumask_any(cs->effective_cpus); in cpuset_cancel_attach()
2276 dl_bw_free(cpu, cs->sum_migrate_dl_bw); in cpuset_cancel_attach()
2277 reset_migrate_dl_data(cs); in cpuset_cancel_attach()
2297 struct cpuset *cs; in cpuset_attach() local
2301 cs = css_cs(css); in cpuset_attach()
2307 if (cs == &top_cpuset) in cpuset_attach()
2310 guarantee_online_cpus(cs, cpus_attach); in cpuset_attach()
2312 guarantee_online_mems(cs, &cpuset_attach_nodemask_to); in cpuset_attach()
2322 cpuset_update_task_spread_flag(cs, task); in cpuset_attach()
2329 cpuset_attach_nodemask_to = cs->effective_mems; in cpuset_attach()
2344 if (is_memory_migrate(cs)) in cpuset_attach()
2345 cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, in cpuset_attach()
2352 cs->old_mems_allowed = cpuset_attach_nodemask_to; in cpuset_attach()
2354 if (cs->nr_migrate_dl_tasks) { in cpuset_attach()
2355 cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; in cpuset_attach()
2356 oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; in cpuset_attach()
2357 reset_migrate_dl_data(cs); in cpuset_attach()
2360 cs->attach_in_progress--; in cpuset_attach()
2361 if (!cs->attach_in_progress) in cpuset_attach()
2391 struct cpuset *cs = css_cs(css); in cpuset_write_u64() local
2392 cpuset_filetype_t type = cft->private; in cpuset_write_u64()
2393 int retval = 0; in cpuset_write_u64()
2397 if (!is_cpuset_online(cs)) { in cpuset_write_u64()
2398 retval = -ENODEV; in cpuset_write_u64()
2404 retval = update_flag(CS_CPU_EXCLUSIVE, cs, val); in cpuset_write_u64()
2407 retval = update_flag(CS_MEM_EXCLUSIVE, cs, val); in cpuset_write_u64()
2410 retval = update_flag(CS_MEM_HARDWALL, cs, val); in cpuset_write_u64()
2413 retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val); in cpuset_write_u64()
2416 retval = update_flag(CS_MEMORY_MIGRATE, cs, val); in cpuset_write_u64()
2422 retval = update_flag(CS_SPREAD_PAGE, cs, val); in cpuset_write_u64()
2425 retval = update_flag(CS_SPREAD_SLAB, cs, val); in cpuset_write_u64()
2428 retval = -EINVAL; in cpuset_write_u64()
2440 struct cpuset *cs = css_cs(css); in cpuset_write_s64() local
2441 cpuset_filetype_t type = cft->private; in cpuset_write_s64()
2442 int retval = -ENODEV; in cpuset_write_s64()
2446 if (!is_cpuset_online(cs)) in cpuset_write_s64()
2451 retval = update_relax_domain_level(cs, val); in cpuset_write_s64()
2454 retval = -EINVAL; in cpuset_write_s64()
2469 struct cpuset *cs = css_cs(of_css(of)); in cpuset_write_resmask() local
2471 int retval = -ENODEV; in cpuset_write_resmask()
2476 * CPU or memory hotunplug may leave @cs w/o any execution in cpuset_write_resmask()
2481 * As writes to "cpus" or "mems" may restore @cs's execution in cpuset_write_resmask()
2490 * protection is okay as we check whether @cs is online after in cpuset_write_resmask()
2494 css_get(&cs->css); in cpuset_write_resmask()
2495 kernfs_break_active_protection(of->kn); in cpuset_write_resmask()
2500 if (!is_cpuset_online(cs)) in cpuset_write_resmask()
2503 trialcs = alloc_trial_cpuset(cs); in cpuset_write_resmask()
2505 retval = -ENOMEM; in cpuset_write_resmask()
2509 switch (of_cft(of)->private) { in cpuset_write_resmask()
2511 retval = update_cpumask(cs, trialcs, buf); in cpuset_write_resmask()
2514 retval = update_nodemask(cs, trialcs, buf); in cpuset_write_resmask()
2517 retval = -EINVAL; in cpuset_write_resmask()
2525 kernfs_unbreak_active_protection(of->kn); in cpuset_write_resmask()
2526 css_put(&cs->css); in cpuset_write_resmask()
2541 struct cpuset *cs = css_cs(seq_css(sf)); in cpuset_common_seq_show() local
2542 cpuset_filetype_t type = seq_cft(sf)->private; in cpuset_common_seq_show()
2543 int ret = 0; in cpuset_common_seq_show()
2549 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_requested)); in cpuset_common_seq_show()
2552 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); in cpuset_common_seq_show()
2555 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); in cpuset_common_seq_show()
2558 seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); in cpuset_common_seq_show()
2561 seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->subparts_cpus)); in cpuset_common_seq_show()
2564 ret = -EINVAL; in cpuset_common_seq_show()
2573 struct cpuset *cs = css_cs(css); in cpuset_read_u64() local
2574 cpuset_filetype_t type = cft->private; in cpuset_read_u64()
2577 return is_cpu_exclusive(cs); in cpuset_read_u64()
2579 return is_mem_exclusive(cs); in cpuset_read_u64()
2581 return is_mem_hardwall(cs); in cpuset_read_u64()
2583 return is_sched_load_balance(cs); in cpuset_read_u64()
2585 return is_memory_migrate(cs); in cpuset_read_u64()
2589 return fmeter_getrate(&cs->fmeter); in cpuset_read_u64()
2591 return is_spread_page(cs); in cpuset_read_u64()
2593 return is_spread_slab(cs); in cpuset_read_u64()
2599 return 0; in cpuset_read_u64()
2604 struct cpuset *cs = css_cs(css); in cpuset_read_s64() local
2605 cpuset_filetype_t type = cft->private; in cpuset_read_s64()
2608 return cs->relax_domain_level; in cpuset_read_s64()
2614 return 0; in cpuset_read_s64()
2619 struct cpuset *cs = css_cs(seq_css(seq)); in sched_partition_show() local
2621 switch (cs->partition_root_state) { in sched_partition_show()
2632 return 0; in sched_partition_show()
2638 struct cpuset *cs = css_cs(of_css(of)); in sched_partition_write() local
2640 int retval = -ENODEV; in sched_partition_write()
2652 return -EINVAL; in sched_partition_write()
2654 css_get(&cs->css); in sched_partition_write()
2657 if (!is_cpuset_online(cs)) in sched_partition_write()
2660 retval = update_prstate(cs, val); in sched_partition_write()
2664 css_put(&cs->css); in sched_partition_write()
2829 * cpuset_css_alloc - allocate a cpuset css
2836 struct cpuset *cs; in cpuset_css_alloc() local
2841 cs = kzalloc(sizeof(*cs), GFP_KERNEL); in cpuset_css_alloc()
2842 if (!cs) in cpuset_css_alloc()
2843 return ERR_PTR(-ENOMEM); in cpuset_css_alloc()
2845 if (alloc_cpumasks(cs, NULL)) { in cpuset_css_alloc()
2846 kfree(cs); in cpuset_css_alloc()
2847 return ERR_PTR(-ENOMEM); in cpuset_css_alloc()
2850 set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); in cpuset_css_alloc()
2851 nodes_clear(cs->mems_allowed); in cpuset_css_alloc()
2852 nodes_clear(cs->effective_mems); in cpuset_css_alloc()
2853 fmeter_init(&cs->fmeter); in cpuset_css_alloc()
2854 cs->relax_domain_level = -1; in cpuset_css_alloc()
2856 return &cs->css; in cpuset_css_alloc()
2861 struct cpuset *cs = css_cs(css); in cpuset_css_online() local
2862 struct cpuset *parent = parent_cs(cs); in cpuset_css_online()
2867 return 0; in cpuset_css_online()
2872 set_bit(CS_ONLINE, &cs->flags); in cpuset_css_online()
2874 set_bit(CS_SPREAD_PAGE, &cs->flags); in cpuset_css_online()
2876 set_bit(CS_SPREAD_SLAB, &cs->flags); in cpuset_css_online()
2882 cpumask_copy(cs->effective_cpus, parent->effective_cpus); in cpuset_css_online()
2883 cs->effective_mems = parent->effective_mems; in cpuset_css_online()
2884 cs->use_parent_ecpus = true; in cpuset_css_online()
2885 parent->child_ecpus_count++; in cpuset_css_online()
2889 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) in cpuset_css_online()
2895 * histrical reasons - the flag may be specified during mount. in cpuset_css_online()
2898 * refuse to clone the configuration - thereby refusing the task to in cpuset_css_online()
2902 * changed to grant parent->cpus_allowed-sibling_cpus_exclusive in cpuset_css_online()
2915 cs->mems_allowed = parent->mems_allowed; in cpuset_css_online()
2916 cs->effective_mems = parent->mems_allowed; in cpuset_css_online()
2917 cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); in cpuset_css_online()
2918 cpumask_copy(cs->cpus_requested, parent->cpus_requested); in cpuset_css_online()
2919 cpumask_copy(cs->effective_cpus, parent->cpus_allowed); in cpuset_css_online()
2924 return 0; in cpuset_css_online()
2940 struct cpuset *cs = css_cs(css); in cpuset_css_offline() local
2945 if (is_partition_root(cs)) in cpuset_css_offline()
2946 update_prstate(cs, 0); in cpuset_css_offline()
2949 is_sched_load_balance(cs)) in cpuset_css_offline()
2950 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); in cpuset_css_offline()
2952 if (cs->use_parent_ecpus) { in cpuset_css_offline()
2953 struct cpuset *parent = parent_cs(cs); in cpuset_css_offline()
2955 cs->use_parent_ecpus = false; in cpuset_css_offline()
2956 parent->child_ecpus_count--; in cpuset_css_offline()
2960 clear_bit(CS_ONLINE, &cs->flags); in cpuset_css_offline()
2968 struct cpuset *cs = css_cs(css); in cpuset_css_free() local
2970 free_cpuset(cs); in cpuset_css_free()
3001 set_cpus_allowed_ptr(task, current->cpus_ptr); in cpuset_fork()
3002 task->mems_allowed = current->mems_allowed; in cpuset_fork()
3023 * cpuset_init - initialize cpusets at system boot
3043 top_cpuset.relax_domain_level = -1; in cpuset_init()
3047 return 0; in cpuset_init()
3055 * cpuset to its next-highest non-empty parent.
3057 static void remove_tasks_in_empty_cpuset(struct cpuset *cs) in remove_tasks_in_empty_cpuset() argument
3062 * Find its next-highest non-empty parent, (top cpuset in remove_tasks_in_empty_cpuset()
3065 parent = parent_cs(cs); in remove_tasks_in_empty_cpuset()
3066 while (cpumask_empty(parent->cpus_allowed) || in remove_tasks_in_empty_cpuset()
3067 nodes_empty(parent->mems_allowed)) in remove_tasks_in_empty_cpuset()
3070 if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { in remove_tasks_in_empty_cpuset()
3072 pr_cont_cgroup_name(cs->css.cgroup); in remove_tasks_in_empty_cpuset()
3078 hotplug_update_tasks_legacy(struct cpuset *cs, in hotplug_update_tasks_legacy() argument
3085 cpumask_copy(cs->cpus_allowed, new_cpus); in hotplug_update_tasks_legacy()
3086 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks_legacy()
3087 cs->mems_allowed = *new_mems; in hotplug_update_tasks_legacy()
3088 cs->effective_mems = *new_mems; in hotplug_update_tasks_legacy()
3095 if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) in hotplug_update_tasks_legacy()
3096 update_tasks_cpumask(cs); in hotplug_update_tasks_legacy()
3097 if (mems_updated && !nodes_empty(cs->mems_allowed)) in hotplug_update_tasks_legacy()
3098 update_tasks_nodemask(cs); in hotplug_update_tasks_legacy()
3100 is_empty = cpumask_empty(cs->cpus_allowed) || in hotplug_update_tasks_legacy()
3101 nodes_empty(cs->mems_allowed); in hotplug_update_tasks_legacy()
3111 remove_tasks_in_empty_cpuset(cs); in hotplug_update_tasks_legacy()
3117 hotplug_update_tasks(struct cpuset *cs, in hotplug_update_tasks() argument
3122 cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); in hotplug_update_tasks()
3124 *new_mems = parent_cs(cs)->effective_mems; in hotplug_update_tasks()
3127 cpumask_copy(cs->effective_cpus, new_cpus); in hotplug_update_tasks()
3128 cs->effective_mems = *new_mems; in hotplug_update_tasks()
3132 update_tasks_cpumask(cs); in hotplug_update_tasks()
3134 update_tasks_nodemask(cs); in hotplug_update_tasks()
3145 * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
3146 * @cs: cpuset in interest
3149 * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
3150 * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
3153 static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) in cpuset_hotplug_update_tasks() argument
3161 wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); in cpuset_hotplug_update_tasks()
3169 if (cs->attach_in_progress) { in cpuset_hotplug_update_tasks()
3174 parent = parent_cs(cs); in cpuset_hotplug_update_tasks()
3175 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3176 nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); in cpuset_hotplug_update_tasks()
3178 if (cs->nr_subparts_cpus) in cpuset_hotplug_update_tasks()
3183 cpumask_andnot(&new_cpus, &new_cpus, cs->subparts_cpus); in cpuset_hotplug_update_tasks()
3185 if (!tmp || !cs->partition_root_state) in cpuset_hotplug_update_tasks()
3193 if (is_partition_root(cs) && (cpumask_empty(&new_cpus) || in cpuset_hotplug_update_tasks()
3194 (parent->partition_root_state == PRS_ERROR))) { in cpuset_hotplug_update_tasks()
3195 if (cs->nr_subparts_cpus) { in cpuset_hotplug_update_tasks()
3197 cs->nr_subparts_cpus = 0; in cpuset_hotplug_update_tasks()
3198 cpumask_clear(cs->subparts_cpus); in cpuset_hotplug_update_tasks()
3200 compute_effective_cpumask(&new_cpus, cs, parent); in cpuset_hotplug_update_tasks()
3209 if ((parent->partition_root_state == PRS_ERROR) || in cpuset_hotplug_update_tasks()
3211 update_parent_subparts_cpumask(cs, partcmd_disable, in cpuset_hotplug_update_tasks()
3214 cs->partition_root_state = PRS_ERROR; in cpuset_hotplug_update_tasks()
3226 ((cs->partition_root_state == PRS_ERROR) || in cpuset_hotplug_update_tasks()
3227 !cpumask_intersects(&new_cpus, parent->subparts_cpus)) && in cpuset_hotplug_update_tasks()
3228 update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp)) in cpuset_hotplug_update_tasks()
3232 cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); in cpuset_hotplug_update_tasks()
3233 mems_updated = !nodes_equal(new_mems, cs->effective_mems); in cpuset_hotplug_update_tasks()
3236 hotplug_update_tasks(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3239 hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems, in cpuset_hotplug_update_tasks()
3246 * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
3254 * Non-root cpusets are only affected by offlining. If any CPUs or memory
3307 top_cpuset.nr_subparts_cpus = 0; in cpuset_hotplug_workfn()
3333 struct cpuset *cs; in cpuset_hotplug_workfn() local
3337 cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { in cpuset_hotplug_workfn()
3338 if (cs == &top_cpuset || !css_tryget_online(&cs->css)) in cpuset_hotplug_workfn()
3342 cpuset_hotplug_update_tasks(cs, ptmp); in cpuset_hotplug_workfn()
3345 css_put(&cs->css); in cpuset_hotplug_workfn()
3392 * cpuset_init_smp - initialize cpus_allowed
3410 cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0); in cpuset_init_smp()
3415 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
3416 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
3420 * attached to the specified @tsk. Guaranteed to return some non-empty
3437 * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
3441 * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
3442 * mode however, this value is the same as task_cs(tsk)->effective_cpus,
3452 task_cs(tsk)->cpus_allowed : cpu_possible_mask); in cpuset_cpus_allowed_fallback()
3456 * We own tsk->cpus_allowed, nobody can change it under us. in cpuset_cpus_allowed_fallback()
3458 * But we used cs && cs->cpus_allowed lockless and thus can in cpuset_cpus_allowed_fallback()
3460 * the wrong tsk->cpus_allowed. However, both cases imply the in cpuset_cpus_allowed_fallback()
3461 * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() in cpuset_cpus_allowed_fallback()
3465 * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary in cpuset_cpus_allowed_fallback()
3476 nodes_setall(current->mems_allowed); in cpuset_init_current_mems_allowed()
3480 * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
3481 * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
3484 * attached to the specified @tsk. Guaranteed to return some non-empty
3504 * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed
3507 * Are any of the nodes in the nodemask allowed in current->mems_allowed?
3511 return nodes_intersects(*nodemask, current->mems_allowed); in cpuset_nodemask_valid_mems_allowed()
3515 * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
3520 static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) in nearest_hardwall_ancestor() argument
3522 while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) in nearest_hardwall_ancestor()
3523 cs = parent_cs(cs); in nearest_hardwall_ancestor()
3524 return cs; in nearest_hardwall_ancestor()
3528 * cpuset_node_allowed - Can we allocate on a memory node?
3561 * in_interrupt - any node ok (current task context irrelevant)
3562 * GFP_ATOMIC - any node ok
3563 * tsk_is_oom_victim - any node ok
3564 * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
3565 * GFP_USER - only nodes in current tasks mems allowed ok.
3569 struct cpuset *cs; /* current cpuset ancestors */ in __cpuset_node_allowed() local
3575 if (node_isset(node, current->mems_allowed)) in __cpuset_node_allowed()
3586 if (current->flags & PF_EXITING) /* Let dying task have memory */ in __cpuset_node_allowed()
3593 cs = nearest_hardwall_ancestor(task_cs(current)); in __cpuset_node_allowed()
3594 allowed = node_isset(node, cs->mems_allowed); in __cpuset_node_allowed()
3602 * cpuset_mem_spread_node() - On which node to begin search for a file page
3603 * cpuset_slab_spread_node() - On which node to begin search for a slab page
3618 * only set nodes in task->mems_allowed that are online. So it
3630 return *rotor = next_node_in(*rotor, current->mems_allowed); in cpuset_spread_node()
3635 if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) in cpuset_mem_spread_node()
3636 current->cpuset_mem_spread_rotor = in cpuset_mem_spread_node()
3637 node_random(¤t->mems_allowed); in cpuset_mem_spread_node()
3639 return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); in cpuset_mem_spread_node()
3644 if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) in cpuset_slab_spread_node()
3645 current->cpuset_slab_spread_rotor = in cpuset_slab_spread_node()
3646 node_random(¤t->mems_allowed); in cpuset_slab_spread_node()
3648 return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); in cpuset_slab_spread_node()
3654 * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
3667 return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); in cpuset_mems_allowed_intersects()
3671 * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
3682 cgrp = task_cs(current)->css.cgroup; in cpuset_print_current_mems_allowed()
3686 nodemask_pr_args(¤t->mems_allowed)); in cpuset_print_current_mems_allowed()
3700 * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
3711 * Display to user space in the per-cpuset read-only file
3720 fmeter_markevent(&task_cs(current)->fmeter); in __cpuset_memory_pressure_bump()
3727 * - Print tasks cpuset path into seq_file.
3728 * - Used for /proc/<pid>/cpuset.
3729 * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
3730 * doesn't really matter if tsk->cpuset changes after we read it,
3741 retval = -ENOMEM; in proc_cpuset_show()
3749 retval = cgroup_path_ns_locked(css->cgroup, buf, PATH_MAX, in proc_cpuset_show()
3750 current->nsproxy->cgroup_ns); in proc_cpuset_show()
3755 retval = -ENAMETOOLONG; in proc_cpuset_show()
3756 if (retval < 0) in proc_cpuset_show()
3760 retval = 0; in proc_cpuset_show()
3772 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()
3774 nodemask_pr_args(&task->mems_allowed)); in cpuset_task_status_allowed()