Lines Matching +full:pd +full:- +full:node
1 // SPDX-License-Identifier: GPL-2.0
38 struct sched_group *group = sd->groups; in sched_domain_debug_one()
39 unsigned long flags = sd->flags; in sched_domain_debug_one()
44 printk(KERN_DEBUG "%*s domain-%d: ", level, "", level); in sched_domain_debug_one()
46 cpumask_pr_args(sched_domain_span(sd)), sd->name); in sched_domain_debug_one()
49 printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu); in sched_domain_debug_one()
52 printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu); in sched_domain_debug_one()
59 if ((meta_flags & SDF_SHARED_CHILD) && sd->child && in sched_domain_debug_one()
60 !(sd->child->flags & flag)) in sched_domain_debug_one()
64 if ((meta_flags & SDF_SHARED_PARENT) && sd->parent && in sched_domain_debug_one()
65 !(sd->parent->flags & flag)) in sched_domain_debug_one()
84 if (!(sd->flags & SD_OVERLAP) && in sched_domain_debug_one()
94 group->sgc->id, in sched_domain_debug_one()
97 if ((sd->flags & SD_OVERLAP) && in sched_domain_debug_one()
103 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) in sched_domain_debug_one()
104 printk(KERN_CONT " cap=%lu", group->sgc->capacity); in sched_domain_debug_one()
106 if (group == sd->groups && sd->child && in sched_domain_debug_one()
107 !cpumask_equal(sched_domain_span(sd->child), in sched_domain_debug_one()
109 printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n"); in sched_domain_debug_one()
114 group = group->next; in sched_domain_debug_one()
116 if (group != sd->groups) in sched_domain_debug_one()
119 } while (group != sd->groups); in sched_domain_debug_one()
123 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); in sched_domain_debug_one()
125 if (sd->parent && in sched_domain_debug_one()
126 !cpumask_subset(groupmask, sched_domain_span(sd->parent))) in sched_domain_debug_one()
127 printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n"); in sched_domain_debug_one()
139 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); in sched_domain_debug()
143 printk(KERN_DEBUG "CPU%d attaching sched-domain(s):\n", cpu); in sched_domain_debug()
149 sd = sd->parent; in sched_domain_debug()
177 if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) && in sd_degenerate()
178 (sd->groups != sd->groups->next)) in sd_degenerate()
182 if (sd->flags & (SD_WAKE_AFFINE)) in sd_degenerate()
191 unsigned long cflags = sd->flags, pflags = parent->flags; in sd_parent_degenerate()
200 if (parent->groups == parent->groups->next) in sd_parent_degenerate()
231 return -EPERM; in sched_energy_aware_handler()
265 static void free_pd(struct perf_domain *pd) in free_pd() argument
269 while (pd) { in free_pd()
270 tmp = pd->next; in free_pd()
271 kfree(pd); in free_pd()
272 pd = tmp; in free_pd()
276 static struct perf_domain *find_pd(struct perf_domain *pd, int cpu) in find_pd() argument
278 while (pd) { in find_pd()
279 if (cpumask_test_cpu(cpu, perf_domain_span(pd))) in find_pd()
280 return pd; in find_pd()
281 pd = pd->next; in find_pd()
290 struct perf_domain *pd; in pd_init() local
298 pd = kzalloc(sizeof(*pd), GFP_KERNEL); in pd_init()
299 if (!pd) in pd_init()
301 pd->em_pd = obj; in pd_init()
303 return pd; in pd_init()
307 struct perf_domain *pd) in perf_domain_debug() argument
309 if (!sched_debug() || !pd) in perf_domain_debug()
314 while (pd) { in perf_domain_debug()
315 printk(KERN_CONT " pd%d:{ cpus=%*pbl nr_pstate=%d }", in perf_domain_debug()
316 cpumask_first(perf_domain_span(pd)), in perf_domain_debug()
317 cpumask_pr_args(perf_domain_span(pd)), in perf_domain_debug()
318 em_pd_nr_perf_states(pd->em_pd)); in perf_domain_debug()
319 pd = pd->next; in perf_domain_debug()
327 struct perf_domain *pd; in destroy_perf_domain_rcu() local
329 pd = container_of(rp, struct perf_domain, rcu); in destroy_perf_domain_rcu()
330 free_pd(pd); in destroy_perf_domain_rcu()
360 * - nr_pd: the number of performance domains
361 * - nr_cpus: the number of CPUs
362 * - nr_ps: the sum of the number of performance states of all performance
366 * It is generally not a good idea to use such a model in the wake-up path on
369 * with per-CPU DVFS and less than 8 performance states each, for example.
377 struct perf_domain *pd = NULL, *tmp; in build_perf_domains() local
379 struct root_domain *rd = cpu_rq(cpu)->rd; in build_perf_domains()
404 pr_warn("rd %*pbl: Disabling EAS: frequency-invariant load tracking not yet supported", in build_perf_domains()
412 if (find_pd(pd, i)) in build_perf_domains()
419 gov = policy->governor; in build_perf_domains()
422 if (rd->pd) in build_perf_domains()
428 /* Create the new pd and add it to the local list. */ in build_perf_domains()
432 tmp->next = pd; in build_perf_domains()
433 pd = tmp; in build_perf_domains()
440 nr_ps += em_pd_nr_perf_states(pd->em_pd); in build_perf_domains()
450 perf_domain_debug(cpu_map, pd); in build_perf_domains()
453 tmp = rd->pd; in build_perf_domains()
454 rcu_assign_pointer(rd->pd, pd); in build_perf_domains()
456 call_rcu(&tmp->rcu, destroy_perf_domain_rcu); in build_perf_domains()
458 return !!pd; in build_perf_domains()
461 free_pd(pd); in build_perf_domains()
462 tmp = rd->pd; in build_perf_domains()
463 rcu_assign_pointer(rd->pd, NULL); in build_perf_domains()
465 call_rcu(&tmp->rcu, destroy_perf_domain_rcu); in build_perf_domains()
470 static void free_pd(struct perf_domain *pd) { } in free_pd() argument
477 cpupri_cleanup(&rd->cpupri); in free_rootdomain()
478 cpudl_cleanup(&rd->cpudl); in free_rootdomain()
479 free_cpumask_var(rd->dlo_mask); in free_rootdomain()
480 free_cpumask_var(rd->rto_mask); in free_rootdomain()
481 free_cpumask_var(rd->online); in free_rootdomain()
482 free_cpumask_var(rd->span); in free_rootdomain()
483 free_pd(rd->pd); in free_rootdomain()
494 if (rq->rd) { in rq_attach_root()
495 old_rd = rq->rd; in rq_attach_root()
497 if (cpumask_test_cpu(rq->cpu, old_rd->online)) in rq_attach_root()
500 cpumask_clear_cpu(rq->cpu, old_rd->span); in rq_attach_root()
507 if (!atomic_dec_and_test(&old_rd->refcount)) in rq_attach_root()
511 atomic_inc(&rd->refcount); in rq_attach_root()
512 rq->rd = rd; in rq_attach_root()
514 cpumask_set_cpu(rq->cpu, rd->span); in rq_attach_root()
515 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) in rq_attach_root()
521 call_rcu(&old_rd->rcu, free_rootdomain); in rq_attach_root()
526 atomic_inc(&rd->refcount); in sched_get_rd()
531 if (!atomic_dec_and_test(&rd->refcount)) in sched_put_rd()
534 call_rcu(&rd->rcu, free_rootdomain); in sched_put_rd()
539 if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL)) in init_rootdomain()
541 if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL)) in init_rootdomain()
543 if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) in init_rootdomain()
545 if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) in init_rootdomain()
549 rd->rto_cpu = -1; in init_rootdomain()
550 raw_spin_lock_init(&rd->rto_lock); in init_rootdomain()
551 rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func); in init_rootdomain()
554 rd->visit_gen = 0; in init_rootdomain()
555 init_dl_bw(&rd->dl_bw); in init_rootdomain()
556 if (cpudl_init(&rd->cpudl) != 0) in init_rootdomain()
559 if (cpupri_init(&rd->cpupri) != 0) in init_rootdomain()
564 cpudl_cleanup(&rd->cpudl); in init_rootdomain()
566 free_cpumask_var(rd->rto_mask); in init_rootdomain()
568 free_cpumask_var(rd->dlo_mask); in init_rootdomain()
570 free_cpumask_var(rd->online); in init_rootdomain()
572 free_cpumask_var(rd->span); in init_rootdomain()
574 return -ENOMEM; in init_rootdomain()
578 * By default the system creates a single root-domain with all CPUs as
615 tmp = sg->next; in free_sched_groups()
617 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) in free_sched_groups()
618 kfree(sg->sgc); in free_sched_groups()
620 if (atomic_dec_and_test(&sg->ref)) in free_sched_groups()
633 free_sched_groups(sd->groups, 1); in destroy_sched_domain()
635 if (sd->shared && atomic_dec_and_test(&sd->shared->ref)) in destroy_sched_domain()
636 kfree(sd->shared); in destroy_sched_domain()
645 struct sched_domain *parent = sd->parent; in destroy_sched_domains_rcu()
654 call_rcu(&sd->rcu, destroy_sched_domains_rcu); in destroy_sched_domains()
686 sds = sd->shared; in update_top_cache_domain()
716 struct sched_domain *parent = tmp->parent; in cpu_attach_domain()
721 tmp->parent = parent->parent; in cpu_attach_domain()
723 if (parent->parent) { in cpu_attach_domain()
724 parent->parent->child = tmp; in cpu_attach_domain()
725 parent->parent->groups->flags = tmp->flags; in cpu_attach_domain()
733 if (parent->flags & SD_PREFER_SIBLING) in cpu_attach_domain()
734 tmp->flags |= SD_PREFER_SIBLING; in cpu_attach_domain()
737 tmp = tmp->parent; in cpu_attach_domain()
742 sd = sd->parent; in cpu_attach_domain()
745 struct sched_group *sg = sd->groups; in cpu_attach_domain()
753 sg->flags = 0; in cpu_attach_domain()
754 } while (sg != sd->groups); in cpu_attach_domain()
756 sd->child = NULL; in cpu_attach_domain()
763 tmp = rq->sd; in cpu_attach_domain()
764 rcu_assign_pointer(rq->sd, sd); in cpu_attach_domain()
801 * Given a node-distance table, for example:
803 * node 0 1 2 3
809 * which represents a 4 node ring topology like:
811 * 0 ----- 1
815 * 3 ----- 2
823 * NUMA-2 0-3 0-3 0-3 0-3
824 * groups: {0-1,3},{1-3} {0-2},{0,2-3} {1-3},{0-1,3} {0,2-3},{0-2}
826 * NUMA-1 0-1,3 0-2 1-3 0,2-3
829 * NUMA-0 0 1 2 3
834 * represented multiple times -- hence the "overlap" naming for this part of
838 * domain. For instance Node-0 NUMA-2 would only get groups: 0-1,3 and 1-3.
842 * - the first group of each domain is its child domain; this
843 * gets us the first 0-1,3
844 * - the only uncovered node is 2, who's child domain is 1-3.
847 * more complicated. Consider for instance the groups of NODE-1 NUMA-2, both
848 * groups include the CPUs of Node-0, while those CPUs would not in fact ever
849 * end up at those groups (they would end up in group: 0-1,3).
864 * node 0 1 2 3
872 * 0 ----- 1
876 * 2 ----- 3
884 * NUMA-2 0-3 0-3
885 * groups: {0-2},{1-3} {1-3},{0-2}
887 * NUMA-1 0-2 0-3 0-3 1-3
889 * NUMA-0 0 1 2 3
907 struct sd_data *sdd = sd->private; in build_balance_mask()
914 sibling = *per_cpu_ptr(sdd->sd, i); in build_balance_mask()
921 if (!sibling->child) in build_balance_mask()
925 if (!cpumask_equal(sg_span, sched_domain_span(sibling->child))) in build_balance_mask()
936 * XXX: This creates per-node group entries; since the load-balancer will
937 * immediately access remote memory to construct this group's load-balance
938 * statistics having the groups node local is of dubious benefit.
953 if (sd->child) { in build_group_from_child_sched_domain()
954 cpumask_copy(sg_span, sched_domain_span(sd->child)); in build_group_from_child_sched_domain()
955 sg->flags = sd->child->flags; in build_group_from_child_sched_domain()
960 atomic_inc(&sg->ref); in build_group_from_child_sched_domain()
968 struct sd_data *sdd = sd->private; in init_overlap_sched_group()
975 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu); in init_overlap_sched_group()
976 if (atomic_inc_return(&sg->sgc->ref) == 1) in init_overlap_sched_group()
982 * Initialize sgc->capacity such that even if we mess up the in init_overlap_sched_group()
987 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); in init_overlap_sched_group()
988 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE; in init_overlap_sched_group()
989 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; in init_overlap_sched_group()
999 while (sibling->child && in find_descended_sibling()
1000 !cpumask_subset(sched_domain_span(sibling->child), in find_descended_sibling()
1002 sibling = sibling->child; in find_descended_sibling()
1009 while (sibling->child && in find_descended_sibling()
1010 cpumask_equal(sched_domain_span(sibling->child), in find_descended_sibling()
1012 sibling = sibling->child; in find_descended_sibling()
1023 struct sd_data *sdd = sd->private; in build_overlap_sched_groups()
1035 sibling = *per_cpu_ptr(sdd->sd, i); in build_overlap_sched_groups()
1038 * Asymmetric node setups can result in situations where the in build_overlap_sched_groups()
1059 * node 0 1 2 3 in build_overlap_sched_groups()
1065 * 0 --- 1 --- 2 --- 3 in build_overlap_sched_groups()
1067 * NUMA-3 0-3 N/A N/A 0-3 in build_overlap_sched_groups()
1068 * groups: {0-2},{1-3} {1-3},{0-2} in build_overlap_sched_groups()
1070 * NUMA-2 0-2 0-3 0-3 1-3 in build_overlap_sched_groups()
1071 * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2} in build_overlap_sched_groups()
1073 * NUMA-1 0-1 0-2 1-3 2-3 in build_overlap_sched_groups()
1076 * NUMA-0 0 1 2 3 in build_overlap_sched_groups()
1078 * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the in build_overlap_sched_groups()
1081 if (sibling->child && in build_overlap_sched_groups()
1082 !cpumask_subset(sched_domain_span(sibling->child), span)) in build_overlap_sched_groups()
1097 last->next = sg; in build_overlap_sched_groups()
1099 last->next = first; in build_overlap_sched_groups()
1101 sd->groups = first; in build_overlap_sched_groups()
1108 return -ENOMEM; in build_overlap_sched_groups()
1113 * Package topology (also see the load-balance blurb in fair.c)
1118 * - Simultaneous multithreading (SMT)
1119 * - Multi-Core Cache (MC)
1120 * - Package (DIE)
1122 * Where the last one more or less denotes everything up to a NUMA node.
1126 * sched_domain -> sched_group -> sched_group_capacity
1128 * `-' `-'
1130 * The sched_domains are per-CPU and have a two way link (parent & child) and
1146 * - or -
1148 * DIE 0-7 0-7 0-7 0-7 0-7 0-7 0-7 0-7
1149 * MC 0-3 0-3 0-3 0-3 4-7 4-7 4-7 4-7
1150 * SMT 0-1 0-1 2-3 2-3 4-5 4-5 6-7 6-7
1163 * - The first is the balance_cpu (see should_we_balance() and the
1164 * load-balance blub in fair.c); for each group we only want 1 CPU to
1167 * - The second is the sched_group_capacity; we want all identical groups
1185 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); in get_group()
1186 struct sched_domain *child = sd->child; in get_group()
1193 sg = *per_cpu_ptr(sdd->sg, cpu); in get_group()
1194 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu); in get_group()
1197 already_visited = atomic_inc_return(&sg->ref) > 1; in get_group()
1199 WARN_ON(already_visited != (atomic_inc_return(&sg->sgc->ref) > 1)); in get_group()
1208 sg->flags = child->flags; in get_group()
1214 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg)); in get_group()
1215 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE; in get_group()
1216 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE; in get_group()
1223 * covered by the given span, will set each group's ->cpumask correctly,
1224 * and will initialize their ->sgc.
1232 struct sd_data *sdd = sd->private; in build_sched_groups()
1255 last->next = sg; in build_sched_groups()
1258 last->next = first; in build_sched_groups()
1259 sd->groups = first; in build_sched_groups()
1276 struct sched_group *sg = sd->groups; in init_sched_groups_capacity()
1282 int cpu, cores = 0, max_cpu = -1; in init_sched_groups_capacity()
1284 sg->group_weight = cpumask_weight(sched_group_span(sg)); in init_sched_groups_capacity()
1293 sg->cores = cores; in init_sched_groups_capacity()
1295 if (!(sd->flags & SD_ASYM_PACKING)) in init_sched_groups_capacity()
1304 sg->asym_prefer_cpu = max_cpu; in init_sched_groups_capacity()
1307 sg = sg->next; in init_sched_groups_capacity()
1308 } while (sg != sd->groups); in init_sched_groups_capacity()
1333 #define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus)
1379 if (capacity == entry->capacity) in asym_cpu_capacity_update_data()
1386 entry->capacity = capacity; in asym_cpu_capacity_update_data()
1387 list_add(&entry->link, &asym_cap_list); in asym_cpu_capacity_update_data()
1393 * Build-up/update list of CPUs grouped by their capacities
1410 list_del(&entry->link); in asym_cpu_capacity_scan()
1421 list_del(&entry->link); in asym_cpu_capacity_scan()
1428 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
1431 static int default_relax_domain_level = -1;
1448 if (!attr || attr->relax_domain_level < 0) { in set_domain_attribute()
1453 request = attr->relax_domain_level; in set_domain_attribute()
1455 if (sd->level > request) { in set_domain_attribute()
1457 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); in set_domain_attribute()
1469 if (!atomic_read(&d->rd->refcount)) in __free_domain_allocs()
1470 free_rootdomain(&d->rd->rcu); in __free_domain_allocs()
1473 free_percpu(d->sd); in __free_domain_allocs()
1490 d->sd = alloc_percpu(struct sched_domain *); in __visit_domain_allocation_hell()
1491 if (!d->sd) in __visit_domain_allocation_hell()
1493 d->rd = alloc_rootdomain(); in __visit_domain_allocation_hell()
1494 if (!d->rd) in __visit_domain_allocation_hell()
1507 struct sd_data *sdd = sd->private; in claim_allocations()
1509 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); in claim_allocations()
1510 *per_cpu_ptr(sdd->sd, cpu) = NULL; in claim_allocations()
1512 if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref)) in claim_allocations()
1513 *per_cpu_ptr(sdd->sds, cpu) = NULL; in claim_allocations()
1515 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) in claim_allocations()
1516 *per_cpu_ptr(sdd->sg, cpu) = NULL; in claim_allocations()
1518 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) in claim_allocations()
1519 *per_cpu_ptr(sdd->sgc, cpu) = NULL; in claim_allocations()
1540 * SD_SHARE_CPUCAPACITY - describes SMT topologies
1541 * SD_SHARE_PKG_RESOURCES - describes shared caches
1542 * SD_NUMA - describes NUMA topologies
1547 * SD_ASYM_PACKING - describes SMT quirks
1560 struct sd_data *sdd = &tl->data; in sd_init()
1561 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); in sd_init()
1569 sched_domains_curr_level = tl->numa_level; in sd_init()
1572 sd_weight = cpumask_weight(tl->mask(cpu)); in sd_init()
1574 if (tl->sd_flags) in sd_init()
1575 sd_flags = (*tl->sd_flags)(); in sd_init()
1607 .name = tl->name, in sd_init()
1612 cpumask_and(sd_span, cpu_map, tl->mask(cpu)); in sd_init()
1615 sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map); in sd_init()
1617 WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) == in sd_init()
1625 if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child) in sd_init()
1626 sd->child->flags &= ~SD_PREFER_SIBLING; in sd_init()
1628 if (sd->flags & SD_SHARE_CPUCAPACITY) { in sd_init()
1629 sd->imbalance_pct = 110; in sd_init()
1631 } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { in sd_init()
1632 sd->imbalance_pct = 117; in sd_init()
1633 sd->cache_nice_tries = 1; in sd_init()
1636 } else if (sd->flags & SD_NUMA) { in sd_init()
1637 sd->cache_nice_tries = 2; in sd_init()
1639 sd->flags &= ~SD_PREFER_SIBLING; in sd_init()
1640 sd->flags |= SD_SERIALIZE; in sd_init()
1641 if (sched_domains_numa_distance[tl->numa_level] > node_reclaim_distance) { in sd_init()
1642 sd->flags &= ~(SD_BALANCE_EXEC | in sd_init()
1649 sd->cache_nice_tries = 1; in sd_init()
1656 if (sd->flags & SD_SHARE_PKG_RESOURCES) { in sd_init()
1657 sd->shared = *per_cpu_ptr(sdd->sds, sd_id); in sd_init()
1658 atomic_inc(&sd->shared->ref); in sd_init()
1659 atomic_set(&sd->shared->nr_busy_cpus, sd_weight); in sd_init()
1662 sd->private = sdd; in sd_init()
1668 * Topology list, bottom-up.
1691 for (tl = sched_domain_topology; tl->mask; tl++)
1777 * - If the maximum distance between any nodes is 1 hop, the system
1779 * - If for two nodes A and B, located N > 1 hops away from each other,
1780 * there is an intermediary node C, which is < N hops away from both
1800 /* Is there an intermediary node between a and b? */ in init_numa_topology_type()
1832 * O(nr_nodes^2) deduplicating selection sort -- in order to find the in sched_init_numa()
1896 * Now for each level, construct a mask per node which contains all in sched_init_numa()
1915 sched_numa_warn("Node-distance not symmetric"); in sched_init_numa()
1941 * Add the NUMA identity distance, aka single NODE. in sched_init_numa()
1946 SD_INIT_NAME(NODE) in sched_init_numa()
1966 WRITE_ONCE(sched_max_numa_distance, sched_domains_numa_distance[nr_levels - 1]); in sched_init_numa()
2011 int node; in sched_update_numa() local
2013 node = cpu_to_node(cpu); in sched_update_numa()
2016 * node is onlined or the last CPU of a node is offlined. in sched_update_numa()
2018 if (cpumask_weight(cpumask_of_node(node)) != 1) in sched_update_numa()
2022 sched_init_numa(online ? NUMA_NO_NODE : node); in sched_update_numa()
2027 int node = cpu_to_node(cpu); in sched_domains_numa_masks_set() local
2035 /* Set ourselves in the remote node's masks */ in sched_domains_numa_masks_set()
2036 if (node_distance(j, node) <= sched_domains_numa_distance[i]) in sched_domains_numa_masks_set()
2055 * sched_numa_find_closest() - given the NUMA topology, find the cpu
2089 int node; member
2099 if (cpumask_weight_and(k->cpus, cur_hop[k->node]) <= k->cpu) in hop_cmp()
2102 if (b == k->masks) { in hop_cmp()
2103 k->w = 0; in hop_cmp()
2107 prev_hop = *((struct cpumask ***)b - 1); in hop_cmp()
2108 k->w = cpumask_weight_and(k->cpus, prev_hop[k->node]); in hop_cmp()
2109 if (k->w <= k->cpu) in hop_cmp()
2112 return -1; in hop_cmp()
2116 * sched_numa_find_nth_cpu() - given the NUMA topology, find the Nth next cpu
2123 int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node) in sched_numa_find_nth_cpu() argument
2131 /* CPU-less node entries are uninitialized in sched_domains_numa_masks */ in sched_numa_find_nth_cpu()
2132 node = numa_nearest_node(node, N_CPU); in sched_numa_find_nth_cpu()
2133 k.node = node; in sched_numa_find_nth_cpu()
2140 hop = hop_masks - k.masks; in sched_numa_find_nth_cpu()
2143 cpumask_nth_and_andnot(cpu - k.w, cpus, k.masks[hop][node], k.masks[hop-1][node]) : in sched_numa_find_nth_cpu()
2144 cpumask_nth_and(cpu, cpus, k.masks[0][node]); in sched_numa_find_nth_cpu()
2152 * sched_numa_hop_mask() - Get the cpumask of CPUs at most @hops hops away from
2153 * @node
2154 * @node: The node to count hops from.
2155 * @hops: Include CPUs up to that many hops away. 0 means local node.
2158 * @node, an error value otherwise.
2161 * read-side section, copy it if required beyond that.
2168 const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops) in sched_numa_hop_mask() argument
2172 if (node >= nr_node_ids || hops >= sched_domains_numa_levels) in sched_numa_hop_mask()
2173 return ERR_PTR(-EINVAL); in sched_numa_hop_mask()
2177 return ERR_PTR(-EBUSY); in sched_numa_hop_mask()
2179 return masks[hops][node]; in sched_numa_hop_mask()
2191 struct sd_data *sdd = &tl->data; in __sdt_alloc()
2193 sdd->sd = alloc_percpu(struct sched_domain *); in __sdt_alloc()
2194 if (!sdd->sd) in __sdt_alloc()
2195 return -ENOMEM; in __sdt_alloc()
2197 sdd->sds = alloc_percpu(struct sched_domain_shared *); in __sdt_alloc()
2198 if (!sdd->sds) in __sdt_alloc()
2199 return -ENOMEM; in __sdt_alloc()
2201 sdd->sg = alloc_percpu(struct sched_group *); in __sdt_alloc()
2202 if (!sdd->sg) in __sdt_alloc()
2203 return -ENOMEM; in __sdt_alloc()
2205 sdd->sgc = alloc_percpu(struct sched_group_capacity *); in __sdt_alloc()
2206 if (!sdd->sgc) in __sdt_alloc()
2207 return -ENOMEM; in __sdt_alloc()
2218 return -ENOMEM; in __sdt_alloc()
2220 *per_cpu_ptr(sdd->sd, j) = sd; in __sdt_alloc()
2225 return -ENOMEM; in __sdt_alloc()
2227 *per_cpu_ptr(sdd->sds, j) = sds; in __sdt_alloc()
2232 return -ENOMEM; in __sdt_alloc()
2234 sg->next = sg; in __sdt_alloc()
2236 *per_cpu_ptr(sdd->sg, j) = sg; in __sdt_alloc()
2241 return -ENOMEM; in __sdt_alloc()
2244 sgc->id = j; in __sdt_alloc()
2247 *per_cpu_ptr(sdd->sgc, j) = sgc; in __sdt_alloc()
2260 struct sd_data *sdd = &tl->data; in __sdt_free()
2265 if (sdd->sd) { in __sdt_free()
2266 sd = *per_cpu_ptr(sdd->sd, j); in __sdt_free()
2267 if (sd && (sd->flags & SD_OVERLAP)) in __sdt_free()
2268 free_sched_groups(sd->groups, 0); in __sdt_free()
2269 kfree(*per_cpu_ptr(sdd->sd, j)); in __sdt_free()
2272 if (sdd->sds) in __sdt_free()
2273 kfree(*per_cpu_ptr(sdd->sds, j)); in __sdt_free()
2274 if (sdd->sg) in __sdt_free()
2275 kfree(*per_cpu_ptr(sdd->sg, j)); in __sdt_free()
2276 if (sdd->sgc) in __sdt_free()
2277 kfree(*per_cpu_ptr(sdd->sgc, j)); in __sdt_free()
2279 free_percpu(sdd->sd); in __sdt_free()
2280 sdd->sd = NULL; in __sdt_free()
2281 free_percpu(sdd->sds); in __sdt_free()
2282 sdd->sds = NULL; in __sdt_free()
2283 free_percpu(sdd->sg); in __sdt_free()
2284 sdd->sg = NULL; in __sdt_free()
2285 free_percpu(sdd->sgc); in __sdt_free()
2286 sdd->sgc = NULL; in __sdt_free()
2297 sd->level = child->level + 1; in build_sched_domain()
2298 sched_domain_level_max = max(sched_domain_level_max, sd->level); in build_sched_domain()
2299 child->parent = sd; in build_sched_domain()
2306 child->name, sd->name); in build_sched_domain()
2322 * any two given CPUs at this (non-NUMA) topology level.
2330 if (tl->flags & SDTL_OVERLAP) in topology_span_sane()
2334 * Non-NUMA levels cannot partially overlap - they must be either in topology_span_sane()
2336 * breaking the sched_group lists - i.e. a later get_group() pass in topology_span_sane()
2348 if (!cpumask_equal(tl->mask(cpu), tl->mask(i)) && in topology_span_sane()
2349 cpumask_intersects(tl->mask(cpu), tl->mask(i))) in topology_span_sane()
2367 int i, ret = -ENOMEM; in build_sched_domains()
2389 has_asym |= sd->flags & SD_ASYM_CPUCAPACITY; in build_sched_domains()
2393 if (tl->flags & SDTL_OVERLAP) in build_sched_domains()
2394 sd->flags |= SD_OVERLAP; in build_sched_domains()
2402 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { in build_sched_domains()
2403 sd->span_weight = cpumask_weight(sched_domain_span(sd)); in build_sched_domains()
2404 if (sd->flags & SD_OVERLAP) { in build_sched_domains()
2422 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { in build_sched_domains()
2423 struct sched_domain *child = sd->child; in build_sched_domains()
2425 if (!(sd->flags & SD_SHARE_PKG_RESOURCES) && child && in build_sched_domains()
2426 (child->flags & SD_SHARE_PKG_RESOURCES)) { in build_sched_domains()
2431 * For a single LLC per node, allow an in build_sched_domains()
2432 * imbalance up to 12.5% of the node. This is in build_sched_domains()
2433 * arbitrary cutoff based two factors -- SMT and in build_sched_domains()
2434 * memory channels. For SMT-2, the intent is to in build_sched_domains()
2436 * SMT-4 or SMT-8 *may* benefit from a different in build_sched_domains()
2444 * on one node while LLCs on another node in build_sched_domains()
2450 nr_llcs = sd->span_weight / child->span_weight; in build_sched_domains()
2452 imb = sd->span_weight >> 3; in build_sched_domains()
2456 sd->imb_numa_nr = imb; in build_sched_domains()
2459 top_p = sd->parent; in build_sched_domains()
2460 while (top_p && !(top_p->flags & SD_NUMA)) { in build_sched_domains()
2461 top_p = top_p->parent; in build_sched_domains()
2463 imb_span = top_p ? top_p->span_weight : sd->span_weight; in build_sched_domains()
2465 int factor = max(1U, (sd->span_weight / imb_span)); in build_sched_domains()
2467 sd->imb_numa_nr = imb * factor; in build_sched_domains()
2473 for (i = nr_cpumask_bits-1; i >= 0; i--) { in build_sched_domains()
2477 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { in build_sched_domains()
2490 if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity)) in build_sched_domains()
2491 WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig); in build_sched_domains()
2502 cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); in build_sched_domains()
2685 * its dl_bw->total_bw needs to be cleared. It in partition_sched_domains_locked()
2689 rd = cpu_rq(cpumask_any(doms_cur[i]))->rd; in partition_sched_domains_locked()
2694 /* No match - a current sched domain not in new doms_new[] */ in partition_sched_domains_locked()
2715 /* No match - add a new doms_new */ in partition_sched_domains_locked()
2726 cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) { in partition_sched_domains_locked()
2731 /* No match - add perf. domains for a new rd */ in partition_sched_domains_locked()