1 /*
2 * arch/arm64/kernel/topology.c
3 *
4 * Copyright (C) 2011,2013,2014 Linaro Limited.
5 *
6 * Based on the arm32 version written by Vincent Guittot in turn based on
7 * arch/sh/kernel/topology.c
8 *
9 * This file is subject to the terms and conditions of the GNU General Public
10 * License. See the file "COPYING" in the main directory of this archive
11 * for more details.
12 */
13
14 #include <linux/arch_topology.h>
15 #include <linux/cpu.h>
16 #include <linux/cpumask.h>
17 #include <linux/init.h>
18 #include <linux/percpu.h>
19 #include <linux/node.h>
20 #include <linux/nodemask.h>
21 #include <linux/of.h>
22 #include <linux/sched.h>
23 #include <linux/sched/topology.h>
24 #include <linux/sched/energy.h>
25 #include <linux/slab.h>
26 #include <linux/string.h>
27
28 #include <asm/cpu.h>
29 #include <asm/cputype.h>
30 #include <asm/topology.h>
31
get_cpu_for_node(struct device_node * node)32 static int __init get_cpu_for_node(struct device_node *node)
33 {
34 struct device_node *cpu_node;
35 int cpu;
36
37 cpu_node = of_parse_phandle(node, "cpu", 0);
38 if (!cpu_node)
39 return -1;
40
41 for_each_possible_cpu(cpu) {
42 if (of_get_cpu_node(cpu, NULL) == cpu_node) {
43 topology_parse_cpu_capacity(cpu_node, cpu);
44 of_node_put(cpu_node);
45 return cpu;
46 }
47 }
48
49 pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
50
51 of_node_put(cpu_node);
52 return -1;
53 }
54
parse_core(struct device_node * core,int cluster_id,int core_id)55 static int __init parse_core(struct device_node *core, int cluster_id,
56 int core_id)
57 {
58 char name[10];
59 bool leaf = true;
60 int i = 0;
61 int cpu;
62 struct device_node *t;
63
64 do {
65 snprintf(name, sizeof(name), "thread%d", i);
66 t = of_get_child_by_name(core, name);
67 if (t) {
68 leaf = false;
69 cpu = get_cpu_for_node(t);
70 if (cpu >= 0) {
71 cpu_topology[cpu].cluster_id = cluster_id;
72 cpu_topology[cpu].core_id = core_id;
73 cpu_topology[cpu].thread_id = i;
74 } else {
75 pr_err("%pOF: Can't get CPU for thread\n",
76 t);
77 of_node_put(t);
78 return -EINVAL;
79 }
80 of_node_put(t);
81 }
82 i++;
83 } while (t);
84
85 cpu = get_cpu_for_node(core);
86 if (cpu >= 0) {
87 if (!leaf) {
88 pr_err("%pOF: Core has both threads and CPU\n",
89 core);
90 return -EINVAL;
91 }
92
93 cpu_topology[cpu].cluster_id = cluster_id;
94 cpu_topology[cpu].core_id = core_id;
95 } else if (leaf) {
96 pr_err("%pOF: Can't get CPU for leaf core\n", core);
97 return -EINVAL;
98 }
99
100 return 0;
101 }
102
parse_cluster(struct device_node * cluster,int depth)103 static int __init parse_cluster(struct device_node *cluster, int depth)
104 {
105 char name[10];
106 bool leaf = true;
107 bool has_cores = false;
108 struct device_node *c;
109 static int cluster_id __initdata;
110 int core_id = 0;
111 int i, ret;
112
113 /*
114 * First check for child clusters; we currently ignore any
115 * information about the nesting of clusters and present the
116 * scheduler with a flat list of them.
117 */
118 i = 0;
119 do {
120 snprintf(name, sizeof(name), "cluster%d", i);
121 c = of_get_child_by_name(cluster, name);
122 if (c) {
123 leaf = false;
124 ret = parse_cluster(c, depth + 1);
125 of_node_put(c);
126 if (ret != 0)
127 return ret;
128 }
129 i++;
130 } while (c);
131
132 /* Now check for cores */
133 i = 0;
134 do {
135 snprintf(name, sizeof(name), "core%d", i);
136 c = of_get_child_by_name(cluster, name);
137 if (c) {
138 has_cores = true;
139
140 if (depth == 0) {
141 pr_err("%pOF: cpu-map children should be clusters\n",
142 c);
143 of_node_put(c);
144 return -EINVAL;
145 }
146
147 if (leaf) {
148 ret = parse_core(c, cluster_id, core_id++);
149 } else {
150 pr_err("%pOF: Non-leaf cluster with core %s\n",
151 cluster, name);
152 ret = -EINVAL;
153 }
154
155 of_node_put(c);
156 if (ret != 0)
157 return ret;
158 }
159 i++;
160 } while (c);
161
162 if (leaf && !has_cores)
163 pr_warn("%pOF: empty cluster\n", cluster);
164
165 if (leaf)
166 cluster_id++;
167
168 return 0;
169 }
170
parse_dt_topology(void)171 static int __init parse_dt_topology(void)
172 {
173 struct device_node *cn, *map;
174 int ret = 0;
175 int cpu;
176
177 cn = of_find_node_by_path("/cpus");
178 if (!cn) {
179 pr_err("No CPU information found in DT\n");
180 return 0;
181 }
182
183 /*
184 * When topology is provided cpu-map is essentially a root
185 * cluster with restricted subnodes.
186 */
187 map = of_get_child_by_name(cn, "cpu-map");
188 if (!map)
189 goto out;
190
191 ret = parse_cluster(map, 0);
192 if (ret != 0)
193 goto out_map;
194
195 topology_normalize_cpu_scale();
196
197 /*
198 * Check that all cores are in the topology; the SMP code will
199 * only mark cores described in the DT as possible.
200 */
201 for_each_possible_cpu(cpu)
202 if (cpu_topology[cpu].cluster_id == -1)
203 ret = -EINVAL;
204
205 out_map:
206 of_node_put(map);
207 out:
208 of_node_put(cn);
209 return ret;
210 }
211
212 /*
213 * cpu topology table
214 */
215 struct cpu_topology cpu_topology[NR_CPUS];
216 EXPORT_SYMBOL_GPL(cpu_topology);
217
cpu_coregroup_mask(int cpu)218 const struct cpumask *cpu_coregroup_mask(int cpu)
219 {
220 return &cpu_topology[cpu].core_sibling;
221 }
222
update_siblings_masks(unsigned int cpuid)223 static void update_siblings_masks(unsigned int cpuid)
224 {
225 struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid];
226 int cpu;
227
228 /* update core and thread sibling masks */
229 for_each_possible_cpu(cpu) {
230 cpu_topo = &cpu_topology[cpu];
231
232 if (cpuid_topo->cluster_id != cpu_topo->cluster_id)
233 continue;
234
235 cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
236 if (cpu != cpuid)
237 cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
238
239 if (cpuid_topo->core_id != cpu_topo->core_id)
240 continue;
241
242 cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
243 if (cpu != cpuid)
244 cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
245 }
246 }
247
store_cpu_topology(unsigned int cpuid)248 void store_cpu_topology(unsigned int cpuid)
249 {
250 struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
251 u64 mpidr;
252
253 if (cpuid_topo->cluster_id != -1)
254 goto topology_populated;
255
256 mpidr = read_cpuid_mpidr();
257
258 /* Uniprocessor systems can rely on default topology values */
259 if (mpidr & MPIDR_UP_BITMASK)
260 return;
261
262 /* Create cpu topology mapping based on MPIDR. */
263 if (mpidr & MPIDR_MT_BITMASK) {
264 /* Multiprocessor system : Multi-threads per core */
265 cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
266 cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
267 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) |
268 MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8;
269 } else {
270 /* Multiprocessor system : Single-thread per core */
271 cpuid_topo->thread_id = -1;
272 cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
273 cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) |
274 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 |
275 MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16;
276 }
277
278 pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n",
279 cpuid, cpuid_topo->cluster_id, cpuid_topo->core_id,
280 cpuid_topo->thread_id, mpidr);
281
282 topology_populated:
283 update_siblings_masks(cpuid);
284 topology_detect_flags();
285 }
286
287 #ifdef CONFIG_SCHED_SMT
smt_flags(void)288 static int smt_flags(void)
289 {
290 return cpu_smt_flags() | topology_smt_flags();
291 }
292 #endif
293
294 #ifdef CONFIG_SCHED_MC
core_flags(void)295 static int core_flags(void)
296 {
297 return cpu_core_flags() | topology_core_flags();
298 }
299 #endif
300
cpu_flags(void)301 static int cpu_flags(void)
302 {
303 return topology_cpu_flags();
304 }
305
306 static inline
cpu_core_energy(int cpu)307 const struct sched_group_energy * const cpu_core_energy(int cpu)
308 {
309 return sge_array[cpu][SD_LEVEL0];
310 }
311
312 static inline
cpu_cluster_energy(int cpu)313 const struct sched_group_energy * const cpu_cluster_energy(int cpu)
314 {
315 return sge_array[cpu][SD_LEVEL1];
316 }
317
318 static inline
cpu_system_energy(int cpu)319 const struct sched_group_energy * const cpu_system_energy(int cpu)
320 {
321 return sge_array[cpu][SD_LEVEL2];
322 }
323
324 static struct sched_domain_topology_level arm64_topology[] = {
325 #ifdef CONFIG_SCHED_SMT
326 { cpu_smt_mask, smt_flags, SD_INIT_NAME(SMT) },
327 #endif
328 #ifdef CONFIG_SCHED_MC
329 { cpu_coregroup_mask, core_flags, cpu_core_energy, SD_INIT_NAME(MC) },
330 #endif
331 { cpu_cpu_mask, cpu_flags, cpu_cluster_energy, SD_INIT_NAME(DIE) },
332 { cpu_cpu_mask, NULL, cpu_system_energy, SD_INIT_NAME(SYS) },
333 { NULL, }
334 };
335
reset_cpu_topology(void)336 static void __init reset_cpu_topology(void)
337 {
338 unsigned int cpu;
339
340 for_each_possible_cpu(cpu) {
341 struct cpu_topology *cpu_topo = &cpu_topology[cpu];
342
343 cpu_topo->thread_id = -1;
344 cpu_topo->core_id = 0;
345 cpu_topo->cluster_id = -1;
346
347 cpumask_clear(&cpu_topo->core_sibling);
348 cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
349 cpumask_clear(&cpu_topo->thread_sibling);
350 cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
351 }
352 }
353
init_cpu_topology(void)354 void __init init_cpu_topology(void)
355 {
356 reset_cpu_topology();
357
358 /*
359 * Discard anything that was parsed if we hit an error so we
360 * don't use partial information.
361 */
362 if (of_have_populated_dt() && parse_dt_topology())
363 reset_cpu_topology();
364 else
365 set_sched_topology(arm64_topology);
366 }
367