1 #include <stddef.h>
2 #include <stdint.h>
3 #include <stdlib.h>
4 #include <string.h>
5
6 #include <arm/linux/api.h>
7 #include <cpuinfo.h>
8 #if defined(__ANDROID__)
9 #include <arm/android/api.h>
10 #endif
11 #include <arm/api.h>
12 #include <arm/midr.h>
13 #include <cpuinfo/internal-api.h>
14 #include <cpuinfo/log.h>
15 #include <linux/api.h>
16
17 struct cpuinfo_arm_isa cpuinfo_isa = {0};
18
19 static struct cpuinfo_package package = {{0}};
20
bitmask_all(uint32_t bitfield,uint32_t mask)21 static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) {
22 return (bitfield & mask) == mask;
23 }
24
min(uint32_t a,uint32_t b)25 static inline uint32_t min(uint32_t a, uint32_t b) {
26 return a < b ? a : b;
27 }
28
cmp(uint32_t a,uint32_t b)29 static inline int cmp(uint32_t a, uint32_t b) {
30 return (a > b) - (a < b);
31 }
32
cluster_siblings_parser(uint32_t processor,uint32_t siblings_start,uint32_t siblings_end,struct cpuinfo_arm_linux_processor * processors)33 static bool cluster_siblings_parser(
34 uint32_t processor,
35 uint32_t siblings_start,
36 uint32_t siblings_end,
37 struct cpuinfo_arm_linux_processor* processors) {
38 processors[processor].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
39 uint32_t package_leader_id = processors[processor].package_leader_id;
40
41 for (uint32_t sibling = siblings_start; sibling < siblings_end; sibling++) {
42 if (!bitmask_all(processors[sibling].flags, CPUINFO_LINUX_FLAG_VALID)) {
43 cpuinfo_log_info(
44 "invalid processor %" PRIu32 " reported as a sibling for processor %" PRIu32,
45 sibling,
46 processor);
47 continue;
48 }
49
50 const uint32_t sibling_package_leader_id = processors[sibling].package_leader_id;
51 if (sibling_package_leader_id < package_leader_id) {
52 package_leader_id = sibling_package_leader_id;
53 }
54
55 processors[sibling].package_leader_id = package_leader_id;
56 processors[sibling].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
57 }
58
59 processors[processor].package_leader_id = package_leader_id;
60
61 return true;
62 }
63
cmp_arm_linux_processor(const void * ptr_a,const void * ptr_b)64 static int cmp_arm_linux_processor(const void* ptr_a, const void* ptr_b) {
65 const struct cpuinfo_arm_linux_processor* processor_a = (const struct cpuinfo_arm_linux_processor*)ptr_a;
66 const struct cpuinfo_arm_linux_processor* processor_b = (const struct cpuinfo_arm_linux_processor*)ptr_b;
67
68 /* Move usable processors towards the start of the array */
69 const bool usable_a = bitmask_all(processor_a->flags, CPUINFO_LINUX_FLAG_VALID);
70 const bool usable_b = bitmask_all(processor_b->flags, CPUINFO_LINUX_FLAG_VALID);
71 if (usable_a != usable_b) {
72 return (int)usable_b - (int)usable_a;
73 }
74
75 /* Compare based on core type (e.g. Cortex-A57 < Cortex-A53) */
76 const uint32_t midr_a = processor_a->midr;
77 const uint32_t midr_b = processor_b->midr;
78 if (midr_a != midr_b) {
79 const uint32_t score_a = midr_score_core(midr_a);
80 const uint32_t score_b = midr_score_core(midr_b);
81 if (score_a != score_b) {
82 return score_a > score_b ? -1 : 1;
83 }
84 }
85
86 /* Compare based on core frequency (e.g. 2.0 GHz < 1.2 GHz) */
87 const uint32_t frequency_a = processor_a->max_frequency;
88 const uint32_t frequency_b = processor_b->max_frequency;
89 if (frequency_a != frequency_b) {
90 return frequency_a > frequency_b ? -1 : 1;
91 }
92
93 /* Compare based on cluster leader id (i.e. cluster 1 < cluster 0) */
94 const uint32_t cluster_a = processor_a->package_leader_id;
95 const uint32_t cluster_b = processor_b->package_leader_id;
96 if (cluster_a != cluster_b) {
97 return cluster_a > cluster_b ? -1 : 1;
98 }
99
100 /* Compare based on system processor id (i.e. processor 0 < processor 1)
101 */
102 const uint32_t id_a = processor_a->system_processor_id;
103 const uint32_t id_b = processor_b->system_processor_id;
104 return cmp(id_a, id_b);
105 }
106
cpuinfo_arm_linux_init(void)107 void cpuinfo_arm_linux_init(void) {
108 struct cpuinfo_arm_linux_processor* arm_linux_processors = NULL;
109 struct cpuinfo_processor* processors = NULL;
110 struct cpuinfo_core* cores = NULL;
111 struct cpuinfo_cluster* clusters = NULL;
112 struct cpuinfo_uarch_info* uarchs = NULL;
113 struct cpuinfo_cache* l1i = NULL;
114 struct cpuinfo_cache* l1d = NULL;
115 struct cpuinfo_cache* l2 = NULL;
116 struct cpuinfo_cache* l3 = NULL;
117 const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL;
118 const struct cpuinfo_core** linux_cpu_to_core_map = NULL;
119 uint32_t* linux_cpu_to_uarch_index_map = NULL;
120
121 const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count();
122 cpuinfo_log_debug("system maximum processors count: %" PRIu32, max_processors_count);
123
124 const uint32_t max_possible_processors_count =
125 1 + cpuinfo_linux_get_max_possible_processor(max_processors_count);
126 cpuinfo_log_debug("maximum possible processors count: %" PRIu32, max_possible_processors_count);
127 const uint32_t max_present_processors_count = 1 + cpuinfo_linux_get_max_present_processor(max_processors_count);
128 cpuinfo_log_debug("maximum present processors count: %" PRIu32, max_present_processors_count);
129
130 uint32_t valid_processor_mask = 0;
131 uint32_t arm_linux_processors_count = max_processors_count;
132 if (max_present_processors_count != 0) {
133 arm_linux_processors_count = min(arm_linux_processors_count, max_present_processors_count);
134 valid_processor_mask = CPUINFO_LINUX_FLAG_PRESENT;
135 }
136 if (max_possible_processors_count != 0) {
137 arm_linux_processors_count = min(arm_linux_processors_count, max_possible_processors_count);
138 valid_processor_mask |= CPUINFO_LINUX_FLAG_POSSIBLE;
139 }
140 if ((max_present_processors_count | max_possible_processors_count) == 0) {
141 cpuinfo_log_error("failed to parse both lists of possible and present processors");
142 return;
143 }
144
145 arm_linux_processors = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_arm_linux_processor));
146 if (arm_linux_processors == NULL) {
147 cpuinfo_log_error(
148 "failed to allocate %zu bytes for descriptions of %" PRIu32 " ARM logical processors",
149 arm_linux_processors_count * sizeof(struct cpuinfo_arm_linux_processor),
150 arm_linux_processors_count);
151 return;
152 }
153
154 if (max_possible_processors_count) {
155 cpuinfo_linux_detect_possible_processors(
156 arm_linux_processors_count,
157 &arm_linux_processors->flags,
158 sizeof(struct cpuinfo_arm_linux_processor),
159 CPUINFO_LINUX_FLAG_POSSIBLE);
160 }
161
162 if (max_present_processors_count) {
163 cpuinfo_linux_detect_present_processors(
164 arm_linux_processors_count,
165 &arm_linux_processors->flags,
166 sizeof(struct cpuinfo_arm_linux_processor),
167 CPUINFO_LINUX_FLAG_PRESENT);
168 }
169
170 #if defined(__ANDROID__)
171 struct cpuinfo_android_properties android_properties;
172 cpuinfo_arm_android_parse_properties(&android_properties);
173 #else
174 char proc_cpuinfo_hardware[CPUINFO_HARDWARE_VALUE_MAX];
175 #endif
176 char proc_cpuinfo_revision[CPUINFO_REVISION_VALUE_MAX];
177
178 if (!cpuinfo_arm_linux_parse_proc_cpuinfo(
179 #if defined(__ANDROID__)
180 android_properties.proc_cpuinfo_hardware,
181 #else
182 proc_cpuinfo_hardware,
183 #endif
184 proc_cpuinfo_revision,
185 arm_linux_processors_count,
186 arm_linux_processors)) {
187 cpuinfo_log_error("failed to parse processor information from /proc/cpuinfo");
188 return;
189 }
190
191 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
192 if (bitmask_all(arm_linux_processors[i].flags, valid_processor_mask)) {
193 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_VALID;
194 cpuinfo_log_debug(
195 "parsed processor %" PRIu32 " MIDR 0x%08" PRIx32, i, arm_linux_processors[i].midr);
196 }
197 }
198
199 uint32_t valid_processors = 0, last_midr = 0;
200 #if CPUINFO_ARCH_ARM
201 uint32_t last_architecture_version = 0, last_architecture_flags = 0;
202 #endif
203 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
204 arm_linux_processors[i].system_processor_id = i;
205 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
206 if (arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR) {
207 /*
208 * Processor is in possible and present lists,
209 * and also reported in /proc/cpuinfo. This
210 * processor is availble for compute.
211 */
212 valid_processors += 1;
213 } else {
214 /*
215 * Processor is in possible and present lists,
216 * but not reported in /proc/cpuinfo. This is
217 * fairly common: high-index processors can be
218 * not reported if they are offline.
219 */
220 cpuinfo_log_info("processor %" PRIu32 " is not listed in /proc/cpuinfo", i);
221 }
222
223 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) {
224 last_midr = arm_linux_processors[i].midr;
225 }
226 #if CPUINFO_ARCH_ARM
227 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ARCHITECTURE)) {
228 last_architecture_version = arm_linux_processors[i].architecture_version;
229 last_architecture_flags = arm_linux_processors[i].architecture_flags;
230 }
231 #endif
232 } else {
233 /* Processor reported in /proc/cpuinfo, but not in
234 * possible and/or present lists: log and ignore */
235 if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) {
236 cpuinfo_log_warning("invalid processor %" PRIu32 " reported in /proc/cpuinfo", i);
237 }
238 }
239 }
240
241 #if defined(__ANDROID__)
242 const struct cpuinfo_arm_chipset chipset =
243 cpuinfo_arm_android_decode_chipset(&android_properties, valid_processors, 0);
244 #else
245 const struct cpuinfo_arm_chipset chipset =
246 cpuinfo_arm_linux_decode_chipset(proc_cpuinfo_hardware, proc_cpuinfo_revision, valid_processors, 0);
247 #endif
248
249 #if CPUINFO_ARCH_ARM
250 uint32_t isa_features = 0;
251 uint64_t isa_features2 = 0;
252 #ifdef __ANDROID__
253 /*
254 * On Android before API 20, libc.so does not provide getauxval
255 * function. Thus, we try to dynamically find it, or use two fallback
256 * mechanisms:
257 * 1. dlopen libc.so, and try to find getauxval
258 * 2. Parse /proc/self/auxv procfs file
259 * 3. Use features reported in /proc/cpuinfo
260 */
261 if (!cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2)) {
262 /* getauxval can't be used, fall back to parsing /proc/self/auxv
263 */
264 if (!cpuinfo_arm_linux_hwcap_from_procfs(&isa_features, &isa_features2)) {
265 /*
266 * Reading /proc/self/auxv failed, probably due to file
267 * permissions. Use information from /proc/cpuinfo to
268 * detect ISA.
269 *
270 * If different processors report different ISA
271 * features, take the intersection.
272 */
273 uint32_t processors_with_features = 0;
274 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
275 if (bitmask_all(
276 arm_linux_processors[i].flags,
277 CPUINFO_LINUX_FLAG_VALID | CPUINFO_ARM_LINUX_VALID_FEATURES)) {
278 if (processors_with_features == 0) {
279 isa_features = arm_linux_processors[i].features;
280 isa_features2 = arm_linux_processors[i].features2;
281 } else {
282 isa_features &= arm_linux_processors[i].features;
283 isa_features2 &= arm_linux_processors[i].features2;
284 }
285 processors_with_features += 1;
286 }
287 }
288 }
289 }
290 #else
291 /* On GNU/Linux getauxval is always available */
292 cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2);
293 #endif
294 cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo(
295 isa_features,
296 isa_features2,
297 last_midr,
298 last_architecture_version,
299 last_architecture_flags,
300 &chipset,
301 &cpuinfo_isa);
302 #elif CPUINFO_ARCH_ARM64
303 uint32_t isa_features = 0;
304 uint64_t isa_features2 = 0;
305 /* getauxval is always available on ARM64 Android */
306 cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2);
307 cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo(
308 isa_features, isa_features2, last_midr, &chipset, &cpuinfo_isa);
309 #endif
310
311 /* Detect min/max frequency and package ID */
312 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
313 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
314 const uint32_t max_frequency = cpuinfo_linux_get_processor_max_frequency(i);
315 if (max_frequency != 0) {
316 arm_linux_processors[i].max_frequency = max_frequency;
317 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY;
318 }
319
320 const uint32_t min_frequency = cpuinfo_linux_get_processor_min_frequency(i);
321 if (min_frequency != 0) {
322 arm_linux_processors[i].min_frequency = min_frequency;
323 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY;
324 }
325
326 if (cpuinfo_linux_get_processor_package_id(i, &arm_linux_processors[i].package_id)) {
327 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_ID;
328 }
329 }
330 }
331
332 /* Initialize topology group IDs */
333 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
334 arm_linux_processors[i].package_leader_id = i;
335 }
336
337 /* Propagate topology group IDs among siblings */
338 bool detected_core_siblings_list_node = false;
339 bool detected_cluster_cpus_list_node = false;
340 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
341 if (!bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
342 continue;
343 }
344
345 if (!bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_PACKAGE_ID)) {
346 continue;
347 }
348
349 /* Use the cluster_cpus_list topology node if available. If not
350 * found, cache the result to avoid repeatedly attempting to
351 * read the non-existent paths.
352 * */
353 if (!detected_core_siblings_list_node && !detected_cluster_cpus_list_node) {
354 if (cpuinfo_linux_detect_cluster_cpus(
355 arm_linux_processors_count,
356 i,
357 (cpuinfo_siblings_callback)cluster_siblings_parser,
358 arm_linux_processors)) {
359 detected_cluster_cpus_list_node = true;
360 continue;
361 } else {
362 detected_core_siblings_list_node = true;
363 }
364 }
365
366 /* The cached result above will guarantee only one of the blocks
367 * below will execute, with a bias towards cluster_cpus_list.
368 **/
369 if (detected_core_siblings_list_node) {
370 cpuinfo_linux_detect_core_siblings(
371 arm_linux_processors_count,
372 i,
373 (cpuinfo_siblings_callback)cluster_siblings_parser,
374 arm_linux_processors);
375 }
376
377 if (detected_cluster_cpus_list_node) {
378 cpuinfo_linux_detect_cluster_cpus(
379 arm_linux_processors_count,
380 i,
381 (cpuinfo_siblings_callback)cluster_siblings_parser,
382 arm_linux_processors);
383 }
384 }
385
386 /* Propagate all cluster IDs */
387 uint32_t clustered_processors = 0;
388 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
389 if (bitmask_all(
390 arm_linux_processors[i].flags,
391 CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) {
392 clustered_processors += 1;
393
394 const uint32_t package_leader_id = arm_linux_processors[i].package_leader_id;
395 if (package_leader_id < i) {
396 arm_linux_processors[i].package_leader_id =
397 arm_linux_processors[package_leader_id].package_leader_id;
398 }
399
400 cpuinfo_log_debug(
401 "processor %" PRIu32 " clustered with processor %" PRIu32
402 " as inferred from system siblings lists",
403 i,
404 arm_linux_processors[i].package_leader_id);
405 }
406 }
407
408 if (clustered_processors != valid_processors) {
409 /*
410 * Topology information about some or all logical processors may
411 * be unavailable, for the following reasons:
412 * - Linux kernel is too old, or configured without support for
413 * topology information in sysfs.
414 * - Core is offline, and Linux kernel is configured to not
415 * report topology for offline cores.
416 *
417 * In this case, we assign processors to clusters using two
418 * methods:
419 * - Try heuristic cluster configurations (e.g. 6-core SoC
420 * usually has 4+2 big.LITTLE configuration).
421 * - If heuristic failed, assign processors to core clusters in
422 * a sequential scan.
423 */
424 if (!cpuinfo_arm_linux_detect_core_clusters_by_heuristic(
425 valid_processors, arm_linux_processors_count, arm_linux_processors)) {
426 cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan(
427 arm_linux_processors_count, arm_linux_processors);
428 }
429 }
430
431 cpuinfo_arm_linux_count_cluster_processors(arm_linux_processors_count, arm_linux_processors);
432
433 const uint32_t cluster_count = cpuinfo_arm_linux_detect_cluster_midr(
434 &chipset, arm_linux_processors_count, valid_processors, arm_linux_processors);
435
436 /* Initialize core vendor, uarch, MIDR, and frequency for every logical
437 * processor */
438 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
439 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
440 const uint32_t cluster_leader = arm_linux_processors[i].package_leader_id;
441 if (cluster_leader == i) {
442 /* Cluster leader: decode core vendor and uarch
443 */
444 cpuinfo_arm_decode_vendor_uarch(
445 arm_linux_processors[cluster_leader].midr,
446 #if CPUINFO_ARCH_ARM
447 !!(arm_linux_processors[cluster_leader].features &
448 CPUINFO_ARM_LINUX_FEATURE_VFPV4),
449 #endif
450 &arm_linux_processors[cluster_leader].vendor,
451 &arm_linux_processors[cluster_leader].uarch);
452 } else {
453 /* Cluster non-leader: copy vendor, uarch, MIDR,
454 * and frequency from cluster leader */
455 arm_linux_processors[i].flags |= arm_linux_processors[cluster_leader].flags &
456 (CPUINFO_ARM_LINUX_VALID_MIDR | CPUINFO_LINUX_FLAG_MAX_FREQUENCY);
457 arm_linux_processors[i].midr = arm_linux_processors[cluster_leader].midr;
458 arm_linux_processors[i].vendor = arm_linux_processors[cluster_leader].vendor;
459 arm_linux_processors[i].uarch = arm_linux_processors[cluster_leader].uarch;
460 arm_linux_processors[i].max_frequency =
461 arm_linux_processors[cluster_leader].max_frequency;
462 }
463 }
464 }
465
466 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
467 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
468 cpuinfo_log_debug(
469 "post-analysis processor %" PRIu32 ": MIDR %08" PRIx32 " frequency %" PRIu32,
470 i,
471 arm_linux_processors[i].midr,
472 arm_linux_processors[i].max_frequency);
473 }
474 }
475
476 qsort(arm_linux_processors,
477 arm_linux_processors_count,
478 sizeof(struct cpuinfo_arm_linux_processor),
479 cmp_arm_linux_processor);
480
481 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
482 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
483 cpuinfo_log_debug(
484 "post-sort processor %" PRIu32 ": system id %" PRIu32 " MIDR %08" PRIx32
485 " frequency %" PRIu32,
486 i,
487 arm_linux_processors[i].system_processor_id,
488 arm_linux_processors[i].midr,
489 arm_linux_processors[i].max_frequency);
490 }
491 }
492
493 uint32_t uarchs_count = 0;
494 enum cpuinfo_uarch last_uarch;
495 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
496 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
497 if (uarchs_count == 0 || arm_linux_processors[i].uarch != last_uarch) {
498 last_uarch = arm_linux_processors[i].uarch;
499 uarchs_count += 1;
500 }
501 arm_linux_processors[i].uarch_index = uarchs_count - 1;
502 }
503 }
504
505 /*
506 * Assumptions:
507 * - No SMP (i.e. each core supports only one hardware thread).
508 * - Level 1 instruction and data caches are private to the core
509 * clusters.
510 * - Level 2 and level 3 cache is shared between cores in the same
511 * cluster.
512 */
513 cpuinfo_arm_chipset_to_string(&chipset, package.name);
514 package.processor_count = valid_processors;
515 package.core_count = valid_processors;
516 package.cluster_count = cluster_count;
517
518 processors = calloc(valid_processors, sizeof(struct cpuinfo_processor));
519 if (processors == NULL) {
520 cpuinfo_log_error(
521 "failed to allocate %zu bytes for descriptions of %" PRIu32 " logical processors",
522 valid_processors * sizeof(struct cpuinfo_processor),
523 valid_processors);
524 goto cleanup;
525 }
526
527 cores = calloc(valid_processors, sizeof(struct cpuinfo_core));
528 if (cores == NULL) {
529 cpuinfo_log_error(
530 "failed to allocate %zu bytes for descriptions of %" PRIu32 " cores",
531 valid_processors * sizeof(struct cpuinfo_core),
532 valid_processors);
533 goto cleanup;
534 }
535
536 clusters = calloc(cluster_count, sizeof(struct cpuinfo_cluster));
537 if (clusters == NULL) {
538 cpuinfo_log_error(
539 "failed to allocate %zu bytes for descriptions of %" PRIu32 " core clusters",
540 cluster_count * sizeof(struct cpuinfo_cluster),
541 cluster_count);
542 goto cleanup;
543 }
544
545 uarchs = calloc(uarchs_count, sizeof(struct cpuinfo_uarch_info));
546 if (uarchs == NULL) {
547 cpuinfo_log_error(
548 "failed to allocate %zu bytes for descriptions of %" PRIu32 " microarchitectures",
549 uarchs_count * sizeof(struct cpuinfo_uarch_info),
550 uarchs_count);
551 goto cleanup;
552 }
553
554 linux_cpu_to_processor_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_processor*));
555 if (linux_cpu_to_processor_map == NULL) {
556 cpuinfo_log_error(
557 "failed to allocate %zu bytes for %" PRIu32 " logical processor mapping entries",
558 arm_linux_processors_count * sizeof(struct cpuinfo_processor*),
559 arm_linux_processors_count);
560 goto cleanup;
561 }
562
563 linux_cpu_to_core_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_core*));
564 if (linux_cpu_to_core_map == NULL) {
565 cpuinfo_log_error(
566 "failed to allocate %zu bytes for %" PRIu32 " core mapping entries",
567 arm_linux_processors_count * sizeof(struct cpuinfo_core*),
568 arm_linux_processors_count);
569 goto cleanup;
570 }
571
572 if (uarchs_count > 1) {
573 linux_cpu_to_uarch_index_map = calloc(arm_linux_processors_count, sizeof(uint32_t));
574 if (linux_cpu_to_uarch_index_map == NULL) {
575 cpuinfo_log_error(
576 "failed to allocate %zu bytes for %" PRIu32 " uarch index mapping entries",
577 arm_linux_processors_count * sizeof(uint32_t),
578 arm_linux_processors_count);
579 goto cleanup;
580 }
581 }
582
583 l1i = calloc(valid_processors, sizeof(struct cpuinfo_cache));
584 if (l1i == NULL) {
585 cpuinfo_log_error(
586 "failed to allocate %zu bytes for descriptions of %" PRIu32 " L1I caches",
587 valid_processors * sizeof(struct cpuinfo_cache),
588 valid_processors);
589 goto cleanup;
590 }
591
592 l1d = calloc(valid_processors, sizeof(struct cpuinfo_cache));
593 if (l1d == NULL) {
594 cpuinfo_log_error(
595 "failed to allocate %zu bytes for descriptions of %" PRIu32 " L1D caches",
596 valid_processors * sizeof(struct cpuinfo_cache),
597 valid_processors);
598 goto cleanup;
599 }
600
601 uint32_t uarchs_index = 0;
602 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
603 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
604 if (uarchs_index == 0 || arm_linux_processors[i].uarch != last_uarch) {
605 last_uarch = arm_linux_processors[i].uarch;
606 uarchs[uarchs_index] = (struct cpuinfo_uarch_info){
607 .uarch = arm_linux_processors[i].uarch,
608 .midr = arm_linux_processors[i].midr,
609 };
610 uarchs_index += 1;
611 }
612 uarchs[uarchs_index - 1].processor_count += 1;
613 uarchs[uarchs_index - 1].core_count += 1;
614 }
615 }
616
617 uint32_t l2_count = 0, l3_count = 0, big_l3_size = 0, cluster_id = UINT32_MAX;
618 /* Indication whether L3 (if it exists) is shared between all cores */
619 bool shared_l3 = true;
620 /* Populate cache information structures in l1i, l1d */
621 for (uint32_t i = 0; i < valid_processors; i++) {
622 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
623 cluster_id += 1;
624 clusters[cluster_id] = (struct cpuinfo_cluster){
625 .processor_start = i,
626 .processor_count = arm_linux_processors[i].package_processor_count,
627 .core_start = i,
628 .core_count = arm_linux_processors[i].package_processor_count,
629 .cluster_id = cluster_id,
630 .package = &package,
631 .vendor = arm_linux_processors[i].vendor,
632 .uarch = arm_linux_processors[i].uarch,
633 .midr = arm_linux_processors[i].midr,
634 };
635 }
636
637 processors[i].smt_id = 0;
638 processors[i].core = cores + i;
639 processors[i].cluster = clusters + cluster_id;
640 processors[i].package = &package;
641 processors[i].linux_id = (int)arm_linux_processors[i].system_processor_id;
642 processors[i].cache.l1i = l1i + i;
643 processors[i].cache.l1d = l1d + i;
644 linux_cpu_to_processor_map[arm_linux_processors[i].system_processor_id] = &processors[i];
645
646 cores[i].processor_start = i;
647 cores[i].processor_count = 1;
648 cores[i].core_id = i;
649 cores[i].cluster = clusters + cluster_id;
650 cores[i].package = &package;
651 cores[i].vendor = arm_linux_processors[i].vendor;
652 cores[i].uarch = arm_linux_processors[i].uarch;
653 cores[i].midr = arm_linux_processors[i].midr;
654 linux_cpu_to_core_map[arm_linux_processors[i].system_processor_id] = &cores[i];
655
656 if (linux_cpu_to_uarch_index_map != NULL) {
657 linux_cpu_to_uarch_index_map[arm_linux_processors[i].system_processor_id] =
658 arm_linux_processors[i].uarch_index;
659 }
660
661 struct cpuinfo_cache temp_l2 = {0}, temp_l3 = {0};
662 cpuinfo_arm_decode_cache(
663 arm_linux_processors[i].uarch,
664 arm_linux_processors[i].package_processor_count,
665 arm_linux_processors[i].midr,
666 &chipset,
667 cluster_id,
668 arm_linux_processors[i].architecture_version,
669 &l1i[i],
670 &l1d[i],
671 &temp_l2,
672 &temp_l3);
673 l1i[i].processor_start = l1d[i].processor_start = i;
674 l1i[i].processor_count = l1d[i].processor_count = 1;
675 #if CPUINFO_ARCH_ARM
676 /* L1I reported in /proc/cpuinfo overrides defaults */
677 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ICACHE)) {
678 l1i[i] = (struct cpuinfo_cache){
679 .size = arm_linux_processors[i].proc_cpuinfo_cache.i_size,
680 .associativity = arm_linux_processors[i].proc_cpuinfo_cache.i_assoc,
681 .sets = arm_linux_processors[i].proc_cpuinfo_cache.i_sets,
682 .partitions = 1,
683 .line_size = arm_linux_processors[i].proc_cpuinfo_cache.i_line_length};
684 }
685 /* L1D reported in /proc/cpuinfo overrides defaults */
686 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_DCACHE)) {
687 l1d[i] = (struct cpuinfo_cache){
688 .size = arm_linux_processors[i].proc_cpuinfo_cache.d_size,
689 .associativity = arm_linux_processors[i].proc_cpuinfo_cache.d_assoc,
690 .sets = arm_linux_processors[i].proc_cpuinfo_cache.d_sets,
691 .partitions = 1,
692 .line_size = arm_linux_processors[i].proc_cpuinfo_cache.d_line_length};
693 }
694 #endif
695
696 if (temp_l3.size != 0) {
697 /*
698 * Assumptions:
699 * - L2 is private to each core
700 * - L3 is shared by cores in the same cluster
701 * - If cores in different clusters report the same L3,
702 * it is shared between all cores.
703 */
704 l2_count += 1;
705 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
706 if (cluster_id == 0) {
707 big_l3_size = temp_l3.size;
708 l3_count = 1;
709 } else if (temp_l3.size != big_l3_size) {
710 /* If some cores have different L3 size,
711 * L3 is not shared between all cores */
712 shared_l3 = false;
713 l3_count += 1;
714 }
715 }
716 } else {
717 /* If some cores don't have L3 cache, L3 is not shared
718 * between all cores
719 */
720 shared_l3 = false;
721 if (temp_l2.size != 0) {
722 /* Assume L2 is shared by cores in the same
723 * cluster */
724 if (arm_linux_processors[i].package_leader_id ==
725 arm_linux_processors[i].system_processor_id) {
726 l2_count += 1;
727 }
728 }
729 }
730 }
731
732 if (l2_count != 0) {
733 l2 = calloc(l2_count, sizeof(struct cpuinfo_cache));
734 if (l2 == NULL) {
735 cpuinfo_log_error(
736 "failed to allocate %zu bytes for descriptions of %" PRIu32 " L2 caches",
737 l2_count * sizeof(struct cpuinfo_cache),
738 l2_count);
739 goto cleanup;
740 }
741
742 if (l3_count != 0) {
743 l3 = calloc(l3_count, sizeof(struct cpuinfo_cache));
744 if (l3 == NULL) {
745 cpuinfo_log_error(
746 "failed to allocate %zu bytes for descriptions of %" PRIu32 " L3 caches",
747 l3_count * sizeof(struct cpuinfo_cache),
748 l3_count);
749 goto cleanup;
750 }
751 }
752 }
753
754 cluster_id = UINT32_MAX;
755 uint32_t l2_index = UINT32_MAX, l3_index = UINT32_MAX;
756 for (uint32_t i = 0; i < valid_processors; i++) {
757 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
758 cluster_id++;
759 }
760
761 struct cpuinfo_cache dummy_l1i, dummy_l1d, temp_l2 = {0}, temp_l3 = {0};
762 cpuinfo_arm_decode_cache(
763 arm_linux_processors[i].uarch,
764 arm_linux_processors[i].package_processor_count,
765 arm_linux_processors[i].midr,
766 &chipset,
767 cluster_id,
768 arm_linux_processors[i].architecture_version,
769 &dummy_l1i,
770 &dummy_l1d,
771 &temp_l2,
772 &temp_l3);
773
774 if (temp_l3.size != 0) {
775 /*
776 * Assumptions:
777 * - L2 is private to each core
778 * - L3 is shared by cores in the same cluster
779 * - If cores in different clusters report the same L3,
780 * it is shared between all cores.
781 */
782 l2_index += 1;
783 l2[l2_index] = (struct cpuinfo_cache){
784 .size = temp_l2.size,
785 .associativity = temp_l2.associativity,
786 .sets = temp_l2.sets,
787 .partitions = 1,
788 .line_size = temp_l2.line_size,
789 .flags = temp_l2.flags,
790 .processor_start = i,
791 .processor_count = 1,
792 };
793 processors[i].cache.l2 = l2 + l2_index;
794 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
795 l3_index += 1;
796 if (l3_index < l3_count) {
797 l3[l3_index] = (struct cpuinfo_cache){
798 .size = temp_l3.size,
799 .associativity = temp_l3.associativity,
800 .sets = temp_l3.sets,
801 .partitions = 1,
802 .line_size = temp_l3.line_size,
803 .flags = temp_l3.flags,
804 .processor_start = i,
805 .processor_count = shared_l3
806 ? valid_processors
807 : arm_linux_processors[i].package_processor_count,
808 };
809 }
810 }
811 if (shared_l3) {
812 processors[i].cache.l3 = l3;
813 } else if (l3_index < l3_count) {
814 processors[i].cache.l3 = l3 + l3_index;
815 }
816 } else if (temp_l2.size != 0) {
817 /* Assume L2 is shared by cores in the same cluster */
818 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
819 l2_index += 1;
820 l2[l2_index] = (struct cpuinfo_cache){
821 .size = temp_l2.size,
822 .associativity = temp_l2.associativity,
823 .sets = temp_l2.sets,
824 .partitions = 1,
825 .line_size = temp_l2.line_size,
826 .flags = temp_l2.flags,
827 .processor_start = i,
828 .processor_count = arm_linux_processors[i].package_processor_count,
829 };
830 }
831 processors[i].cache.l2 = l2 + l2_index;
832 }
833 }
834
835 /* Commit */
836 cpuinfo_processors = processors;
837 cpuinfo_cores = cores;
838 cpuinfo_clusters = clusters;
839 cpuinfo_packages = &package;
840 cpuinfo_uarchs = uarchs;
841 cpuinfo_cache[cpuinfo_cache_level_1i] = l1i;
842 cpuinfo_cache[cpuinfo_cache_level_1d] = l1d;
843 cpuinfo_cache[cpuinfo_cache_level_2] = l2;
844 cpuinfo_cache[cpuinfo_cache_level_3] = l3;
845
846 cpuinfo_processors_count = valid_processors;
847 cpuinfo_cores_count = valid_processors;
848 cpuinfo_clusters_count = cluster_count;
849 cpuinfo_packages_count = 1;
850 cpuinfo_uarchs_count = uarchs_count;
851 cpuinfo_cache_count[cpuinfo_cache_level_1i] = valid_processors;
852 cpuinfo_cache_count[cpuinfo_cache_level_1d] = valid_processors;
853 cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count;
854 cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count;
855 cpuinfo_max_cache_size = cpuinfo_arm_compute_max_cache_size(&processors[0]);
856
857 cpuinfo_linux_cpu_max = arm_linux_processors_count;
858 cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map;
859 cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map;
860 cpuinfo_linux_cpu_to_uarch_index_map = linux_cpu_to_uarch_index_map;
861
862 __sync_synchronize();
863
864 cpuinfo_is_initialized = true;
865
866 processors = NULL;
867 cores = NULL;
868 clusters = NULL;
869 uarchs = NULL;
870 l1i = l1d = l2 = l3 = NULL;
871 linux_cpu_to_processor_map = NULL;
872 linux_cpu_to_core_map = NULL;
873 linux_cpu_to_uarch_index_map = NULL;
874
875 cleanup:
876 free(arm_linux_processors);
877 free(processors);
878 free(cores);
879 free(clusters);
880 free(uarchs);
881 free(l1i);
882 free(l1d);
883 free(l2);
884 free(l3);
885 free(linux_cpu_to_processor_map);
886 free(linux_cpu_to_core_map);
887 free(linux_cpu_to_uarch_index_map);
888 }
889