1 #include <stdint.h>
2 #include <stddef.h>
3 #include <stdlib.h>
4 #include <string.h>
5
6 #include <cpuinfo.h>
7 #include <arm/linux/api.h>
8 #if defined(__ANDROID__)
9 #include <arm/android/api.h>
10 #endif
11 #include <arm/api.h>
12 #include <arm/midr.h>
13 #include <linux/api.h>
14 #include <cpuinfo/internal-api.h>
15 #include <cpuinfo/log.h>
16
17
18 struct cpuinfo_arm_isa cpuinfo_isa = { 0 };
19
20 static struct cpuinfo_package package = { { 0 } };
21
bitmask_all(uint32_t bitfield,uint32_t mask)22 static inline bool bitmask_all(uint32_t bitfield, uint32_t mask) {
23 return (bitfield & mask) == mask;
24 }
25
min(uint32_t a,uint32_t b)26 static inline uint32_t min(uint32_t a, uint32_t b) {
27 return a < b ? a : b;
28 }
29
cmp(uint32_t a,uint32_t b)30 static inline int cmp(uint32_t a, uint32_t b) {
31 return (a > b) - (a < b);
32 }
33
cluster_siblings_parser(uint32_t processor,uint32_t siblings_start,uint32_t siblings_end,struct cpuinfo_arm_linux_processor * processors)34 static bool cluster_siblings_parser(
35 uint32_t processor, uint32_t siblings_start, uint32_t siblings_end,
36 struct cpuinfo_arm_linux_processor* processors)
37 {
38 processors[processor].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
39 uint32_t package_leader_id = processors[processor].package_leader_id;
40
41 for (uint32_t sibling = siblings_start; sibling < siblings_end; sibling++) {
42 if (!bitmask_all(processors[sibling].flags, CPUINFO_LINUX_FLAG_VALID)) {
43 cpuinfo_log_info("invalid processor %"PRIu32" reported as a sibling for processor %"PRIu32,
44 sibling, processor);
45 continue;
46 }
47
48 const uint32_t sibling_package_leader_id = processors[sibling].package_leader_id;
49 if (sibling_package_leader_id < package_leader_id) {
50 package_leader_id = sibling_package_leader_id;
51 }
52
53 processors[sibling].package_leader_id = package_leader_id;
54 processors[sibling].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;
55 }
56
57 processors[processor].package_leader_id = package_leader_id;
58
59 return true;
60 }
61
cmp_arm_linux_processor(const void * ptr_a,const void * ptr_b)62 static int cmp_arm_linux_processor(const void* ptr_a, const void* ptr_b) {
63 const struct cpuinfo_arm_linux_processor* processor_a = (const struct cpuinfo_arm_linux_processor*) ptr_a;
64 const struct cpuinfo_arm_linux_processor* processor_b = (const struct cpuinfo_arm_linux_processor*) ptr_b;
65
66 /* Move usable processors towards the start of the array */
67 const bool usable_a = bitmask_all(processor_a->flags, CPUINFO_LINUX_FLAG_VALID);
68 const bool usable_b = bitmask_all(processor_b->flags, CPUINFO_LINUX_FLAG_VALID);
69 if (usable_a != usable_b) {
70 return (int) usable_b - (int) usable_a;
71 }
72
73 /* Compare based on core type (e.g. Cortex-A57 < Cortex-A53) */
74 const uint32_t midr_a = processor_a->midr;
75 const uint32_t midr_b = processor_b->midr;
76 if (midr_a != midr_b) {
77 const uint32_t score_a = midr_score_core(midr_a);
78 const uint32_t score_b = midr_score_core(midr_b);
79 if (score_a != score_b) {
80 return score_a > score_b ? -1 : 1;
81 }
82 }
83
84 /* Compare based on core frequency (e.g. 2.0 GHz < 1.2 GHz) */
85 const uint32_t frequency_a = processor_a->max_frequency;
86 const uint32_t frequency_b = processor_b->max_frequency;
87 if (frequency_a != frequency_b) {
88 return frequency_a > frequency_b ? -1 : 1;
89 }
90
91 /* Compare based on cluster leader id (i.e. cluster 1 < cluster 0) */
92 const uint32_t cluster_a = processor_a->package_leader_id;
93 const uint32_t cluster_b = processor_b->package_leader_id;
94 if (cluster_a != cluster_b) {
95 return cluster_a > cluster_b ? -1 : 1;
96 }
97
98 /* Compare based on system processor id (i.e. processor 0 < processor 1) */
99 const uint32_t id_a = processor_a->system_processor_id;
100 const uint32_t id_b = processor_b->system_processor_id;
101 return cmp(id_a, id_b);
102 }
103
cpuinfo_arm_linux_init(void)104 void cpuinfo_arm_linux_init(void) {
105 struct cpuinfo_arm_linux_processor* arm_linux_processors = NULL;
106 struct cpuinfo_processor* processors = NULL;
107 struct cpuinfo_core* cores = NULL;
108 struct cpuinfo_cluster* clusters = NULL;
109 const struct cpuinfo_processor** linux_cpu_to_processor_map = NULL;
110 const struct cpuinfo_core** linux_cpu_to_core_map = NULL;
111 struct cpuinfo_cache* l1i = NULL;
112 struct cpuinfo_cache* l1d = NULL;
113 struct cpuinfo_cache* l2 = NULL;
114 struct cpuinfo_cache* l3 = NULL;
115
116 const uint32_t max_processors_count = cpuinfo_linux_get_max_processors_count();
117 cpuinfo_log_debug("system maximum processors count: %"PRIu32, max_processors_count);
118
119 const uint32_t max_possible_processors_count = 1 +
120 cpuinfo_linux_get_max_possible_processor(max_processors_count);
121 cpuinfo_log_debug("maximum possible processors count: %"PRIu32, max_possible_processors_count);
122 const uint32_t max_present_processors_count = 1 +
123 cpuinfo_linux_get_max_present_processor(max_processors_count);
124 cpuinfo_log_debug("maximum present processors count: %"PRIu32, max_present_processors_count);
125
126 uint32_t valid_processor_mask = 0;
127 uint32_t arm_linux_processors_count = max_processors_count;
128 if (max_present_processors_count != 0) {
129 arm_linux_processors_count = min(arm_linux_processors_count, max_present_processors_count);
130 valid_processor_mask = CPUINFO_LINUX_FLAG_PRESENT;
131 }
132 if (max_possible_processors_count != 0) {
133 arm_linux_processors_count = min(arm_linux_processors_count, max_possible_processors_count);
134 valid_processor_mask |= CPUINFO_LINUX_FLAG_POSSIBLE;
135 }
136 if ((max_present_processors_count | max_possible_processors_count) == 0) {
137 cpuinfo_log_error("failed to parse both lists of possible and present processors");
138 return;
139 }
140
141 arm_linux_processors = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_arm_linux_processor));
142 if (arm_linux_processors == NULL) {
143 cpuinfo_log_error(
144 "failed to allocate %zu bytes for descriptions of %"PRIu32" ARM logical processors",
145 arm_linux_processors_count * sizeof(struct cpuinfo_arm_linux_processor),
146 arm_linux_processors_count);
147 return;
148 }
149
150 if (max_possible_processors_count) {
151 cpuinfo_linux_detect_possible_processors(
152 arm_linux_processors_count, &arm_linux_processors->flags,
153 sizeof(struct cpuinfo_arm_linux_processor),
154 CPUINFO_LINUX_FLAG_POSSIBLE);
155 }
156
157 if (max_present_processors_count) {
158 cpuinfo_linux_detect_present_processors(
159 arm_linux_processors_count, &arm_linux_processors->flags,
160 sizeof(struct cpuinfo_arm_linux_processor),
161 CPUINFO_LINUX_FLAG_PRESENT);
162 }
163
164 #if defined(__ANDROID__)
165 struct cpuinfo_android_properties android_properties;
166 cpuinfo_arm_android_parse_properties(&android_properties);
167 #else
168 char proc_cpuinfo_hardware[CPUINFO_HARDWARE_VALUE_MAX] = { 0 };
169 #endif
170
171 if (!cpuinfo_arm_linux_parse_proc_cpuinfo(
172 #if defined(__ANDROID__)
173 android_properties.proc_cpuinfo_hardware,
174 #else
175 proc_cpuinfo_hardware,
176 #endif
177 arm_linux_processors_count,
178 arm_linux_processors)) {
179 cpuinfo_log_error("failed to parse processor information from /proc/cpuinfo");
180 return;
181 }
182
183 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
184 if (bitmask_all(arm_linux_processors[i].flags, valid_processor_mask)) {
185 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_VALID;
186 cpuinfo_log_debug("parsed processor %"PRIu32" MIDR 0x%08"PRIx32,
187 i, arm_linux_processors[i].midr);
188 }
189 }
190
191 uint32_t valid_processors = 0, last_midr = 0;
192 #if CPUINFO_ARCH_ARM
193 uint32_t last_architecture_version = 0, last_architecture_flags = 0;
194 #endif
195 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
196 arm_linux_processors[i].system_processor_id = i;
197 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
198 valid_processors += 1;
199
200 if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) {
201 /*
202 * Processor is in possible and present lists, but not reported in /proc/cpuinfo.
203 * This is fairly common: high-index processors can be not reported if they are offline.
204 */
205 cpuinfo_log_info("processor %"PRIu32" is not listed in /proc/cpuinfo", i);
206 }
207
208 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_MIDR)) {
209 last_midr = arm_linux_processors[i].midr;
210 }
211 #if CPUINFO_ARCH_ARM
212 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ARCHITECTURE)) {
213 last_architecture_version = arm_linux_processors[i].architecture_version;
214 last_architecture_flags = arm_linux_processors[i].architecture_flags;
215 }
216 #endif
217 } else {
218 /* Processor reported in /proc/cpuinfo, but not in possible and/or present lists: log and ignore */
219 if (!(arm_linux_processors[i].flags & CPUINFO_ARM_LINUX_VALID_PROCESSOR)) {
220 cpuinfo_log_warning("invalid processor %"PRIu32" reported in /proc/cpuinfo", i);
221 }
222 }
223 }
224
225 #if defined(__ANDROID__)
226 const struct cpuinfo_arm_chipset chipset =
227 cpuinfo_arm_android_decode_chipset(&android_properties, valid_processors, 0);
228 #else
229 const struct cpuinfo_arm_chipset chipset = {
230 .vendor = cpuinfo_arm_chipset_vendor_unknown,
231 .series = cpuinfo_arm_chipset_series_unknown,
232 };
233 #endif
234
235 #if CPUINFO_ARCH_ARM
236 uint32_t isa_features = 0, isa_features2 = 0;
237 #ifdef __ANDROID__
238 /*
239 * On Android before API 20, libc.so does not provide getauxval function.
240 * Thus, we try to dynamically find it, or use two fallback mechanisms:
241 * 1. dlopen libc.so, and try to find getauxval
242 * 2. Parse /proc/self/auxv procfs file
243 * 3. Use features reported in /proc/cpuinfo
244 */
245 if (!cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2)) {
246 /* getauxval can't be used, fall back to parsing /proc/self/auxv */
247 if (!cpuinfo_arm_linux_hwcap_from_procfs(&isa_features, &isa_features2)) {
248 /*
249 * Reading /proc/self/auxv failed, probably due to file permissions.
250 * Use information from /proc/cpuinfo to detect ISA.
251 *
252 * If different processors report different ISA features, take the intersection.
253 */
254 uint32_t processors_with_features = 0;
255 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
256 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID | CPUINFO_ARM_LINUX_VALID_FEATURES)) {
257 if (processors_with_features == 0) {
258 isa_features = arm_linux_processors[i].features;
259 isa_features2 = arm_linux_processors[i].features2;
260 } else {
261 isa_features &= arm_linux_processors[i].features;
262 isa_features2 &= arm_linux_processors[i].features2;
263 }
264 processors_with_features += 1;
265 }
266 }
267 }
268 }
269 #else
270 /* On GNU/Linux getauxval is always available */
271 cpuinfo_arm_linux_hwcap_from_getauxval(&isa_features, &isa_features2);
272 #endif
273 cpuinfo_arm_linux_decode_isa_from_proc_cpuinfo(
274 isa_features, isa_features2,
275 last_midr, last_architecture_version, last_architecture_flags,
276 &chipset, &cpuinfo_isa);
277 #elif CPUINFO_ARCH_ARM64
278 /* getauxval is always available on ARM64 Android */
279 const uint32_t isa_features = cpuinfo_arm_linux_hwcap_from_getauxval();
280 cpuinfo_arm64_linux_decode_isa_from_proc_cpuinfo(
281 isa_features, last_midr, &chipset, &cpuinfo_isa);
282 #endif
283
284 /* Detect min/max frequency and package ID */
285 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
286 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
287 const uint32_t max_frequency = cpuinfo_linux_get_processor_max_frequency(i);
288 if (max_frequency != 0) {
289 arm_linux_processors[i].max_frequency = max_frequency;
290 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MAX_FREQUENCY;
291 }
292
293 const uint32_t min_frequency = cpuinfo_linux_get_processor_min_frequency(i);
294 if (min_frequency != 0) {
295 arm_linux_processors[i].min_frequency = min_frequency;
296 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_MIN_FREQUENCY;
297 }
298
299 if (cpuinfo_linux_get_processor_package_id(i, &arm_linux_processors[i].package_id)) {
300 arm_linux_processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_ID;
301 }
302 }
303 }
304
305 /* Initialize topology group IDs */
306 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
307 arm_linux_processors[i].package_leader_id = i;
308 }
309
310 /* Propagate topology group IDs among siblings */
311 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
312 if (!bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
313 continue;
314 }
315
316 if (arm_linux_processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_ID) {
317 cpuinfo_linux_detect_core_siblings(
318 arm_linux_processors_count, i,
319 (cpuinfo_siblings_callback) cluster_siblings_parser,
320 arm_linux_processors);
321 }
322 }
323
324 /* Propagate all cluster IDs */
325 uint32_t clustered_processors = 0;
326 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
327 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) {
328 clustered_processors += 1;
329
330 const uint32_t package_leader_id = arm_linux_processors[i].package_leader_id;
331 if (package_leader_id < i) {
332 arm_linux_processors[i].package_leader_id = arm_linux_processors[package_leader_id].package_leader_id;
333 }
334
335 cpuinfo_log_debug("processor %"PRIu32" clustered with processor %"PRIu32" as inferred from system siblings lists",
336 i, arm_linux_processors[i].package_leader_id);
337 }
338 }
339
340 if (clustered_processors != valid_processors) {
341 /*
342 * Topology information about some or all logical processors may be unavailable, for the following reasons:
343 * - Linux kernel is too old, or configured without support for topology information in sysfs.
344 * - Core is offline, and Linux kernel is configured to not report topology for offline cores.
345 *
346 * In this case, we assign processors to clusters using two methods:
347 * - Try heuristic cluster configurations (e.g. 6-core SoC usually has 4+2 big.LITTLE configuration).
348 * - If heuristic failed, assign processors to core clusters in a sequential scan.
349 */
350 if (!cpuinfo_arm_linux_detect_core_clusters_by_heuristic(valid_processors, arm_linux_processors_count, arm_linux_processors)) {
351 cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan(arm_linux_processors_count, arm_linux_processors);
352 }
353 }
354
355 cpuinfo_arm_linux_count_cluster_processors(arm_linux_processors_count, arm_linux_processors);
356
357 const uint32_t cluster_count = cpuinfo_arm_linux_detect_cluster_midr(
358 &chipset,
359 arm_linux_processors_count, valid_processors, arm_linux_processors);
360
361 /* Initialize core vendor, uarch, MIDR, and frequency for every logical processor */
362 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
363 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
364 const uint32_t cluster_leader = arm_linux_processors[i].package_leader_id;
365 if (cluster_leader == i) {
366 /* Cluster leader: decode core vendor and uarch */
367 cpuinfo_arm_decode_vendor_uarch(
368 arm_linux_processors[cluster_leader].midr,
369 #if CPUINFO_ARCH_ARM
370 !!(arm_linux_processors[cluster_leader].features & CPUINFO_ARM_LINUX_FEATURE_VFPV4),
371 #endif
372 &arm_linux_processors[cluster_leader].vendor,
373 &arm_linux_processors[cluster_leader].uarch);
374 } else {
375 /* Cluster non-leader: copy vendor, uarch, MIDR, and frequency from cluster leader */
376 arm_linux_processors[i].flags |= arm_linux_processors[cluster_leader].flags &
377 (CPUINFO_ARM_LINUX_VALID_MIDR | CPUINFO_LINUX_FLAG_MAX_FREQUENCY);
378 arm_linux_processors[i].midr = arm_linux_processors[cluster_leader].midr;
379 arm_linux_processors[i].vendor = arm_linux_processors[cluster_leader].vendor;
380 arm_linux_processors[i].uarch = arm_linux_processors[cluster_leader].uarch;
381 arm_linux_processors[i].max_frequency = arm_linux_processors[cluster_leader].max_frequency;
382 }
383 }
384 }
385
386 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
387 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
388 cpuinfo_log_debug("post-analysis processor %"PRIu32": MIDR %08"PRIx32" frequency %"PRIu32,
389 i, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency);
390 }
391 }
392
393 qsort(arm_linux_processors, arm_linux_processors_count,
394 sizeof(struct cpuinfo_arm_linux_processor), cmp_arm_linux_processor);
395
396 for (uint32_t i = 0; i < arm_linux_processors_count; i++) {
397 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {
398 cpuinfo_log_debug("post-sort processor %"PRIu32": system id %"PRIu32" MIDR %08"PRIx32" frequency %"PRIu32,
399 i, arm_linux_processors[i].system_processor_id, arm_linux_processors[i].midr, arm_linux_processors[i].max_frequency);
400 }
401 }
402
403 /*
404 * Assumptions:
405 * - No SMP (i.e. each core supports only one hardware thread).
406 * - Level 1 instruction and data caches are private to the core clusters.
407 * - Level 2 and level 3 cache is shared between cores in the same cluster.
408 */
409 cpuinfo_arm_chipset_to_string(&chipset, package.name);
410 package.processor_count = valid_processors;
411 package.core_count = valid_processors;
412 package.cluster_count = cluster_count;
413
414 processors = calloc(valid_processors, sizeof(struct cpuinfo_processor));
415 if (processors == NULL) {
416 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" logical processors",
417 valid_processors * sizeof(struct cpuinfo_processor), valid_processors);
418 goto cleanup;
419 }
420
421 cores = calloc(valid_processors, sizeof(struct cpuinfo_core));
422 if (cores == NULL) {
423 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" cores",
424 valid_processors * sizeof(struct cpuinfo_core), valid_processors);
425 goto cleanup;
426 }
427
428 clusters = calloc(cluster_count, sizeof(struct cpuinfo_cluster));
429 if (clusters == NULL) {
430 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" core clusters",
431 cluster_count * sizeof(struct cpuinfo_cluster), cluster_count);
432 goto cleanup;
433 }
434
435 linux_cpu_to_processor_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_processor*));
436 if (linux_cpu_to_processor_map == NULL) {
437 cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" logical processor mapping entries",
438 arm_linux_processors_count * sizeof(struct cpuinfo_processor*), arm_linux_processors_count);
439 goto cleanup;
440 }
441
442 linux_cpu_to_core_map = calloc(arm_linux_processors_count, sizeof(struct cpuinfo_core*));
443 if (linux_cpu_to_core_map == NULL) {
444 cpuinfo_log_error("failed to allocate %zu bytes for %"PRIu32" core mapping entries",
445 arm_linux_processors_count * sizeof(struct cpuinfo_core*), arm_linux_processors_count);
446 goto cleanup;
447 }
448
449 l1i = calloc(valid_processors, sizeof(struct cpuinfo_cache));
450 if (l1i == NULL) {
451 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1I caches",
452 valid_processors * sizeof(struct cpuinfo_cache), valid_processors);
453 goto cleanup;
454 }
455
456 l1d = calloc(valid_processors, sizeof(struct cpuinfo_cache));
457 if (l1d == NULL) {
458 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L1D caches",
459 valid_processors * sizeof(struct cpuinfo_cache), valid_processors);
460 goto cleanup;
461 }
462
463 uint32_t l2_count = 0, l3_count = 0, big_l3_size = 0, cluster_id = UINT32_MAX;
464 /* Indication whether L3 (if it exists) is shared between all cores */
465 bool shared_l3 = true;
466 /* Populate cache infromation structures in l1i, l1d */
467 for (uint32_t i = 0; i < valid_processors; i++) {
468 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
469 cluster_id += 1;
470 clusters[cluster_id] = (struct cpuinfo_cluster) {
471 .processor_start = i,
472 .processor_count = arm_linux_processors[i].package_processor_count,
473 .core_start = i,
474 .core_count = arm_linux_processors[i].package_processor_count,
475 .cluster_id = cluster_id,
476 .package = &package,
477 .vendor = arm_linux_processors[i].vendor,
478 .uarch = arm_linux_processors[i].uarch,
479 .midr = arm_linux_processors[i].midr,
480 };
481 }
482
483 processors[i].smt_id = 0;
484 processors[i].core = cores + i;
485 processors[i].cluster = clusters + cluster_id;
486 processors[i].package = &package;
487 processors[i].linux_id = (int) arm_linux_processors[i].system_processor_id;
488 processors[i].cache.l1i = l1i + i;
489 processors[i].cache.l1d = l1d + i;
490 linux_cpu_to_processor_map[arm_linux_processors[i].system_processor_id] = &processors[i];
491
492 cores[i].processor_start = i;
493 cores[i].processor_count = 1;
494 cores[i].core_id = i;
495 cores[i].cluster = clusters + cluster_id;
496 cores[i].package = &package;
497 cores[i].vendor = arm_linux_processors[i].vendor;
498 cores[i].uarch = arm_linux_processors[i].uarch;
499 cores[i].midr = arm_linux_processors[i].midr;
500 linux_cpu_to_core_map[arm_linux_processors[i].system_processor_id] = &cores[i];
501
502 struct cpuinfo_cache temp_l2 = { 0 }, temp_l3 = { 0 };
503 cpuinfo_arm_decode_cache(
504 arm_linux_processors[i].uarch,
505 arm_linux_processors[i].package_processor_count,
506 arm_linux_processors[i].midr,
507 &chipset,
508 cluster_id,
509 arm_linux_processors[i].architecture_version,
510 &l1i[i], &l1d[i], &temp_l2, &temp_l3);
511 l1i[i].processor_start = l1d[i].processor_start = i;
512 l1i[i].processor_count = l1d[i].processor_count = 1;
513 #if CPUINFO_ARCH_ARM
514 /* L1I reported in /proc/cpuinfo overrides defaults */
515 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_ICACHE)) {
516 l1i[i] = (struct cpuinfo_cache) {
517 .size = arm_linux_processors[i].proc_cpuinfo_cache.i_size,
518 .associativity = arm_linux_processors[i].proc_cpuinfo_cache.i_assoc,
519 .sets = arm_linux_processors[i].proc_cpuinfo_cache.i_sets,
520 .partitions = 1,
521 .line_size = arm_linux_processors[i].proc_cpuinfo_cache.i_line_length
522 };
523 }
524 /* L1D reported in /proc/cpuinfo overrides defaults */
525 if (bitmask_all(arm_linux_processors[i].flags, CPUINFO_ARM_LINUX_VALID_DCACHE)) {
526 l1d[i] = (struct cpuinfo_cache) {
527 .size = arm_linux_processors[i].proc_cpuinfo_cache.d_size,
528 .associativity = arm_linux_processors[i].proc_cpuinfo_cache.d_assoc,
529 .sets = arm_linux_processors[i].proc_cpuinfo_cache.d_sets,
530 .partitions = 1,
531 .line_size = arm_linux_processors[i].proc_cpuinfo_cache.d_line_length
532 };
533 }
534 #endif
535
536 if (temp_l3.size != 0) {
537 /*
538 * Assumptions:
539 * - L2 is private to each core
540 * - L3 is shared by cores in the same cluster
541 * - If cores in different clusters report the same L3, it is shared between all cores.
542 */
543 l2_count += 1;
544 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
545 if (cluster_id == 0) {
546 big_l3_size = temp_l3.size;
547 l3_count = 1;
548 } else if (temp_l3.size != big_l3_size) {
549 /* If some cores have different L3 size, L3 is not shared between all cores */
550 shared_l3 = false;
551 l3_count += 1;
552 }
553 }
554 } else {
555 /* If some cores don't have L3 cache, L3 is not shared between all cores */
556 shared_l3 = false;
557 if (temp_l2.size != 0) {
558 /* Assume L2 is shared by cores in the same cluster */
559 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
560 l2_count += 1;
561 }
562 }
563 }
564 }
565
566 if (l2_count != 0) {
567 l2 = calloc(l2_count, sizeof(struct cpuinfo_cache));
568 if (l2 == NULL) {
569 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L2 caches",
570 l2_count * sizeof(struct cpuinfo_cache), l2_count);
571 goto cleanup;
572 }
573
574 if (l3_count != 0) {
575 l3 = calloc(l3_count, sizeof(struct cpuinfo_cache));
576 if (l3 == NULL) {
577 cpuinfo_log_error("failed to allocate %zu bytes for descriptions of %"PRIu32" L3 caches",
578 l3_count * sizeof(struct cpuinfo_cache), l3_count);
579 goto cleanup;
580 }
581 }
582 }
583
584 cluster_id = UINT32_MAX;
585 uint32_t l2_index = UINT32_MAX, l3_index = UINT32_MAX;
586 for (uint32_t i = 0; i < valid_processors; i++) {
587 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
588 cluster_id++;
589 }
590
591 struct cpuinfo_cache dummy_l1i, dummy_l1d, temp_l2 = { 0 }, temp_l3 = { 0 };
592 cpuinfo_arm_decode_cache(
593 arm_linux_processors[i].uarch,
594 arm_linux_processors[i].package_processor_count,
595 arm_linux_processors[i].midr,
596 &chipset,
597 cluster_id,
598 arm_linux_processors[i].architecture_version,
599 &dummy_l1i, &dummy_l1d, &temp_l2, &temp_l3);
600
601 if (temp_l3.size != 0) {
602 /*
603 * Assumptions:
604 * - L2 is private to each core
605 * - L3 is shared by cores in the same cluster
606 * - If cores in different clusters report the same L3, it is shared between all cores.
607 */
608 l2_index += 1;
609 l2[l2_index] = (struct cpuinfo_cache) {
610 .size = temp_l2.size,
611 .associativity = temp_l2.associativity,
612 .sets = temp_l2.sets,
613 .partitions = 1,
614 .line_size = temp_l2.line_size,
615 .flags = temp_l2.flags,
616 .processor_start = i,
617 .processor_count = 1,
618 };
619 processors[i].cache.l2 = l2 + l2_index;
620 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
621 l3_index += 1;
622 if (l3_index < l3_count) {
623 l3[l3_index] = (struct cpuinfo_cache) {
624 .size = temp_l3.size,
625 .associativity = temp_l3.associativity,
626 .sets = temp_l3.sets,
627 .partitions = 1,
628 .line_size = temp_l3.line_size,
629 .flags = temp_l3.flags,
630 .processor_start = i,
631 .processor_count =
632 shared_l3 ? valid_processors : arm_linux_processors[i].package_processor_count,
633 };
634 }
635 }
636 if (shared_l3) {
637 processors[i].cache.l3 = l3;
638 } else if (l3_index < l3_count) {
639 processors[i].cache.l3 = l3 + l3_index;
640 }
641 } else if (temp_l2.size != 0) {
642 /* Assume L2 is shared by cores in the same cluster */
643 if (arm_linux_processors[i].package_leader_id == arm_linux_processors[i].system_processor_id) {
644 l2_index += 1;
645 l2[l2_index] = (struct cpuinfo_cache) {
646 .size = temp_l2.size,
647 .associativity = temp_l2.associativity,
648 .sets = temp_l2.sets,
649 .partitions = 1,
650 .line_size = temp_l2.line_size,
651 .flags = temp_l2.flags,
652 .processor_start = i,
653 .processor_count = arm_linux_processors[i].package_processor_count,
654 };
655 }
656 processors[i].cache.l2 = l2 + l2_index;
657 }
658 }
659
660 /* Commit */
661 cpuinfo_linux_cpu_to_processor_map = linux_cpu_to_processor_map;
662 cpuinfo_linux_cpu_to_core_map = linux_cpu_to_core_map;
663 cpuinfo_processors = processors;
664 cpuinfo_cores = cores;
665 cpuinfo_clusters = clusters;
666 cpuinfo_packages = &package;
667 cpuinfo_cache[cpuinfo_cache_level_1i] = l1i;
668 cpuinfo_cache[cpuinfo_cache_level_1d] = l1d;
669 cpuinfo_cache[cpuinfo_cache_level_2] = l2;
670 cpuinfo_cache[cpuinfo_cache_level_3] = l3;
671
672 cpuinfo_processors_count = valid_processors;
673 cpuinfo_cores_count = valid_processors;
674 cpuinfo_clusters_count = cluster_count;
675 cpuinfo_packages_count = 1;
676 cpuinfo_cache_count[cpuinfo_cache_level_1i] = valid_processors;
677 cpuinfo_cache_count[cpuinfo_cache_level_1d] = valid_processors;
678 cpuinfo_cache_count[cpuinfo_cache_level_2] = l2_count;
679 cpuinfo_cache_count[cpuinfo_cache_level_3] = l3_count;
680
681 cpuinfo_max_cache_size = cpuinfo_arm_compute_max_cache_size(&processors[0]);
682
683 __sync_synchronize();
684
685 cpuinfo_is_initialized = true;
686
687 linux_cpu_to_processor_map = NULL;
688 linux_cpu_to_core_map = NULL;
689 processors = NULL;
690 cores = NULL;
691 clusters = NULL;
692 l1i = l1d = l2 = l3 = NULL;
693
694 cleanup:
695 free(arm_linux_processors);
696 free(linux_cpu_to_processor_map);
697 free(linux_cpu_to_core_map);
698 free(processors);
699 free(cores);
700 free(clusters);
701 free(l1i);
702 free(l1d);
703 free(l2);
704 free(l3);
705 }
706