22 …* Assigns logical processors to clusters of cores using heuristic based on the typical configurati…
34 …*     configuration of logical processors 4-7 is not reported (all remaining processors 4-7 form c…
36 …*     of logical processors 4-9 is not reported (processors 4-7 form cluster 1, and processors 8-9…
38 …* Heuristic assignment of processors to the above pre-defined clusters fails if such assignment wo…
41 …* - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have k…
43 …* - Processors in a heuristic cluster have no OS-provided cluster siblings information, but have k…
46 …* If the heuristic assignment of processors to clusters of cores fails, all processors' clusters a…
48 …* @param usable_processors - number of processors in the @p processors array with CPUINFO_LINUX_FL…
49  * @param max_processors - number of elements in the @p processors array.
50 …* @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, mi…
53  * @retval true if the heuristic successfully assigned all processors into clusters of cores.
54 …* @retval false if known details about processors contradict the heuristic configuration of core c…
59 	struct cpuinfo_arm_linux_processor processors[restrict static max_processors])  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
91 	 * Assignment of processors to core clusters is done in two passes:  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
92 … Verify that the clusters proposed by heuristic are compatible with known details about processors.  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
93 	 * 2. If verification passed, update core clusters for the processors.  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
101 		if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
105 				expected_cluster_exists = !!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
107 					if (processors[i].package_leader_id != i) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
112 							i, cluster, cluster_processors[cluster], processors[i].package_leader_id);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
127 					 * For all processors we expect in the cluster, check that:  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
133 					if ((processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) == 0) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
141 					if (processors[i].package_leader_id != cluster_start) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
152 					 * For all processors we expect in the cluster, check that:  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
158 					if (processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
163 							i, processors[i].package_leader_id);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
167 					if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
169 							if (cluster_min_frequency != processors[i].min_frequency) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
173 									i, processors[i].min_frequency, cluster_min_frequency);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
177 							cluster_min_frequency = processors[i].min_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
182 					if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
184 							if (cluster_max_frequency != processors[i].max_frequency) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
188 									i, processors[i].max_frequency, cluster_max_frequency);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
192 							cluster_max_frequency = processors[i].max_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
197 					if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
199 …if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_I…  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
203 									i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr));  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
207 							cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
212 					if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
214 …if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIA…  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
218 									i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr));  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
222 							cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
227 					if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
229 …if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MAS…  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
233 									i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr));  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
237 							cluster_midr = midr_copy_part(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
242 					if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
244 …if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVI…  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
248 									i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr));  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
252 							cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
262 	/* Verification passed, assign all processors to new clusters */  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
266 		if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
275 				if (!(processors[i].flags & CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) {  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
280 				processors[i].package_leader_id = cluster_start;  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
281 				processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;  in cpuinfo_arm_linux_detect_core_clusters_by_heuristic()
290  * Assigns logical processors to clusters of cores in sequential manner:
292  *   - Processors assigned to these clusters stay assigned to the same clusters
293  *   - No new processors are added to these clusters
294  * - Processors without pre-assigned cluster are clustered in one sequential scan:
301 …* if heuristic assignment of processors to clusters (cpuinfo_arm_linux_cluster_processors_by_heuri…
307 …* Sequential assignment of processors to clusters always succeeds, and upon exit, all usable proce…
308  * @p processors array have cluster information.
310  * @param max_processors - number of elements in the @p processors array.
311 …* @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags, mi…
314  * @retval true if the heuristic successfully assigned all processors into clusters of cores.
315 …* @retval false if known details about processors contradict the heuristic configuration of core c…
319 	struct cpuinfo_arm_linux_processor processors[restrict static max_processors])  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
325 …if ((processors[i].flags & (CPUINFO_LINUX_FLAG_VALID | CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER)) == CPU…  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
330 			if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
332 					if (cluster_min_frequency != processors[i].min_frequency) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
336 							i, processors[i].min_frequency, cluster_min_frequency, i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
340 					cluster_min_frequency = processors[i].min_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
345 			if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
347 					if (cluster_max_frequency != processors[i].max_frequency) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
351 							i, processors[i].max_frequency, cluster_max_frequency, i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
355 					cluster_max_frequency = processors[i].max_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
360 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
362 …if ((cluster_midr & CPUINFO_ARM_MIDR_IMPLEMENTER_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_I…  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
366 							i, midr_get_implementer(processors[i].midr), midr_get_implementer(cluster_midr), i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
370 					cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
375 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
377 …if ((cluster_midr & CPUINFO_ARM_MIDR_VARIANT_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_VARIA…  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
381 							i, midr_get_variant(processors[i].midr), midr_get_variant(cluster_midr), i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
385 					cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
390 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
392 …if ((cluster_midr & CPUINFO_ARM_MIDR_PART_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_PART_MAS…  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
396 							i, midr_get_part(processors[i].midr), midr_get_part(cluster_midr), i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
400 					cluster_midr = midr_copy_part(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
405 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
407 …if ((cluster_midr & CPUINFO_ARM_MIDR_REVISION_MASK) != (processors[i].midr & CPUINFO_ARM_MIDR_REVI…  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
411 							i, midr_get_revision(cluster_midr), midr_get_revision(processors[i].midr), i);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
415 					cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
422 			processors[i].package_leader_id = cluster_start;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
423 			processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
430 			processors[i].package_leader_id = i;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
431 			processors[i].flags |= CPUINFO_LINUX_FLAG_PACKAGE_CLUSTER;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
436 			if (processors[i].flags & CPUINFO_LINUX_FLAG_MIN_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
437 				cluster_min_frequency = processors[i].min_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
440 			if (processors[i].flags & CPUINFO_LINUX_FLAG_MAX_FREQUENCY) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
441 				cluster_max_frequency = processors[i].max_frequency;  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
444 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_IMPLEMENTER) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
445 				cluster_midr = midr_copy_implementer(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
448 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_VARIANT) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
449 				cluster_midr = midr_copy_variant(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
452 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_PART) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
453 				cluster_midr = midr_copy_part(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
456 			if (processors[i].flags & CPUINFO_ARM_LINUX_VALID_REVISION) {  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
457 				cluster_midr = midr_copy_revision(cluster_midr, processors[i].midr);  in cpuinfo_arm_linux_detect_core_clusters_by_sequential_scan()
465  * Counts the number of logical processors in each core cluster.
466  * This function should be called after all processors are assigned to core clusters.
468  * @param max_processors - number of elements in the @p processors array.
469  * @param[in,out] processors - processor descriptors with pre-parsed POSSIBLE and PRESENT flags,
471 …*                             The function expects the value of processors[i].package_processor_co…
472 …*                             Upon return, processors[i].package_processor_count will contain the …
473  *                             processors in the respective core cluster.
477 	struct cpuinfo_arm_linux_processor processors[restrict static max_processors])  in cpuinfo_arm_linux_count_cluster_processors()
479 	/* First pass: accumulate the number of processors at the group leader's package_processor_count */  in cpuinfo_arm_linux_count_cluster_processors()
481 		if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {  in cpuinfo_arm_linux_count_cluster_processors()
482 			const uint32_t package_leader_id = processors[i].package_leader_id;  in cpuinfo_arm_linux_count_cluster_processors()
483 			processors[package_leader_id].package_processor_count += 1;  in cpuinfo_arm_linux_count_cluster_processors()
488 		if (bitmask_all(processors[i].flags, CPUINFO_LINUX_FLAG_VALID)) {  in cpuinfo_arm_linux_count_cluster_processors()
489 			const uint32_t package_leader_id = processors[i].package_leader_id;  in cpuinfo_arm_linux_count_cluster_processors()
490 			processors[i].package_processor_count = processors[package_leader_id].package_processor_count;  in cpuinfo_arm_linux_count_cluster_processors()