1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * turbostat -- show CPU frequency and C-state residency
4 * on modern Intel and AMD processors.
5 *
6 * Copyright (c) 2024 Intel Corporation.
7 * Len Brown <len.brown@intel.com>
8 */
9
10 #define _GNU_SOURCE
11 #include MSRHEADER
12
13 // copied from arch/x86/include/asm/cpu_device_id.h
14 #define VFM_MODEL_BIT 0
15 #define VFM_FAMILY_BIT 8
16 #define VFM_VENDOR_BIT 16
17 #define VFM_RSVD_BIT 24
18
19 #define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT)
20 #define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT)
21 #define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT)
22
23 #define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT)
24 #define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT)
25 #define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT)
26
27 #define VFM_MAKE(_vendor, _family, _model) ( \
28 ((_model) << VFM_MODEL_BIT) | \
29 ((_family) << VFM_FAMILY_BIT) | \
30 ((_vendor) << VFM_VENDOR_BIT) \
31 )
32 // end copied section
33
34 #define CPUID_LEAF_MODEL_ID 0x1A
35 #define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24
36
37 #define X86_VENDOR_INTEL 0
38
39 #include INTEL_FAMILY_HEADER
40 #include BUILD_BUG_HEADER
41 #include <stdarg.h>
42 #include <stdio.h>
43 #include <err.h>
44 #include <unistd.h>
45 #include <sys/types.h>
46 #include <sys/wait.h>
47 #include <sys/stat.h>
48 #include <sys/select.h>
49 #include <sys/resource.h>
50 #include <sys/mman.h>
51 #include <fcntl.h>
52 #include <signal.h>
53 #include <sys/time.h>
54 #include <stdlib.h>
55 #include <getopt.h>
56 #include <dirent.h>
57 #include <string.h>
58 #include <ctype.h>
59 #include <sched.h>
60 #include <time.h>
61 #include <cpuid.h>
62 #include <sys/capability.h>
63 #include <errno.h>
64 #include <math.h>
65 #include <linux/perf_event.h>
66 #include <asm/unistd.h>
67 #include <stdbool.h>
68 #include <assert.h>
69 #include <linux/kernel.h>
70 #include <limits.h>
71
72 #define UNUSED(x) (void)(x)
73
74 /*
75 * This list matches the column headers, except
76 * 1. built-in only, the sysfs counters are not here -- we learn of those at run-time
77 * 2. Core and CPU are moved to the end, we can't have strings that contain them
78 * matching on them for --show and --hide.
79 */
80
81 /*
82 * buffer size used by sscanf() for added column names
83 * Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
84 */
85 #define NAME_BYTES 20
86 #define PATH_BYTES 128
87 #define PERF_NAME_BYTES 128
88
89 #define MAX_NOFILE 0x8000
90
91 #define COUNTER_KIND_PERF_PREFIX "perf/"
92 #define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX)
93 #define PERF_DEV_NAME_BYTES 32
94 #define PERF_EVT_NAME_BYTES 32
95
96 #define INTEL_ECORE_TYPE 0x20
97 #define INTEL_PCORE_TYPE 0x40
98
99 #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
100
101 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
102 enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
103 enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
104 enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR };
105
106 struct perf_counter_info {
107 struct perf_counter_info *next;
108
109 /* How to open the counter / What counter it is. */
110 char device[PERF_DEV_NAME_BYTES];
111 char event[PERF_EVT_NAME_BYTES];
112
113 /* How to show/format the counter. */
114 char name[PERF_NAME_BYTES];
115 unsigned int width;
116 enum counter_scope scope;
117 enum counter_type type;
118 enum counter_format format;
119 double scale;
120
121 /* For reading the counter. */
122 int *fd_perf_per_domain;
123 size_t num_domains;
124 };
125
126 struct sysfs_path {
127 char path[PATH_BYTES];
128 int id;
129 struct sysfs_path *next;
130 };
131
132 struct msr_counter {
133 unsigned int msr_num;
134 char name[NAME_BYTES];
135 struct sysfs_path *sp;
136 unsigned int width;
137 enum counter_type type;
138 enum counter_format format;
139 struct msr_counter *next;
140 unsigned int flags;
141 #define FLAGS_HIDE (1 << 0)
142 #define FLAGS_SHOW (1 << 1)
143 #define SYSFS_PERCPU (1 << 1)
144 };
145
146 struct msr_counter bic[] = {
147 { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 },
148 { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 },
149 { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 },
150 { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 },
151 { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 },
152 { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 },
153 { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 },
154 { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 },
155 { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 },
156 { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 },
157 { 0x0, "sysfs", NULL, 0, 0, 0, NULL, 0 },
158 { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 },
159 { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 },
160 { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 },
161 { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 },
162 { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 },
163 { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 },
164 { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 },
165 { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 },
166 { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 },
167 { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 },
168 { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 },
169 { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 },
170 { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 },
171 { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 },
172 { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 },
173 { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 },
174 { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 },
175 { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 },
176 { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 },
177 { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 },
178 { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 },
179 { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 },
180 { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 },
181 { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 },
182 { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 },
183 { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 },
184 { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 },
185 { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 },
186 { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 },
187 { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 },
188 { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 },
189 { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 },
190 { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 },
191 { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 },
192 { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 },
193 { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 },
194 { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 },
195 { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 },
196 { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 },
197 { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 },
198 { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 },
199 { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 },
200 { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 },
201 { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 },
202 { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 },
203 { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 },
204 { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 },
205 { 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 },
206 };
207
208 #define MAX_BIC (sizeof(bic) / sizeof(struct msr_counter))
209 #define BIC_USEC (1ULL << 0)
210 #define BIC_TOD (1ULL << 1)
211 #define BIC_Package (1ULL << 2)
212 #define BIC_Node (1ULL << 3)
213 #define BIC_Avg_MHz (1ULL << 4)
214 #define BIC_Busy (1ULL << 5)
215 #define BIC_Bzy_MHz (1ULL << 6)
216 #define BIC_TSC_MHz (1ULL << 7)
217 #define BIC_IRQ (1ULL << 8)
218 #define BIC_SMI (1ULL << 9)
219 #define BIC_sysfs (1ULL << 10)
220 #define BIC_CPU_c1 (1ULL << 11)
221 #define BIC_CPU_c3 (1ULL << 12)
222 #define BIC_CPU_c6 (1ULL << 13)
223 #define BIC_CPU_c7 (1ULL << 14)
224 #define BIC_ThreadC (1ULL << 15)
225 #define BIC_CoreTmp (1ULL << 16)
226 #define BIC_CoreCnt (1ULL << 17)
227 #define BIC_PkgTmp (1ULL << 18)
228 #define BIC_GFX_rc6 (1ULL << 19)
229 #define BIC_GFXMHz (1ULL << 20)
230 #define BIC_Pkgpc2 (1ULL << 21)
231 #define BIC_Pkgpc3 (1ULL << 22)
232 #define BIC_Pkgpc6 (1ULL << 23)
233 #define BIC_Pkgpc7 (1ULL << 24)
234 #define BIC_Pkgpc8 (1ULL << 25)
235 #define BIC_Pkgpc9 (1ULL << 26)
236 #define BIC_Pkgpc10 (1ULL << 27)
237 #define BIC_CPU_LPI (1ULL << 28)
238 #define BIC_SYS_LPI (1ULL << 29)
239 #define BIC_PkgWatt (1ULL << 30)
240 #define BIC_CorWatt (1ULL << 31)
241 #define BIC_GFXWatt (1ULL << 32)
242 #define BIC_PkgCnt (1ULL << 33)
243 #define BIC_RAMWatt (1ULL << 34)
244 #define BIC_PKG__ (1ULL << 35)
245 #define BIC_RAM__ (1ULL << 36)
246 #define BIC_Pkg_J (1ULL << 37)
247 #define BIC_Cor_J (1ULL << 38)
248 #define BIC_GFX_J (1ULL << 39)
249 #define BIC_RAM_J (1ULL << 40)
250 #define BIC_Mod_c6 (1ULL << 41)
251 #define BIC_Totl_c0 (1ULL << 42)
252 #define BIC_Any_c0 (1ULL << 43)
253 #define BIC_GFX_c0 (1ULL << 44)
254 #define BIC_CPUGFX (1ULL << 45)
255 #define BIC_Core (1ULL << 46)
256 #define BIC_CPU (1ULL << 47)
257 #define BIC_APIC (1ULL << 48)
258 #define BIC_X2APIC (1ULL << 49)
259 #define BIC_Die (1ULL << 50)
260 #define BIC_GFXACTMHz (1ULL << 51)
261 #define BIC_IPC (1ULL << 52)
262 #define BIC_CORE_THROT_CNT (1ULL << 53)
263 #define BIC_UNCORE_MHZ (1ULL << 54)
264 #define BIC_SAM_mc6 (1ULL << 55)
265 #define BIC_SAMMHz (1ULL << 56)
266 #define BIC_SAMACTMHz (1ULL << 57)
267 #define BIC_Diec6 (1ULL << 58)
268
269 #define BIC_TOPOLOGY (BIC_Package | BIC_Node | BIC_CoreCnt | BIC_PkgCnt | BIC_Core | BIC_CPU | BIC_Die )
270 #define BIC_THERMAL_PWR ( BIC_CoreTmp | BIC_PkgTmp | BIC_PkgWatt | BIC_CorWatt | BIC_GFXWatt | BIC_RAMWatt | BIC_PKG__ | BIC_RAM__)
271 #define BIC_FREQUENCY (BIC_Avg_MHz | BIC_Busy | BIC_Bzy_MHz | BIC_TSC_MHz | BIC_GFXMHz | BIC_GFXACTMHz | BIC_SAMMHz | BIC_SAMACTMHz | BIC_UNCORE_MHZ)
272 #define BIC_IDLE (BIC_sysfs | BIC_CPU_c1 | BIC_CPU_c3 | BIC_CPU_c6 | BIC_CPU_c7 | BIC_GFX_rc6 | BIC_Pkgpc2 | BIC_Pkgpc3 | BIC_Pkgpc6 | BIC_Pkgpc7 | BIC_Pkgpc8 | BIC_Pkgpc9 | BIC_Pkgpc10 | BIC_CPU_LPI | BIC_SYS_LPI | BIC_Mod_c6 | BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_SAM_mc6 | BIC_Diec6)
273 #define BIC_OTHER ( BIC_IRQ | BIC_SMI | BIC_ThreadC | BIC_CoreTmp | BIC_IPC)
274
275 #define BIC_DISABLED_BY_DEFAULT (BIC_USEC | BIC_TOD | BIC_APIC | BIC_X2APIC)
276
277 unsigned long long bic_enabled = (0xFFFFFFFFFFFFFFFFULL & ~BIC_DISABLED_BY_DEFAULT);
278 unsigned long long bic_present = BIC_USEC | BIC_TOD | BIC_sysfs | BIC_APIC | BIC_X2APIC;
279
280 #define DO_BIC(COUNTER_NAME) (bic_enabled & bic_present & COUNTER_NAME)
281 #define DO_BIC_READ(COUNTER_NAME) (bic_present & COUNTER_NAME)
282 #define ENABLE_BIC(COUNTER_NAME) (bic_enabled |= COUNTER_NAME)
283 #define BIC_PRESENT(COUNTER_BIT) (bic_present |= COUNTER_BIT)
284 #define BIC_NOT_PRESENT(COUNTER_BIT) (bic_present &= ~COUNTER_BIT)
285 #define BIC_IS_ENABLED(COUNTER_BIT) (bic_enabled & COUNTER_BIT)
286
287 /*
288 * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit:
289 * If you change the values, note they are used both in comparisons
290 * (>= PCL__7) and to index pkg_cstate_limit_strings[].
291 */
292 #define PCLUKN 0 /* Unknown */
293 #define PCLRSV 1 /* Reserved */
294 #define PCL__0 2 /* PC0 */
295 #define PCL__1 3 /* PC1 */
296 #define PCL__2 4 /* PC2 */
297 #define PCL__3 5 /* PC3 */
298 #define PCL__4 6 /* PC4 */
299 #define PCL__6 7 /* PC6 */
300 #define PCL_6N 8 /* PC6 No Retention */
301 #define PCL_6R 9 /* PC6 Retention */
302 #define PCL__7 10 /* PC7 */
303 #define PCL_7S 11 /* PC7 Shrink */
304 #define PCL__8 12 /* PC8 */
305 #define PCL__9 13 /* PC9 */
306 #define PCL_10 14 /* PC10 */
307 #define PCLUNL 15 /* Unlimited */
308
309 struct amperf_group_fd;
310
311 char *proc_stat = "/proc/stat";
312 FILE *outf;
313 int *fd_percpu;
314 int *fd_instr_count_percpu;
315 struct timeval interval_tv = { 5, 0 };
316 struct timespec interval_ts = { 5, 0 };
317
318 unsigned int num_iterations;
319 unsigned int header_iterations;
320 unsigned int debug;
321 unsigned int quiet;
322 unsigned int shown;
323 unsigned int sums_need_wide_columns;
324 unsigned int rapl_joules;
325 unsigned int summary_only;
326 unsigned int list_header_only;
327 unsigned int dump_only;
328 unsigned int has_aperf;
329 unsigned int has_aperf_access;
330 unsigned int has_epb;
331 unsigned int has_turbo;
332 unsigned int is_hybrid;
333 unsigned int units = 1000000; /* MHz etc */
334 unsigned int genuine_intel;
335 unsigned int authentic_amd;
336 unsigned int hygon_genuine;
337 unsigned int max_level, max_extended_level;
338 unsigned int has_invariant_tsc;
339 unsigned int aperf_mperf_multiplier = 1;
340 double bclk;
341 double base_hz;
342 unsigned int has_base_hz;
343 double tsc_tweak = 1.0;
344 unsigned int show_pkg_only;
345 unsigned int show_core_only;
346 char *output_buffer, *outp;
347 unsigned int do_dts;
348 unsigned int do_ptm;
349 unsigned int do_ipc;
350 unsigned long long cpuidle_cur_cpu_lpi_us;
351 unsigned long long cpuidle_cur_sys_lpi_us;
352 unsigned int tj_max;
353 unsigned int tj_max_override;
354 double rapl_power_units, rapl_time_units;
355 double rapl_dram_energy_units, rapl_energy_units;
356 double rapl_joule_counter_range;
357 unsigned int crystal_hz;
358 unsigned long long tsc_hz;
359 int base_cpu;
360 unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
361 /* IA32_HWP_REQUEST, IA32_HWP_STATUS */
362 unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */
363 unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
364 unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
365 unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
366 unsigned int first_counter_read = 1;
367 int ignore_stdin;
368 bool no_msr;
369 bool no_perf;
370
371 enum gfx_sysfs_idx {
372 GFX_rc6,
373 GFX_MHz,
374 GFX_ACTMHz,
375 SAM_mc6,
376 SAM_MHz,
377 SAM_ACTMHz,
378 GFX_MAX
379 };
380
381 struct gfx_sysfs_info {
382 const char *path;
383 FILE *fp;
384 unsigned int val;
385 unsigned long long val_ull;
386 };
387
388 static struct gfx_sysfs_info gfx_info[GFX_MAX];
389
390 int get_msr(int cpu, off_t offset, unsigned long long *msr);
391 int add_counter(unsigned int msr_num, char *path, char *name,
392 unsigned int width, enum counter_scope scope,
393 enum counter_type type, enum counter_format format, int flags, int package_num);
394
395 /* Model specific support Start */
396
397 /* List of features that may diverge among different platforms */
398 struct platform_features {
399 bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */
400 bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */
401 bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */
402 bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */
403 int bclk_freq; /* CPU base clock */
404 int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */
405 int supported_cstates; /* Core cstates and Package cstates supported */
406 int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */
407 bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */
408 bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */
409 bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */
410 bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */
411 bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */
412 bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */
413 bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */
414 bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */
415 bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */
416 int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */
417 int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */
418 int rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */
419 bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */
420 bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */
421 bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */
422 int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */
423 int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */
424 bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */
425 bool need_perf_multiplier; /* mperf/aperf multiplier */
426 };
427
428 struct platform_data {
429 unsigned int vfm;
430 const struct platform_features *features;
431 };
432
433 /* For BCLK */
434 enum bclk_freq {
435 BCLK_100MHZ = 1,
436 BCLK_133MHZ,
437 BCLK_SLV,
438 };
439
440 #define SLM_BCLK_FREQS 5
441 double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 };
442
slm_bclk(void)443 double slm_bclk(void)
444 {
445 unsigned long long msr = 3;
446 unsigned int i;
447 double freq;
448
449 if (get_msr(base_cpu, MSR_FSB_FREQ, &msr))
450 fprintf(outf, "SLM BCLK: unknown\n");
451
452 i = msr & 0xf;
453 if (i >= SLM_BCLK_FREQS) {
454 fprintf(outf, "SLM BCLK[%d] invalid\n", i);
455 i = 3;
456 }
457 freq = slm_freq_table[i];
458
459 if (!quiet)
460 fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
461
462 return freq;
463 }
464
465 /* For Package cstate limit */
466 enum package_cstate_limit {
467 CST_LIMIT_NHM = 1,
468 CST_LIMIT_SNB,
469 CST_LIMIT_HSW,
470 CST_LIMIT_SKX,
471 CST_LIMIT_ICX,
472 CST_LIMIT_SLV,
473 CST_LIMIT_AMT,
474 CST_LIMIT_KNL,
475 CST_LIMIT_GMT,
476 };
477
478 /* For Turbo Ratio Limit MSRs */
479 enum turbo_ratio_limit_msrs {
480 TRL_BASE = BIT(0),
481 TRL_LIMIT1 = BIT(1),
482 TRL_LIMIT2 = BIT(2),
483 TRL_ATOM = BIT(3),
484 TRL_KNL = BIT(4),
485 TRL_CORECOUNT = BIT(5),
486 };
487
488 /* For Perf Limit Reason MSRs */
489 enum perf_limit_reason_msrs {
490 PLR_CORE = BIT(0),
491 PLR_GFX = BIT(1),
492 PLR_RING = BIT(2),
493 };
494
495 /* For RAPL MSRs */
496 enum rapl_msrs {
497 RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */
498 RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */
499 RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */
500 RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */
501 RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */
502 RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */
503 RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */
504 RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */
505 RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */
506 RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */
507 RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */
508 RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */
509 RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */
510 RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */
511 RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */
512 RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */
513 RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */
514 };
515
516 #define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT)
517 #define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT)
518 #define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT)
519 #define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS)
520
521 #define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO)
522 #define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO)
523 #define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY)
524 #define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLIGY)
525
526 #define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT)
527
528 /* For Cstates */
529 enum cstates {
530 CC1 = BIT(0),
531 CC3 = BIT(1),
532 CC6 = BIT(2),
533 CC7 = BIT(3),
534 PC2 = BIT(4),
535 PC3 = BIT(5),
536 PC6 = BIT(6),
537 PC7 = BIT(7),
538 PC8 = BIT(8),
539 PC9 = BIT(9),
540 PC10 = BIT(10),
541 };
542
543 static const struct platform_features nhm_features = {
544 .has_msr_misc_pwr_mgmt = 1,
545 .has_nhm_msrs = 1,
546 .bclk_freq = BCLK_133MHZ,
547 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
548 .cst_limit = CST_LIMIT_NHM,
549 .trl_msrs = TRL_BASE,
550 };
551
552 static const struct platform_features nhx_features = {
553 .has_msr_misc_pwr_mgmt = 1,
554 .has_nhm_msrs = 1,
555 .bclk_freq = BCLK_133MHZ,
556 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
557 .cst_limit = CST_LIMIT_NHM,
558 };
559
560 static const struct platform_features snb_features = {
561 .has_msr_misc_feature_control = 1,
562 .has_msr_misc_pwr_mgmt = 1,
563 .has_nhm_msrs = 1,
564 .bclk_freq = BCLK_100MHZ,
565 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
566 .cst_limit = CST_LIMIT_SNB,
567 .has_irtl_msrs = 1,
568 .trl_msrs = TRL_BASE,
569 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
570 };
571
572 static const struct platform_features snx_features = {
573 .has_msr_misc_feature_control = 1,
574 .has_msr_misc_pwr_mgmt = 1,
575 .has_nhm_msrs = 1,
576 .bclk_freq = BCLK_100MHZ,
577 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
578 .cst_limit = CST_LIMIT_SNB,
579 .has_irtl_msrs = 1,
580 .trl_msrs = TRL_BASE,
581 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
582 };
583
584 static const struct platform_features ivb_features = {
585 .has_msr_misc_feature_control = 1,
586 .has_msr_misc_pwr_mgmt = 1,
587 .has_nhm_msrs = 1,
588 .has_config_tdp = 1,
589 .bclk_freq = BCLK_100MHZ,
590 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
591 .cst_limit = CST_LIMIT_SNB,
592 .has_irtl_msrs = 1,
593 .trl_msrs = TRL_BASE,
594 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
595 };
596
597 static const struct platform_features ivx_features = {
598 .has_msr_misc_feature_control = 1,
599 .has_msr_misc_pwr_mgmt = 1,
600 .has_nhm_msrs = 1,
601 .bclk_freq = BCLK_100MHZ,
602 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
603 .cst_limit = CST_LIMIT_SNB,
604 .has_irtl_msrs = 1,
605 .trl_msrs = TRL_BASE | TRL_LIMIT1,
606 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
607 };
608
609 static const struct platform_features hsw_features = {
610 .has_msr_misc_feature_control = 1,
611 .has_msr_misc_pwr_mgmt = 1,
612 .has_nhm_msrs = 1,
613 .has_config_tdp = 1,
614 .bclk_freq = BCLK_100MHZ,
615 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
616 .cst_limit = CST_LIMIT_HSW,
617 .has_irtl_msrs = 1,
618 .trl_msrs = TRL_BASE,
619 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
620 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
621 };
622
623 static const struct platform_features hsx_features = {
624 .has_msr_misc_feature_control = 1,
625 .has_msr_misc_pwr_mgmt = 1,
626 .has_nhm_msrs = 1,
627 .has_config_tdp = 1,
628 .bclk_freq = BCLK_100MHZ,
629 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
630 .cst_limit = CST_LIMIT_HSW,
631 .has_irtl_msrs = 1,
632 .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2,
633 .plr_msrs = PLR_CORE | PLR_RING,
634 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
635 .has_fixed_rapl_unit = 1,
636 };
637
638 static const struct platform_features hswl_features = {
639 .has_msr_misc_feature_control = 1,
640 .has_msr_misc_pwr_mgmt = 1,
641 .has_nhm_msrs = 1,
642 .has_config_tdp = 1,
643 .bclk_freq = BCLK_100MHZ,
644 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
645 .cst_limit = CST_LIMIT_HSW,
646 .has_irtl_msrs = 1,
647 .trl_msrs = TRL_BASE,
648 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
649 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
650 };
651
652 static const struct platform_features hswg_features = {
653 .has_msr_misc_feature_control = 1,
654 .has_msr_misc_pwr_mgmt = 1,
655 .has_nhm_msrs = 1,
656 .has_config_tdp = 1,
657 .bclk_freq = BCLK_100MHZ,
658 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
659 .cst_limit = CST_LIMIT_HSW,
660 .has_irtl_msrs = 1,
661 .trl_msrs = TRL_BASE,
662 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
663 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
664 };
665
666 static const struct platform_features bdw_features = {
667 .has_msr_misc_feature_control = 1,
668 .has_msr_misc_pwr_mgmt = 1,
669 .has_nhm_msrs = 1,
670 .has_config_tdp = 1,
671 .bclk_freq = BCLK_100MHZ,
672 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
673 .cst_limit = CST_LIMIT_HSW,
674 .has_irtl_msrs = 1,
675 .trl_msrs = TRL_BASE,
676 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
677 };
678
679 static const struct platform_features bdwg_features = {
680 .has_msr_misc_feature_control = 1,
681 .has_msr_misc_pwr_mgmt = 1,
682 .has_nhm_msrs = 1,
683 .has_config_tdp = 1,
684 .bclk_freq = BCLK_100MHZ,
685 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
686 .cst_limit = CST_LIMIT_HSW,
687 .has_irtl_msrs = 1,
688 .trl_msrs = TRL_BASE,
689 .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
690 };
691
692 static const struct platform_features bdx_features = {
693 .has_msr_misc_feature_control = 1,
694 .has_msr_misc_pwr_mgmt = 1,
695 .has_nhm_msrs = 1,
696 .has_config_tdp = 1,
697 .bclk_freq = BCLK_100MHZ,
698 .supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6,
699 .cst_limit = CST_LIMIT_HSW,
700 .has_irtl_msrs = 1,
701 .has_cst_auto_convension = 1,
702 .trl_msrs = TRL_BASE,
703 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
704 .has_fixed_rapl_unit = 1,
705 };
706
707 static const struct platform_features skl_features = {
708 .has_msr_misc_feature_control = 1,
709 .has_msr_misc_pwr_mgmt = 1,
710 .has_nhm_msrs = 1,
711 .has_config_tdp = 1,
712 .bclk_freq = BCLK_100MHZ,
713 .crystal_freq = 24000000,
714 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
715 .cst_limit = CST_LIMIT_HSW,
716 .has_irtl_msrs = 1,
717 .has_ext_cst_msrs = 1,
718 .trl_msrs = TRL_BASE,
719 .tcc_offset_bits = 6,
720 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
721 .enable_tsc_tweak = 1,
722 };
723
724 static const struct platform_features cnl_features = {
725 .has_msr_misc_feature_control = 1,
726 .has_msr_misc_pwr_mgmt = 1,
727 .has_nhm_msrs = 1,
728 .has_config_tdp = 1,
729 .bclk_freq = BCLK_100MHZ,
730 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
731 .cst_limit = CST_LIMIT_HSW,
732 .has_irtl_msrs = 1,
733 .has_msr_core_c1_res = 1,
734 .has_ext_cst_msrs = 1,
735 .trl_msrs = TRL_BASE,
736 .tcc_offset_bits = 6,
737 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
738 .enable_tsc_tweak = 1,
739 };
740
741 static const struct platform_features adl_features = {
742 .has_msr_misc_feature_control = 1,
743 .has_msr_misc_pwr_mgmt = 1,
744 .has_nhm_msrs = 1,
745 .has_config_tdp = 1,
746 .bclk_freq = BCLK_100MHZ,
747 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10,
748 .cst_limit = CST_LIMIT_HSW,
749 .has_irtl_msrs = 1,
750 .has_msr_core_c1_res = 1,
751 .has_ext_cst_msrs = 1,
752 .trl_msrs = TRL_BASE,
753 .tcc_offset_bits = 6,
754 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
755 .enable_tsc_tweak = 1,
756 };
757
758 static const struct platform_features arl_features = {
759 .has_msr_misc_feature_control = 1,
760 .has_msr_misc_pwr_mgmt = 1,
761 .has_nhm_msrs = 1,
762 .has_config_tdp = 1,
763 .bclk_freq = BCLK_100MHZ,
764 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC10,
765 .cst_limit = CST_LIMIT_HSW,
766 .has_irtl_msrs = 1,
767 .has_msr_core_c1_res = 1,
768 .has_ext_cst_msrs = 1,
769 .trl_msrs = TRL_BASE,
770 .tcc_offset_bits = 6,
771 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
772 .enable_tsc_tweak = 1,
773 };
774
775 static const struct platform_features skx_features = {
776 .has_msr_misc_feature_control = 1,
777 .has_msr_misc_pwr_mgmt = 1,
778 .has_nhm_msrs = 1,
779 .has_config_tdp = 1,
780 .bclk_freq = BCLK_100MHZ,
781 .supported_cstates = CC1 | CC6 | PC2 | PC6,
782 .cst_limit = CST_LIMIT_SKX,
783 .has_irtl_msrs = 1,
784 .has_cst_auto_convension = 1,
785 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
786 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
787 .has_fixed_rapl_unit = 1,
788 };
789
790 static const struct platform_features icx_features = {
791 .has_msr_misc_feature_control = 1,
792 .has_msr_misc_pwr_mgmt = 1,
793 .has_nhm_msrs = 1,
794 .has_config_tdp = 1,
795 .bclk_freq = BCLK_100MHZ,
796 .supported_cstates = CC1 | CC6 | PC2 | PC6,
797 .cst_limit = CST_LIMIT_ICX,
798 .has_msr_core_c1_res = 1,
799 .has_irtl_msrs = 1,
800 .has_cst_prewake_bit = 1,
801 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
802 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
803 .has_fixed_rapl_unit = 1,
804 };
805
806 static const struct platform_features spr_features = {
807 .has_msr_misc_feature_control = 1,
808 .has_msr_misc_pwr_mgmt = 1,
809 .has_nhm_msrs = 1,
810 .has_config_tdp = 1,
811 .bclk_freq = BCLK_100MHZ,
812 .supported_cstates = CC1 | CC6 | PC2 | PC6,
813 .cst_limit = CST_LIMIT_SKX,
814 .has_msr_core_c1_res = 1,
815 .has_irtl_msrs = 1,
816 .has_cst_prewake_bit = 1,
817 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
818 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
819 };
820
821 static const struct platform_features srf_features = {
822 .has_msr_misc_feature_control = 1,
823 .has_msr_misc_pwr_mgmt = 1,
824 .has_nhm_msrs = 1,
825 .has_config_tdp = 1,
826 .bclk_freq = BCLK_100MHZ,
827 .supported_cstates = CC1 | CC6 | PC2 | PC6,
828 .cst_limit = CST_LIMIT_SKX,
829 .has_msr_core_c1_res = 1,
830 .has_msr_module_c6_res_ms = 1,
831 .has_irtl_msrs = 1,
832 .has_cst_prewake_bit = 1,
833 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
834 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
835 };
836
837 static const struct platform_features grr_features = {
838 .has_msr_misc_feature_control = 1,
839 .has_msr_misc_pwr_mgmt = 1,
840 .has_nhm_msrs = 1,
841 .has_config_tdp = 1,
842 .bclk_freq = BCLK_100MHZ,
843 .supported_cstates = CC1 | CC6,
844 .cst_limit = CST_LIMIT_SKX,
845 .has_msr_core_c1_res = 1,
846 .has_msr_module_c6_res_ms = 1,
847 .has_irtl_msrs = 1,
848 .has_cst_prewake_bit = 1,
849 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
850 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
851 };
852
853 static const struct platform_features slv_features = {
854 .has_nhm_msrs = 1,
855 .bclk_freq = BCLK_SLV,
856 .supported_cstates = CC1 | CC6 | PC6,
857 .cst_limit = CST_LIMIT_SLV,
858 .has_msr_core_c1_res = 1,
859 .has_msr_module_c6_res_ms = 1,
860 .has_msr_c6_demotion_policy_config = 1,
861 .has_msr_atom_pkg_c6_residency = 1,
862 .trl_msrs = TRL_ATOM,
863 .rapl_msrs = RAPL_PKG | RAPL_CORE,
864 .has_rapl_divisor = 1,
865 .rapl_quirk_tdp = 30,
866 };
867
868 static const struct platform_features slvd_features = {
869 .has_msr_misc_pwr_mgmt = 1,
870 .has_nhm_msrs = 1,
871 .bclk_freq = BCLK_SLV,
872 .supported_cstates = CC1 | CC6 | PC3 | PC6,
873 .cst_limit = CST_LIMIT_SLV,
874 .has_msr_atom_pkg_c6_residency = 1,
875 .trl_msrs = TRL_BASE,
876 .rapl_msrs = RAPL_PKG | RAPL_CORE,
877 .rapl_quirk_tdp = 30,
878 };
879
880 static const struct platform_features amt_features = {
881 .has_nhm_msrs = 1,
882 .bclk_freq = BCLK_133MHZ,
883 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
884 .cst_limit = CST_LIMIT_AMT,
885 .trl_msrs = TRL_BASE,
886 };
887
888 static const struct platform_features gmt_features = {
889 .has_msr_misc_pwr_mgmt = 1,
890 .has_nhm_msrs = 1,
891 .bclk_freq = BCLK_100MHZ,
892 .crystal_freq = 19200000,
893 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
894 .cst_limit = CST_LIMIT_GMT,
895 .has_irtl_msrs = 1,
896 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
897 .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
898 };
899
900 static const struct platform_features gmtd_features = {
901 .has_msr_misc_pwr_mgmt = 1,
902 .has_nhm_msrs = 1,
903 .bclk_freq = BCLK_100MHZ,
904 .crystal_freq = 25000000,
905 .supported_cstates = CC1 | CC6 | PC2 | PC6,
906 .cst_limit = CST_LIMIT_GMT,
907 .has_irtl_msrs = 1,
908 .has_msr_core_c1_res = 1,
909 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
910 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS,
911 };
912
913 static const struct platform_features gmtp_features = {
914 .has_msr_misc_pwr_mgmt = 1,
915 .has_nhm_msrs = 1,
916 .bclk_freq = BCLK_100MHZ,
917 .crystal_freq = 19200000,
918 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
919 .cst_limit = CST_LIMIT_GMT,
920 .has_irtl_msrs = 1,
921 .trl_msrs = TRL_BASE,
922 .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
923 };
924
925 static const struct platform_features tmt_features = {
926 .has_msr_misc_pwr_mgmt = 1,
927 .has_nhm_msrs = 1,
928 .bclk_freq = BCLK_100MHZ,
929 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
930 .cst_limit = CST_LIMIT_GMT,
931 .has_irtl_msrs = 1,
932 .trl_msrs = TRL_BASE,
933 .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
934 .enable_tsc_tweak = 1,
935 };
936
937 static const struct platform_features tmtd_features = {
938 .has_msr_misc_pwr_mgmt = 1,
939 .has_nhm_msrs = 1,
940 .bclk_freq = BCLK_100MHZ,
941 .supported_cstates = CC1 | CC6,
942 .cst_limit = CST_LIMIT_GMT,
943 .has_irtl_msrs = 1,
944 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
945 .rapl_msrs = RAPL_PKG_ALL,
946 };
947
948 static const struct platform_features knl_features = {
949 .has_msr_misc_pwr_mgmt = 1,
950 .has_nhm_msrs = 1,
951 .has_config_tdp = 1,
952 .bclk_freq = BCLK_100MHZ,
953 .supported_cstates = CC1 | CC6 | PC3 | PC6,
954 .cst_limit = CST_LIMIT_KNL,
955 .has_msr_knl_core_c6_residency = 1,
956 .trl_msrs = TRL_KNL,
957 .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
958 .has_fixed_rapl_unit = 1,
959 .need_perf_multiplier = 1,
960 };
961
962 static const struct platform_features default_features = {
963 };
964
965 static const struct platform_features amd_features_with_rapl = {
966 .rapl_msrs = RAPL_AMD_F17H,
967 .has_per_core_rapl = 1,
968 .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */
969 };
970
971 static const struct platform_data turbostat_pdata[] = {
972 { INTEL_NEHALEM, &nhm_features },
973 { INTEL_NEHALEM_G, &nhm_features },
974 { INTEL_NEHALEM_EP, &nhm_features },
975 { INTEL_NEHALEM_EX, &nhx_features },
976 { INTEL_WESTMERE, &nhm_features },
977 { INTEL_WESTMERE_EP, &nhm_features },
978 { INTEL_WESTMERE_EX, &nhx_features },
979 { INTEL_SANDYBRIDGE, &snb_features },
980 { INTEL_SANDYBRIDGE_X, &snx_features },
981 { INTEL_IVYBRIDGE, &ivb_features },
982 { INTEL_IVYBRIDGE_X, &ivx_features },
983 { INTEL_HASWELL, &hsw_features },
984 { INTEL_HASWELL_X, &hsx_features },
985 { INTEL_HASWELL_L, &hswl_features },
986 { INTEL_HASWELL_G, &hswg_features },
987 { INTEL_BROADWELL, &bdw_features },
988 { INTEL_BROADWELL_G, &bdwg_features },
989 { INTEL_BROADWELL_X, &bdx_features },
990 { INTEL_BROADWELL_D, &bdx_features },
991 { INTEL_SKYLAKE_L, &skl_features },
992 { INTEL_SKYLAKE, &skl_features },
993 { INTEL_SKYLAKE_X, &skx_features },
994 { INTEL_KABYLAKE_L, &skl_features },
995 { INTEL_KABYLAKE, &skl_features },
996 { INTEL_COMETLAKE, &skl_features },
997 { INTEL_COMETLAKE_L, &skl_features },
998 { INTEL_CANNONLAKE_L, &cnl_features },
999 { INTEL_ICELAKE_X, &icx_features },
1000 { INTEL_ICELAKE_D, &icx_features },
1001 { INTEL_ICELAKE_L, &cnl_features },
1002 { INTEL_ICELAKE_NNPI, &cnl_features },
1003 { INTEL_ROCKETLAKE, &cnl_features },
1004 { INTEL_TIGERLAKE_L, &cnl_features },
1005 { INTEL_TIGERLAKE, &cnl_features },
1006 { INTEL_SAPPHIRERAPIDS_X, &spr_features },
1007 { INTEL_EMERALDRAPIDS_X, &spr_features },
1008 { INTEL_GRANITERAPIDS_X, &spr_features },
1009 { INTEL_LAKEFIELD, &cnl_features },
1010 { INTEL_ALDERLAKE, &adl_features },
1011 { INTEL_ALDERLAKE_L, &adl_features },
1012 { INTEL_RAPTORLAKE, &adl_features },
1013 { INTEL_RAPTORLAKE_P, &adl_features },
1014 { INTEL_RAPTORLAKE_S, &adl_features },
1015 { INTEL_METEORLAKE, &cnl_features },
1016 { INTEL_METEORLAKE_L, &cnl_features },
1017 { INTEL_ARROWLAKE_H, &arl_features },
1018 { INTEL_ARROWLAKE_U, &arl_features },
1019 { INTEL_ARROWLAKE, &arl_features },
1020 { INTEL_LUNARLAKE_M, &arl_features },
1021 { INTEL_ATOM_SILVERMONT, &slv_features },
1022 { INTEL_ATOM_SILVERMONT_D, &slvd_features },
1023 { INTEL_ATOM_AIRMONT, &amt_features },
1024 { INTEL_ATOM_GOLDMONT, &gmt_features },
1025 { INTEL_ATOM_GOLDMONT_D, &gmtd_features },
1026 { INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features },
1027 { INTEL_ATOM_TREMONT_D, &tmtd_features },
1028 { INTEL_ATOM_TREMONT, &tmt_features },
1029 { INTEL_ATOM_TREMONT_L, &tmt_features },
1030 { INTEL_ATOM_GRACEMONT, &adl_features },
1031 { INTEL_ATOM_CRESTMONT_X, &srf_features },
1032 { INTEL_ATOM_CRESTMONT, &grr_features },
1033 { INTEL_XEON_PHI_KNL, &knl_features },
1034 { INTEL_XEON_PHI_KNM, &knl_features },
1035 /*
1036 * Missing support for
1037 * INTEL_ICELAKE
1038 * INTEL_ATOM_SILVERMONT_MID
1039 * INTEL_ATOM_AIRMONT_MID
1040 * INTEL_ATOM_AIRMONT_NP
1041 */
1042 { 0, NULL },
1043 };
1044
1045 static const struct platform_features *platform;
1046
probe_platform_features(unsigned int family,unsigned int model)1047 void probe_platform_features(unsigned int family, unsigned int model)
1048 {
1049 int i;
1050
1051 platform = &default_features;
1052
1053 if (authentic_amd || hygon_genuine) {
1054 if (max_extended_level >= 0x80000007) {
1055 unsigned int eax, ebx, ecx, edx;
1056
1057 __cpuid(0x80000007, eax, ebx, ecx, edx);
1058 /* RAPL (Fam 17h+) */
1059 if ((edx & (1 << 14)) && family >= 0x17)
1060 platform = &amd_features_with_rapl;
1061 }
1062 return;
1063 }
1064
1065 if (!genuine_intel)
1066 return;
1067
1068 for (i = 0; turbostat_pdata[i].features; i++) {
1069 if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) {
1070 platform = turbostat_pdata[i].features;
1071 return;
1072 }
1073 }
1074 }
1075
1076 /* Model specific support End */
1077
1078 #define TJMAX_DEFAULT 100
1079
1080 /* MSRs that are not yet in the kernel-provided header. */
1081 #define MSR_RAPL_PWR_UNIT 0xc0010299
1082 #define MSR_CORE_ENERGY_STAT 0xc001029a
1083 #define MSR_PKG_ENERGY_STAT 0xc001029b
1084
1085 #define MAX(a, b) ((a) > (b) ? (a) : (b))
1086
1087 int backwards_count;
1088 char *progname;
1089
1090 #define CPU_SUBSET_MAXCPUS 1024 /* need to use before probe... */
1091 cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
1092 size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
1093 #define MAX_ADDED_THREAD_COUNTERS 24
1094 #define MAX_ADDED_CORE_COUNTERS 8
1095 #define MAX_ADDED_PACKAGE_COUNTERS 16
1096 #define PMT_MAX_ADDED_THREAD_COUNTERS 24
1097 #define PMT_MAX_ADDED_CORE_COUNTERS 8
1098 #define PMT_MAX_ADDED_PACKAGE_COUNTERS 16
1099 #define BITMASK_SIZE 32
1100
1101 #define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr))
1102
1103 /* Indexes used to map data read from perf and MSRs into global variables */
1104 enum rapl_rci_index {
1105 RAPL_RCI_INDEX_ENERGY_PKG = 0,
1106 RAPL_RCI_INDEX_ENERGY_CORES = 1,
1107 RAPL_RCI_INDEX_DRAM = 2,
1108 RAPL_RCI_INDEX_GFX = 3,
1109 RAPL_RCI_INDEX_PKG_PERF_STATUS = 4,
1110 RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5,
1111 RAPL_RCI_INDEX_CORE_ENERGY = 6,
1112 NUM_RAPL_COUNTERS,
1113 };
1114
1115 enum rapl_unit {
1116 RAPL_UNIT_INVALID,
1117 RAPL_UNIT_JOULES,
1118 RAPL_UNIT_WATTS,
1119 };
1120
1121 struct rapl_counter_info_t {
1122 unsigned long long data[NUM_RAPL_COUNTERS];
1123 enum counter_source source[NUM_RAPL_COUNTERS];
1124 unsigned long long flags[NUM_RAPL_COUNTERS];
1125 double scale[NUM_RAPL_COUNTERS];
1126 enum rapl_unit unit[NUM_RAPL_COUNTERS];
1127 unsigned long long msr[NUM_RAPL_COUNTERS];
1128 unsigned long long msr_mask[NUM_RAPL_COUNTERS];
1129 int msr_shift[NUM_RAPL_COUNTERS];
1130
1131 int fd_perf;
1132 };
1133
1134 /* struct rapl_counter_info_t for each RAPL domain */
1135 struct rapl_counter_info_t *rapl_counter_info_perdomain;
1136 unsigned int rapl_counter_info_perdomain_size;
1137
1138 #define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1)
1139
1140 struct rapl_counter_arch_info {
1141 int feature_mask; /* Mask for testing if the counter is supported on host */
1142 const char *perf_subsys;
1143 const char *perf_name;
1144 unsigned long long msr;
1145 unsigned long long msr_mask;
1146 int msr_shift; /* Positive mean shift right, negative mean shift left */
1147 double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */
1148 unsigned int rci_index; /* Maps data from perf counters to global variables */
1149 unsigned long long bic;
1150 double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */
1151 unsigned long long flags;
1152 };
1153
1154 static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = {
1155 {
1156 .feature_mask = RAPL_PKG,
1157 .perf_subsys = "power",
1158 .perf_name = "energy-pkg",
1159 .msr = MSR_PKG_ENERGY_STATUS,
1160 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1161 .msr_shift = 0,
1162 .platform_rapl_msr_scale = &rapl_energy_units,
1163 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1164 .bic = BIC_PkgWatt | BIC_Pkg_J,
1165 .compat_scale = 1.0,
1166 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1167 },
1168 {
1169 .feature_mask = RAPL_AMD_F17H,
1170 .perf_subsys = "power",
1171 .perf_name = "energy-pkg",
1172 .msr = MSR_PKG_ENERGY_STAT,
1173 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1174 .msr_shift = 0,
1175 .platform_rapl_msr_scale = &rapl_energy_units,
1176 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1177 .bic = BIC_PkgWatt | BIC_Pkg_J,
1178 .compat_scale = 1.0,
1179 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1180 },
1181 {
1182 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1183 .perf_subsys = "power",
1184 .perf_name = "energy-cores",
1185 .msr = MSR_PP0_ENERGY_STATUS,
1186 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1187 .msr_shift = 0,
1188 .platform_rapl_msr_scale = &rapl_energy_units,
1189 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1190 .bic = BIC_CorWatt | BIC_Cor_J,
1191 .compat_scale = 1.0,
1192 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1193 },
1194 {
1195 .feature_mask = RAPL_DRAM,
1196 .perf_subsys = "power",
1197 .perf_name = "energy-ram",
1198 .msr = MSR_DRAM_ENERGY_STATUS,
1199 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1200 .msr_shift = 0,
1201 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1202 .rci_index = RAPL_RCI_INDEX_DRAM,
1203 .bic = BIC_RAMWatt | BIC_RAM_J,
1204 .compat_scale = 1.0,
1205 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1206 },
1207 {
1208 .feature_mask = RAPL_GFX,
1209 .perf_subsys = "power",
1210 .perf_name = "energy-gpu",
1211 .msr = MSR_PP1_ENERGY_STATUS,
1212 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1213 .msr_shift = 0,
1214 .platform_rapl_msr_scale = &rapl_energy_units,
1215 .rci_index = RAPL_RCI_INDEX_GFX,
1216 .bic = BIC_GFXWatt | BIC_GFX_J,
1217 .compat_scale = 1.0,
1218 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1219 },
1220 {
1221 .feature_mask = RAPL_PKG_PERF_STATUS,
1222 .perf_subsys = NULL,
1223 .perf_name = NULL,
1224 .msr = MSR_PKG_PERF_STATUS,
1225 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1226 .msr_shift = 0,
1227 .platform_rapl_msr_scale = &rapl_time_units,
1228 .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS,
1229 .bic = BIC_PKG__,
1230 .compat_scale = 100.0,
1231 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1232 },
1233 {
1234 .feature_mask = RAPL_DRAM_PERF_STATUS,
1235 .perf_subsys = NULL,
1236 .perf_name = NULL,
1237 .msr = MSR_DRAM_PERF_STATUS,
1238 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1239 .msr_shift = 0,
1240 .platform_rapl_msr_scale = &rapl_time_units,
1241 .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS,
1242 .bic = BIC_RAM__,
1243 .compat_scale = 100.0,
1244 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1245 },
1246 {
1247 .feature_mask = RAPL_AMD_F17H,
1248 .perf_subsys = NULL,
1249 .perf_name = NULL,
1250 .msr = MSR_CORE_ENERGY_STAT,
1251 .msr_mask = 0xFFFFFFFF,
1252 .msr_shift = 0,
1253 .platform_rapl_msr_scale = &rapl_energy_units,
1254 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1255 .bic = BIC_CorWatt | BIC_Cor_J,
1256 .compat_scale = 1.0,
1257 .flags = 0,
1258 },
1259 };
1260
1261 struct rapl_counter {
1262 unsigned long long raw_value;
1263 enum rapl_unit unit;
1264 double scale;
1265 };
1266
1267 /* Indexes used to map data read from perf and MSRs into global variables */
1268 enum ccstate_rci_index {
1269 CCSTATE_RCI_INDEX_C1_RESIDENCY = 0,
1270 CCSTATE_RCI_INDEX_C3_RESIDENCY = 1,
1271 CCSTATE_RCI_INDEX_C6_RESIDENCY = 2,
1272 CCSTATE_RCI_INDEX_C7_RESIDENCY = 3,
1273 PCSTATE_RCI_INDEX_C2_RESIDENCY = 4,
1274 PCSTATE_RCI_INDEX_C3_RESIDENCY = 5,
1275 PCSTATE_RCI_INDEX_C6_RESIDENCY = 6,
1276 PCSTATE_RCI_INDEX_C7_RESIDENCY = 7,
1277 PCSTATE_RCI_INDEX_C8_RESIDENCY = 8,
1278 PCSTATE_RCI_INDEX_C9_RESIDENCY = 9,
1279 PCSTATE_RCI_INDEX_C10_RESIDENCY = 10,
1280 NUM_CSTATE_COUNTERS,
1281 };
1282
1283 struct cstate_counter_info_t {
1284 unsigned long long data[NUM_CSTATE_COUNTERS];
1285 enum counter_source source[NUM_CSTATE_COUNTERS];
1286 unsigned long long msr[NUM_CSTATE_COUNTERS];
1287 int fd_perf_core;
1288 int fd_perf_pkg;
1289 };
1290
1291 struct cstate_counter_info_t *ccstate_counter_info;
1292 unsigned int ccstate_counter_info_size;
1293
1294 #define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0)
1295 #define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE)
1296 #define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2)
1297
1298 struct cstate_counter_arch_info {
1299 int feature_mask; /* Mask for testing if the counter is supported on host */
1300 const char *perf_subsys;
1301 const char *perf_name;
1302 unsigned long long msr;
1303 unsigned int rci_index; /* Maps data from perf counters to global variables */
1304 unsigned long long bic;
1305 unsigned long long flags;
1306 int pkg_cstate_limit;
1307 };
1308
1309 static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = {
1310 {
1311 .feature_mask = CC1,
1312 .perf_subsys = "cstate_core",
1313 .perf_name = "c1-residency",
1314 .msr = MSR_CORE_C1_RES,
1315 .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY,
1316 .bic = BIC_CPU_c1,
1317 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD,
1318 .pkg_cstate_limit = 0,
1319 },
1320 {
1321 .feature_mask = CC3,
1322 .perf_subsys = "cstate_core",
1323 .perf_name = "c3-residency",
1324 .msr = MSR_CORE_C3_RESIDENCY,
1325 .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY,
1326 .bic = BIC_CPU_c3,
1327 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1328 .pkg_cstate_limit = 0,
1329 },
1330 {
1331 .feature_mask = CC6,
1332 .perf_subsys = "cstate_core",
1333 .perf_name = "c6-residency",
1334 .msr = MSR_CORE_C6_RESIDENCY,
1335 .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY,
1336 .bic = BIC_CPU_c6,
1337 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1338 .pkg_cstate_limit = 0,
1339 },
1340 {
1341 .feature_mask = CC7,
1342 .perf_subsys = "cstate_core",
1343 .perf_name = "c7-residency",
1344 .msr = MSR_CORE_C7_RESIDENCY,
1345 .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY,
1346 .bic = BIC_CPU_c7,
1347 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1348 .pkg_cstate_limit = 0,
1349 },
1350 {
1351 .feature_mask = PC2,
1352 .perf_subsys = "cstate_pkg",
1353 .perf_name = "c2-residency",
1354 .msr = MSR_PKG_C2_RESIDENCY,
1355 .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY,
1356 .bic = BIC_Pkgpc2,
1357 .flags = 0,
1358 .pkg_cstate_limit = PCL__2,
1359 },
1360 {
1361 .feature_mask = PC3,
1362 .perf_subsys = "cstate_pkg",
1363 .perf_name = "c3-residency",
1364 .msr = MSR_PKG_C3_RESIDENCY,
1365 .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY,
1366 .bic = BIC_Pkgpc3,
1367 .flags = 0,
1368 .pkg_cstate_limit = PCL__3,
1369 },
1370 {
1371 .feature_mask = PC6,
1372 .perf_subsys = "cstate_pkg",
1373 .perf_name = "c6-residency",
1374 .msr = MSR_PKG_C6_RESIDENCY,
1375 .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY,
1376 .bic = BIC_Pkgpc6,
1377 .flags = 0,
1378 .pkg_cstate_limit = PCL__6,
1379 },
1380 {
1381 .feature_mask = PC7,
1382 .perf_subsys = "cstate_pkg",
1383 .perf_name = "c7-residency",
1384 .msr = MSR_PKG_C7_RESIDENCY,
1385 .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY,
1386 .bic = BIC_Pkgpc7,
1387 .flags = 0,
1388 .pkg_cstate_limit = PCL__7,
1389 },
1390 {
1391 .feature_mask = PC8,
1392 .perf_subsys = "cstate_pkg",
1393 .perf_name = "c8-residency",
1394 .msr = MSR_PKG_C8_RESIDENCY,
1395 .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY,
1396 .bic = BIC_Pkgpc8,
1397 .flags = 0,
1398 .pkg_cstate_limit = PCL__8,
1399 },
1400 {
1401 .feature_mask = PC9,
1402 .perf_subsys = "cstate_pkg",
1403 .perf_name = "c9-residency",
1404 .msr = MSR_PKG_C9_RESIDENCY,
1405 .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY,
1406 .bic = BIC_Pkgpc9,
1407 .flags = 0,
1408 .pkg_cstate_limit = PCL__9,
1409 },
1410 {
1411 .feature_mask = PC10,
1412 .perf_subsys = "cstate_pkg",
1413 .perf_name = "c10-residency",
1414 .msr = MSR_PKG_C10_RESIDENCY,
1415 .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY,
1416 .bic = BIC_Pkgpc10,
1417 .flags = 0,
1418 .pkg_cstate_limit = PCL_10,
1419 },
1420 };
1421
1422 /* Indexes used to map data read from perf and MSRs into global variables */
1423 enum msr_rci_index {
1424 MSR_RCI_INDEX_APERF = 0,
1425 MSR_RCI_INDEX_MPERF = 1,
1426 MSR_RCI_INDEX_SMI = 2,
1427 NUM_MSR_COUNTERS,
1428 };
1429
1430 struct msr_counter_info_t {
1431 unsigned long long data[NUM_MSR_COUNTERS];
1432 enum counter_source source[NUM_MSR_COUNTERS];
1433 unsigned long long msr[NUM_MSR_COUNTERS];
1434 unsigned long long msr_mask[NUM_MSR_COUNTERS];
1435 int fd_perf;
1436 };
1437
1438 struct msr_counter_info_t *msr_counter_info;
1439 unsigned int msr_counter_info_size;
1440
1441 struct msr_counter_arch_info {
1442 const char *perf_subsys;
1443 const char *perf_name;
1444 unsigned long long msr;
1445 unsigned long long msr_mask;
1446 unsigned int rci_index; /* Maps data from perf counters to global variables */
1447 bool needed;
1448 bool present;
1449 };
1450
1451 enum msr_arch_info_index {
1452 MSR_ARCH_INFO_APERF_INDEX = 0,
1453 MSR_ARCH_INFO_MPERF_INDEX = 1,
1454 MSR_ARCH_INFO_SMI_INDEX = 2,
1455 };
1456
1457 static struct msr_counter_arch_info msr_counter_arch_infos[] = {
1458 [MSR_ARCH_INFO_APERF_INDEX] = {
1459 .perf_subsys = "msr",
1460 .perf_name = "aperf",
1461 .msr = MSR_IA32_APERF,
1462 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1463 .rci_index = MSR_RCI_INDEX_APERF,
1464 },
1465
1466 [MSR_ARCH_INFO_MPERF_INDEX] = {
1467 .perf_subsys = "msr",
1468 .perf_name = "mperf",
1469 .msr = MSR_IA32_MPERF,
1470 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1471 .rci_index = MSR_RCI_INDEX_MPERF,
1472 },
1473
1474 [MSR_ARCH_INFO_SMI_INDEX] = {
1475 .perf_subsys = "msr",
1476 .perf_name = "smi",
1477 .msr = MSR_SMI_COUNT,
1478 .msr_mask = 0xFFFFFFFF,
1479 .rci_index = MSR_RCI_INDEX_SMI,
1480 },
1481 };
1482
1483 /* Can be redefined when compiling, useful for testing. */
1484 #ifndef SYSFS_TELEM_PATH
1485 #define SYSFS_TELEM_PATH "/sys/class/intel_pmt"
1486 #endif
1487
1488 #define PMT_COUNTER_MTL_DC6_OFFSET 120
1489 #define PMT_COUNTER_MTL_DC6_LSB 0
1490 #define PMT_COUNTER_MTL_DC6_MSB 63
1491 #define PMT_MTL_DC6_GUID 0x1a067102
1492
1493 #define PMT_COUNTER_NAME_SIZE_BYTES 16
1494 #define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32
1495
1496 struct pmt_mmio {
1497 struct pmt_mmio *next;
1498
1499 unsigned int guid;
1500 unsigned int size;
1501
1502 /* Base pointer to the mmaped memory. */
1503 void *mmio_base;
1504
1505 /*
1506 * Offset to be applied to the mmio_base
1507 * to get the beginning of the PMT counters for given GUID.
1508 */
1509 unsigned long pmt_offset;
1510 } *pmt_mmios;
1511
1512 enum pmt_datatype {
1513 PMT_TYPE_RAW,
1514 PMT_TYPE_XTAL_TIME,
1515 };
1516
1517 struct pmt_domain_info {
1518 /*
1519 * Pointer to the MMIO obtained by applying a counter offset
1520 * to the mmio_base of the mmaped region for the given GUID.
1521 *
1522 * This is where to read the raw value of the counter from.
1523 */
1524 unsigned long *pcounter;
1525 };
1526
1527 struct pmt_counter {
1528 struct pmt_counter *next;
1529
1530 /* PMT metadata */
1531 char name[PMT_COUNTER_NAME_SIZE_BYTES];
1532 enum pmt_datatype type;
1533 enum counter_scope scope;
1534 unsigned int lsb;
1535 unsigned int msb;
1536
1537 /* BIC-like metadata */
1538 enum counter_format format;
1539
1540 unsigned int num_domains;
1541 struct pmt_domain_info *domains;
1542 };
1543
pmt_counter_get_width(const struct pmt_counter * p)1544 unsigned int pmt_counter_get_width(const struct pmt_counter *p)
1545 {
1546 return (p->msb - p->lsb) + 1;
1547 }
1548
pmt_counter_resize_(struct pmt_counter * pcounter,unsigned int new_size)1549 void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size)
1550 {
1551 struct pmt_domain_info *new_mem;
1552
1553 new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains));
1554 if (!new_mem) {
1555 fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__);
1556 exit(1);
1557 }
1558
1559 /* Zero initialize just allocated memory. */
1560 const size_t num_new_domains = new_size - pcounter->num_domains;
1561
1562 memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains));
1563
1564 pcounter->num_domains = new_size;
1565 pcounter->domains = new_mem;
1566 }
1567
pmt_counter_resize(struct pmt_counter * pcounter,unsigned int new_size)1568 void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size)
1569 {
1570 /*
1571 * Allocate more memory ahead of time.
1572 *
1573 * Always allocate space for at least 8 elements
1574 * and double the size when growing.
1575 */
1576 if (new_size < 8)
1577 new_size = 8;
1578 new_size = MAX(new_size, pcounter->num_domains * 2);
1579
1580 pmt_counter_resize_(pcounter, new_size);
1581 }
1582
1583 struct thread_data {
1584 struct timeval tv_begin;
1585 struct timeval tv_end;
1586 struct timeval tv_delta;
1587 unsigned long long tsc;
1588 unsigned long long aperf;
1589 unsigned long long mperf;
1590 unsigned long long c1;
1591 unsigned long long instr_count;
1592 unsigned long long irq_count;
1593 unsigned int smi_count;
1594 unsigned int cpu_id;
1595 unsigned int apic_id;
1596 unsigned int x2apic_id;
1597 unsigned int flags;
1598 bool is_atom;
1599 unsigned long long counter[MAX_ADDED_THREAD_COUNTERS];
1600 unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS];
1601 unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS];
1602 } *thread_even, *thread_odd;
1603
1604 struct core_data {
1605 int base_cpu;
1606 unsigned long long c3;
1607 unsigned long long c6;
1608 unsigned long long c7;
1609 unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
1610 unsigned int core_temp_c;
1611 struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */
1612 unsigned int core_id;
1613 unsigned long long core_throt_cnt;
1614 unsigned long long counter[MAX_ADDED_CORE_COUNTERS];
1615 unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS];
1616 unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS];
1617 } *core_even, *core_odd;
1618
1619 struct pkg_data {
1620 int base_cpu;
1621 unsigned long long pc2;
1622 unsigned long long pc3;
1623 unsigned long long pc6;
1624 unsigned long long pc7;
1625 unsigned long long pc8;
1626 unsigned long long pc9;
1627 unsigned long long pc10;
1628 long long cpu_lpi;
1629 long long sys_lpi;
1630 unsigned long long pkg_wtd_core_c0;
1631 unsigned long long pkg_any_core_c0;
1632 unsigned long long pkg_any_gfxe_c0;
1633 unsigned long long pkg_both_core_gfxe_c0;
1634 long long gfx_rc6_ms;
1635 unsigned int gfx_mhz;
1636 unsigned int gfx_act_mhz;
1637 long long sam_mc6_ms;
1638 unsigned int sam_mhz;
1639 unsigned int sam_act_mhz;
1640 unsigned int package_id;
1641 struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */
1642 struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */
1643 struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */
1644 struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */
1645 struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
1646 struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
1647 unsigned int pkg_temp_c;
1648 unsigned int uncore_mhz;
1649 unsigned long long die_c6;
1650 unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS];
1651 unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS];
1652 unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS];
1653 } *package_even, *package_odd;
1654
1655 #define ODD_COUNTERS thread_odd, core_odd, package_odd
1656 #define EVEN_COUNTERS thread_even, core_even, package_even
1657
1658 #define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \
1659 ((thread_base) + \
1660 ((pkg_no) * \
1661 topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \
1662 ((node_no) * topo.cores_per_node * topo.threads_per_core) + \
1663 ((core_no) * topo.threads_per_core) + \
1664 (thread_no))
1665
1666 #define GET_CORE(core_base, core_no, node_no, pkg_no) \
1667 ((core_base) + \
1668 ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \
1669 ((node_no) * topo.cores_per_node) + \
1670 (core_no))
1671
1672 #define GET_PKG(pkg_base, pkg_no) (pkg_base + pkg_no)
1673
1674 /*
1675 * The accumulated sum of MSR is defined as a monotonic
1676 * increasing MSR, it will be accumulated periodically,
1677 * despite its register's bit width.
1678 */
1679 enum {
1680 IDX_PKG_ENERGY,
1681 IDX_DRAM_ENERGY,
1682 IDX_PP0_ENERGY,
1683 IDX_PP1_ENERGY,
1684 IDX_PKG_PERF,
1685 IDX_DRAM_PERF,
1686 IDX_COUNT,
1687 };
1688
1689 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr);
1690
1691 struct msr_sum_array {
1692 /* get_msr_sum() = sum + (get_msr() - last) */
1693 struct {
1694 /*The accumulated MSR value is updated by the timer */
1695 unsigned long long sum;
1696 /*The MSR footprint recorded in last timer */
1697 unsigned long long last;
1698 } entries[IDX_COUNT];
1699 };
1700
1701 /* The percpu MSR sum array.*/
1702 struct msr_sum_array *per_cpu_msr_sum;
1703
idx_to_offset(int idx)1704 off_t idx_to_offset(int idx)
1705 {
1706 off_t offset;
1707
1708 switch (idx) {
1709 case IDX_PKG_ENERGY:
1710 if (platform->rapl_msrs & RAPL_AMD_F17H)
1711 offset = MSR_PKG_ENERGY_STAT;
1712 else
1713 offset = MSR_PKG_ENERGY_STATUS;
1714 break;
1715 case IDX_DRAM_ENERGY:
1716 offset = MSR_DRAM_ENERGY_STATUS;
1717 break;
1718 case IDX_PP0_ENERGY:
1719 offset = MSR_PP0_ENERGY_STATUS;
1720 break;
1721 case IDX_PP1_ENERGY:
1722 offset = MSR_PP1_ENERGY_STATUS;
1723 break;
1724 case IDX_PKG_PERF:
1725 offset = MSR_PKG_PERF_STATUS;
1726 break;
1727 case IDX_DRAM_PERF:
1728 offset = MSR_DRAM_PERF_STATUS;
1729 break;
1730 default:
1731 offset = -1;
1732 }
1733 return offset;
1734 }
1735
offset_to_idx(off_t offset)1736 int offset_to_idx(off_t offset)
1737 {
1738 int idx;
1739
1740 switch (offset) {
1741 case MSR_PKG_ENERGY_STATUS:
1742 case MSR_PKG_ENERGY_STAT:
1743 idx = IDX_PKG_ENERGY;
1744 break;
1745 case MSR_DRAM_ENERGY_STATUS:
1746 idx = IDX_DRAM_ENERGY;
1747 break;
1748 case MSR_PP0_ENERGY_STATUS:
1749 idx = IDX_PP0_ENERGY;
1750 break;
1751 case MSR_PP1_ENERGY_STATUS:
1752 idx = IDX_PP1_ENERGY;
1753 break;
1754 case MSR_PKG_PERF_STATUS:
1755 idx = IDX_PKG_PERF;
1756 break;
1757 case MSR_DRAM_PERF_STATUS:
1758 idx = IDX_DRAM_PERF;
1759 break;
1760 default:
1761 idx = -1;
1762 }
1763 return idx;
1764 }
1765
idx_valid(int idx)1766 int idx_valid(int idx)
1767 {
1768 switch (idx) {
1769 case IDX_PKG_ENERGY:
1770 return platform->rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H);
1771 case IDX_DRAM_ENERGY:
1772 return platform->rapl_msrs & RAPL_DRAM;
1773 case IDX_PP0_ENERGY:
1774 return platform->rapl_msrs & RAPL_CORE_ENERGY_STATUS;
1775 case IDX_PP1_ENERGY:
1776 return platform->rapl_msrs & RAPL_GFX;
1777 case IDX_PKG_PERF:
1778 return platform->rapl_msrs & RAPL_PKG_PERF_STATUS;
1779 case IDX_DRAM_PERF:
1780 return platform->rapl_msrs & RAPL_DRAM_PERF_STATUS;
1781 default:
1782 return 0;
1783 }
1784 }
1785
1786 struct sys_counters {
1787 /* MSR added counters */
1788 unsigned int added_thread_counters;
1789 unsigned int added_core_counters;
1790 unsigned int added_package_counters;
1791 struct msr_counter *tp;
1792 struct msr_counter *cp;
1793 struct msr_counter *pp;
1794
1795 /* perf added counters */
1796 unsigned int added_thread_perf_counters;
1797 unsigned int added_core_perf_counters;
1798 unsigned int added_package_perf_counters;
1799 struct perf_counter_info *perf_tp;
1800 struct perf_counter_info *perf_cp;
1801 struct perf_counter_info *perf_pp;
1802
1803 struct pmt_counter *pmt_tp;
1804 struct pmt_counter *pmt_cp;
1805 struct pmt_counter *pmt_pp;
1806 } sys;
1807
free_msr_counters_(struct msr_counter ** pp)1808 static size_t free_msr_counters_(struct msr_counter **pp)
1809 {
1810 struct msr_counter *p = NULL;
1811 size_t num_freed = 0;
1812
1813 while (*pp) {
1814 p = *pp;
1815
1816 if (p->msr_num != 0) {
1817 *pp = p->next;
1818
1819 free(p);
1820 ++num_freed;
1821
1822 continue;
1823 }
1824
1825 pp = &p->next;
1826 }
1827
1828 return num_freed;
1829 }
1830
1831 /*
1832 * Free all added counters accessed via msr.
1833 */
free_sys_msr_counters(void)1834 static void free_sys_msr_counters(void)
1835 {
1836 /* Thread counters */
1837 sys.added_thread_counters -= free_msr_counters_(&sys.tp);
1838
1839 /* Core counters */
1840 sys.added_core_counters -= free_msr_counters_(&sys.cp);
1841
1842 /* Package counters */
1843 sys.added_package_counters -= free_msr_counters_(&sys.pp);
1844 }
1845
1846 struct system_summary {
1847 struct thread_data threads;
1848 struct core_data cores;
1849 struct pkg_data packages;
1850 } average;
1851
1852 struct cpu_topology {
1853 int physical_package_id;
1854 int die_id;
1855 int logical_cpu_id;
1856 int physical_node_id;
1857 int logical_node_id; /* 0-based count within the package */
1858 int physical_core_id;
1859 int thread_id;
1860 int type;
1861 cpu_set_t *put_ids; /* Processing Unit/Thread IDs */
1862 } *cpus;
1863
1864 struct topo_params {
1865 int num_packages;
1866 int num_die;
1867 int num_cpus;
1868 int num_cores;
1869 int allowed_packages;
1870 int allowed_cpus;
1871 int allowed_cores;
1872 int max_cpu_num;
1873 int max_core_id;
1874 int max_package_id;
1875 int max_die_id;
1876 int max_node_num;
1877 int nodes_per_pkg;
1878 int cores_per_node;
1879 int threads_per_core;
1880 } topo;
1881
1882 struct timeval tv_even, tv_odd, tv_delta;
1883
1884 int *irq_column_2_cpu; /* /proc/interrupts column numbers */
1885 int *irqs_per_cpu; /* indexed by cpu_num */
1886
1887 void setup_all_buffers(bool startup);
1888
1889 char *sys_lpi_file;
1890 char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us";
1891 char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec";
1892
cpu_is_not_present(int cpu)1893 int cpu_is_not_present(int cpu)
1894 {
1895 return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
1896 }
1897
cpu_is_not_allowed(int cpu)1898 int cpu_is_not_allowed(int cpu)
1899 {
1900 return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
1901 }
1902
1903 /*
1904 * run func(thread, core, package) in topology order
1905 * skip non-present cpus
1906 */
1907
for_all_cpus(int (func)(struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base)1908 int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *),
1909 struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
1910 {
1911 int retval, pkg_no, core_no, thread_no, node_no;
1912
1913 for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
1914 for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) {
1915 for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
1916 for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
1917 struct thread_data *t;
1918 struct core_data *c;
1919 struct pkg_data *p;
1920 t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
1921
1922 if (cpu_is_not_allowed(t->cpu_id))
1923 continue;
1924
1925 c = GET_CORE(core_base, core_no, node_no, pkg_no);
1926 p = GET_PKG(pkg_base, pkg_no);
1927
1928 retval = func(t, c, p);
1929 if (retval)
1930 return retval;
1931 }
1932 }
1933 }
1934 }
1935 return 0;
1936 }
1937
is_cpu_first_thread_in_core(struct thread_data * t,struct core_data * c,struct pkg_data * p)1938 int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c, struct pkg_data *p)
1939 {
1940 UNUSED(p);
1941
1942 return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0);
1943 }
1944
is_cpu_first_core_in_package(struct thread_data * t,struct core_data * c,struct pkg_data * p)1945 int is_cpu_first_core_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
1946 {
1947 UNUSED(c);
1948
1949 return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0);
1950 }
1951
is_cpu_first_thread_in_package(struct thread_data * t,struct core_data * c,struct pkg_data * p)1952 int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
1953 {
1954 return is_cpu_first_thread_in_core(t, c, p) && is_cpu_first_core_in_package(t, c, p);
1955 }
1956
cpu_migrate(int cpu)1957 int cpu_migrate(int cpu)
1958 {
1959 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
1960 CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
1961 if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
1962 return -1;
1963 else
1964 return 0;
1965 }
1966
get_msr_fd(int cpu)1967 int get_msr_fd(int cpu)
1968 {
1969 char pathname[32];
1970 int fd;
1971
1972 fd = fd_percpu[cpu];
1973
1974 if (fd)
1975 return fd;
1976
1977 sprintf(pathname, "/dev/cpu/%d/msr", cpu);
1978 fd = open(pathname, O_RDONLY);
1979 if (fd < 0)
1980 err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, "
1981 "or run with --no-msr, or run as root", pathname);
1982
1983 fd_percpu[cpu] = fd;
1984
1985 return fd;
1986 }
1987
bic_disable_msr_access(void)1988 static void bic_disable_msr_access(void)
1989 {
1990 const unsigned long bic_msrs = BIC_Mod_c6 | BIC_CoreTmp |
1991 BIC_Totl_c0 | BIC_Any_c0 | BIC_GFX_c0 | BIC_CPUGFX | BIC_PkgTmp;
1992
1993 bic_enabled &= ~bic_msrs;
1994
1995 free_sys_msr_counters();
1996 }
1997
perf_event_open(struct perf_event_attr * hw_event,pid_t pid,int cpu,int group_fd,unsigned long flags)1998 static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
1999 {
2000 assert(!no_perf);
2001
2002 return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
2003 }
2004
open_perf_counter(int cpu,unsigned int type,unsigned int config,int group_fd,__u64 read_format)2005 static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format)
2006 {
2007 struct perf_event_attr attr;
2008 const pid_t pid = -1;
2009 const unsigned long flags = 0;
2010
2011 assert(!no_perf);
2012
2013 memset(&attr, 0, sizeof(struct perf_event_attr));
2014
2015 attr.type = type;
2016 attr.size = sizeof(struct perf_event_attr);
2017 attr.config = config;
2018 attr.disabled = 0;
2019 attr.sample_type = PERF_SAMPLE_IDENTIFIER;
2020 attr.read_format = read_format;
2021
2022 const int fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
2023
2024 return fd;
2025 }
2026
get_instr_count_fd(int cpu)2027 int get_instr_count_fd(int cpu)
2028 {
2029 if (fd_instr_count_percpu[cpu])
2030 return fd_instr_count_percpu[cpu];
2031
2032 fd_instr_count_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
2033
2034 return fd_instr_count_percpu[cpu];
2035 }
2036
get_msr(int cpu,off_t offset,unsigned long long * msr)2037 int get_msr(int cpu, off_t offset, unsigned long long *msr)
2038 {
2039 ssize_t retval;
2040
2041 assert(!no_msr);
2042
2043 retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
2044
2045 if (retval != sizeof *msr)
2046 err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
2047
2048 return 0;
2049 }
2050
probe_msr(int cpu,off_t offset)2051 int probe_msr(int cpu, off_t offset)
2052 {
2053 ssize_t retval;
2054 unsigned long long dummy;
2055
2056 assert(!no_msr);
2057
2058 retval = pread(get_msr_fd(cpu), &dummy, sizeof(dummy), offset);
2059
2060 if (retval != sizeof(dummy))
2061 return 1;
2062
2063 return 0;
2064 }
2065
2066 /* Convert CPU ID to domain ID for given added perf counter. */
cpu_to_domain(const struct perf_counter_info * pc,int cpu)2067 unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu)
2068 {
2069 switch (pc->scope) {
2070 case SCOPE_CPU:
2071 return cpu;
2072
2073 case SCOPE_CORE:
2074 return cpus[cpu].physical_core_id;
2075
2076 case SCOPE_PACKAGE:
2077 return cpus[cpu].physical_package_id;
2078 }
2079
2080 __builtin_unreachable();
2081 }
2082
2083 #define MAX_DEFERRED 16
2084 char *deferred_add_names[MAX_DEFERRED];
2085 char *deferred_skip_names[MAX_DEFERRED];
2086 int deferred_add_index;
2087 int deferred_skip_index;
2088
2089 /*
2090 * HIDE_LIST - hide this list of counters, show the rest [default]
2091 * SHOW_LIST - show this list of counters, hide the rest
2092 */
2093 enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
2094
help(void)2095 void help(void)
2096 {
2097 fprintf(outf,
2098 "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
2099 "\n"
2100 "Turbostat forks the specified COMMAND and prints statistics\n"
2101 "when COMMAND completes.\n"
2102 "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
2103 "to print statistics, until interrupted.\n"
2104 " -a, --add add a counter\n"
2105 " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
2106 " eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n"
2107 " eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n"
2108 " -c, --cpu cpu-set limit output to summary plus cpu-set:\n"
2109 " {core | package | j,k,l..m,n-p }\n"
2110 " -d, --debug displays usec, Time_Of_Day_Seconds and more debugging\n"
2111 " debug messages are printed to stderr\n"
2112 " -D, --Dump displays the raw counter values\n"
2113 " -e, --enable [all | column]\n"
2114 " shows all or the specified disabled column\n"
2115 " -H, --hide [column|column,column,...]\n"
2116 " hide the specified column(s)\n"
2117 " -i, --interval sec.subsec\n"
2118 " Override default 5-second measurement interval\n"
2119 " -J, --Joules displays energy in Joules instead of Watts\n"
2120 " -l, --list list column headers only\n"
2121 " -M, --no-msr Disable all uses of the MSR driver\n"
2122 " -P, --no-perf Disable all uses of the perf API\n"
2123 " -n, --num_iterations num\n"
2124 " number of the measurement iterations\n"
2125 " -N, --header_iterations num\n"
2126 " print header every num iterations\n"
2127 " -o, --out file\n"
2128 " create or truncate \"file\" for all output\n"
2129 " -q, --quiet skip decoding system configuration header\n"
2130 " -s, --show [column|column,column,...]\n"
2131 " show only the specified column(s)\n"
2132 " -S, --Summary\n"
2133 " limits output to 1-line system summary per interval\n"
2134 " -T, --TCC temperature\n"
2135 " sets the Thermal Control Circuit temperature in\n"
2136 " degrees Celsius\n"
2137 " -h, --help print this help message\n"
2138 " -v, --version print version information\n" "\n" "For more help, run \"man turbostat\"\n");
2139 }
2140
2141 /*
2142 * bic_lookup
2143 * for all the strings in comma separate name_list,
2144 * set the approprate bit in return value.
2145 */
bic_lookup(char * name_list,enum show_hide_mode mode)2146 unsigned long long bic_lookup(char *name_list, enum show_hide_mode mode)
2147 {
2148 unsigned int i;
2149 unsigned long long retval = 0;
2150
2151 while (name_list) {
2152 char *comma;
2153
2154 comma = strchr(name_list, ',');
2155
2156 if (comma)
2157 *comma = '\0';
2158
2159 for (i = 0; i < MAX_BIC; ++i) {
2160 if (!strcmp(name_list, bic[i].name)) {
2161 retval |= (1ULL << i);
2162 break;
2163 }
2164 if (!strcmp(name_list, "all")) {
2165 retval |= ~0;
2166 break;
2167 } else if (!strcmp(name_list, "topology")) {
2168 retval |= BIC_TOPOLOGY;
2169 break;
2170 } else if (!strcmp(name_list, "power")) {
2171 retval |= BIC_THERMAL_PWR;
2172 break;
2173 } else if (!strcmp(name_list, "idle")) {
2174 retval |= BIC_IDLE;
2175 break;
2176 } else if (!strcmp(name_list, "frequency")) {
2177 retval |= BIC_FREQUENCY;
2178 break;
2179 } else if (!strcmp(name_list, "other")) {
2180 retval |= BIC_OTHER;
2181 break;
2182 }
2183
2184 }
2185 if (i == MAX_BIC) {
2186 if (mode == SHOW_LIST) {
2187 deferred_add_names[deferred_add_index++] = name_list;
2188 if (deferred_add_index >= MAX_DEFERRED) {
2189 fprintf(stderr, "More than max %d un-recognized --add options '%s'\n",
2190 MAX_DEFERRED, name_list);
2191 help();
2192 exit(1);
2193 }
2194 } else {
2195 deferred_skip_names[deferred_skip_index++] = name_list;
2196 if (debug)
2197 fprintf(stderr, "deferred \"%s\"\n", name_list);
2198 if (deferred_skip_index >= MAX_DEFERRED) {
2199 fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n",
2200 MAX_DEFERRED, name_list);
2201 help();
2202 exit(1);
2203 }
2204 }
2205 }
2206
2207 name_list = comma;
2208 if (name_list)
2209 name_list++;
2210
2211 }
2212 return retval;
2213 }
2214
print_header(char * delim)2215 void print_header(char *delim)
2216 {
2217 struct msr_counter *mp;
2218 struct perf_counter_info *pp;
2219 struct pmt_counter *ppmt;
2220 int printed = 0;
2221
2222 if (DO_BIC(BIC_USEC))
2223 outp += sprintf(outp, "%susec", (printed++ ? delim : ""));
2224 if (DO_BIC(BIC_TOD))
2225 outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : ""));
2226 if (DO_BIC(BIC_Package))
2227 outp += sprintf(outp, "%sPackage", (printed++ ? delim : ""));
2228 if (DO_BIC(BIC_Die))
2229 outp += sprintf(outp, "%sDie", (printed++ ? delim : ""));
2230 if (DO_BIC(BIC_Node))
2231 outp += sprintf(outp, "%sNode", (printed++ ? delim : ""));
2232 if (DO_BIC(BIC_Core))
2233 outp += sprintf(outp, "%sCore", (printed++ ? delim : ""));
2234 if (DO_BIC(BIC_CPU))
2235 outp += sprintf(outp, "%sCPU", (printed++ ? delim : ""));
2236 if (DO_BIC(BIC_APIC))
2237 outp += sprintf(outp, "%sAPIC", (printed++ ? delim : ""));
2238 if (DO_BIC(BIC_X2APIC))
2239 outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : ""));
2240 if (DO_BIC(BIC_Avg_MHz))
2241 outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : ""));
2242 if (DO_BIC(BIC_Busy))
2243 outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : ""));
2244 if (DO_BIC(BIC_Bzy_MHz))
2245 outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : ""));
2246 if (DO_BIC(BIC_TSC_MHz))
2247 outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : ""));
2248
2249 if (DO_BIC(BIC_IPC))
2250 outp += sprintf(outp, "%sIPC", (printed++ ? delim : ""));
2251
2252 if (DO_BIC(BIC_IRQ)) {
2253 if (sums_need_wide_columns)
2254 outp += sprintf(outp, "%s IRQ", (printed++ ? delim : ""));
2255 else
2256 outp += sprintf(outp, "%sIRQ", (printed++ ? delim : ""));
2257 }
2258
2259 if (DO_BIC(BIC_SMI))
2260 outp += sprintf(outp, "%sSMI", (printed++ ? delim : ""));
2261
2262 for (mp = sys.tp; mp; mp = mp->next) {
2263
2264 if (mp->format == FORMAT_RAW) {
2265 if (mp->width == 64)
2266 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), mp->name);
2267 else
2268 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), mp->name);
2269 } else {
2270 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2271 outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), mp->name);
2272 else
2273 outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), mp->name);
2274 }
2275 }
2276
2277 for (pp = sys.perf_tp; pp; pp = pp->next) {
2278
2279 if (pp->format == FORMAT_RAW) {
2280 if (pp->width == 64)
2281 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name);
2282 else
2283 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name);
2284 } else {
2285 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2286 outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name);
2287 else
2288 outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name);
2289 }
2290 }
2291
2292 ppmt = sys.pmt_tp;
2293 while (ppmt) {
2294 switch (ppmt->type) {
2295 case PMT_TYPE_RAW:
2296 if (pmt_counter_get_width(ppmt) <= 32)
2297 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name);
2298 else
2299 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name);
2300
2301 break;
2302
2303 case PMT_TYPE_XTAL_TIME:
2304 outp += sprintf(outp, "%s%s", delim, ppmt->name);
2305 break;
2306 }
2307
2308 ppmt = ppmt->next;
2309 }
2310
2311 if (DO_BIC(BIC_CPU_c1))
2312 outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : ""));
2313 if (DO_BIC(BIC_CPU_c3))
2314 outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : ""));
2315 if (DO_BIC(BIC_CPU_c6))
2316 outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : ""));
2317 if (DO_BIC(BIC_CPU_c7))
2318 outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : ""));
2319
2320 if (DO_BIC(BIC_Mod_c6))
2321 outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : ""));
2322
2323 if (DO_BIC(BIC_CoreTmp))
2324 outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : ""));
2325
2326 if (DO_BIC(BIC_CORE_THROT_CNT))
2327 outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : ""));
2328
2329 if (platform->rapl_msrs && !rapl_joules) {
2330 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
2331 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
2332 } else if (platform->rapl_msrs && rapl_joules) {
2333 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
2334 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
2335 }
2336
2337 for (mp = sys.cp; mp; mp = mp->next) {
2338 if (mp->format == FORMAT_RAW) {
2339 if (mp->width == 64)
2340 outp += sprintf(outp, "%s%18.18s", delim, mp->name);
2341 else
2342 outp += sprintf(outp, "%s%10.10s", delim, mp->name);
2343 } else {
2344 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2345 outp += sprintf(outp, "%s%8s", delim, mp->name);
2346 else
2347 outp += sprintf(outp, "%s%s", delim, mp->name);
2348 }
2349 }
2350
2351 for (pp = sys.perf_cp; pp; pp = pp->next) {
2352
2353 if (pp->format == FORMAT_RAW) {
2354 if (pp->width == 64)
2355 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name);
2356 else
2357 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name);
2358 } else {
2359 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2360 outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name);
2361 else
2362 outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name);
2363 }
2364 }
2365
2366 ppmt = sys.pmt_cp;
2367 while (ppmt) {
2368 switch (ppmt->type) {
2369 case PMT_TYPE_RAW:
2370 if (pmt_counter_get_width(ppmt) <= 32)
2371 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name);
2372 else
2373 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name);
2374
2375 break;
2376
2377 case PMT_TYPE_XTAL_TIME:
2378 outp += sprintf(outp, "%s%s", delim, ppmt->name);
2379 break;
2380 }
2381
2382 ppmt = ppmt->next;
2383 }
2384
2385 if (DO_BIC(BIC_PkgTmp))
2386 outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : ""));
2387
2388 if (DO_BIC(BIC_GFX_rc6))
2389 outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : ""));
2390
2391 if (DO_BIC(BIC_GFXMHz))
2392 outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : ""));
2393
2394 if (DO_BIC(BIC_GFXACTMHz))
2395 outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : ""));
2396
2397 if (DO_BIC(BIC_SAM_mc6))
2398 outp += sprintf(outp, "%sSAM%%mc6", (printed++ ? delim : ""));
2399
2400 if (DO_BIC(BIC_SAMMHz))
2401 outp += sprintf(outp, "%sSAMMHz", (printed++ ? delim : ""));
2402
2403 if (DO_BIC(BIC_SAMACTMHz))
2404 outp += sprintf(outp, "%sSAMAMHz", (printed++ ? delim : ""));
2405
2406 if (DO_BIC(BIC_Totl_c0))
2407 outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : ""));
2408 if (DO_BIC(BIC_Any_c0))
2409 outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : ""));
2410 if (DO_BIC(BIC_GFX_c0))
2411 outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : ""));
2412 if (DO_BIC(BIC_CPUGFX))
2413 outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : ""));
2414
2415 if (DO_BIC(BIC_Pkgpc2))
2416 outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : ""));
2417 if (DO_BIC(BIC_Pkgpc3))
2418 outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : ""));
2419 if (DO_BIC(BIC_Pkgpc6))
2420 outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : ""));
2421 if (DO_BIC(BIC_Pkgpc7))
2422 outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : ""));
2423 if (DO_BIC(BIC_Pkgpc8))
2424 outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : ""));
2425 if (DO_BIC(BIC_Pkgpc9))
2426 outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : ""));
2427 if (DO_BIC(BIC_Pkgpc10))
2428 outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : ""));
2429 if (DO_BIC(BIC_Diec6))
2430 outp += sprintf(outp, "%sDie%%c6", (printed++ ? delim : ""));
2431 if (DO_BIC(BIC_CPU_LPI))
2432 outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : ""));
2433 if (DO_BIC(BIC_SYS_LPI))
2434 outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : ""));
2435
2436 if (platform->rapl_msrs && !rapl_joules) {
2437 if (DO_BIC(BIC_PkgWatt))
2438 outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : ""));
2439 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
2440 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
2441 if (DO_BIC(BIC_GFXWatt))
2442 outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : ""));
2443 if (DO_BIC(BIC_RAMWatt))
2444 outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : ""));
2445 if (DO_BIC(BIC_PKG__))
2446 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
2447 if (DO_BIC(BIC_RAM__))
2448 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
2449 } else if (platform->rapl_msrs && rapl_joules) {
2450 if (DO_BIC(BIC_Pkg_J))
2451 outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : ""));
2452 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
2453 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
2454 if (DO_BIC(BIC_GFX_J))
2455 outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : ""));
2456 if (DO_BIC(BIC_RAM_J))
2457 outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : ""));
2458 if (DO_BIC(BIC_PKG__))
2459 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
2460 if (DO_BIC(BIC_RAM__))
2461 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
2462 }
2463 if (DO_BIC(BIC_UNCORE_MHZ))
2464 outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : ""));
2465
2466 for (mp = sys.pp; mp; mp = mp->next) {
2467 if (mp->format == FORMAT_RAW) {
2468 if (mp->width == 64)
2469 outp += sprintf(outp, "%s%18.18s", delim, mp->name);
2470 else if (mp->width == 32)
2471 outp += sprintf(outp, "%s%10.10s", delim, mp->name);
2472 else
2473 outp += sprintf(outp, "%s%7.7s", delim, mp->name);
2474 } else {
2475 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2476 outp += sprintf(outp, "%s%8s", delim, mp->name);
2477 else
2478 outp += sprintf(outp, "%s%7.7s", delim, mp->name);
2479 }
2480 }
2481
2482 for (pp = sys.perf_pp; pp; pp = pp->next) {
2483
2484 if (pp->format == FORMAT_RAW) {
2485 if (pp->width == 64)
2486 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name);
2487 else
2488 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name);
2489 } else {
2490 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2491 outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name);
2492 else
2493 outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name);
2494 }
2495 }
2496
2497 ppmt = sys.pmt_pp;
2498 while (ppmt) {
2499 switch (ppmt->type) {
2500 case PMT_TYPE_RAW:
2501 if (pmt_counter_get_width(ppmt) <= 32)
2502 outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name);
2503 else
2504 outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name);
2505
2506 break;
2507
2508 case PMT_TYPE_XTAL_TIME:
2509 outp += sprintf(outp, "%s%s", delim, ppmt->name);
2510 break;
2511 }
2512
2513 ppmt = ppmt->next;
2514 }
2515
2516 outp += sprintf(outp, "\n");
2517 }
2518
dump_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)2519 int dump_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2520 {
2521 int i;
2522 struct msr_counter *mp;
2523
2524 outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p);
2525
2526 if (t) {
2527 outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags);
2528 outp += sprintf(outp, "TSC: %016llX\n", t->tsc);
2529 outp += sprintf(outp, "aperf: %016llX\n", t->aperf);
2530 outp += sprintf(outp, "mperf: %016llX\n", t->mperf);
2531 outp += sprintf(outp, "c1: %016llX\n", t->c1);
2532
2533 if (DO_BIC(BIC_IPC))
2534 outp += sprintf(outp, "IPC: %lld\n", t->instr_count);
2535
2536 if (DO_BIC(BIC_IRQ))
2537 outp += sprintf(outp, "IRQ: %lld\n", t->irq_count);
2538 if (DO_BIC(BIC_SMI))
2539 outp += sprintf(outp, "SMI: %d\n", t->smi_count);
2540
2541 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
2542 outp +=
2543 sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num,
2544 t->counter[i], mp->sp->path);
2545 }
2546 }
2547
2548 if (c && is_cpu_first_thread_in_core(t, c, p)) {
2549 outp += sprintf(outp, "core: %d\n", c->core_id);
2550 outp += sprintf(outp, "c3: %016llX\n", c->c3);
2551 outp += sprintf(outp, "c6: %016llX\n", c->c6);
2552 outp += sprintf(outp, "c7: %016llX\n", c->c7);
2553 outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c);
2554 outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt);
2555
2556 const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale;
2557 const double energy_scale = c->core_energy.scale;
2558
2559 if (c->core_energy.unit == RAPL_UNIT_JOULES)
2560 outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale);
2561
2562 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
2563 outp +=
2564 sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num,
2565 c->counter[i], mp->sp->path);
2566 }
2567 outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us);
2568 }
2569
2570 if (p && is_cpu_first_core_in_package(t, c, p)) {
2571 outp += sprintf(outp, "package: %d\n", p->package_id);
2572
2573 outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0);
2574 outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0);
2575 outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0);
2576 outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
2577
2578 outp += sprintf(outp, "pc2: %016llX\n", p->pc2);
2579 if (DO_BIC(BIC_Pkgpc3))
2580 outp += sprintf(outp, "pc3: %016llX\n", p->pc3);
2581 if (DO_BIC(BIC_Pkgpc6))
2582 outp += sprintf(outp, "pc6: %016llX\n", p->pc6);
2583 if (DO_BIC(BIC_Pkgpc7))
2584 outp += sprintf(outp, "pc7: %016llX\n", p->pc7);
2585 outp += sprintf(outp, "pc8: %016llX\n", p->pc8);
2586 outp += sprintf(outp, "pc9: %016llX\n", p->pc9);
2587 outp += sprintf(outp, "pc10: %016llX\n", p->pc10);
2588 outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi);
2589 outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi);
2590 outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg.raw_value);
2591 outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores.raw_value);
2592 outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx.raw_value);
2593 outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram.raw_value);
2594 outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value);
2595 outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value);
2596 outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c);
2597
2598 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
2599 outp +=
2600 sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num,
2601 p->counter[i], mp->sp->path);
2602 }
2603 }
2604
2605 outp += sprintf(outp, "\n");
2606
2607 return 0;
2608 }
2609
rapl_counter_get_value(const struct rapl_counter * c,enum rapl_unit desired_unit,double interval)2610 double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
2611 {
2612 assert(desired_unit != RAPL_UNIT_INVALID);
2613
2614 /*
2615 * For now we don't expect anything other than joules,
2616 * so just simplify the logic.
2617 */
2618 assert(c->unit == RAPL_UNIT_JOULES);
2619
2620 const double scaled = c->raw_value * c->scale;
2621
2622 if (desired_unit == RAPL_UNIT_WATTS)
2623 return scaled / interval;
2624 return scaled;
2625 }
2626
2627 /*
2628 * column formatting convention & formats
2629 */
format_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)2630 int format_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2631 {
2632 double interval_float, tsc;
2633 char *fmt8;
2634 int i;
2635 struct msr_counter *mp;
2636 struct perf_counter_info *pp;
2637 struct pmt_counter *ppmt;
2638 char *delim = "\t";
2639 int printed = 0;
2640
2641 /* if showing only 1st thread in core and this isn't one, bail out */
2642 if (show_core_only && !is_cpu_first_thread_in_core(t, c, p))
2643 return 0;
2644
2645 /* if showing only 1st thread in pkg and this isn't one, bail out */
2646 if (show_pkg_only && !is_cpu_first_core_in_package(t, c, p))
2647 return 0;
2648
2649 /*if not summary line and --cpu is used */
2650 if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
2651 return 0;
2652
2653 if (DO_BIC(BIC_USEC)) {
2654 /* on each row, print how many usec each timestamp took to gather */
2655 struct timeval tv;
2656
2657 timersub(&t->tv_end, &t->tv_begin, &tv);
2658 outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec);
2659 }
2660
2661 /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */
2662 if (DO_BIC(BIC_TOD))
2663 outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec);
2664
2665 interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0;
2666
2667 tsc = t->tsc * tsc_tweak;
2668
2669 /* topo columns, print blanks on 1st (average) line */
2670 if (t == &average.threads) {
2671 if (DO_BIC(BIC_Package))
2672 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2673 if (DO_BIC(BIC_Die))
2674 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2675 if (DO_BIC(BIC_Node))
2676 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2677 if (DO_BIC(BIC_Core))
2678 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2679 if (DO_BIC(BIC_CPU))
2680 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2681 if (DO_BIC(BIC_APIC))
2682 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2683 if (DO_BIC(BIC_X2APIC))
2684 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2685 } else {
2686 if (DO_BIC(BIC_Package)) {
2687 if (p)
2688 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id);
2689 else
2690 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2691 }
2692 if (DO_BIC(BIC_Die)) {
2693 if (c)
2694 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id);
2695 else
2696 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2697 }
2698 if (DO_BIC(BIC_Node)) {
2699 if (t)
2700 outp += sprintf(outp, "%s%d",
2701 (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id);
2702 else
2703 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2704 }
2705 if (DO_BIC(BIC_Core)) {
2706 if (c)
2707 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id);
2708 else
2709 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
2710 }
2711 if (DO_BIC(BIC_CPU))
2712 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id);
2713 if (DO_BIC(BIC_APIC))
2714 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id);
2715 if (DO_BIC(BIC_X2APIC))
2716 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id);
2717 }
2718
2719 if (DO_BIC(BIC_Avg_MHz))
2720 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
2721
2722 if (DO_BIC(BIC_Busy))
2723 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->mperf / tsc);
2724
2725 if (DO_BIC(BIC_Bzy_MHz)) {
2726 if (has_base_hz)
2727 outp +=
2728 sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
2729 else
2730 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""),
2731 tsc / units * t->aperf / t->mperf / interval_float);
2732 }
2733
2734 if (DO_BIC(BIC_TSC_MHz))
2735 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float);
2736
2737 if (DO_BIC(BIC_IPC))
2738 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf);
2739
2740 /* IRQ */
2741 if (DO_BIC(BIC_IRQ)) {
2742 if (sums_need_wide_columns)
2743 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count);
2744 else
2745 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count);
2746 }
2747
2748 /* SMI */
2749 if (DO_BIC(BIC_SMI))
2750 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count);
2751
2752 /* Added counters */
2753 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
2754 if (mp->format == FORMAT_RAW) {
2755 if (mp->width == 32)
2756 outp +=
2757 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)t->counter[i]);
2758 else
2759 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]);
2760 } else if (mp->format == FORMAT_DELTA) {
2761 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2762 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]);
2763 else
2764 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->counter[i]);
2765 } else if (mp->format == FORMAT_PERCENT) {
2766 if (mp->type == COUNTER_USEC)
2767 outp +=
2768 sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2769 t->counter[i] / interval_float / 10000);
2770 else
2771 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->counter[i] / tsc);
2772 }
2773 }
2774
2775 /* Added perf counters */
2776 for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) {
2777 if (pp->format == FORMAT_RAW) {
2778 if (pp->width == 32)
2779 outp +=
2780 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
2781 (unsigned int)t->perf_counter[i]);
2782 else
2783 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->perf_counter[i]);
2784 } else if (pp->format == FORMAT_DELTA) {
2785 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2786 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->perf_counter[i]);
2787 else
2788 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->perf_counter[i]);
2789 } else if (pp->format == FORMAT_PERCENT) {
2790 if (pp->type == COUNTER_USEC)
2791 outp +=
2792 sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2793 t->perf_counter[i] / interval_float / 10000);
2794 else
2795 outp +=
2796 sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->perf_counter[i] / tsc);
2797 }
2798 }
2799
2800 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
2801 switch (ppmt->type) {
2802 case PMT_TYPE_RAW:
2803 if (pmt_counter_get_width(ppmt) <= 32)
2804 outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
2805 (unsigned int)t->pmt_counter[i]);
2806 else
2807 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->pmt_counter[i]);
2808
2809 break;
2810
2811 case PMT_TYPE_XTAL_TIME:
2812 const unsigned long value_raw = t->pmt_counter[i];
2813 const double value_converted = 100.0 * value_raw / crystal_hz / interval_float;
2814
2815 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
2816 break;
2817 }
2818 }
2819
2820 /* C1 */
2821 if (DO_BIC(BIC_CPU_c1))
2822 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->c1 / tsc);
2823
2824 /* print per-core data only for 1st thread in core */
2825 if (!is_cpu_first_thread_in_core(t, c, p))
2826 goto done;
2827
2828 if (DO_BIC(BIC_CPU_c3))
2829 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c3 / tsc);
2830 if (DO_BIC(BIC_CPU_c6))
2831 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c6 / tsc);
2832 if (DO_BIC(BIC_CPU_c7))
2833 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c7 / tsc);
2834
2835 /* Mod%c6 */
2836 if (DO_BIC(BIC_Mod_c6))
2837 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->mc6_us / tsc);
2838
2839 if (DO_BIC(BIC_CoreTmp))
2840 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
2841
2842 /* Core throttle count */
2843 if (DO_BIC(BIC_CORE_THROT_CNT))
2844 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt);
2845
2846 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
2847 if (mp->format == FORMAT_RAW) {
2848 if (mp->width == 32)
2849 outp +=
2850 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)c->counter[i]);
2851 else
2852 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]);
2853 } else if (mp->format == FORMAT_DELTA) {
2854 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2855 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]);
2856 else
2857 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->counter[i]);
2858 } else if (mp->format == FORMAT_PERCENT) {
2859 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->counter[i] / tsc);
2860 }
2861 }
2862
2863 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
2864 if (pp->format == FORMAT_RAW) {
2865 if (pp->width == 32)
2866 outp +=
2867 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
2868 (unsigned int)c->perf_counter[i]);
2869 else
2870 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->perf_counter[i]);
2871 } else if (pp->format == FORMAT_DELTA) {
2872 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
2873 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->perf_counter[i]);
2874 else
2875 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->perf_counter[i]);
2876 } else if (pp->format == FORMAT_PERCENT) {
2877 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->perf_counter[i] / tsc);
2878 }
2879 }
2880
2881 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
2882 switch (ppmt->type) {
2883 case PMT_TYPE_RAW:
2884 if (pmt_counter_get_width(ppmt) <= 32)
2885 outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
2886 (unsigned int)c->pmt_counter[i]);
2887 else
2888 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->pmt_counter[i]);
2889
2890 break;
2891
2892 case PMT_TYPE_XTAL_TIME:
2893 const unsigned long value_raw = c->pmt_counter[i];
2894 const double value_converted = 100.0 * value_raw / crystal_hz / interval_float;
2895
2896 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
2897 break;
2898 }
2899 }
2900
2901 fmt8 = "%s%.2f";
2902
2903 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
2904 outp +=
2905 sprintf(outp, fmt8, (printed++ ? delim : ""),
2906 rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float));
2907 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
2908 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
2909 rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float));
2910
2911 /* print per-package data only for 1st core in package */
2912 if (!is_cpu_first_core_in_package(t, c, p))
2913 goto done;
2914
2915 /* PkgTmp */
2916 if (DO_BIC(BIC_PkgTmp))
2917 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
2918
2919 /* GFXrc6 */
2920 if (DO_BIC(BIC_GFX_rc6)) {
2921 if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
2922 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
2923 } else {
2924 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2925 p->gfx_rc6_ms / 10.0 / interval_float);
2926 }
2927 }
2928
2929 /* GFXMHz */
2930 if (DO_BIC(BIC_GFXMHz))
2931 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
2932
2933 /* GFXACTMHz */
2934 if (DO_BIC(BIC_GFXACTMHz))
2935 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz);
2936
2937 /* SAMmc6 */
2938 if (DO_BIC(BIC_SAM_mc6)) {
2939 if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */
2940 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
2941 } else {
2942 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2943 p->sam_mc6_ms / 10.0 / interval_float);
2944 }
2945 }
2946
2947 /* SAMMHz */
2948 if (DO_BIC(BIC_SAMMHz))
2949 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_mhz);
2950
2951 /* SAMACTMHz */
2952 if (DO_BIC(BIC_SAMACTMHz))
2953 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_act_mhz);
2954
2955 /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
2956 if (DO_BIC(BIC_Totl_c0))
2957 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_wtd_core_c0 / tsc);
2958 if (DO_BIC(BIC_Any_c0))
2959 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_core_c0 / tsc);
2960 if (DO_BIC(BIC_GFX_c0))
2961 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_gfxe_c0 / tsc);
2962 if (DO_BIC(BIC_CPUGFX))
2963 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_both_core_gfxe_c0 / tsc);
2964
2965 if (DO_BIC(BIC_Pkgpc2))
2966 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc2 / tsc);
2967 if (DO_BIC(BIC_Pkgpc3))
2968 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc3 / tsc);
2969 if (DO_BIC(BIC_Pkgpc6))
2970 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc6 / tsc);
2971 if (DO_BIC(BIC_Pkgpc7))
2972 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc7 / tsc);
2973 if (DO_BIC(BIC_Pkgpc8))
2974 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc8 / tsc);
2975 if (DO_BIC(BIC_Pkgpc9))
2976 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc9 / tsc);
2977 if (DO_BIC(BIC_Pkgpc10))
2978 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc10 / tsc);
2979
2980 if (DO_BIC(BIC_Diec6))
2981 outp +=
2982 sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->die_c6 / crystal_hz / interval_float);
2983
2984 if (DO_BIC(BIC_CPU_LPI)) {
2985 if (p->cpu_lpi >= 0)
2986 outp +=
2987 sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2988 100.0 * p->cpu_lpi / 1000000.0 / interval_float);
2989 else
2990 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
2991 }
2992 if (DO_BIC(BIC_SYS_LPI)) {
2993 if (p->sys_lpi >= 0)
2994 outp +=
2995 sprintf(outp, "%s%.2f", (printed++ ? delim : ""),
2996 100.0 * p->sys_lpi / 1000000.0 / interval_float);
2997 else
2998 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
2999 }
3000
3001 if (DO_BIC(BIC_PkgWatt))
3002 outp +=
3003 sprintf(outp, fmt8, (printed++ ? delim : ""),
3004 rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float));
3005 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3006 outp +=
3007 sprintf(outp, fmt8, (printed++ ? delim : ""),
3008 rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float));
3009 if (DO_BIC(BIC_GFXWatt))
3010 outp +=
3011 sprintf(outp, fmt8, (printed++ ? delim : ""),
3012 rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float));
3013 if (DO_BIC(BIC_RAMWatt))
3014 outp +=
3015 sprintf(outp, fmt8, (printed++ ? delim : ""),
3016 rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float));
3017 if (DO_BIC(BIC_Pkg_J))
3018 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3019 rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float));
3020 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3021 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3022 rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float));
3023 if (DO_BIC(BIC_GFX_J))
3024 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3025 rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float));
3026 if (DO_BIC(BIC_RAM_J))
3027 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3028 rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float));
3029 if (DO_BIC(BIC_PKG__))
3030 outp +=
3031 sprintf(outp, fmt8, (printed++ ? delim : ""),
3032 rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float));
3033 if (DO_BIC(BIC_RAM__))
3034 outp +=
3035 sprintf(outp, fmt8, (printed++ ? delim : ""),
3036 rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float));
3037 /* UncMHz */
3038 if (DO_BIC(BIC_UNCORE_MHZ))
3039 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz);
3040
3041 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3042 if (mp->format == FORMAT_RAW) {
3043 if (mp->width == 32)
3044 outp +=
3045 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)p->counter[i]);
3046 else
3047 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->counter[i]);
3048 } else if (mp->format == FORMAT_DELTA) {
3049 if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns)
3050 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->counter[i]);
3051 else
3052 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]);
3053 } else if (mp->format == FORMAT_PERCENT) {
3054 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i] / tsc);
3055 } else if (mp->type == COUNTER_K2M)
3056 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000);
3057 }
3058
3059 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3060 if (pp->format == FORMAT_RAW) {
3061 if (pp->width == 32)
3062 outp +=
3063 sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
3064 (unsigned int)p->perf_counter[i]);
3065 else
3066 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->perf_counter[i]);
3067 } else if (pp->format == FORMAT_DELTA) {
3068 if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns)
3069 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->perf_counter[i]);
3070 else
3071 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->perf_counter[i]);
3072 } else if (pp->format == FORMAT_PERCENT) {
3073 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->perf_counter[i] / tsc);
3074 } else if (pp->type == COUNTER_K2M) {
3075 outp +=
3076 sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000);
3077 }
3078 }
3079
3080 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3081 switch (ppmt->type) {
3082 case PMT_TYPE_RAW:
3083 if (pmt_counter_get_width(ppmt) <= 32)
3084 outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""),
3085 (unsigned int)p->pmt_counter[i]);
3086 else
3087 outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->pmt_counter[i]);
3088
3089 break;
3090
3091 case PMT_TYPE_XTAL_TIME:
3092 const unsigned long value_raw = p->pmt_counter[i];
3093 const double value_converted = 100.0 * value_raw / crystal_hz / interval_float;
3094
3095 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
3096 break;
3097 }
3098 }
3099
3100 done:
3101 if (*(outp - 1) != '\n')
3102 outp += sprintf(outp, "\n");
3103
3104 return 0;
3105 }
3106
flush_output_stdout(void)3107 void flush_output_stdout(void)
3108 {
3109 FILE *filep;
3110
3111 if (outf == stderr)
3112 filep = stdout;
3113 else
3114 filep = outf;
3115
3116 fputs(output_buffer, filep);
3117 fflush(filep);
3118
3119 outp = output_buffer;
3120 }
3121
flush_output_stderr(void)3122 void flush_output_stderr(void)
3123 {
3124 fputs(output_buffer, outf);
3125 fflush(outf);
3126 outp = output_buffer;
3127 }
3128
format_all_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)3129 void format_all_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
3130 {
3131 static int count;
3132
3133 if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only)
3134 print_header("\t");
3135
3136 format_counters(&average.threads, &average.cores, &average.packages);
3137
3138 count++;
3139
3140 if (summary_only)
3141 return;
3142
3143 for_all_cpus(format_counters, t, c, p);
3144 }
3145
3146 #define DELTA_WRAP32(new, old) \
3147 old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32);
3148
delta_package(struct pkg_data * new,struct pkg_data * old)3149 int delta_package(struct pkg_data *new, struct pkg_data *old)
3150 {
3151 int i;
3152 struct msr_counter *mp;
3153 struct perf_counter_info *pp;
3154 struct pmt_counter *ppmt;
3155
3156 if (DO_BIC(BIC_Totl_c0))
3157 old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0;
3158 if (DO_BIC(BIC_Any_c0))
3159 old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0;
3160 if (DO_BIC(BIC_GFX_c0))
3161 old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0;
3162 if (DO_BIC(BIC_CPUGFX))
3163 old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
3164
3165 old->pc2 = new->pc2 - old->pc2;
3166 if (DO_BIC(BIC_Pkgpc3))
3167 old->pc3 = new->pc3 - old->pc3;
3168 if (DO_BIC(BIC_Pkgpc6))
3169 old->pc6 = new->pc6 - old->pc6;
3170 if (DO_BIC(BIC_Pkgpc7))
3171 old->pc7 = new->pc7 - old->pc7;
3172 old->pc8 = new->pc8 - old->pc8;
3173 old->pc9 = new->pc9 - old->pc9;
3174 old->pc10 = new->pc10 - old->pc10;
3175 old->die_c6 = new->die_c6 - old->die_c6;
3176 old->cpu_lpi = new->cpu_lpi - old->cpu_lpi;
3177 old->sys_lpi = new->sys_lpi - old->sys_lpi;
3178 old->pkg_temp_c = new->pkg_temp_c;
3179
3180 /* flag an error when rc6 counter resets/wraps */
3181 if (old->gfx_rc6_ms > new->gfx_rc6_ms)
3182 old->gfx_rc6_ms = -1;
3183 else
3184 old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms;
3185
3186 old->uncore_mhz = new->uncore_mhz;
3187 old->gfx_mhz = new->gfx_mhz;
3188 old->gfx_act_mhz = new->gfx_act_mhz;
3189
3190 /* flag an error when mc6 counter resets/wraps */
3191 if (old->sam_mc6_ms > new->sam_mc6_ms)
3192 old->sam_mc6_ms = -1;
3193 else
3194 old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms;
3195
3196 old->sam_mhz = new->sam_mhz;
3197 old->sam_act_mhz = new->sam_act_mhz;
3198
3199 old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value;
3200 old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value;
3201 old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value;
3202 old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value;
3203 old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value;
3204 old->rapl_dram_perf_status.raw_value =
3205 new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value;
3206
3207 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3208 if (mp->format == FORMAT_RAW)
3209 old->counter[i] = new->counter[i];
3210 else if (mp->format == FORMAT_AVERAGE)
3211 old->counter[i] = new->counter[i];
3212 else
3213 old->counter[i] = new->counter[i] - old->counter[i];
3214 }
3215
3216 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3217 if (pp->format == FORMAT_RAW)
3218 old->perf_counter[i] = new->perf_counter[i];
3219 else if (pp->format == FORMAT_AVERAGE)
3220 old->perf_counter[i] = new->perf_counter[i];
3221 else
3222 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3223 }
3224
3225 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3226 if (ppmt->format == FORMAT_RAW)
3227 old->pmt_counter[i] = new->pmt_counter[i];
3228 else
3229 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3230 }
3231
3232 return 0;
3233 }
3234
delta_core(struct core_data * new,struct core_data * old)3235 void delta_core(struct core_data *new, struct core_data *old)
3236 {
3237 int i;
3238 struct msr_counter *mp;
3239 struct perf_counter_info *pp;
3240 struct pmt_counter *ppmt;
3241
3242 old->c3 = new->c3 - old->c3;
3243 old->c6 = new->c6 - old->c6;
3244 old->c7 = new->c7 - old->c7;
3245 old->core_temp_c = new->core_temp_c;
3246 old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;
3247 old->mc6_us = new->mc6_us - old->mc6_us;
3248
3249 DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value);
3250
3251 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3252 if (mp->format == FORMAT_RAW)
3253 old->counter[i] = new->counter[i];
3254 else
3255 old->counter[i] = new->counter[i] - old->counter[i];
3256 }
3257
3258 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3259 if (pp->format == FORMAT_RAW)
3260 old->perf_counter[i] = new->perf_counter[i];
3261 else
3262 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3263 }
3264
3265 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3266 if (ppmt->format == FORMAT_RAW)
3267 old->pmt_counter[i] = new->pmt_counter[i];
3268 else
3269 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3270 }
3271 }
3272
soft_c1_residency_display(int bic)3273 int soft_c1_residency_display(int bic)
3274 {
3275 if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res)
3276 return 0;
3277
3278 return DO_BIC_READ(bic);
3279 }
3280
3281 /*
3282 * old = new - old
3283 */
delta_thread(struct thread_data * new,struct thread_data * old,struct core_data * core_delta)3284 int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta)
3285 {
3286 int i;
3287 struct msr_counter *mp;
3288 struct perf_counter_info *pp;
3289 struct pmt_counter *ppmt;
3290
3291 /* we run cpuid just the 1st time, copy the results */
3292 if (DO_BIC(BIC_APIC))
3293 new->apic_id = old->apic_id;
3294 if (DO_BIC(BIC_X2APIC))
3295 new->x2apic_id = old->x2apic_id;
3296
3297 /*
3298 * the timestamps from start of measurement interval are in "old"
3299 * the timestamp from end of measurement interval are in "new"
3300 * over-write old w/ new so we can print end of interval values
3301 */
3302
3303 timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta);
3304 old->tv_begin = new->tv_begin;
3305 old->tv_end = new->tv_end;
3306
3307 old->tsc = new->tsc - old->tsc;
3308
3309 /* check for TSC < 1 Mcycles over interval */
3310 if (old->tsc < (1000 * 1000))
3311 errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n"
3312 "You can disable all c-states by booting with \"idle=poll\"\n"
3313 "or just the deep ones with \"processor.max_cstate=1\"");
3314
3315 old->c1 = new->c1 - old->c1;
3316
3317 if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC)
3318 || soft_c1_residency_display(BIC_Avg_MHz)) {
3319 if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
3320 old->aperf = new->aperf - old->aperf;
3321 old->mperf = new->mperf - old->mperf;
3322 } else {
3323 return -1;
3324 }
3325 }
3326
3327 if (platform->has_msr_core_c1_res) {
3328 /*
3329 * Some models have a dedicated C1 residency MSR,
3330 * which should be more accurate than the derivation below.
3331 */
3332 } else {
3333 /*
3334 * As counter collection is not atomic,
3335 * it is possible for mperf's non-halted cycles + idle states
3336 * to exceed TSC's all cycles: show c1 = 0% in that case.
3337 */
3338 if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak))
3339 old->c1 = 0;
3340 else {
3341 /* normal case, derive c1 */
3342 old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3
3343 - core_delta->c6 - core_delta->c7;
3344 }
3345 }
3346
3347 if (old->mperf == 0) {
3348 if (debug > 1)
3349 fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id);
3350 old->mperf = 1; /* divide by 0 protection */
3351 }
3352
3353 if (DO_BIC(BIC_IPC))
3354 old->instr_count = new->instr_count - old->instr_count;
3355
3356 if (DO_BIC(BIC_IRQ))
3357 old->irq_count = new->irq_count - old->irq_count;
3358
3359 if (DO_BIC(BIC_SMI))
3360 old->smi_count = new->smi_count - old->smi_count;
3361
3362 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3363 if (mp->format == FORMAT_RAW)
3364 old->counter[i] = new->counter[i];
3365 else
3366 old->counter[i] = new->counter[i] - old->counter[i];
3367 }
3368
3369 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
3370 if (pp->format == FORMAT_RAW)
3371 old->perf_counter[i] = new->perf_counter[i];
3372 else
3373 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3374 }
3375
3376 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3377 if (ppmt->format == FORMAT_RAW)
3378 old->pmt_counter[i] = new->pmt_counter[i];
3379 else
3380 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3381 }
3382
3383 return 0;
3384 }
3385
delta_cpu(struct thread_data * t,struct core_data * c,struct pkg_data * p,struct thread_data * t2,struct core_data * c2,struct pkg_data * p2)3386 int delta_cpu(struct thread_data *t, struct core_data *c,
3387 struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2)
3388 {
3389 int retval = 0;
3390
3391 /* calculate core delta only for 1st thread in core */
3392 if (is_cpu_first_thread_in_core(t, c, p))
3393 delta_core(c, c2);
3394
3395 /* always calculate thread delta */
3396 retval = delta_thread(t, t2, c2); /* c2 is core delta */
3397 if (retval)
3398 return retval;
3399
3400 /* calculate package delta only for 1st core in package */
3401 if (is_cpu_first_core_in_package(t, c, p))
3402 retval = delta_package(p, p2);
3403
3404 return retval;
3405 }
3406
rapl_counter_clear(struct rapl_counter * c)3407 void rapl_counter_clear(struct rapl_counter *c)
3408 {
3409 c->raw_value = 0;
3410 c->scale = 0.0;
3411 c->unit = RAPL_UNIT_INVALID;
3412 }
3413
clear_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)3414 void clear_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
3415 {
3416 int i;
3417 struct msr_counter *mp;
3418
3419 t->tv_begin.tv_sec = 0;
3420 t->tv_begin.tv_usec = 0;
3421 t->tv_end.tv_sec = 0;
3422 t->tv_end.tv_usec = 0;
3423 t->tv_delta.tv_sec = 0;
3424 t->tv_delta.tv_usec = 0;
3425
3426 t->tsc = 0;
3427 t->aperf = 0;
3428 t->mperf = 0;
3429 t->c1 = 0;
3430
3431 t->instr_count = 0;
3432
3433 t->irq_count = 0;
3434 t->smi_count = 0;
3435
3436 c->c3 = 0;
3437 c->c6 = 0;
3438 c->c7 = 0;
3439 c->mc6_us = 0;
3440 c->core_temp_c = 0;
3441 rapl_counter_clear(&c->core_energy);
3442 c->core_throt_cnt = 0;
3443
3444 p->pkg_wtd_core_c0 = 0;
3445 p->pkg_any_core_c0 = 0;
3446 p->pkg_any_gfxe_c0 = 0;
3447 p->pkg_both_core_gfxe_c0 = 0;
3448
3449 p->pc2 = 0;
3450 if (DO_BIC(BIC_Pkgpc3))
3451 p->pc3 = 0;
3452 if (DO_BIC(BIC_Pkgpc6))
3453 p->pc6 = 0;
3454 if (DO_BIC(BIC_Pkgpc7))
3455 p->pc7 = 0;
3456 p->pc8 = 0;
3457 p->pc9 = 0;
3458 p->pc10 = 0;
3459 p->die_c6 = 0;
3460 p->cpu_lpi = 0;
3461 p->sys_lpi = 0;
3462
3463 rapl_counter_clear(&p->energy_pkg);
3464 rapl_counter_clear(&p->energy_dram);
3465 rapl_counter_clear(&p->energy_cores);
3466 rapl_counter_clear(&p->energy_gfx);
3467 rapl_counter_clear(&p->rapl_pkg_perf_status);
3468 rapl_counter_clear(&p->rapl_dram_perf_status);
3469 p->pkg_temp_c = 0;
3470
3471 p->gfx_rc6_ms = 0;
3472 p->uncore_mhz = 0;
3473 p->gfx_mhz = 0;
3474 p->gfx_act_mhz = 0;
3475 p->sam_mc6_ms = 0;
3476 p->sam_mhz = 0;
3477 p->sam_act_mhz = 0;
3478 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
3479 t->counter[i] = 0;
3480
3481 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next)
3482 c->counter[i] = 0;
3483
3484 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next)
3485 p->counter[i] = 0;
3486
3487 memset(&t->perf_counter[0], 0, sizeof(t->perf_counter));
3488 memset(&c->perf_counter[0], 0, sizeof(c->perf_counter));
3489 memset(&p->perf_counter[0], 0, sizeof(p->perf_counter));
3490
3491 memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter));
3492 memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter));
3493 memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter));
3494 }
3495
rapl_counter_accumulate(struct rapl_counter * dst,const struct rapl_counter * src)3496 void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src)
3497 {
3498 /* Copy unit and scale from src if dst is not initialized */
3499 if (dst->unit == RAPL_UNIT_INVALID) {
3500 dst->unit = src->unit;
3501 dst->scale = src->scale;
3502 }
3503
3504 assert(dst->unit == src->unit);
3505 assert(dst->scale == src->scale);
3506
3507 dst->raw_value += src->raw_value;
3508 }
3509
sum_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)3510 int sum_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
3511 {
3512 int i;
3513 struct msr_counter *mp;
3514 struct perf_counter_info *pp;
3515 struct pmt_counter *ppmt;
3516
3517 /* copy un-changing apic_id's */
3518 if (DO_BIC(BIC_APIC))
3519 average.threads.apic_id = t->apic_id;
3520 if (DO_BIC(BIC_X2APIC))
3521 average.threads.x2apic_id = t->x2apic_id;
3522
3523 /* remember first tv_begin */
3524 if (average.threads.tv_begin.tv_sec == 0)
3525 average.threads.tv_begin = t->tv_begin;
3526
3527 /* remember last tv_end */
3528 average.threads.tv_end = t->tv_end;
3529
3530 average.threads.tsc += t->tsc;
3531 average.threads.aperf += t->aperf;
3532 average.threads.mperf += t->mperf;
3533 average.threads.c1 += t->c1;
3534
3535 average.threads.instr_count += t->instr_count;
3536
3537 average.threads.irq_count += t->irq_count;
3538 average.threads.smi_count += t->smi_count;
3539
3540 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3541 if (mp->format == FORMAT_RAW)
3542 continue;
3543 average.threads.counter[i] += t->counter[i];
3544 }
3545
3546 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
3547 if (pp->format == FORMAT_RAW)
3548 continue;
3549 average.threads.perf_counter[i] += t->perf_counter[i];
3550 }
3551
3552 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3553 average.threads.pmt_counter[i] += t->pmt_counter[i];
3554 }
3555
3556 /* sum per-core values only for 1st thread in core */
3557 if (!is_cpu_first_thread_in_core(t, c, p))
3558 return 0;
3559
3560 average.cores.c3 += c->c3;
3561 average.cores.c6 += c->c6;
3562 average.cores.c7 += c->c7;
3563 average.cores.mc6_us += c->mc6_us;
3564
3565 average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c);
3566 average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt);
3567
3568 rapl_counter_accumulate(&average.cores.core_energy, &c->core_energy);
3569
3570 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3571 if (mp->format == FORMAT_RAW)
3572 continue;
3573 average.cores.counter[i] += c->counter[i];
3574 }
3575
3576 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3577 if (pp->format == FORMAT_RAW)
3578 continue;
3579 average.cores.perf_counter[i] += c->perf_counter[i];
3580 }
3581
3582 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3583 average.cores.pmt_counter[i] += c->pmt_counter[i];
3584 }
3585
3586 /* sum per-pkg values only for 1st core in pkg */
3587 if (!is_cpu_first_core_in_package(t, c, p))
3588 return 0;
3589
3590 if (DO_BIC(BIC_Totl_c0))
3591 average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0;
3592 if (DO_BIC(BIC_Any_c0))
3593 average.packages.pkg_any_core_c0 += p->pkg_any_core_c0;
3594 if (DO_BIC(BIC_GFX_c0))
3595 average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0;
3596 if (DO_BIC(BIC_CPUGFX))
3597 average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0;
3598
3599 average.packages.pc2 += p->pc2;
3600 if (DO_BIC(BIC_Pkgpc3))
3601 average.packages.pc3 += p->pc3;
3602 if (DO_BIC(BIC_Pkgpc6))
3603 average.packages.pc6 += p->pc6;
3604 if (DO_BIC(BIC_Pkgpc7))
3605 average.packages.pc7 += p->pc7;
3606 average.packages.pc8 += p->pc8;
3607 average.packages.pc9 += p->pc9;
3608 average.packages.pc10 += p->pc10;
3609 average.packages.die_c6 += p->die_c6;
3610
3611 average.packages.cpu_lpi = p->cpu_lpi;
3612 average.packages.sys_lpi = p->sys_lpi;
3613
3614 rapl_counter_accumulate(&average.packages.energy_pkg, &p->energy_pkg);
3615 rapl_counter_accumulate(&average.packages.energy_dram, &p->energy_dram);
3616 rapl_counter_accumulate(&average.packages.energy_cores, &p->energy_cores);
3617 rapl_counter_accumulate(&average.packages.energy_gfx, &p->energy_gfx);
3618
3619 average.packages.gfx_rc6_ms = p->gfx_rc6_ms;
3620 average.packages.uncore_mhz = p->uncore_mhz;
3621 average.packages.gfx_mhz = p->gfx_mhz;
3622 average.packages.gfx_act_mhz = p->gfx_act_mhz;
3623 average.packages.sam_mc6_ms = p->sam_mc6_ms;
3624 average.packages.sam_mhz = p->sam_mhz;
3625 average.packages.sam_act_mhz = p->sam_act_mhz;
3626
3627 average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c);
3628
3629 rapl_counter_accumulate(&average.packages.rapl_pkg_perf_status, &p->rapl_pkg_perf_status);
3630 rapl_counter_accumulate(&average.packages.rapl_dram_perf_status, &p->rapl_dram_perf_status);
3631
3632 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3633 if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0))
3634 average.packages.counter[i] = p->counter[i];
3635 else
3636 average.packages.counter[i] += p->counter[i];
3637 }
3638
3639 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3640 if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0))
3641 average.packages.perf_counter[i] = p->perf_counter[i];
3642 else
3643 average.packages.perf_counter[i] += p->perf_counter[i];
3644 }
3645
3646 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3647 average.packages.pmt_counter[i] += p->pmt_counter[i];
3648 }
3649
3650 return 0;
3651 }
3652
3653 /*
3654 * sum the counters for all cpus in the system
3655 * compute the weighted average
3656 */
compute_average(struct thread_data * t,struct core_data * c,struct pkg_data * p)3657 void compute_average(struct thread_data *t, struct core_data *c, struct pkg_data *p)
3658 {
3659 int i;
3660 struct msr_counter *mp;
3661 struct perf_counter_info *pp;
3662 struct pmt_counter *ppmt;
3663
3664 clear_counters(&average.threads, &average.cores, &average.packages);
3665
3666 for_all_cpus(sum_counters, t, c, p);
3667
3668 /* Use the global time delta for the average. */
3669 average.threads.tv_delta = tv_delta;
3670
3671 average.threads.tsc /= topo.allowed_cpus;
3672 average.threads.aperf /= topo.allowed_cpus;
3673 average.threads.mperf /= topo.allowed_cpus;
3674 average.threads.instr_count /= topo.allowed_cpus;
3675 average.threads.c1 /= topo.allowed_cpus;
3676
3677 if (average.threads.irq_count > 9999999)
3678 sums_need_wide_columns = 1;
3679
3680 average.cores.c3 /= topo.allowed_cores;
3681 average.cores.c6 /= topo.allowed_cores;
3682 average.cores.c7 /= topo.allowed_cores;
3683 average.cores.mc6_us /= topo.allowed_cores;
3684
3685 if (DO_BIC(BIC_Totl_c0))
3686 average.packages.pkg_wtd_core_c0 /= topo.allowed_packages;
3687 if (DO_BIC(BIC_Any_c0))
3688 average.packages.pkg_any_core_c0 /= topo.allowed_packages;
3689 if (DO_BIC(BIC_GFX_c0))
3690 average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages;
3691 if (DO_BIC(BIC_CPUGFX))
3692 average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages;
3693
3694 average.packages.pc2 /= topo.allowed_packages;
3695 if (DO_BIC(BIC_Pkgpc3))
3696 average.packages.pc3 /= topo.allowed_packages;
3697 if (DO_BIC(BIC_Pkgpc6))
3698 average.packages.pc6 /= topo.allowed_packages;
3699 if (DO_BIC(BIC_Pkgpc7))
3700 average.packages.pc7 /= topo.allowed_packages;
3701
3702 average.packages.pc8 /= topo.allowed_packages;
3703 average.packages.pc9 /= topo.allowed_packages;
3704 average.packages.pc10 /= topo.allowed_packages;
3705 average.packages.die_c6 /= topo.allowed_packages;
3706
3707 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3708 if (mp->format == FORMAT_RAW)
3709 continue;
3710 if (mp->type == COUNTER_ITEMS) {
3711 if (average.threads.counter[i] > 9999999)
3712 sums_need_wide_columns = 1;
3713 continue;
3714 }
3715 average.threads.counter[i] /= topo.allowed_cpus;
3716 }
3717 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3718 if (mp->format == FORMAT_RAW)
3719 continue;
3720 if (mp->type == COUNTER_ITEMS) {
3721 if (average.cores.counter[i] > 9999999)
3722 sums_need_wide_columns = 1;
3723 }
3724 average.cores.counter[i] /= topo.allowed_cores;
3725 }
3726 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3727 if (mp->format == FORMAT_RAW)
3728 continue;
3729 if (mp->type == COUNTER_ITEMS) {
3730 if (average.packages.counter[i] > 9999999)
3731 sums_need_wide_columns = 1;
3732 }
3733 average.packages.counter[i] /= topo.allowed_packages;
3734 }
3735
3736 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
3737 if (pp->format == FORMAT_RAW)
3738 continue;
3739 if (pp->type == COUNTER_ITEMS) {
3740 if (average.threads.perf_counter[i] > 9999999)
3741 sums_need_wide_columns = 1;
3742 continue;
3743 }
3744 average.threads.perf_counter[i] /= topo.allowed_cpus;
3745 }
3746 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3747 if (pp->format == FORMAT_RAW)
3748 continue;
3749 if (pp->type == COUNTER_ITEMS) {
3750 if (average.cores.perf_counter[i] > 9999999)
3751 sums_need_wide_columns = 1;
3752 }
3753 average.cores.perf_counter[i] /= topo.allowed_cores;
3754 }
3755 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3756 if (pp->format == FORMAT_RAW)
3757 continue;
3758 if (pp->type == COUNTER_ITEMS) {
3759 if (average.packages.perf_counter[i] > 9999999)
3760 sums_need_wide_columns = 1;
3761 }
3762 average.packages.perf_counter[i] /= topo.allowed_packages;
3763 }
3764
3765 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3766 average.threads.pmt_counter[i] /= topo.allowed_cpus;
3767 }
3768 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3769 average.cores.pmt_counter[i] /= topo.allowed_cores;
3770 }
3771 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3772 average.packages.pmt_counter[i] /= topo.allowed_packages;
3773 }
3774 }
3775
rdtsc(void)3776 static unsigned long long rdtsc(void)
3777 {
3778 unsigned int low, high;
3779
3780 asm volatile ("rdtsc":"=a" (low), "=d"(high));
3781
3782 return low | ((unsigned long long)high) << 32;
3783 }
3784
3785 /*
3786 * Open a file, and exit on failure
3787 */
fopen_or_die(const char * path,const char * mode)3788 FILE *fopen_or_die(const char *path, const char *mode)
3789 {
3790 FILE *filep = fopen(path, mode);
3791
3792 if (!filep)
3793 err(1, "%s: open failed", path);
3794 return filep;
3795 }
3796
3797 /*
3798 * snapshot_sysfs_counter()
3799 *
3800 * return snapshot of given counter
3801 */
snapshot_sysfs_counter(char * path)3802 unsigned long long snapshot_sysfs_counter(char *path)
3803 {
3804 FILE *fp;
3805 int retval;
3806 unsigned long long counter;
3807
3808 fp = fopen_or_die(path, "r");
3809
3810 retval = fscanf(fp, "%lld", &counter);
3811 if (retval != 1)
3812 err(1, "snapshot_sysfs_counter(%s)", path);
3813
3814 fclose(fp);
3815
3816 return counter;
3817 }
3818
get_mp(int cpu,struct msr_counter * mp,unsigned long long * counterp,char * counter_path)3819 int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path)
3820 {
3821 if (mp->msr_num != 0) {
3822 assert(!no_msr);
3823 if (get_msr(cpu, mp->msr_num, counterp))
3824 return -1;
3825 } else {
3826 char path[128 + PATH_BYTES];
3827
3828 if (mp->flags & SYSFS_PERCPU) {
3829 sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path);
3830
3831 *counterp = snapshot_sysfs_counter(path);
3832 } else {
3833 *counterp = snapshot_sysfs_counter(counter_path);
3834 }
3835 }
3836
3837 return 0;
3838 }
3839
get_legacy_uncore_mhz(int package)3840 unsigned long long get_legacy_uncore_mhz(int package)
3841 {
3842 char path[128];
3843 int die;
3844 static int warn_once;
3845
3846 /*
3847 * for this package, use the first die_id that exists
3848 */
3849 for (die = 0; die <= topo.max_die_id; ++die) {
3850
3851 sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz",
3852 package, die);
3853
3854 if (access(path, R_OK) == 0)
3855 return (snapshot_sysfs_counter(path) / 1000);
3856 }
3857 if (!warn_once) {
3858 warnx("BUG: %s: No %s", __func__, path);
3859 warn_once = 1;
3860 }
3861
3862 return 0;
3863 }
3864
get_epb(int cpu)3865 int get_epb(int cpu)
3866 {
3867 char path[128 + PATH_BYTES];
3868 unsigned long long msr;
3869 int ret, epb = -1;
3870 FILE *fp;
3871
3872 sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu);
3873
3874 fp = fopen(path, "r");
3875 if (!fp)
3876 goto msr_fallback;
3877
3878 ret = fscanf(fp, "%d", &epb);
3879 if (ret != 1)
3880 err(1, "%s(%s)", __func__, path);
3881
3882 fclose(fp);
3883
3884 return epb;
3885
3886 msr_fallback:
3887 if (no_msr)
3888 return -1;
3889
3890 get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr);
3891
3892 return msr & 0xf;
3893 }
3894
get_apic_id(struct thread_data * t)3895 void get_apic_id(struct thread_data *t)
3896 {
3897 unsigned int eax, ebx, ecx, edx;
3898
3899 if (DO_BIC(BIC_APIC)) {
3900 eax = ebx = ecx = edx = 0;
3901 __cpuid(1, eax, ebx, ecx, edx);
3902
3903 t->apic_id = (ebx >> 24) & 0xff;
3904 }
3905
3906 if (!DO_BIC(BIC_X2APIC))
3907 return;
3908
3909 if (authentic_amd || hygon_genuine) {
3910 unsigned int topology_extensions;
3911
3912 if (max_extended_level < 0x8000001e)
3913 return;
3914
3915 eax = ebx = ecx = edx = 0;
3916 __cpuid(0x80000001, eax, ebx, ecx, edx);
3917 topology_extensions = ecx & (1 << 22);
3918
3919 if (topology_extensions == 0)
3920 return;
3921
3922 eax = ebx = ecx = edx = 0;
3923 __cpuid(0x8000001e, eax, ebx, ecx, edx);
3924
3925 t->x2apic_id = eax;
3926 return;
3927 }
3928
3929 if (!genuine_intel)
3930 return;
3931
3932 if (max_level < 0xb)
3933 return;
3934
3935 ecx = 0;
3936 __cpuid(0xb, eax, ebx, ecx, edx);
3937 t->x2apic_id = edx;
3938
3939 if (debug && (t->apic_id != (t->x2apic_id & 0xff)))
3940 fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id);
3941 }
3942
get_core_throt_cnt(int cpu,unsigned long long * cnt)3943 int get_core_throt_cnt(int cpu, unsigned long long *cnt)
3944 {
3945 char path[128 + PATH_BYTES];
3946 unsigned long long tmp;
3947 FILE *fp;
3948 int ret;
3949
3950 sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu);
3951 fp = fopen(path, "r");
3952 if (!fp)
3953 return -1;
3954 ret = fscanf(fp, "%lld", &tmp);
3955 fclose(fp);
3956 if (ret != 1)
3957 return -1;
3958 *cnt = tmp;
3959
3960 return 0;
3961 }
3962
3963 struct amperf_group_fd {
3964 int aperf; /* Also the group descriptor */
3965 int mperf;
3966 };
3967
read_perf_counter_info(const char * const path,const char * const parse_format,void * value_ptr)3968 static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr)
3969 {
3970 int fdmt;
3971 int bytes_read;
3972 char buf[64];
3973 int ret = -1;
3974
3975 fdmt = open(path, O_RDONLY, 0);
3976 if (fdmt == -1) {
3977 if (debug)
3978 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
3979 ret = -1;
3980 goto cleanup_and_exit;
3981 }
3982
3983 bytes_read = read(fdmt, buf, sizeof(buf) - 1);
3984 if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) {
3985 if (debug)
3986 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
3987 ret = -1;
3988 goto cleanup_and_exit;
3989 }
3990
3991 buf[bytes_read] = '\0';
3992
3993 if (sscanf(buf, parse_format, value_ptr) != 1) {
3994 if (debug)
3995 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
3996 ret = -1;
3997 goto cleanup_and_exit;
3998 }
3999
4000 ret = 0;
4001
4002 cleanup_and_exit:
4003 close(fdmt);
4004 return ret;
4005 }
4006
read_perf_counter_info_n(const char * const path,const char * const parse_format)4007 static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format)
4008 {
4009 unsigned int v;
4010 int status;
4011
4012 status = read_perf_counter_info(path, parse_format, &v);
4013 if (status)
4014 v = -1;
4015
4016 return v;
4017 }
4018
read_perf_type(const char * subsys)4019 static unsigned int read_perf_type(const char *subsys)
4020 {
4021 const char *const path_format = "/sys/bus/event_source/devices/%s/type";
4022 const char *const format = "%u";
4023 char path[128];
4024
4025 snprintf(path, sizeof(path), path_format, subsys);
4026
4027 return read_perf_counter_info_n(path, format);
4028 }
4029
read_perf_config(const char * subsys,const char * event_name)4030 static unsigned int read_perf_config(const char *subsys, const char *event_name)
4031 {
4032 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s";
4033 FILE *fconfig = NULL;
4034 char path[128];
4035 char config_str[64];
4036 unsigned int config;
4037 unsigned int umask;
4038 bool has_config = false;
4039 bool has_umask = false;
4040 unsigned int ret = -1;
4041
4042 snprintf(path, sizeof(path), path_format, subsys, event_name);
4043
4044 fconfig = fopen(path, "r");
4045 if (!fconfig)
4046 return -1;
4047
4048 if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str)
4049 goto cleanup_and_exit;
4050
4051 for (char *pconfig_str = &config_str[0]; pconfig_str;) {
4052 if (sscanf(pconfig_str, "event=%x", &config) == 1) {
4053 has_config = true;
4054 goto next;
4055 }
4056
4057 if (sscanf(pconfig_str, "umask=%x", &umask) == 1) {
4058 has_umask = true;
4059 goto next;
4060 }
4061
4062 next:
4063 pconfig_str = strchr(pconfig_str, ',');
4064 if (pconfig_str) {
4065 *pconfig_str = '\0';
4066 ++pconfig_str;
4067 }
4068 }
4069
4070 if (!has_umask)
4071 umask = 0;
4072
4073 if (has_config)
4074 ret = (umask << 8) | config;
4075
4076 cleanup_and_exit:
4077 fclose(fconfig);
4078 return ret;
4079 }
4080
read_perf_rapl_unit(const char * subsys,const char * event_name)4081 static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name)
4082 {
4083 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit";
4084 const char *const format = "%s";
4085 char path[128];
4086 char unit_buffer[16];
4087
4088 snprintf(path, sizeof(path), path_format, subsys, event_name);
4089
4090 read_perf_counter_info(path, format, &unit_buffer);
4091 if (strcmp("Joules", unit_buffer) == 0)
4092 return RAPL_UNIT_JOULES;
4093
4094 return RAPL_UNIT_INVALID;
4095 }
4096
read_perf_scale(const char * subsys,const char * event_name)4097 static double read_perf_scale(const char *subsys, const char *event_name)
4098 {
4099 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale";
4100 const char *const format = "%lf";
4101 char path[128];
4102 double scale;
4103
4104 snprintf(path, sizeof(path), path_format, subsys, event_name);
4105
4106 if (read_perf_counter_info(path, format, &scale))
4107 return 0.0;
4108
4109 return scale;
4110 }
4111
rapl_counter_info_count_perf(const struct rapl_counter_info_t * rci)4112 size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci)
4113 {
4114 size_t ret = 0;
4115
4116 for (int i = 0; i < NUM_RAPL_COUNTERS; ++i)
4117 if (rci->source[i] == COUNTER_SOURCE_PERF)
4118 ++ret;
4119
4120 return ret;
4121 }
4122
cstate_counter_info_count_perf(const struct cstate_counter_info_t * cci)4123 static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci)
4124 {
4125 size_t ret = 0;
4126
4127 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i)
4128 if (cci->source[i] == COUNTER_SOURCE_PERF)
4129 ++ret;
4130
4131 return ret;
4132 }
4133
write_rapl_counter(struct rapl_counter * rc,struct rapl_counter_info_t * rci,unsigned int idx)4134 void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx)
4135 {
4136 rc->raw_value = rci->data[idx];
4137 rc->unit = rci->unit[idx];
4138 rc->scale = rci->scale[idx];
4139 }
4140
get_rapl_counters(int cpu,unsigned int domain,struct core_data * c,struct pkg_data * p)4141 int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p)
4142 {
4143 unsigned long long perf_data[NUM_RAPL_COUNTERS + 1];
4144 struct rapl_counter_info_t *rci;
4145
4146 if (debug >= 2)
4147 fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain);
4148
4149 assert(rapl_counter_info_perdomain);
4150 assert(domain < rapl_counter_info_perdomain_size);
4151
4152 rci = &rapl_counter_info_perdomain[domain];
4153
4154 /*
4155 * If we have any perf counters to read, read them all now, in bulk
4156 */
4157 if (rci->fd_perf != -1) {
4158 size_t num_perf_counters = rapl_counter_info_count_perf(rci);
4159 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4160 const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data));
4161
4162 if (actual_read_size != expected_read_size)
4163 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size,
4164 actual_read_size);
4165 }
4166
4167 for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) {
4168 switch (rci->source[i]) {
4169 case COUNTER_SOURCE_NONE:
4170 break;
4171
4172 case COUNTER_SOURCE_PERF:
4173 assert(pi < ARRAY_SIZE(perf_data));
4174 assert(rci->fd_perf != -1);
4175
4176 if (debug >= 2)
4177 fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n",
4178 i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]);
4179
4180 rci->data[i] = perf_data[pi];
4181
4182 ++pi;
4183 break;
4184
4185 case COUNTER_SOURCE_MSR:
4186 if (debug >= 2)
4187 fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
4188
4189 assert(!no_msr);
4190 if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
4191 if (get_msr_sum(cpu, rci->msr[i], &rci->data[i]))
4192 return -13 - i;
4193 } else {
4194 if (get_msr(cpu, rci->msr[i], &rci->data[i]))
4195 return -13 - i;
4196 }
4197
4198 rci->data[i] &= rci->msr_mask[i];
4199 if (rci->msr_shift[i] >= 0)
4200 rci->data[i] >>= abs(rci->msr_shift[i]);
4201 else
4202 rci->data[i] <<= abs(rci->msr_shift[i]);
4203
4204 break;
4205 }
4206 }
4207
4208 BUILD_BUG_ON(NUM_RAPL_COUNTERS != 7);
4209 write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG);
4210 write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES);
4211 write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM);
4212 write_rapl_counter(&p->energy_gfx, rci, RAPL_RCI_INDEX_GFX);
4213 write_rapl_counter(&p->rapl_pkg_perf_status, rci, RAPL_RCI_INDEX_PKG_PERF_STATUS);
4214 write_rapl_counter(&p->rapl_dram_perf_status, rci, RAPL_RCI_INDEX_DRAM_PERF_STATUS);
4215 write_rapl_counter(&c->core_energy, rci, RAPL_RCI_INDEX_CORE_ENERGY);
4216
4217 return 0;
4218 }
4219
find_sysfs_path_by_id(struct sysfs_path * sp,int id)4220 char *find_sysfs_path_by_id(struct sysfs_path *sp, int id)
4221 {
4222 while (sp) {
4223 if (sp->id == id)
4224 return (sp->path);
4225 sp = sp->next;
4226 }
4227 if (debug)
4228 warnx("%s: id%d not found", __func__, id);
4229 return NULL;
4230 }
4231
get_cstate_counters(unsigned int cpu,struct thread_data * t,struct core_data * c,struct pkg_data * p)4232 int get_cstate_counters(unsigned int cpu, struct thread_data *t, struct core_data *c, struct pkg_data *p)
4233 {
4234 /*
4235 * Overcommit memory a little bit here,
4236 * but skip calculating exact sizes for the buffers.
4237 */
4238 unsigned long long perf_data[NUM_CSTATE_COUNTERS];
4239 unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1];
4240 unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1];
4241
4242 struct cstate_counter_info_t *cci;
4243
4244 if (debug >= 2)
4245 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4246
4247 assert(ccstate_counter_info);
4248 assert(cpu <= ccstate_counter_info_size);
4249
4250 ZERO_ARRAY(perf_data);
4251 ZERO_ARRAY(perf_data_core);
4252 ZERO_ARRAY(perf_data_pkg);
4253
4254 cci = &ccstate_counter_info[cpu];
4255
4256 /*
4257 * If we have any perf counters to read, read them all now, in bulk
4258 */
4259 const size_t num_perf_counters = cstate_counter_info_count_perf(cci);
4260 ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long);
4261 ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0;
4262
4263 if (cci->fd_perf_core != -1) {
4264 /* Each descriptor read begins with number of counters read. */
4265 expected_read_size += sizeof(unsigned long long);
4266
4267 actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core));
4268
4269 if (actual_read_size_core <= 0)
4270 err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core);
4271 }
4272
4273 if (cci->fd_perf_pkg != -1) {
4274 /* Each descriptor read begins with number of counters read. */
4275 expected_read_size += sizeof(unsigned long long);
4276
4277 actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg));
4278
4279 if (actual_read_size_pkg <= 0)
4280 err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg);
4281 }
4282
4283 const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg;
4284
4285 if (actual_read_size_total != expected_read_size)
4286 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total);
4287
4288 /*
4289 * Copy ccstate and pcstate data into unified buffer.
4290 *
4291 * Skip first element from core and pkg buffers.
4292 * Kernel puts there how many counters were read.
4293 */
4294 const size_t num_core_counters = perf_data_core[0];
4295 const size_t num_pkg_counters = perf_data_pkg[0];
4296
4297 assert(num_perf_counters == num_core_counters + num_pkg_counters);
4298
4299 /* Copy ccstate perf data */
4300 memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long));
4301
4302 /* Copy pcstate perf data */
4303 memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long));
4304
4305 for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) {
4306 switch (cci->source[i]) {
4307 case COUNTER_SOURCE_NONE:
4308 break;
4309
4310 case COUNTER_SOURCE_PERF:
4311 assert(pi < ARRAY_SIZE(perf_data));
4312 assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1);
4313
4314 if (debug >= 2)
4315 fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]);
4316
4317 cci->data[i] = perf_data[pi];
4318
4319 ++pi;
4320 break;
4321
4322 case COUNTER_SOURCE_MSR:
4323 assert(!no_msr);
4324 if (get_msr(cpu, cci->msr[i], &cci->data[i]))
4325 return -13 - i;
4326
4327 if (debug >= 2)
4328 fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
4329
4330 break;
4331 }
4332 }
4333
4334 /*
4335 * Helper to write the data only if the source of
4336 * the counter for the current cpu is not none.
4337 *
4338 * Otherwise we would overwrite core data with 0 (default value),
4339 * when invoked for the thread sibling.
4340 */
4341 #define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \
4342 if (cci->source[index] != COUNTER_SOURCE_NONE) \
4343 out_counter = cci->data[index]; \
4344 } while (0)
4345
4346 BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11);
4347
4348 PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY);
4349 PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY);
4350 PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY);
4351 PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY);
4352
4353 PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY);
4354 PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY);
4355 PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY);
4356 PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY);
4357 PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY);
4358 PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY);
4359 PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY);
4360
4361 #undef PERF_COUNTER_WRITE_DATA
4362
4363 return 0;
4364 }
4365
msr_counter_info_count_perf(const struct msr_counter_info_t * mci)4366 size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci)
4367 {
4368 size_t ret = 0;
4369
4370 for (int i = 0; i < NUM_MSR_COUNTERS; ++i)
4371 if (mci->source[i] == COUNTER_SOURCE_PERF)
4372 ++ret;
4373
4374 return ret;
4375 }
4376
get_smi_aperf_mperf(unsigned int cpu,struct thread_data * t)4377 int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
4378 {
4379 unsigned long long perf_data[NUM_MSR_COUNTERS + 1];
4380
4381 struct msr_counter_info_t *mci;
4382
4383 if (debug >= 2)
4384 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4385
4386 assert(msr_counter_info);
4387 assert(cpu <= msr_counter_info_size);
4388
4389 mci = &msr_counter_info[cpu];
4390
4391 ZERO_ARRAY(perf_data);
4392 ZERO_ARRAY(mci->data);
4393
4394 if (mci->fd_perf != -1) {
4395 const size_t num_perf_counters = msr_counter_info_count_perf(mci);
4396 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4397 const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data));
4398
4399 if (actual_read_size != expected_read_size)
4400 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size,
4401 actual_read_size);
4402 }
4403
4404 for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) {
4405 switch (mci->source[i]) {
4406 case COUNTER_SOURCE_NONE:
4407 break;
4408
4409 case COUNTER_SOURCE_PERF:
4410 assert(pi < ARRAY_SIZE(perf_data));
4411 assert(mci->fd_perf != -1);
4412
4413 if (debug >= 2)
4414 fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]);
4415
4416 mci->data[i] = perf_data[pi];
4417
4418 ++pi;
4419 break;
4420
4421 case COUNTER_SOURCE_MSR:
4422 assert(!no_msr);
4423
4424 if (get_msr(cpu, mci->msr[i], &mci->data[i]))
4425 return -2 - i;
4426
4427 mci->data[i] &= mci->msr_mask[i];
4428
4429 if (debug >= 2)
4430 fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]);
4431
4432 break;
4433 }
4434 }
4435
4436 BUILD_BUG_ON(NUM_MSR_COUNTERS != 3);
4437 t->aperf = mci->data[MSR_RCI_INDEX_APERF];
4438 t->mperf = mci->data[MSR_RCI_INDEX_MPERF];
4439 t->smi_count = mci->data[MSR_RCI_INDEX_SMI];
4440
4441 return 0;
4442 }
4443
perf_counter_info_read_values(struct perf_counter_info * pp,int cpu,unsigned long long * out,size_t out_size)4444 int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size)
4445 {
4446 unsigned int domain;
4447 unsigned long long value;
4448 int fd_counter;
4449
4450 for (size_t i = 0; pp; ++i, pp = pp->next) {
4451 domain = cpu_to_domain(pp, cpu);
4452 assert(domain < pp->num_domains);
4453
4454 fd_counter = pp->fd_perf_per_domain[domain];
4455
4456 if (fd_counter == -1)
4457 continue;
4458
4459 if (read(fd_counter, &value, sizeof(value)) != sizeof(value))
4460 return 1;
4461
4462 assert(i < out_size);
4463 out[i] = value * pp->scale;
4464 }
4465
4466 return 0;
4467 }
4468
pmt_gen_value_mask(unsigned int lsb,unsigned int msb)4469 unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb)
4470 {
4471 unsigned long mask;
4472
4473 if (msb == 63)
4474 mask = 0xffffffffffffffff;
4475 else
4476 mask = ((1 << (msb + 1)) - 1);
4477
4478 mask -= (1 << lsb) - 1;
4479
4480 return mask;
4481 }
4482
pmt_read_counter(struct pmt_counter * ppmt,unsigned int domain_id)4483 unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id)
4484 {
4485 assert(domain_id < ppmt->num_domains);
4486
4487 const unsigned long *pmmio = ppmt->domains[domain_id].pcounter;
4488 const unsigned long value = pmmio ? *pmmio : 0;
4489 const unsigned long value_mask = pmt_gen_value_mask(ppmt->lsb, ppmt->msb);
4490 const unsigned long value_shift = ppmt->lsb;
4491
4492 return (value & value_mask) >> value_shift;
4493 }
4494
4495
4496 /* Rapl domain enumeration helpers */
get_rapl_num_domains(void)4497 static inline int get_rapl_num_domains(void)
4498 {
4499 int num_packages = topo.max_package_id + 1;
4500 int num_cores_per_package;
4501 int num_cores;
4502
4503 if (!platform->has_per_core_rapl)
4504 return num_packages;
4505
4506 num_cores_per_package = topo.max_core_id + 1;
4507 num_cores = num_cores_per_package * num_packages;
4508
4509 return num_cores;
4510 }
4511
get_rapl_domain_id(int cpu)4512 static inline int get_rapl_domain_id(int cpu)
4513 {
4514 int nr_cores_per_package = topo.max_core_id + 1;
4515 int rapl_core_id;
4516
4517 if (!platform->has_per_core_rapl)
4518 return cpus[cpu].physical_package_id;
4519
4520 /* Compute the system-wide unique core-id for @cpu */
4521 rapl_core_id = cpus[cpu].physical_core_id;
4522 rapl_core_id += cpus[cpu].physical_package_id * nr_cores_per_package;
4523
4524 return rapl_core_id;
4525 }
4526
4527 /*
4528 * get_counters(...)
4529 * migrate to cpu
4530 * acquire and record local counters for that cpu
4531 */
get_counters(struct thread_data * t,struct core_data * c,struct pkg_data * p)4532 int get_counters(struct thread_data *t, struct core_data *c, struct pkg_data *p)
4533 {
4534 int cpu = t->cpu_id;
4535 unsigned long long msr;
4536 struct msr_counter *mp;
4537 struct pmt_counter *pp;
4538 int i;
4539 int status;
4540
4541 if (cpu_migrate(cpu)) {
4542 fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu);
4543 return -1;
4544 }
4545
4546 gettimeofday(&t->tv_begin, (struct timezone *)NULL);
4547
4548 if (first_counter_read)
4549 get_apic_id(t);
4550
4551 t->tsc = rdtsc(); /* we are running on local CPU of interest */
4552
4553 get_smi_aperf_mperf(cpu, t);
4554
4555 if (DO_BIC(BIC_IPC))
4556 if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long))
4557 return -4;
4558
4559 if (DO_BIC(BIC_IRQ))
4560 t->irq_count = irqs_per_cpu[cpu];
4561
4562 get_cstate_counters(cpu, t, c, p);
4563
4564 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4565 if (get_mp(cpu, mp, &t->counter[i], mp->sp->path))
4566 return -10;
4567 }
4568
4569 if (perf_counter_info_read_values(sys.perf_tp, cpu, t->perf_counter, MAX_ADDED_THREAD_COUNTERS))
4570 return -10;
4571
4572 for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next)
4573 t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id);
4574
4575 /* collect core counters only for 1st thread in core */
4576 if (!is_cpu_first_thread_in_core(t, c, p))
4577 goto done;
4578
4579 if (platform->has_per_core_rapl) {
4580 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
4581 if (status != 0)
4582 return status;
4583 }
4584
4585 if (DO_BIC(BIC_CPU_c7) && t->is_atom) {
4586 /*
4587 * For Atom CPUs that has core cstate deeper than c6,
4588 * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper.
4589 * Minus CC7 (and deeper cstates) residency to get
4590 * accturate cc6 residency.
4591 */
4592 c->c6 -= c->c7;
4593 }
4594
4595 if (DO_BIC(BIC_Mod_c6))
4596 if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
4597 return -8;
4598
4599 if (DO_BIC(BIC_CoreTmp)) {
4600 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
4601 return -9;
4602 c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
4603 }
4604
4605 if (DO_BIC(BIC_CORE_THROT_CNT))
4606 get_core_throt_cnt(cpu, &c->core_throt_cnt);
4607
4608 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4609 if (get_mp(cpu, mp, &c->counter[i], mp->sp->path))
4610 return -10;
4611 }
4612
4613 if (perf_counter_info_read_values(sys.perf_cp, cpu, c->perf_counter, MAX_ADDED_CORE_COUNTERS))
4614 return -10;
4615
4616 for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next)
4617 c->pmt_counter[i] = pmt_read_counter(pp, c->core_id);
4618
4619 /* collect package counters only for 1st core in package */
4620 if (!is_cpu_first_core_in_package(t, c, p))
4621 goto done;
4622
4623 if (DO_BIC(BIC_Totl_c0)) {
4624 if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0))
4625 return -10;
4626 }
4627 if (DO_BIC(BIC_Any_c0)) {
4628 if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0))
4629 return -11;
4630 }
4631 if (DO_BIC(BIC_GFX_c0)) {
4632 if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0))
4633 return -12;
4634 }
4635 if (DO_BIC(BIC_CPUGFX)) {
4636 if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
4637 return -13;
4638 }
4639
4640 if (DO_BIC(BIC_CPU_LPI))
4641 p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
4642 if (DO_BIC(BIC_SYS_LPI))
4643 p->sys_lpi = cpuidle_cur_sys_lpi_us;
4644
4645 if (!platform->has_per_core_rapl) {
4646 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
4647 if (status != 0)
4648 return status;
4649 }
4650
4651 if (DO_BIC(BIC_PkgTmp)) {
4652 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
4653 return -17;
4654 p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
4655 }
4656
4657 if (DO_BIC(BIC_UNCORE_MHZ))
4658 p->uncore_mhz = get_legacy_uncore_mhz(p->package_id);
4659
4660 if (DO_BIC(BIC_GFX_rc6))
4661 p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull;
4662
4663 if (DO_BIC(BIC_GFXMHz))
4664 p->gfx_mhz = gfx_info[GFX_MHz].val;
4665
4666 if (DO_BIC(BIC_GFXACTMHz))
4667 p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val;
4668
4669 if (DO_BIC(BIC_SAM_mc6))
4670 p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull;
4671
4672 if (DO_BIC(BIC_SAMMHz))
4673 p->sam_mhz = gfx_info[SAM_MHz].val;
4674
4675 if (DO_BIC(BIC_SAMACTMHz))
4676 p->sam_act_mhz = gfx_info[SAM_ACTMHz].val;
4677
4678 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4679 char *path = NULL;
4680
4681 if (mp->msr_num == 0) {
4682 path = find_sysfs_path_by_id(mp->sp, p->package_id);
4683 if (path == NULL) {
4684 warnx("%s: package_id %d not found", __func__, p->package_id);
4685 return -10;
4686 }
4687 }
4688 if (get_mp(cpu, mp, &p->counter[i], path))
4689 return -10;
4690 }
4691
4692 if (perf_counter_info_read_values(sys.perf_pp, cpu, p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS))
4693 return -10;
4694
4695 for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next)
4696 p->pmt_counter[i] = pmt_read_counter(pp, p->package_id);
4697
4698 done:
4699 gettimeofday(&t->tv_end, (struct timezone *)NULL);
4700
4701 return 0;
4702 }
4703
4704 int pkg_cstate_limit = PCLUKN;
4705 char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2",
4706 "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited"
4707 };
4708
4709 int nhm_pkg_cstate_limits[16] =
4710 { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4711 PCLRSV, PCLRSV
4712 };
4713
4714 int snb_pkg_cstate_limits[16] =
4715 { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4716 PCLRSV, PCLRSV
4717 };
4718
4719 int hsw_pkg_cstate_limits[16] =
4720 { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4721 PCLRSV, PCLRSV
4722 };
4723
4724 int slv_pkg_cstate_limits[16] =
4725 { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4726 PCL__6, PCL__7
4727 };
4728
4729 int amt_pkg_cstate_limits[16] =
4730 { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4731 PCLRSV, PCLRSV
4732 };
4733
4734 int phi_pkg_cstate_limits[16] =
4735 { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4736 PCLRSV, PCLRSV
4737 };
4738
4739 int glm_pkg_cstate_limits[16] =
4740 { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4741 PCLRSV, PCLRSV
4742 };
4743
4744 int skx_pkg_cstate_limits[16] =
4745 { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4746 PCLRSV, PCLRSV
4747 };
4748
4749 int icx_pkg_cstate_limits[16] =
4750 { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
4751 PCLRSV, PCLRSV
4752 };
4753
probe_cst_limit(void)4754 void probe_cst_limit(void)
4755 {
4756 unsigned long long msr;
4757 int *pkg_cstate_limits;
4758
4759 if (!platform->has_nhm_msrs || no_msr)
4760 return;
4761
4762 switch (platform->cst_limit) {
4763 case CST_LIMIT_NHM:
4764 pkg_cstate_limits = nhm_pkg_cstate_limits;
4765 break;
4766 case CST_LIMIT_SNB:
4767 pkg_cstate_limits = snb_pkg_cstate_limits;
4768 break;
4769 case CST_LIMIT_HSW:
4770 pkg_cstate_limits = hsw_pkg_cstate_limits;
4771 break;
4772 case CST_LIMIT_SKX:
4773 pkg_cstate_limits = skx_pkg_cstate_limits;
4774 break;
4775 case CST_LIMIT_ICX:
4776 pkg_cstate_limits = icx_pkg_cstate_limits;
4777 break;
4778 case CST_LIMIT_SLV:
4779 pkg_cstate_limits = slv_pkg_cstate_limits;
4780 break;
4781 case CST_LIMIT_AMT:
4782 pkg_cstate_limits = amt_pkg_cstate_limits;
4783 break;
4784 case CST_LIMIT_KNL:
4785 pkg_cstate_limits = phi_pkg_cstate_limits;
4786 break;
4787 case CST_LIMIT_GMT:
4788 pkg_cstate_limits = glm_pkg_cstate_limits;
4789 break;
4790 default:
4791 return;
4792 }
4793
4794 get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
4795 pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
4796 }
4797
dump_platform_info(void)4798 static void dump_platform_info(void)
4799 {
4800 unsigned long long msr;
4801 unsigned int ratio;
4802
4803 if (!platform->has_nhm_msrs || no_msr)
4804 return;
4805
4806 get_msr(base_cpu, MSR_PLATFORM_INFO, &msr);
4807
4808 fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr);
4809
4810 ratio = (msr >> 40) & 0xFF;
4811 fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk);
4812
4813 ratio = (msr >> 8) & 0xFF;
4814 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
4815 }
4816
dump_power_ctl(void)4817 static void dump_power_ctl(void)
4818 {
4819 unsigned long long msr;
4820
4821 if (!platform->has_nhm_msrs || no_msr)
4822 return;
4823
4824 get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr);
4825 fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n",
4826 base_cpu, msr, msr & 0x2 ? "EN" : "DIS");
4827
4828 /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */
4829 if (platform->has_cst_prewake_bit)
4830 fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN");
4831
4832 return;
4833 }
4834
dump_turbo_ratio_limit2(void)4835 static void dump_turbo_ratio_limit2(void)
4836 {
4837 unsigned long long msr;
4838 unsigned int ratio;
4839
4840 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr);
4841
4842 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr);
4843
4844 ratio = (msr >> 8) & 0xFF;
4845 if (ratio)
4846 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk);
4847
4848 ratio = (msr >> 0) & 0xFF;
4849 if (ratio)
4850 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk);
4851 return;
4852 }
4853
dump_turbo_ratio_limit1(void)4854 static void dump_turbo_ratio_limit1(void)
4855 {
4856 unsigned long long msr;
4857 unsigned int ratio;
4858
4859 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr);
4860
4861 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr);
4862
4863 ratio = (msr >> 56) & 0xFF;
4864 if (ratio)
4865 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk);
4866
4867 ratio = (msr >> 48) & 0xFF;
4868 if (ratio)
4869 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk);
4870
4871 ratio = (msr >> 40) & 0xFF;
4872 if (ratio)
4873 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk);
4874
4875 ratio = (msr >> 32) & 0xFF;
4876 if (ratio)
4877 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk);
4878
4879 ratio = (msr >> 24) & 0xFF;
4880 if (ratio)
4881 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk);
4882
4883 ratio = (msr >> 16) & 0xFF;
4884 if (ratio)
4885 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk);
4886
4887 ratio = (msr >> 8) & 0xFF;
4888 if (ratio)
4889 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk);
4890
4891 ratio = (msr >> 0) & 0xFF;
4892 if (ratio)
4893 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk);
4894 return;
4895 }
4896
dump_turbo_ratio_limits(int trl_msr_offset)4897 static void dump_turbo_ratio_limits(int trl_msr_offset)
4898 {
4899 unsigned long long msr, core_counts;
4900 int shift;
4901
4902 get_msr(base_cpu, trl_msr_offset, &msr);
4903 fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n",
4904 base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr);
4905
4906 if (platform->trl_msrs & TRL_CORECOUNT) {
4907 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts);
4908 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts);
4909 } else {
4910 core_counts = 0x0807060504030201;
4911 }
4912
4913 for (shift = 56; shift >= 0; shift -= 8) {
4914 unsigned int ratio, group_size;
4915
4916 ratio = (msr >> shift) & 0xFF;
4917 group_size = (core_counts >> shift) & 0xFF;
4918 if (ratio)
4919 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n",
4920 ratio, bclk, ratio * bclk, group_size);
4921 }
4922
4923 return;
4924 }
4925
dump_atom_turbo_ratio_limits(void)4926 static void dump_atom_turbo_ratio_limits(void)
4927 {
4928 unsigned long long msr;
4929 unsigned int ratio;
4930
4931 get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr);
4932 fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
4933
4934 ratio = (msr >> 0) & 0x3F;
4935 if (ratio)
4936 fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk);
4937
4938 ratio = (msr >> 8) & 0x3F;
4939 if (ratio)
4940 fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk);
4941
4942 ratio = (msr >> 16) & 0x3F;
4943 if (ratio)
4944 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
4945
4946 get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr);
4947 fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
4948
4949 ratio = (msr >> 24) & 0x3F;
4950 if (ratio)
4951 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk);
4952
4953 ratio = (msr >> 16) & 0x3F;
4954 if (ratio)
4955 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk);
4956
4957 ratio = (msr >> 8) & 0x3F;
4958 if (ratio)
4959 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk);
4960
4961 ratio = (msr >> 0) & 0x3F;
4962 if (ratio)
4963 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk);
4964 }
4965
dump_knl_turbo_ratio_limits(void)4966 static void dump_knl_turbo_ratio_limits(void)
4967 {
4968 const unsigned int buckets_no = 7;
4969
4970 unsigned long long msr;
4971 int delta_cores, delta_ratio;
4972 int i, b_nr;
4973 unsigned int cores[buckets_no];
4974 unsigned int ratio[buckets_no];
4975
4976 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
4977
4978 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr);
4979
4980 /*
4981 * Turbo encoding in KNL is as follows:
4982 * [0] -- Reserved
4983 * [7:1] -- Base value of number of active cores of bucket 1.
4984 * [15:8] -- Base value of freq ratio of bucket 1.
4985 * [20:16] -- +ve delta of number of active cores of bucket 2.
4986 * i.e. active cores of bucket 2 =
4987 * active cores of bucket 1 + delta
4988 * [23:21] -- Negative delta of freq ratio of bucket 2.
4989 * i.e. freq ratio of bucket 2 =
4990 * freq ratio of bucket 1 - delta
4991 * [28:24]-- +ve delta of number of active cores of bucket 3.
4992 * [31:29]-- -ve delta of freq ratio of bucket 3.
4993 * [36:32]-- +ve delta of number of active cores of bucket 4.
4994 * [39:37]-- -ve delta of freq ratio of bucket 4.
4995 * [44:40]-- +ve delta of number of active cores of bucket 5.
4996 * [47:45]-- -ve delta of freq ratio of bucket 5.
4997 * [52:48]-- +ve delta of number of active cores of bucket 6.
4998 * [55:53]-- -ve delta of freq ratio of bucket 6.
4999 * [60:56]-- +ve delta of number of active cores of bucket 7.
5000 * [63:61]-- -ve delta of freq ratio of bucket 7.
5001 */
5002
5003 b_nr = 0;
5004 cores[b_nr] = (msr & 0xFF) >> 1;
5005 ratio[b_nr] = (msr >> 8) & 0xFF;
5006
5007 for (i = 16; i < 64; i += 8) {
5008 delta_cores = (msr >> i) & 0x1F;
5009 delta_ratio = (msr >> (i + 5)) & 0x7;
5010
5011 cores[b_nr + 1] = cores[b_nr] + delta_cores;
5012 ratio[b_nr + 1] = ratio[b_nr] - delta_ratio;
5013 b_nr++;
5014 }
5015
5016 for (i = buckets_no - 1; i >= 0; i--)
5017 if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
5018 fprintf(outf,
5019 "%d * %.1f = %.1f MHz max turbo %d active cores\n",
5020 ratio[i], bclk, ratio[i] * bclk, cores[i]);
5021 }
5022
dump_cst_cfg(void)5023 static void dump_cst_cfg(void)
5024 {
5025 unsigned long long msr;
5026
5027 if (!platform->has_nhm_msrs || no_msr)
5028 return;
5029
5030 get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5031
5032 fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr);
5033
5034 fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)",
5035 (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
5036 (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "",
5037 (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "",
5038 (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "",
5039 (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]);
5040
5041 #define AUTOMATIC_CSTATE_CONVERSION (1UL << 16)
5042 if (platform->has_cst_auto_convension) {
5043 fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off");
5044 }
5045
5046 fprintf(outf, ")\n");
5047
5048 return;
5049 }
5050
dump_config_tdp(void)5051 static void dump_config_tdp(void)
5052 {
5053 unsigned long long msr;
5054
5055 get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr);
5056 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr);
5057 fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF);
5058
5059 get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr);
5060 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr);
5061 if (msr) {
5062 fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5063 fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5064 fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5065 fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF);
5066 }
5067 fprintf(outf, ")\n");
5068
5069 get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr);
5070 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr);
5071 if (msr) {
5072 fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5073 fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5074 fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5075 fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF);
5076 }
5077 fprintf(outf, ")\n");
5078
5079 get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr);
5080 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr);
5081 if ((msr) & 0x3)
5082 fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3);
5083 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5084 fprintf(outf, ")\n");
5085
5086 get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr);
5087 fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr);
5088 fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF);
5089 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5090 fprintf(outf, ")\n");
5091 }
5092
5093 unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
5094
print_irtl(void)5095 void print_irtl(void)
5096 {
5097 unsigned long long msr;
5098
5099 if (!platform->has_irtl_msrs || no_msr)
5100 return;
5101
5102 if (platform->supported_cstates & PC3) {
5103 get_msr(base_cpu, MSR_PKGC3_IRTL, &msr);
5104 fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr);
5105 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5106 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5107 }
5108
5109 if (platform->supported_cstates & PC6) {
5110 get_msr(base_cpu, MSR_PKGC6_IRTL, &msr);
5111 fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr);
5112 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5113 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5114 }
5115
5116 if (platform->supported_cstates & PC7) {
5117 get_msr(base_cpu, MSR_PKGC7_IRTL, &msr);
5118 fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr);
5119 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5120 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5121 }
5122
5123 if (platform->supported_cstates & PC8) {
5124 get_msr(base_cpu, MSR_PKGC8_IRTL, &msr);
5125 fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr);
5126 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5127 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5128 }
5129
5130 if (platform->supported_cstates & PC9) {
5131 get_msr(base_cpu, MSR_PKGC9_IRTL, &msr);
5132 fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr);
5133 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5134 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5135 }
5136
5137 if (platform->supported_cstates & PC10) {
5138 get_msr(base_cpu, MSR_PKGC10_IRTL, &msr);
5139 fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr);
5140 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT",
5141 (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5142 }
5143 }
5144
free_fd_percpu(void)5145 void free_fd_percpu(void)
5146 {
5147 int i;
5148
5149 if (!fd_percpu)
5150 return;
5151
5152 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
5153 if (fd_percpu[i] != 0)
5154 close(fd_percpu[i]);
5155 }
5156
5157 free(fd_percpu);
5158 fd_percpu = NULL;
5159 }
5160
free_fd_instr_count_percpu(void)5161 void free_fd_instr_count_percpu(void)
5162 {
5163 if (!fd_instr_count_percpu)
5164 return;
5165
5166 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5167 if (fd_instr_count_percpu[i] != 0)
5168 close(fd_instr_count_percpu[i]);
5169 }
5170
5171 free(fd_instr_count_percpu);
5172 fd_instr_count_percpu = NULL;
5173 }
5174
free_fd_cstate(void)5175 void free_fd_cstate(void)
5176 {
5177 if (!ccstate_counter_info)
5178 return;
5179
5180 const int counter_info_num = ccstate_counter_info_size;
5181
5182 for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) {
5183 if (ccstate_counter_info[counter_id].fd_perf_core != -1)
5184 close(ccstate_counter_info[counter_id].fd_perf_core);
5185
5186 if (ccstate_counter_info[counter_id].fd_perf_pkg != -1)
5187 close(ccstate_counter_info[counter_id].fd_perf_pkg);
5188 }
5189
5190 free(ccstate_counter_info);
5191 ccstate_counter_info = NULL;
5192 ccstate_counter_info_size = 0;
5193 }
5194
free_fd_msr(void)5195 void free_fd_msr(void)
5196 {
5197 if (!msr_counter_info)
5198 return;
5199
5200 for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
5201 if (msr_counter_info[cpu].fd_perf != -1)
5202 close(msr_counter_info[cpu].fd_perf);
5203 }
5204
5205 free(msr_counter_info);
5206 msr_counter_info = NULL;
5207 msr_counter_info_size = 0;
5208 }
5209
free_fd_rapl_percpu(void)5210 void free_fd_rapl_percpu(void)
5211 {
5212 if (!rapl_counter_info_perdomain)
5213 return;
5214
5215 const int num_domains = rapl_counter_info_perdomain_size;
5216
5217 for (int domain_id = 0; domain_id < num_domains; ++domain_id) {
5218 if (rapl_counter_info_perdomain[domain_id].fd_perf != -1)
5219 close(rapl_counter_info_perdomain[domain_id].fd_perf);
5220 }
5221
5222 free(rapl_counter_info_perdomain);
5223 rapl_counter_info_perdomain = NULL;
5224 rapl_counter_info_perdomain_size = 0;
5225 }
5226
free_fd_added_perf_counters_(struct perf_counter_info * pp)5227 void free_fd_added_perf_counters_(struct perf_counter_info *pp)
5228 {
5229 if (!pp)
5230 return;
5231
5232 if (!pp->fd_perf_per_domain)
5233 return;
5234
5235 while (pp) {
5236 for (size_t domain = 0; domain < pp->num_domains; ++domain) {
5237 if (pp->fd_perf_per_domain[domain] != -1) {
5238 close(pp->fd_perf_per_domain[domain]);
5239 pp->fd_perf_per_domain[domain] = -1;
5240 }
5241 }
5242
5243 free(pp->fd_perf_per_domain);
5244 pp->fd_perf_per_domain = NULL;
5245
5246 pp = pp->next;
5247 }
5248 }
5249
free_fd_added_perf_counters(void)5250 void free_fd_added_perf_counters(void)
5251 {
5252 free_fd_added_perf_counters_(sys.perf_tp);
5253 free_fd_added_perf_counters_(sys.perf_cp);
5254 free_fd_added_perf_counters_(sys.perf_pp);
5255 }
5256
free_all_buffers(void)5257 void free_all_buffers(void)
5258 {
5259 int i;
5260
5261 CPU_FREE(cpu_present_set);
5262 cpu_present_set = NULL;
5263 cpu_present_setsize = 0;
5264
5265 CPU_FREE(cpu_effective_set);
5266 cpu_effective_set = NULL;
5267 cpu_effective_setsize = 0;
5268
5269 CPU_FREE(cpu_allowed_set);
5270 cpu_allowed_set = NULL;
5271 cpu_allowed_setsize = 0;
5272
5273 CPU_FREE(cpu_affinity_set);
5274 cpu_affinity_set = NULL;
5275 cpu_affinity_setsize = 0;
5276
5277 free(thread_even);
5278 free(core_even);
5279 free(package_even);
5280
5281 thread_even = NULL;
5282 core_even = NULL;
5283 package_even = NULL;
5284
5285 free(thread_odd);
5286 free(core_odd);
5287 free(package_odd);
5288
5289 thread_odd = NULL;
5290 core_odd = NULL;
5291 package_odd = NULL;
5292
5293 free(output_buffer);
5294 output_buffer = NULL;
5295 outp = NULL;
5296
5297 free_fd_percpu();
5298 free_fd_instr_count_percpu();
5299 free_fd_msr();
5300 free_fd_rapl_percpu();
5301 free_fd_cstate();
5302 free_fd_added_perf_counters();
5303
5304 free(irq_column_2_cpu);
5305 free(irqs_per_cpu);
5306
5307 for (i = 0; i <= topo.max_cpu_num; ++i) {
5308 if (cpus[i].put_ids)
5309 CPU_FREE(cpus[i].put_ids);
5310 }
5311 free(cpus);
5312 }
5313
5314 /*
5315 * Parse a file containing a single int.
5316 * Return 0 if file can not be opened
5317 * Exit if file can be opened, but can not be parsed
5318 */
parse_int_file(const char * fmt,...)5319 int parse_int_file(const char *fmt, ...)
5320 {
5321 va_list args;
5322 char path[PATH_MAX];
5323 FILE *filep;
5324 int value;
5325
5326 va_start(args, fmt);
5327 vsnprintf(path, sizeof(path), fmt, args);
5328 va_end(args);
5329 filep = fopen(path, "r");
5330 if (!filep)
5331 return 0;
5332 if (fscanf(filep, "%d", &value) != 1)
5333 err(1, "%s: failed to parse number from file", path);
5334 fclose(filep);
5335 return value;
5336 }
5337
5338 /*
5339 * cpu_is_first_core_in_package(cpu)
5340 * return 1 if given CPU is 1st core in package
5341 */
cpu_is_first_core_in_package(int cpu)5342 int cpu_is_first_core_in_package(int cpu)
5343 {
5344 return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu);
5345 }
5346
get_physical_package_id(int cpu)5347 int get_physical_package_id(int cpu)
5348 {
5349 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
5350 }
5351
get_die_id(int cpu)5352 int get_die_id(int cpu)
5353 {
5354 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu);
5355 }
5356
get_core_id(int cpu)5357 int get_core_id(int cpu)
5358 {
5359 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu);
5360 }
5361
set_node_data(void)5362 void set_node_data(void)
5363 {
5364 int pkg, node, lnode, cpu, cpux;
5365 int cpu_count;
5366
5367 /* initialize logical_node_id */
5368 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu)
5369 cpus[cpu].logical_node_id = -1;
5370
5371 cpu_count = 0;
5372 for (pkg = 0; pkg < topo.num_packages; pkg++) {
5373 lnode = 0;
5374 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
5375 if (cpus[cpu].physical_package_id != pkg)
5376 continue;
5377 /* find a cpu with an unset logical_node_id */
5378 if (cpus[cpu].logical_node_id != -1)
5379 continue;
5380 cpus[cpu].logical_node_id = lnode;
5381 node = cpus[cpu].physical_node_id;
5382 cpu_count++;
5383 /*
5384 * find all matching cpus on this pkg and set
5385 * the logical_node_id
5386 */
5387 for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) {
5388 if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) {
5389 cpus[cpux].logical_node_id = lnode;
5390 cpu_count++;
5391 }
5392 }
5393 lnode++;
5394 if (lnode > topo.nodes_per_pkg)
5395 topo.nodes_per_pkg = lnode;
5396 }
5397 if (cpu_count >= topo.max_cpu_num)
5398 break;
5399 }
5400 }
5401
get_physical_node_id(struct cpu_topology * thiscpu)5402 int get_physical_node_id(struct cpu_topology *thiscpu)
5403 {
5404 char path[80];
5405 FILE *filep;
5406 int i;
5407 int cpu = thiscpu->logical_cpu_id;
5408
5409 for (i = 0; i <= topo.max_cpu_num; i++) {
5410 sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i);
5411 filep = fopen(path, "r");
5412 if (!filep)
5413 continue;
5414 fclose(filep);
5415 return i;
5416 }
5417 return -1;
5418 }
5419
parse_cpu_str(char * cpu_str,cpu_set_t * cpu_set,int cpu_set_size)5420 static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
5421 {
5422 unsigned int start, end;
5423 char *next = cpu_str;
5424
5425 while (next && *next) {
5426
5427 if (*next == '-') /* no negative cpu numbers */
5428 return 1;
5429
5430 if (*next == '\0' || *next == '\n')
5431 break;
5432
5433 start = strtoul(next, &next, 10);
5434
5435 if (start >= CPU_SUBSET_MAXCPUS)
5436 return 1;
5437 CPU_SET_S(start, cpu_set_size, cpu_set);
5438
5439 if (*next == '\0' || *next == '\n')
5440 break;
5441
5442 if (*next == ',') {
5443 next += 1;
5444 continue;
5445 }
5446
5447 if (*next == '-') {
5448 next += 1; /* start range */
5449 } else if (*next == '.') {
5450 next += 1;
5451 if (*next == '.')
5452 next += 1; /* start range */
5453 else
5454 return 1;
5455 }
5456
5457 end = strtoul(next, &next, 10);
5458 if (end <= start)
5459 return 1;
5460
5461 while (++start <= end) {
5462 if (start >= CPU_SUBSET_MAXCPUS)
5463 return 1;
5464 CPU_SET_S(start, cpu_set_size, cpu_set);
5465 }
5466
5467 if (*next == ',')
5468 next += 1;
5469 else if (*next != '\0' && *next != '\n')
5470 return 1;
5471 }
5472
5473 return 0;
5474 }
5475
get_thread_siblings(struct cpu_topology * thiscpu)5476 int get_thread_siblings(struct cpu_topology *thiscpu)
5477 {
5478 char path[80], character;
5479 FILE *filep;
5480 unsigned long map;
5481 int so, shift, sib_core;
5482 int cpu = thiscpu->logical_cpu_id;
5483 int offset = topo.max_cpu_num + 1;
5484 size_t size;
5485 int thread_id = 0;
5486
5487 thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1));
5488 if (thiscpu->thread_id < 0)
5489 thiscpu->thread_id = thread_id++;
5490 if (!thiscpu->put_ids)
5491 return -1;
5492
5493 size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
5494 CPU_ZERO_S(size, thiscpu->put_ids);
5495
5496 sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu);
5497 filep = fopen(path, "r");
5498
5499 if (!filep) {
5500 warnx("%s: open failed", path);
5501 return -1;
5502 }
5503 do {
5504 offset -= BITMASK_SIZE;
5505 if (fscanf(filep, "%lx%c", &map, &character) != 2)
5506 err(1, "%s: failed to parse file", path);
5507 for (shift = 0; shift < BITMASK_SIZE; shift++) {
5508 if ((map >> shift) & 0x1) {
5509 so = shift + offset;
5510 sib_core = get_core_id(so);
5511 if (sib_core == thiscpu->physical_core_id) {
5512 CPU_SET_S(so, size, thiscpu->put_ids);
5513 if ((so != cpu) && (cpus[so].thread_id < 0))
5514 cpus[so].thread_id = thread_id++;
5515 }
5516 }
5517 }
5518 } while (character == ',');
5519 fclose(filep);
5520
5521 return CPU_COUNT_S(size, thiscpu->put_ids);
5522 }
5523
5524 /*
5525 * run func(thread, core, package) in topology order
5526 * skip non-present cpus
5527 */
5528
for_all_cpus_2(int (func)(struct thread_data *,struct core_data *,struct pkg_data *,struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,struct thread_data * thread_base2,struct core_data * core_base2,struct pkg_data * pkg_base2)5529 int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *,
5530 struct pkg_data *, struct thread_data *, struct core_data *,
5531 struct pkg_data *), struct thread_data *thread_base,
5532 struct core_data *core_base, struct pkg_data *pkg_base,
5533 struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2)
5534 {
5535 int retval, pkg_no, node_no, core_no, thread_no;
5536
5537 for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
5538 for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) {
5539 for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
5540 for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
5541 struct thread_data *t, *t2;
5542 struct core_data *c, *c2;
5543 struct pkg_data *p, *p2;
5544
5545 t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
5546
5547 if (cpu_is_not_allowed(t->cpu_id))
5548 continue;
5549
5550 t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no);
5551
5552 c = GET_CORE(core_base, core_no, node_no, pkg_no);
5553 c2 = GET_CORE(core_base2, core_no, node_no, pkg_no);
5554
5555 p = GET_PKG(pkg_base, pkg_no);
5556 p2 = GET_PKG(pkg_base2, pkg_no);
5557
5558 retval = func(t, c, p, t2, c2, p2);
5559 if (retval)
5560 return retval;
5561 }
5562 }
5563 }
5564 }
5565 return 0;
5566 }
5567
5568 /*
5569 * run func(cpu) on every cpu in /proc/stat
5570 * return max_cpu number
5571 */
for_all_proc_cpus(int (func)(int))5572 int for_all_proc_cpus(int (func) (int))
5573 {
5574 FILE *fp;
5575 int cpu_num;
5576 int retval;
5577
5578 fp = fopen_or_die(proc_stat, "r");
5579
5580 retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
5581 if (retval != 0)
5582 err(1, "%s: failed to parse format", proc_stat);
5583
5584 while (1) {
5585 retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
5586 if (retval != 1)
5587 break;
5588
5589 retval = func(cpu_num);
5590 if (retval) {
5591 fclose(fp);
5592 return (retval);
5593 }
5594 }
5595 fclose(fp);
5596 return 0;
5597 }
5598
5599 #define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective"
5600
5601 static char cpu_effective_str[1024];
5602
update_effective_str(bool startup)5603 static int update_effective_str(bool startup)
5604 {
5605 FILE *fp;
5606 char *pos;
5607 char buf[1024];
5608 int ret;
5609
5610 if (cpu_effective_str[0] == '\0' && !startup)
5611 return 0;
5612
5613 fp = fopen(PATH_EFFECTIVE_CPUS, "r");
5614 if (!fp)
5615 return 0;
5616
5617 pos = fgets(buf, 1024, fp);
5618 if (!pos)
5619 err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS);
5620
5621 fclose(fp);
5622
5623 ret = strncmp(cpu_effective_str, buf, 1024);
5624 if (!ret)
5625 return 0;
5626
5627 strncpy(cpu_effective_str, buf, 1024);
5628 return 1;
5629 }
5630
update_effective_set(bool startup)5631 static void update_effective_set(bool startup)
5632 {
5633 update_effective_str(startup);
5634
5635 if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
5636 err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str);
5637 }
5638
5639 void linux_perf_init(void);
5640 void msr_perf_init(void);
5641 void rapl_perf_init(void);
5642 void cstate_perf_init(void);
5643 void added_perf_counters_init(void);
5644 void pmt_init(void);
5645
re_initialize(void)5646 void re_initialize(void)
5647 {
5648 free_all_buffers();
5649 setup_all_buffers(false);
5650 linux_perf_init();
5651 msr_perf_init();
5652 rapl_perf_init();
5653 cstate_perf_init();
5654 added_perf_counters_init();
5655 pmt_init();
5656 fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus,
5657 topo.allowed_cpus);
5658 }
5659
set_max_cpu_num(void)5660 void set_max_cpu_num(void)
5661 {
5662 FILE *filep;
5663 int base_cpu;
5664 unsigned long dummy;
5665 char pathname[64];
5666
5667 base_cpu = sched_getcpu();
5668 if (base_cpu < 0)
5669 err(1, "cannot find calling cpu ID");
5670 sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu);
5671
5672 filep = fopen_or_die(pathname, "r");
5673 topo.max_cpu_num = 0;
5674 while (fscanf(filep, "%lx,", &dummy) == 1)
5675 topo.max_cpu_num += BITMASK_SIZE;
5676 fclose(filep);
5677 topo.max_cpu_num--; /* 0 based */
5678 }
5679
5680 /*
5681 * count_cpus()
5682 * remember the last one seen, it will be the max
5683 */
count_cpus(int cpu)5684 int count_cpus(int cpu)
5685 {
5686 UNUSED(cpu);
5687
5688 topo.num_cpus++;
5689 return 0;
5690 }
5691
mark_cpu_present(int cpu)5692 int mark_cpu_present(int cpu)
5693 {
5694 CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
5695 return 0;
5696 }
5697
init_thread_id(int cpu)5698 int init_thread_id(int cpu)
5699 {
5700 cpus[cpu].thread_id = -1;
5701 return 0;
5702 }
5703
set_my_cpu_type(void)5704 int set_my_cpu_type(void)
5705 {
5706 unsigned int eax, ebx, ecx, edx;
5707 unsigned int max_level;
5708
5709 __cpuid(0, max_level, ebx, ecx, edx);
5710
5711 if (max_level < CPUID_LEAF_MODEL_ID)
5712 return 0;
5713
5714 __cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx);
5715
5716 return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT);
5717 }
5718
set_cpu_hybrid_type(int cpu)5719 int set_cpu_hybrid_type(int cpu)
5720 {
5721 if (cpu_migrate(cpu))
5722 return -1;
5723
5724 int type = set_my_cpu_type();
5725
5726 cpus[cpu].type = type;
5727 return 0;
5728 }
5729
5730 /*
5731 * snapshot_proc_interrupts()
5732 *
5733 * read and record summary of /proc/interrupts
5734 *
5735 * return 1 if config change requires a restart, else return 0
5736 */
snapshot_proc_interrupts(void)5737 int snapshot_proc_interrupts(void)
5738 {
5739 static FILE *fp;
5740 int column, retval;
5741
5742 if (fp == NULL)
5743 fp = fopen_or_die("/proc/interrupts", "r");
5744 else
5745 rewind(fp);
5746
5747 /* read 1st line of /proc/interrupts to get cpu* name for each column */
5748 for (column = 0; column < topo.num_cpus; ++column) {
5749 int cpu_number;
5750
5751 retval = fscanf(fp, " CPU%d", &cpu_number);
5752 if (retval != 1)
5753 break;
5754
5755 if (cpu_number > topo.max_cpu_num) {
5756 warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num);
5757 return 1;
5758 }
5759
5760 irq_column_2_cpu[column] = cpu_number;
5761 irqs_per_cpu[cpu_number] = 0;
5762 }
5763
5764 /* read /proc/interrupt count lines and sum up irqs per cpu */
5765 while (1) {
5766 int column;
5767 char buf[64];
5768
5769 retval = fscanf(fp, " %s:", buf); /* flush irq# "N:" */
5770 if (retval != 1)
5771 break;
5772
5773 /* read the count per cpu */
5774 for (column = 0; column < topo.num_cpus; ++column) {
5775
5776 int cpu_number, irq_count;
5777
5778 retval = fscanf(fp, " %d", &irq_count);
5779 if (retval != 1)
5780 break;
5781
5782 cpu_number = irq_column_2_cpu[column];
5783 irqs_per_cpu[cpu_number] += irq_count;
5784
5785 }
5786
5787 while (getc(fp) != '\n') ; /* flush interrupt description */
5788
5789 }
5790 return 0;
5791 }
5792
5793 /*
5794 * snapshot_graphics()
5795 *
5796 * record snapshot of specified graphics sysfs knob
5797 *
5798 * return 1 if config change requires a restart, else return 0
5799 */
snapshot_graphics(int idx)5800 int snapshot_graphics(int idx)
5801 {
5802 FILE *fp;
5803 int retval;
5804
5805 switch (idx) {
5806 case GFX_rc6:
5807 case SAM_mc6:
5808 fp = fopen_or_die(gfx_info[idx].path, "r");
5809 retval = fscanf(fp, "%lld", &gfx_info[idx].val_ull);
5810 if (retval != 1)
5811 err(1, "rc6");
5812 fclose(fp);
5813 return 0;
5814 case GFX_MHz:
5815 case GFX_ACTMHz:
5816 case SAM_MHz:
5817 case SAM_ACTMHz:
5818 if (gfx_info[idx].fp == NULL) {
5819 gfx_info[idx].fp = fopen_or_die(gfx_info[idx].path, "r");
5820 } else {
5821 rewind(gfx_info[idx].fp);
5822 fflush(gfx_info[idx].fp);
5823 }
5824 retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val);
5825 if (retval != 1)
5826 err(1, "MHz");
5827 return 0;
5828 default:
5829 return -EINVAL;
5830 }
5831 }
5832
5833 /*
5834 * snapshot_cpu_lpi()
5835 *
5836 * record snapshot of
5837 * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us
5838 */
snapshot_cpu_lpi_us(void)5839 int snapshot_cpu_lpi_us(void)
5840 {
5841 FILE *fp;
5842 int retval;
5843
5844 fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r");
5845
5846 retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us);
5847 if (retval != 1) {
5848 fprintf(stderr, "Disabling Low Power Idle CPU output\n");
5849 BIC_NOT_PRESENT(BIC_CPU_LPI);
5850 fclose(fp);
5851 return -1;
5852 }
5853
5854 fclose(fp);
5855
5856 return 0;
5857 }
5858
5859 /*
5860 * snapshot_sys_lpi()
5861 *
5862 * record snapshot of sys_lpi_file
5863 */
snapshot_sys_lpi_us(void)5864 int snapshot_sys_lpi_us(void)
5865 {
5866 FILE *fp;
5867 int retval;
5868
5869 fp = fopen_or_die(sys_lpi_file, "r");
5870
5871 retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us);
5872 if (retval != 1) {
5873 fprintf(stderr, "Disabling Low Power Idle System output\n");
5874 BIC_NOT_PRESENT(BIC_SYS_LPI);
5875 fclose(fp);
5876 return -1;
5877 }
5878 fclose(fp);
5879
5880 return 0;
5881 }
5882
5883 /*
5884 * snapshot /proc and /sys files
5885 *
5886 * return 1 if configuration restart needed, else return 0
5887 */
snapshot_proc_sysfs_files(void)5888 int snapshot_proc_sysfs_files(void)
5889 {
5890 if (DO_BIC(BIC_IRQ))
5891 if (snapshot_proc_interrupts())
5892 return 1;
5893
5894 if (DO_BIC(BIC_GFX_rc6))
5895 snapshot_graphics(GFX_rc6);
5896
5897 if (DO_BIC(BIC_GFXMHz))
5898 snapshot_graphics(GFX_MHz);
5899
5900 if (DO_BIC(BIC_GFXACTMHz))
5901 snapshot_graphics(GFX_ACTMHz);
5902
5903 if (DO_BIC(BIC_SAM_mc6))
5904 snapshot_graphics(SAM_mc6);
5905
5906 if (DO_BIC(BIC_SAMMHz))
5907 snapshot_graphics(SAM_MHz);
5908
5909 if (DO_BIC(BIC_SAMACTMHz))
5910 snapshot_graphics(SAM_ACTMHz);
5911
5912 if (DO_BIC(BIC_CPU_LPI))
5913 snapshot_cpu_lpi_us();
5914
5915 if (DO_BIC(BIC_SYS_LPI))
5916 snapshot_sys_lpi_us();
5917
5918 return 0;
5919 }
5920
5921 int exit_requested;
5922
signal_handler(int signal)5923 static void signal_handler(int signal)
5924 {
5925 switch (signal) {
5926 case SIGINT:
5927 exit_requested = 1;
5928 if (debug)
5929 fprintf(stderr, " SIGINT\n");
5930 break;
5931 case SIGUSR1:
5932 if (debug > 1)
5933 fprintf(stderr, "SIGUSR1\n");
5934 break;
5935 }
5936 }
5937
setup_signal_handler(void)5938 void setup_signal_handler(void)
5939 {
5940 struct sigaction sa;
5941
5942 memset(&sa, 0, sizeof(sa));
5943
5944 sa.sa_handler = &signal_handler;
5945
5946 if (sigaction(SIGINT, &sa, NULL) < 0)
5947 err(1, "sigaction SIGINT");
5948 if (sigaction(SIGUSR1, &sa, NULL) < 0)
5949 err(1, "sigaction SIGUSR1");
5950 }
5951
do_sleep(void)5952 void do_sleep(void)
5953 {
5954 struct timeval tout;
5955 struct timespec rest;
5956 fd_set readfds;
5957 int retval;
5958
5959 FD_ZERO(&readfds);
5960 FD_SET(0, &readfds);
5961
5962 if (ignore_stdin) {
5963 nanosleep(&interval_ts, NULL);
5964 return;
5965 }
5966
5967 tout = interval_tv;
5968 retval = select(1, &readfds, NULL, NULL, &tout);
5969
5970 if (retval == 1) {
5971 switch (getc(stdin)) {
5972 case 'q':
5973 exit_requested = 1;
5974 break;
5975 case EOF:
5976 /*
5977 * 'stdin' is a pipe closed on the other end. There
5978 * won't be any further input.
5979 */
5980 ignore_stdin = 1;
5981 /* Sleep the rest of the time */
5982 rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000);
5983 rest.tv_nsec = (tout.tv_usec % 1000000) * 1000;
5984 nanosleep(&rest, NULL);
5985 }
5986 }
5987 }
5988
get_msr_sum(int cpu,off_t offset,unsigned long long * msr)5989 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
5990 {
5991 int ret, idx;
5992 unsigned long long msr_cur, msr_last;
5993
5994 assert(!no_msr);
5995
5996 if (!per_cpu_msr_sum)
5997 return 1;
5998
5999 idx = offset_to_idx(offset);
6000 if (idx < 0)
6001 return idx;
6002 /* get_msr_sum() = sum + (get_msr() - last) */
6003 ret = get_msr(cpu, offset, &msr_cur);
6004 if (ret)
6005 return ret;
6006 msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
6007 DELTA_WRAP32(msr_cur, msr_last);
6008 *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
6009
6010 return 0;
6011 }
6012
6013 timer_t timerid;
6014
6015 /* Timer callback, update the sum of MSRs periodically. */
update_msr_sum(struct thread_data * t,struct core_data * c,struct pkg_data * p)6016 static int update_msr_sum(struct thread_data *t, struct core_data *c, struct pkg_data *p)
6017 {
6018 int i, ret;
6019 int cpu = t->cpu_id;
6020
6021 UNUSED(c);
6022 UNUSED(p);
6023
6024 assert(!no_msr);
6025
6026 for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) {
6027 unsigned long long msr_cur, msr_last;
6028 off_t offset;
6029
6030 if (!idx_valid(i))
6031 continue;
6032 offset = idx_to_offset(i);
6033 if (offset < 0)
6034 continue;
6035 ret = get_msr(cpu, offset, &msr_cur);
6036 if (ret) {
6037 fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
6038 continue;
6039 }
6040
6041 msr_last = per_cpu_msr_sum[cpu].entries[i].last;
6042 per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
6043
6044 DELTA_WRAP32(msr_cur, msr_last);
6045 per_cpu_msr_sum[cpu].entries[i].sum += msr_last;
6046 }
6047 return 0;
6048 }
6049
msr_record_handler(union sigval v)6050 static void msr_record_handler(union sigval v)
6051 {
6052 UNUSED(v);
6053
6054 for_all_cpus(update_msr_sum, EVEN_COUNTERS);
6055 }
6056
msr_sum_record(void)6057 void msr_sum_record(void)
6058 {
6059 struct itimerspec its;
6060 struct sigevent sev;
6061
6062 per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array));
6063 if (!per_cpu_msr_sum) {
6064 fprintf(outf, "Can not allocate memory for long time MSR.\n");
6065 return;
6066 }
6067 /*
6068 * Signal handler might be restricted, so use thread notifier instead.
6069 */
6070 memset(&sev, 0, sizeof(struct sigevent));
6071 sev.sigev_notify = SIGEV_THREAD;
6072 sev.sigev_notify_function = msr_record_handler;
6073
6074 sev.sigev_value.sival_ptr = &timerid;
6075 if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) {
6076 fprintf(outf, "Can not create timer.\n");
6077 goto release_msr;
6078 }
6079
6080 its.it_value.tv_sec = 0;
6081 its.it_value.tv_nsec = 1;
6082 /*
6083 * A wraparound time has been calculated early.
6084 * Some sources state that the peak power for a
6085 * microprocessor is usually 1.5 times the TDP rating,
6086 * use 2 * TDP for safety.
6087 */
6088 its.it_interval.tv_sec = rapl_joule_counter_range / 2;
6089 its.it_interval.tv_nsec = 0;
6090
6091 if (timer_settime(timerid, 0, &its, NULL) == -1) {
6092 fprintf(outf, "Can not set timer.\n");
6093 goto release_timer;
6094 }
6095 return;
6096
6097 release_timer:
6098 timer_delete(timerid);
6099 release_msr:
6100 free(per_cpu_msr_sum);
6101 }
6102
6103 /*
6104 * set_my_sched_priority(pri)
6105 * return previous priority on success
6106 * return value < -20 on failure
6107 */
set_my_sched_priority(int priority)6108 int set_my_sched_priority(int priority)
6109 {
6110 int retval;
6111 int original_priority;
6112
6113 errno = 0;
6114 original_priority = getpriority(PRIO_PROCESS, 0);
6115 if (errno && (original_priority == -1))
6116 return -21;
6117
6118 retval = setpriority(PRIO_PROCESS, 0, priority);
6119 if (retval)
6120 return -21;
6121
6122 errno = 0;
6123 retval = getpriority(PRIO_PROCESS, 0);
6124 if (retval != priority)
6125 return -21;
6126
6127 return original_priority;
6128 }
6129
turbostat_loop()6130 void turbostat_loop()
6131 {
6132 int retval;
6133 int restarted = 0;
6134 unsigned int done_iters = 0;
6135
6136 setup_signal_handler();
6137
6138 /*
6139 * elevate own priority for interval mode
6140 *
6141 * ignore on error - we probably don't have permission to set it, but
6142 * it's not a big deal
6143 */
6144 set_my_sched_priority(-20);
6145
6146 restart:
6147 restarted++;
6148
6149 snapshot_proc_sysfs_files();
6150 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6151 first_counter_read = 0;
6152 if (retval < -1) {
6153 exit(retval);
6154 } else if (retval == -1) {
6155 if (restarted > 10) {
6156 exit(retval);
6157 }
6158 re_initialize();
6159 goto restart;
6160 }
6161 restarted = 0;
6162 done_iters = 0;
6163 gettimeofday(&tv_even, (struct timezone *)NULL);
6164
6165 while (1) {
6166 if (for_all_proc_cpus(cpu_is_not_present)) {
6167 re_initialize();
6168 goto restart;
6169 }
6170 if (update_effective_str(false)) {
6171 re_initialize();
6172 goto restart;
6173 }
6174 do_sleep();
6175 if (snapshot_proc_sysfs_files())
6176 goto restart;
6177 retval = for_all_cpus(get_counters, ODD_COUNTERS);
6178 if (retval < -1) {
6179 exit(retval);
6180 } else if (retval == -1) {
6181 re_initialize();
6182 goto restart;
6183 }
6184 gettimeofday(&tv_odd, (struct timezone *)NULL);
6185 timersub(&tv_odd, &tv_even, &tv_delta);
6186 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) {
6187 re_initialize();
6188 goto restart;
6189 }
6190 compute_average(EVEN_COUNTERS);
6191 format_all_counters(EVEN_COUNTERS);
6192 flush_output_stdout();
6193 if (exit_requested)
6194 break;
6195 if (num_iterations && ++done_iters >= num_iterations)
6196 break;
6197 do_sleep();
6198 if (snapshot_proc_sysfs_files())
6199 goto restart;
6200 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6201 if (retval < -1) {
6202 exit(retval);
6203 } else if (retval == -1) {
6204 re_initialize();
6205 goto restart;
6206 }
6207 gettimeofday(&tv_even, (struct timezone *)NULL);
6208 timersub(&tv_even, &tv_odd, &tv_delta);
6209 if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) {
6210 re_initialize();
6211 goto restart;
6212 }
6213 compute_average(ODD_COUNTERS);
6214 format_all_counters(ODD_COUNTERS);
6215 flush_output_stdout();
6216 if (exit_requested)
6217 break;
6218 if (num_iterations && ++done_iters >= num_iterations)
6219 break;
6220 }
6221 }
6222
check_dev_msr()6223 void check_dev_msr()
6224 {
6225 struct stat sb;
6226 char pathname[32];
6227
6228 if (no_msr)
6229 return;
6230
6231 sprintf(pathname, "/dev/cpu/%d/msr", base_cpu);
6232 if (stat(pathname, &sb))
6233 if (system("/sbin/modprobe msr > /dev/null 2>&1"))
6234 no_msr = 1;
6235 }
6236
6237 /*
6238 * check for CAP_SYS_RAWIO
6239 * return 0 on success
6240 * return 1 on fail
6241 */
check_for_cap_sys_rawio(void)6242 int check_for_cap_sys_rawio(void)
6243 {
6244 cap_t caps;
6245 cap_flag_value_t cap_flag_value;
6246 int ret = 0;
6247
6248 caps = cap_get_proc();
6249 if (caps == NULL) {
6250 /*
6251 * CONFIG_MULTIUSER=n kernels have no cap_get_proc()
6252 * Allow them to continue and attempt to access MSRs
6253 */
6254 if (errno == ENOSYS)
6255 return 0;
6256
6257 return 1;
6258 }
6259
6260 if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) {
6261 ret = 1;
6262 goto free_and_exit;
6263 }
6264
6265 if (cap_flag_value != CAP_SET) {
6266 ret = 1;
6267 goto free_and_exit;
6268 }
6269
6270 free_and_exit:
6271 if (cap_free(caps) == -1)
6272 err(-6, "cap_free\n");
6273
6274 return ret;
6275 }
6276
check_msr_permission(void)6277 void check_msr_permission(void)
6278 {
6279 int failed = 0;
6280 char pathname[32];
6281
6282 if (no_msr)
6283 return;
6284
6285 /* check for CAP_SYS_RAWIO */
6286 failed += check_for_cap_sys_rawio();
6287
6288 /* test file permissions */
6289 sprintf(pathname, "/dev/cpu/%d/msr", base_cpu);
6290 if (euidaccess(pathname, R_OK)) {
6291 failed++;
6292 }
6293
6294 if (failed) {
6295 warnx("Failed to access %s. Some of the counters may not be available\n"
6296 "\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr");
6297 no_msr = 1;
6298 }
6299 }
6300
probe_bclk(void)6301 void probe_bclk(void)
6302 {
6303 unsigned long long msr;
6304 unsigned int base_ratio;
6305
6306 if (!platform->has_nhm_msrs || no_msr)
6307 return;
6308
6309 if (platform->bclk_freq == BCLK_100MHZ)
6310 bclk = 100.00;
6311 else if (platform->bclk_freq == BCLK_133MHZ)
6312 bclk = 133.33;
6313 else if (platform->bclk_freq == BCLK_SLV)
6314 bclk = slm_bclk();
6315 else
6316 return;
6317
6318 get_msr(base_cpu, MSR_PLATFORM_INFO, &msr);
6319 base_ratio = (msr >> 8) & 0xFF;
6320
6321 base_hz = base_ratio * bclk * 1000000;
6322 has_base_hz = 1;
6323
6324 if (platform->enable_tsc_tweak)
6325 tsc_tweak = base_hz / tsc_hz;
6326 }
6327
remove_underbar(char * s)6328 static void remove_underbar(char *s)
6329 {
6330 char *to = s;
6331
6332 while (*s) {
6333 if (*s != '_')
6334 *to++ = *s;
6335 s++;
6336 }
6337
6338 *to = 0;
6339 }
6340
dump_turbo_ratio_info(void)6341 static void dump_turbo_ratio_info(void)
6342 {
6343 if (!has_turbo)
6344 return;
6345
6346 if (!platform->has_nhm_msrs || no_msr)
6347 return;
6348
6349 if (platform->trl_msrs & TRL_LIMIT2)
6350 dump_turbo_ratio_limit2();
6351
6352 if (platform->trl_msrs & TRL_LIMIT1)
6353 dump_turbo_ratio_limit1();
6354
6355 if (platform->trl_msrs & TRL_BASE) {
6356 dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT);
6357
6358 if (is_hybrid)
6359 dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT);
6360 }
6361
6362 if (platform->trl_msrs & TRL_ATOM)
6363 dump_atom_turbo_ratio_limits();
6364
6365 if (platform->trl_msrs & TRL_KNL)
6366 dump_knl_turbo_ratio_limits();
6367
6368 if (platform->has_config_tdp)
6369 dump_config_tdp();
6370 }
6371
read_sysfs_int(char * path)6372 static int read_sysfs_int(char *path)
6373 {
6374 FILE *input;
6375 int retval = -1;
6376
6377 input = fopen(path, "r");
6378 if (input == NULL) {
6379 if (debug)
6380 fprintf(outf, "NSFOD %s\n", path);
6381 return (-1);
6382 }
6383 if (fscanf(input, "%d", &retval) != 1)
6384 err(1, "%s: failed to read int from file", path);
6385 fclose(input);
6386
6387 return (retval);
6388 }
6389
dump_sysfs_file(char * path)6390 static void dump_sysfs_file(char *path)
6391 {
6392 FILE *input;
6393 char cpuidle_buf[64];
6394
6395 input = fopen(path, "r");
6396 if (input == NULL) {
6397 if (debug)
6398 fprintf(outf, "NSFOD %s\n", path);
6399 return;
6400 }
6401 if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input))
6402 err(1, "%s: failed to read file", path);
6403 fclose(input);
6404
6405 fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf);
6406 }
6407
probe_intel_uncore_frequency_legacy(void)6408 static void probe_intel_uncore_frequency_legacy(void)
6409 {
6410 int i, j;
6411 char path[256];
6412
6413 for (i = 0; i < topo.num_packages; ++i) {
6414 for (j = 0; j <= topo.max_die_id; ++j) {
6415 int k, l;
6416 char path_base[128];
6417
6418 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i,
6419 j);
6420
6421 sprintf(path, "%s/current_freq_khz", path_base);
6422 if (access(path, R_OK))
6423 continue;
6424
6425 BIC_PRESENT(BIC_UNCORE_MHZ);
6426
6427 if (quiet)
6428 return;
6429
6430 sprintf(path, "%s/min_freq_khz", path_base);
6431 k = read_sysfs_int(path);
6432 sprintf(path, "%s/max_freq_khz", path_base);
6433 l = read_sysfs_int(path);
6434 fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000);
6435
6436 sprintf(path, "%s/initial_min_freq_khz", path_base);
6437 k = read_sysfs_int(path);
6438 sprintf(path, "%s/initial_max_freq_khz", path_base);
6439 l = read_sysfs_int(path);
6440 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
6441
6442 sprintf(path, "%s/current_freq_khz", path_base);
6443 k = read_sysfs_int(path);
6444 fprintf(outf, " %d MHz\n", k / 1000);
6445 }
6446 }
6447 }
6448
probe_intel_uncore_frequency_cluster(void)6449 static void probe_intel_uncore_frequency_cluster(void)
6450 {
6451 int i, uncore_max_id;
6452 char path[256];
6453 char path_base[128];
6454
6455 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK))
6456 return;
6457
6458 for (uncore_max_id = 0;; ++uncore_max_id) {
6459
6460 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id);
6461
6462 /* uncore## start at 00 and skips no numbers, so stop upon first missing */
6463 if (access(path_base, R_OK)) {
6464 uncore_max_id -= 1;
6465 break;
6466 }
6467 }
6468 for (i = uncore_max_id; i >= 0; --i) {
6469 int k, l;
6470 int package_id, domain_id, cluster_id;
6471 char name_buf[16];
6472
6473 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i);
6474
6475 if (access(path_base, R_OK))
6476 err(1, "%s: %s\n", __func__, path_base);
6477
6478 sprintf(path, "%s/package_id", path_base);
6479 package_id = read_sysfs_int(path);
6480
6481 sprintf(path, "%s/domain_id", path_base);
6482 domain_id = read_sysfs_int(path);
6483
6484 sprintf(path, "%s/fabric_cluster_id", path_base);
6485 cluster_id = read_sysfs_int(path);
6486
6487 sprintf(path, "%s/current_freq_khz", path_base);
6488 sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id);
6489
6490 /*
6491 * Once add_couter() is called, that counter is always read
6492 * and reported -- So it is effectively (enabled & present).
6493 * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)
6494 * is (enabled). Since we are in this routine, we
6495 * know we will not probe and set (present) the legacy counter.
6496 *
6497 * This allows "--show/--hide UncMHz" to be effective for
6498 * the clustered MHz counters, as a group.
6499 */
6500 if BIC_IS_ENABLED(BIC_UNCORE_MHZ)
6501 add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id);
6502
6503 if (quiet)
6504 continue;
6505
6506 sprintf(path, "%s/min_freq_khz", path_base);
6507 k = read_sysfs_int(path);
6508 sprintf(path, "%s/max_freq_khz", path_base);
6509 l = read_sysfs_int(path);
6510 fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id,
6511 cluster_id, k / 1000, l / 1000);
6512
6513 sprintf(path, "%s/initial_min_freq_khz", path_base);
6514 k = read_sysfs_int(path);
6515 sprintf(path, "%s/initial_max_freq_khz", path_base);
6516 l = read_sysfs_int(path);
6517 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
6518
6519 sprintf(path, "%s/current_freq_khz", path_base);
6520 k = read_sysfs_int(path);
6521 fprintf(outf, " %d MHz\n", k / 1000);
6522 }
6523 }
6524
probe_intel_uncore_frequency(void)6525 static void probe_intel_uncore_frequency(void)
6526 {
6527 if (!genuine_intel)
6528 return;
6529
6530 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0)
6531 probe_intel_uncore_frequency_cluster();
6532 else
6533 probe_intel_uncore_frequency_legacy();
6534 }
6535
probe_graphics(void)6536 static void probe_graphics(void)
6537 {
6538 /* Xe graphics sysfs knobs */
6539 if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) {
6540 FILE *fp;
6541 char buf[8];
6542 bool gt0_is_gt;
6543 int idx;
6544
6545 fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r");
6546 if (!fp)
6547 goto next;
6548
6549 if (!fread(buf, sizeof(char), 7, fp)) {
6550 fclose(fp);
6551 goto next;
6552 }
6553 fclose(fp);
6554
6555 if (!strncmp(buf, "gt0-rc", strlen("gt0-rc")))
6556 gt0_is_gt = true;
6557 else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc")))
6558 gt0_is_gt = false;
6559 else
6560 goto next;
6561
6562 idx = gt0_is_gt ? GFX_rc6 : SAM_mc6;
6563 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms";
6564
6565 idx = gt0_is_gt ? GFX_MHz : SAM_MHz;
6566 if (!access("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", R_OK))
6567 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq";
6568
6569 idx = gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz;
6570 if (!access("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", R_OK))
6571 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq";
6572
6573 idx = gt0_is_gt ? SAM_mc6 : GFX_rc6;
6574 if (!access("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", R_OK))
6575 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms";
6576
6577 idx = gt0_is_gt ? SAM_MHz : GFX_MHz;
6578 if (!access("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", R_OK))
6579 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq";
6580
6581 idx = gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz;
6582 if (!access("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", R_OK))
6583 gfx_info[idx].path = "/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq";
6584
6585 goto end;
6586 }
6587
6588 next:
6589 /* New i915 graphics sysfs knobs */
6590 if (!access("/sys/class/drm/card0/gt/gt0/rc6_residency_ms", R_OK)) {
6591 gfx_info[GFX_rc6].path = "/sys/class/drm/card0/gt/gt0/rc6_residency_ms";
6592
6593 if (!access("/sys/class/drm/card0/gt/gt0/rps_cur_freq_mhz", R_OK))
6594 gfx_info[GFX_MHz].path = "/sys/class/drm/card0/gt/gt0/rps_cur_freq_mhz";
6595
6596 if (!access("/sys/class/drm/card0/gt/gt0/rps_act_freq_mhz", R_OK))
6597 gfx_info[GFX_ACTMHz].path = "/sys/class/drm/card0/gt/gt0/rps_act_freq_mhz";
6598
6599 if (!access("/sys/class/drm/card0/gt/gt1/rc6_residency_ms", R_OK))
6600 gfx_info[SAM_mc6].path = "/sys/class/drm/card0/gt/gt1/rc6_residency_ms";
6601
6602 if (!access("/sys/class/drm/card0/gt/gt1/rps_cur_freq_mhz", R_OK))
6603 gfx_info[SAM_MHz].path = "/sys/class/drm/card0/gt/gt1/rps_cur_freq_mhz";
6604
6605 if (!access("/sys/class/drm/card0/gt/gt1/rps_act_freq_mhz", R_OK))
6606 gfx_info[SAM_ACTMHz].path = "/sys/class/drm/card0/gt/gt1/rps_act_freq_mhz";
6607
6608 goto end;
6609 }
6610
6611 /* Fall back to traditional i915 graphics sysfs knobs */
6612 if (!access("/sys/class/drm/card0/power/rc6_residency_ms", R_OK))
6613 gfx_info[GFX_rc6].path = "/sys/class/drm/card0/power/rc6_residency_ms";
6614
6615 if (!access("/sys/class/drm/card0/gt_cur_freq_mhz", R_OK))
6616 gfx_info[GFX_MHz].path = "/sys/class/drm/card0/gt_cur_freq_mhz";
6617 else if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", R_OK))
6618 gfx_info[GFX_MHz].path = "/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz";
6619
6620 if (!access("/sys/class/drm/card0/gt_act_freq_mhz", R_OK))
6621 gfx_info[GFX_ACTMHz].path = "/sys/class/drm/card0/gt_act_freq_mhz";
6622 else if (!access("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", R_OK))
6623 gfx_info[GFX_ACTMHz].path = "/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz";
6624
6625 end:
6626 if (gfx_info[GFX_rc6].path)
6627 BIC_PRESENT(BIC_GFX_rc6);
6628 if (gfx_info[GFX_MHz].path)
6629 BIC_PRESENT(BIC_GFXMHz);
6630 if (gfx_info[GFX_ACTMHz].path)
6631 BIC_PRESENT(BIC_GFXACTMHz);
6632 if (gfx_info[SAM_mc6].path)
6633 BIC_PRESENT(BIC_SAM_mc6);
6634 if (gfx_info[SAM_MHz].path)
6635 BIC_PRESENT(BIC_SAMMHz);
6636 if (gfx_info[SAM_ACTMHz].path)
6637 BIC_PRESENT(BIC_SAMACTMHz);
6638 }
6639
dump_sysfs_cstate_config(void)6640 static void dump_sysfs_cstate_config(void)
6641 {
6642 char path[64];
6643 char name_buf[16];
6644 char desc[64];
6645 FILE *input;
6646 int state;
6647 char *sp;
6648
6649 if (access("/sys/devices/system/cpu/cpuidle", R_OK)) {
6650 fprintf(outf, "cpuidle not loaded\n");
6651 return;
6652 }
6653
6654 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver");
6655 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor");
6656 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro");
6657
6658 for (state = 0; state < 10; ++state) {
6659
6660 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
6661 input = fopen(path, "r");
6662 if (input == NULL)
6663 continue;
6664 if (!fgets(name_buf, sizeof(name_buf), input))
6665 err(1, "%s: failed to read file", path);
6666
6667 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
6668 sp = strchr(name_buf, '-');
6669 if (!sp)
6670 sp = strchrnul(name_buf, '\n');
6671 *sp = '\0';
6672 fclose(input);
6673
6674 remove_underbar(name_buf);
6675
6676 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state);
6677 input = fopen(path, "r");
6678 if (input == NULL)
6679 continue;
6680 if (!fgets(desc, sizeof(desc), input))
6681 err(1, "%s: failed to read file", path);
6682
6683 fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc);
6684 fclose(input);
6685 }
6686 }
6687
dump_sysfs_pstate_config(void)6688 static void dump_sysfs_pstate_config(void)
6689 {
6690 char path[64];
6691 char driver_buf[64];
6692 char governor_buf[64];
6693 FILE *input;
6694 int turbo;
6695
6696 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu);
6697 input = fopen(path, "r");
6698 if (input == NULL) {
6699 fprintf(outf, "NSFOD %s\n", path);
6700 return;
6701 }
6702 if (!fgets(driver_buf, sizeof(driver_buf), input))
6703 err(1, "%s: failed to read file", path);
6704 fclose(input);
6705
6706 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu);
6707 input = fopen(path, "r");
6708 if (input == NULL) {
6709 fprintf(outf, "NSFOD %s\n", path);
6710 return;
6711 }
6712 if (!fgets(governor_buf, sizeof(governor_buf), input))
6713 err(1, "%s: failed to read file", path);
6714 fclose(input);
6715
6716 fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf);
6717 fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf);
6718
6719 sprintf(path, "/sys/devices/system/cpu/cpufreq/boost");
6720 input = fopen(path, "r");
6721 if (input != NULL) {
6722 if (fscanf(input, "%d", &turbo) != 1)
6723 err(1, "%s: failed to parse number from file", path);
6724 fprintf(outf, "cpufreq boost: %d\n", turbo);
6725 fclose(input);
6726 }
6727
6728 sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo");
6729 input = fopen(path, "r");
6730 if (input != NULL) {
6731 if (fscanf(input, "%d", &turbo) != 1)
6732 err(1, "%s: failed to parse number from file", path);
6733 fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
6734 fclose(input);
6735 }
6736 }
6737
6738 /*
6739 * print_epb()
6740 * Decode the ENERGY_PERF_BIAS MSR
6741 */
print_epb(struct thread_data * t,struct core_data * c,struct pkg_data * p)6742 int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p)
6743 {
6744 char *epb_string;
6745 int cpu, epb;
6746
6747 UNUSED(c);
6748 UNUSED(p);
6749
6750 if (!has_epb)
6751 return 0;
6752
6753 cpu = t->cpu_id;
6754
6755 /* EPB is per-package */
6756 if (!is_cpu_first_thread_in_package(t, c, p))
6757 return 0;
6758
6759 if (cpu_migrate(cpu)) {
6760 fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu);
6761 return -1;
6762 }
6763
6764 epb = get_epb(cpu);
6765 if (epb < 0)
6766 return 0;
6767
6768 switch (epb) {
6769 case ENERGY_PERF_BIAS_PERFORMANCE:
6770 epb_string = "performance";
6771 break;
6772 case ENERGY_PERF_BIAS_NORMAL:
6773 epb_string = "balanced";
6774 break;
6775 case ENERGY_PERF_BIAS_POWERSAVE:
6776 epb_string = "powersave";
6777 break;
6778 default:
6779 epb_string = "custom";
6780 break;
6781 }
6782 fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string);
6783
6784 return 0;
6785 }
6786
6787 /*
6788 * print_hwp()
6789 * Decode the MSR_HWP_CAPABILITIES
6790 */
print_hwp(struct thread_data * t,struct core_data * c,struct pkg_data * p)6791 int print_hwp(struct thread_data *t, struct core_data *c, struct pkg_data *p)
6792 {
6793 unsigned long long msr;
6794 int cpu;
6795
6796 UNUSED(c);
6797 UNUSED(p);
6798
6799 if (no_msr)
6800 return 0;
6801
6802 if (!has_hwp)
6803 return 0;
6804
6805 cpu = t->cpu_id;
6806
6807 /* MSR_HWP_CAPABILITIES is per-package */
6808 if (!is_cpu_first_thread_in_package(t, c, p))
6809 return 0;
6810
6811 if (cpu_migrate(cpu)) {
6812 fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu);
6813 return -1;
6814 }
6815
6816 if (get_msr(cpu, MSR_PM_ENABLE, &msr))
6817 return 0;
6818
6819 fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
6820
6821 /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
6822 if ((msr & (1 << 0)) == 0)
6823 return 0;
6824
6825 if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr))
6826 return 0;
6827
6828 fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
6829 "(high %d guar %d eff %d low %d)\n",
6830 cpu, msr,
6831 (unsigned int)HWP_HIGHEST_PERF(msr),
6832 (unsigned int)HWP_GUARANTEED_PERF(msr),
6833 (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
6834
6835 if (get_msr(cpu, MSR_HWP_REQUEST, &msr))
6836 return 0;
6837
6838 fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
6839 "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
6840 cpu, msr,
6841 (unsigned int)(((msr) >> 0) & 0xff),
6842 (unsigned int)(((msr) >> 8) & 0xff),
6843 (unsigned int)(((msr) >> 16) & 0xff),
6844 (unsigned int)(((msr) >> 24) & 0xff),
6845 (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
6846
6847 if (has_hwp_pkg) {
6848 if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr))
6849 return 0;
6850
6851 fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
6852 "(min %d max %d des %d epp 0x%x window 0x%x)\n",
6853 cpu, msr,
6854 (unsigned int)(((msr) >> 0) & 0xff),
6855 (unsigned int)(((msr) >> 8) & 0xff),
6856 (unsigned int)(((msr) >> 16) & 0xff),
6857 (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
6858 }
6859 if (has_hwp_notify) {
6860 if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr))
6861 return 0;
6862
6863 fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
6864 "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n",
6865 cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
6866 }
6867 if (get_msr(cpu, MSR_HWP_STATUS, &msr))
6868 return 0;
6869
6870 fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
6871 "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n",
6872 cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
6873
6874 return 0;
6875 }
6876
6877 /*
6878 * print_perf_limit()
6879 */
print_perf_limit(struct thread_data * t,struct core_data * c,struct pkg_data * p)6880 int print_perf_limit(struct thread_data *t, struct core_data *c, struct pkg_data *p)
6881 {
6882 unsigned long long msr;
6883 int cpu;
6884
6885 UNUSED(c);
6886 UNUSED(p);
6887
6888 if (no_msr)
6889 return 0;
6890
6891 cpu = t->cpu_id;
6892
6893 /* per-package */
6894 if (!is_cpu_first_thread_in_package(t, c, p))
6895 return 0;
6896
6897 if (cpu_migrate(cpu)) {
6898 fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu);
6899 return -1;
6900 }
6901
6902 if (platform->plr_msrs & PLR_CORE) {
6903 get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr);
6904 fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
6905 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)",
6906 (msr & 1 << 15) ? "bit15, " : "",
6907 (msr & 1 << 14) ? "bit14, " : "",
6908 (msr & 1 << 13) ? "Transitions, " : "",
6909 (msr & 1 << 12) ? "MultiCoreTurbo, " : "",
6910 (msr & 1 << 11) ? "PkgPwrL2, " : "",
6911 (msr & 1 << 10) ? "PkgPwrL1, " : "",
6912 (msr & 1 << 9) ? "CorePwr, " : "",
6913 (msr & 1 << 8) ? "Amps, " : "",
6914 (msr & 1 << 6) ? "VR-Therm, " : "",
6915 (msr & 1 << 5) ? "Auto-HWP, " : "",
6916 (msr & 1 << 4) ? "Graphics, " : "",
6917 (msr & 1 << 2) ? "bit2, " : "",
6918 (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : "");
6919 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
6920 (msr & 1 << 31) ? "bit31, " : "",
6921 (msr & 1 << 30) ? "bit30, " : "",
6922 (msr & 1 << 29) ? "Transitions, " : "",
6923 (msr & 1 << 28) ? "MultiCoreTurbo, " : "",
6924 (msr & 1 << 27) ? "PkgPwrL2, " : "",
6925 (msr & 1 << 26) ? "PkgPwrL1, " : "",
6926 (msr & 1 << 25) ? "CorePwr, " : "",
6927 (msr & 1 << 24) ? "Amps, " : "",
6928 (msr & 1 << 22) ? "VR-Therm, " : "",
6929 (msr & 1 << 21) ? "Auto-HWP, " : "",
6930 (msr & 1 << 20) ? "Graphics, " : "",
6931 (msr & 1 << 18) ? "bit18, " : "",
6932 (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : "");
6933
6934 }
6935 if (platform->plr_msrs & PLR_GFX) {
6936 get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr);
6937 fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
6938 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)",
6939 (msr & 1 << 0) ? "PROCHOT, " : "",
6940 (msr & 1 << 1) ? "ThermStatus, " : "",
6941 (msr & 1 << 4) ? "Graphics, " : "",
6942 (msr & 1 << 6) ? "VR-Therm, " : "",
6943 (msr & 1 << 8) ? "Amps, " : "",
6944 (msr & 1 << 9) ? "GFXPwr, " : "",
6945 (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
6946 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n",
6947 (msr & 1 << 16) ? "PROCHOT, " : "",
6948 (msr & 1 << 17) ? "ThermStatus, " : "",
6949 (msr & 1 << 20) ? "Graphics, " : "",
6950 (msr & 1 << 22) ? "VR-Therm, " : "",
6951 (msr & 1 << 24) ? "Amps, " : "",
6952 (msr & 1 << 25) ? "GFXPwr, " : "",
6953 (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
6954 }
6955 if (platform->plr_msrs & PLR_RING) {
6956 get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr);
6957 fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
6958 fprintf(outf, " (Active: %s%s%s%s%s%s)",
6959 (msr & 1 << 0) ? "PROCHOT, " : "",
6960 (msr & 1 << 1) ? "ThermStatus, " : "",
6961 (msr & 1 << 6) ? "VR-Therm, " : "",
6962 (msr & 1 << 8) ? "Amps, " : "",
6963 (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
6964 fprintf(outf, " (Logged: %s%s%s%s%s%s)\n",
6965 (msr & 1 << 16) ? "PROCHOT, " : "",
6966 (msr & 1 << 17) ? "ThermStatus, " : "",
6967 (msr & 1 << 22) ? "VR-Therm, " : "",
6968 (msr & 1 << 24) ? "Amps, " : "",
6969 (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
6970 }
6971 return 0;
6972 }
6973
6974 #define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */
6975 #define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */
6976
get_quirk_tdp(void)6977 double get_quirk_tdp(void)
6978 {
6979 if (platform->rapl_quirk_tdp)
6980 return platform->rapl_quirk_tdp;
6981
6982 return 135.0;
6983 }
6984
get_tdp_intel(void)6985 double get_tdp_intel(void)
6986 {
6987 unsigned long long msr;
6988
6989 if (platform->rapl_msrs & RAPL_PKG_POWER_INFO)
6990 if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr))
6991 return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
6992 return get_quirk_tdp();
6993 }
6994
get_tdp_amd(void)6995 double get_tdp_amd(void)
6996 {
6997 return get_quirk_tdp();
6998 }
6999
rapl_probe_intel(void)7000 void rapl_probe_intel(void)
7001 {
7002 unsigned long long msr;
7003 unsigned int time_unit;
7004 double tdp;
7005 const unsigned long long bic_watt_bits = BIC_PkgWatt | BIC_CorWatt | BIC_RAMWatt | BIC_GFXWatt;
7006 const unsigned long long bic_joules_bits = BIC_Pkg_J | BIC_Cor_J | BIC_RAM_J | BIC_GFX_J;
7007
7008 if (rapl_joules)
7009 bic_enabled &= ~bic_watt_bits;
7010 else
7011 bic_enabled &= ~bic_joules_bits;
7012
7013 if (!(platform->rapl_msrs & RAPL_PKG_PERF_STATUS))
7014 bic_enabled &= ~BIC_PKG__;
7015 if (!(platform->rapl_msrs & RAPL_DRAM_PERF_STATUS))
7016 bic_enabled &= ~BIC_RAM__;
7017
7018 /* units on package 0, verify later other packages match */
7019 if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr))
7020 return;
7021
7022 rapl_power_units = 1.0 / (1 << (msr & 0xF));
7023 if (platform->has_rapl_divisor)
7024 rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000;
7025 else
7026 rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F));
7027
7028 if (platform->has_fixed_rapl_unit)
7029 rapl_dram_energy_units = (15.3 / 1000000);
7030 else
7031 rapl_dram_energy_units = rapl_energy_units;
7032
7033 time_unit = msr >> 16 & 0xF;
7034 if (time_unit == 0)
7035 time_unit = 0xA;
7036
7037 rapl_time_units = 1.0 / (1 << (time_unit));
7038
7039 tdp = get_tdp_intel();
7040
7041 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7042 if (!quiet)
7043 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7044 }
7045
rapl_probe_amd(void)7046 void rapl_probe_amd(void)
7047 {
7048 unsigned long long msr;
7049 double tdp;
7050 const unsigned long long bic_watt_bits = BIC_PkgWatt | BIC_CorWatt;
7051 const unsigned long long bic_joules_bits = BIC_Pkg_J | BIC_Cor_J;
7052
7053 if (rapl_joules)
7054 bic_enabled &= ~bic_watt_bits;
7055 else
7056 bic_enabled &= ~bic_joules_bits;
7057
7058 if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr))
7059 return;
7060
7061 rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
7062 rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
7063 rapl_power_units = ldexp(1.0, -(msr & 0xf));
7064
7065 tdp = get_tdp_amd();
7066
7067 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7068 if (!quiet)
7069 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7070 }
7071
print_power_limit_msr(int cpu,unsigned long long msr,char * label)7072 void print_power_limit_msr(int cpu, unsigned long long msr, char *label)
7073 {
7074 fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n",
7075 cpu, label,
7076 ((msr >> 15) & 1) ? "EN" : "DIS",
7077 ((msr >> 0) & 0x7FFF) * rapl_power_units,
7078 (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units,
7079 (((msr >> 16) & 1) ? "EN" : "DIS"));
7080
7081 return;
7082 }
7083
print_rapl(struct thread_data * t,struct core_data * c,struct pkg_data * p)7084 int print_rapl(struct thread_data *t, struct core_data *c, struct pkg_data *p)
7085 {
7086 unsigned long long msr;
7087 const char *msr_name;
7088 int cpu;
7089
7090 UNUSED(c);
7091 UNUSED(p);
7092
7093 if (!platform->rapl_msrs)
7094 return 0;
7095
7096 /* RAPL counters are per package, so print only for 1st thread/package */
7097 if (!is_cpu_first_thread_in_package(t, c, p))
7098 return 0;
7099
7100 cpu = t->cpu_id;
7101 if (cpu_migrate(cpu)) {
7102 fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu);
7103 return -1;
7104 }
7105
7106 if (platform->rapl_msrs & RAPL_AMD_F17H) {
7107 msr_name = "MSR_RAPL_PWR_UNIT";
7108 if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr))
7109 return -1;
7110 } else {
7111 msr_name = "MSR_RAPL_POWER_UNIT";
7112 if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
7113 return -1;
7114 }
7115
7116 fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr,
7117 rapl_power_units, rapl_energy_units, rapl_time_units);
7118
7119 if (platform->rapl_msrs & RAPL_PKG_POWER_INFO) {
7120
7121 if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
7122 return -5;
7123
7124 fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7125 cpu, msr,
7126 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7127 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7128 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7129 ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7130
7131 }
7132 if (platform->rapl_msrs & RAPL_PKG) {
7133
7134 if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr))
7135 return -9;
7136
7137 fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n",
7138 cpu, msr, (msr >> 63) & 1 ? "" : "UN");
7139
7140 print_power_limit_msr(cpu, msr, "PKG Limit #1");
7141 fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n",
7142 cpu,
7143 ((msr >> 47) & 1) ? "EN" : "DIS",
7144 ((msr >> 32) & 0x7FFF) * rapl_power_units,
7145 (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units,
7146 ((msr >> 48) & 1) ? "EN" : "DIS");
7147
7148 if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr))
7149 return -9;
7150
7151 fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
7152 fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n",
7153 cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
7154 }
7155
7156 if (platform->rapl_msrs & RAPL_DRAM_POWER_INFO) {
7157 if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr))
7158 return -6;
7159
7160 fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7161 cpu, msr,
7162 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7163 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7164 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7165 ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7166 }
7167 if (platform->rapl_msrs & RAPL_DRAM) {
7168 if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
7169 return -9;
7170 fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n",
7171 cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7172
7173 print_power_limit_msr(cpu, msr, "DRAM Limit");
7174 }
7175 if (platform->rapl_msrs & RAPL_CORE_POLICY) {
7176 if (get_msr(cpu, MSR_PP0_POLICY, &msr))
7177 return -7;
7178
7179 fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
7180 }
7181 if (platform->rapl_msrs & RAPL_CORE_POWER_LIMIT) {
7182 if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
7183 return -9;
7184 fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n",
7185 cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7186 print_power_limit_msr(cpu, msr, "Cores Limit");
7187 }
7188 if (platform->rapl_msrs & RAPL_GFX) {
7189 if (get_msr(cpu, MSR_PP1_POLICY, &msr))
7190 return -8;
7191
7192 fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
7193
7194 if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
7195 return -9;
7196 fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n",
7197 cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7198 print_power_limit_msr(cpu, msr, "GFX Limit");
7199 }
7200 return 0;
7201 }
7202
7203 /*
7204 * probe_rapl()
7205 *
7206 * sets rapl_power_units, rapl_energy_units, rapl_time_units
7207 */
probe_rapl(void)7208 void probe_rapl(void)
7209 {
7210 if (!platform->rapl_msrs || no_msr)
7211 return;
7212
7213 if (genuine_intel)
7214 rapl_probe_intel();
7215 if (authentic_amd || hygon_genuine)
7216 rapl_probe_amd();
7217
7218 if (quiet)
7219 return;
7220
7221 for_all_cpus(print_rapl, ODD_COUNTERS);
7222 }
7223
7224 /*
7225 * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where
7226 * the Thermal Control Circuit (TCC) activates.
7227 * This is usually equal to tjMax.
7228 *
7229 * Older processors do not have this MSR, so there we guess,
7230 * but also allow cmdline over-ride with -T.
7231 *
7232 * Several MSR temperature values are in units of degrees-C
7233 * below this value, including the Digital Thermal Sensor (DTS),
7234 * Package Thermal Management Sensor (PTM), and thermal event thresholds.
7235 */
set_temperature_target(struct thread_data * t,struct core_data * c,struct pkg_data * p)7236 int set_temperature_target(struct thread_data *t, struct core_data *c, struct pkg_data *p)
7237 {
7238 unsigned long long msr;
7239 unsigned int tcc_default, tcc_offset;
7240 int cpu;
7241
7242 UNUSED(c);
7243 UNUSED(p);
7244
7245 /* tj_max is used only for dts or ptm */
7246 if (!(do_dts || do_ptm))
7247 return 0;
7248
7249 /* this is a per-package concept */
7250 if (!is_cpu_first_thread_in_package(t, c, p))
7251 return 0;
7252
7253 cpu = t->cpu_id;
7254 if (cpu_migrate(cpu)) {
7255 fprintf(outf, "Could not migrate to CPU %d\n", cpu);
7256 return -1;
7257 }
7258
7259 if (tj_max_override != 0) {
7260 tj_max = tj_max_override;
7261 fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max);
7262 return 0;
7263 }
7264
7265 /* Temperature Target MSR is Nehalem and newer only */
7266 if (!platform->has_nhm_msrs || no_msr)
7267 goto guess;
7268
7269 if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr))
7270 goto guess;
7271
7272 tcc_default = (msr >> 16) & 0xFF;
7273
7274 if (!quiet) {
7275 int bits = platform->tcc_offset_bits;
7276 unsigned long long enabled = 0;
7277
7278 if (bits && !get_msr(base_cpu, MSR_PLATFORM_INFO, &enabled))
7279 enabled = (enabled >> 30) & 1;
7280
7281 if (bits && enabled) {
7282 tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
7283 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n",
7284 cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset);
7285 } else {
7286 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default);
7287 }
7288 }
7289
7290 if (!tcc_default)
7291 goto guess;
7292
7293 tj_max = tcc_default;
7294
7295 return 0;
7296
7297 guess:
7298 tj_max = TJMAX_DEFAULT;
7299 fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max);
7300
7301 return 0;
7302 }
7303
print_thermal(struct thread_data * t,struct core_data * c,struct pkg_data * p)7304 int print_thermal(struct thread_data *t, struct core_data *c, struct pkg_data *p)
7305 {
7306 unsigned long long msr;
7307 unsigned int dts, dts2;
7308 int cpu;
7309
7310 UNUSED(c);
7311 UNUSED(p);
7312
7313 if (no_msr)
7314 return 0;
7315
7316 if (!(do_dts || do_ptm))
7317 return 0;
7318
7319 cpu = t->cpu_id;
7320
7321 /* DTS is per-core, no need to print for each thread */
7322 if (!is_cpu_first_thread_in_core(t, c, p))
7323 return 0;
7324
7325 if (cpu_migrate(cpu)) {
7326 fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu);
7327 return -1;
7328 }
7329
7330 if (do_ptm && is_cpu_first_core_in_package(t, c, p)) {
7331 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
7332 return 0;
7333
7334 dts = (msr >> 16) & 0x7F;
7335 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
7336
7337 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr))
7338 return 0;
7339
7340 dts = (msr >> 16) & 0x7F;
7341 dts2 = (msr >> 8) & 0x7F;
7342 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n",
7343 cpu, msr, tj_max - dts, tj_max - dts2);
7344 }
7345
7346 if (do_dts && debug) {
7347 unsigned int resolution;
7348
7349 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
7350 return 0;
7351
7352 dts = (msr >> 16) & 0x7F;
7353 resolution = (msr >> 27) & 0xF;
7354 fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n",
7355 cpu, msr, tj_max - dts, resolution);
7356
7357 if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr))
7358 return 0;
7359
7360 dts = (msr >> 16) & 0x7F;
7361 dts2 = (msr >> 8) & 0x7F;
7362 fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n",
7363 cpu, msr, tj_max - dts, tj_max - dts2);
7364 }
7365
7366 return 0;
7367 }
7368
probe_thermal(void)7369 void probe_thermal(void)
7370 {
7371 if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK))
7372 BIC_PRESENT(BIC_CORE_THROT_CNT);
7373 else
7374 BIC_NOT_PRESENT(BIC_CORE_THROT_CNT);
7375
7376 for_all_cpus(set_temperature_target, ODD_COUNTERS);
7377
7378 if (quiet)
7379 return;
7380
7381 for_all_cpus(print_thermal, ODD_COUNTERS);
7382 }
7383
get_cpu_type(struct thread_data * t,struct core_data * c,struct pkg_data * p)7384 int get_cpu_type(struct thread_data *t, struct core_data *c, struct pkg_data *p)
7385 {
7386 unsigned int eax, ebx, ecx, edx;
7387
7388 UNUSED(c);
7389 UNUSED(p);
7390
7391 if (!genuine_intel)
7392 return 0;
7393
7394 if (cpu_migrate(t->cpu_id)) {
7395 fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id);
7396 return -1;
7397 }
7398
7399 if (max_level < 0x1a)
7400 return 0;
7401
7402 __cpuid(0x1a, eax, ebx, ecx, edx);
7403 eax = (eax >> 24) & 0xFF;
7404 if (eax == 0x20)
7405 t->is_atom = true;
7406 return 0;
7407 }
7408
decode_feature_control_msr(void)7409 void decode_feature_control_msr(void)
7410 {
7411 unsigned long long msr;
7412
7413 if (no_msr)
7414 return;
7415
7416 if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr))
7417 fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
7418 base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
7419 }
7420
decode_misc_enable_msr(void)7421 void decode_misc_enable_msr(void)
7422 {
7423 unsigned long long msr;
7424
7425 if (no_msr)
7426 return;
7427
7428 if (!genuine_intel)
7429 return;
7430
7431 if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr))
7432 fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
7433 base_cpu, msr,
7434 msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
7435 msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
7436 msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
7437 msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "",
7438 msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
7439 }
7440
decode_misc_feature_control(void)7441 void decode_misc_feature_control(void)
7442 {
7443 unsigned long long msr;
7444
7445 if (no_msr)
7446 return;
7447
7448 if (!platform->has_msr_misc_feature_control)
7449 return;
7450
7451 if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
7452 fprintf(outf,
7453 "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
7454 base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "",
7455 msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
7456 }
7457
7458 /*
7459 * Decode MSR_MISC_PWR_MGMT
7460 *
7461 * Decode the bits according to the Nehalem documentation
7462 * bit[0] seems to continue to have same meaning going forward
7463 * bit[1] less so...
7464 */
decode_misc_pwr_mgmt_msr(void)7465 void decode_misc_pwr_mgmt_msr(void)
7466 {
7467 unsigned long long msr;
7468
7469 if (no_msr)
7470 return;
7471
7472 if (!platform->has_msr_misc_pwr_mgmt)
7473 return;
7474
7475 if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr))
7476 fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
7477 base_cpu, msr,
7478 msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
7479 }
7480
7481 /*
7482 * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
7483 *
7484 * This MSRs are present on Silvermont processors,
7485 * Intel Atom processor E3000 series (Baytrail), and friends.
7486 */
decode_c6_demotion_policy_msr(void)7487 void decode_c6_demotion_policy_msr(void)
7488 {
7489 unsigned long long msr;
7490
7491 if (no_msr)
7492 return;
7493
7494 if (!platform->has_msr_c6_demotion_policy_config)
7495 return;
7496
7497 if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
7498 fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n",
7499 base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
7500
7501 if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
7502 fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n",
7503 base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
7504 }
7505
print_dev_latency(void)7506 void print_dev_latency(void)
7507 {
7508 char *path = "/dev/cpu_dma_latency";
7509 int fd;
7510 int value;
7511 int retval;
7512
7513 fd = open(path, O_RDONLY);
7514 if (fd < 0) {
7515 if (debug)
7516 warnx("Read %s failed", path);
7517 return;
7518 }
7519
7520 retval = read(fd, (void *)&value, sizeof(int));
7521 if (retval != sizeof(int)) {
7522 warn("read failed %s", path);
7523 close(fd);
7524 return;
7525 }
7526 fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained");
7527
7528 close(fd);
7529 }
7530
has_instr_count_access(void)7531 static int has_instr_count_access(void)
7532 {
7533 int fd;
7534 int has_access;
7535
7536 if (no_perf)
7537 return 0;
7538
7539 fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
7540 has_access = fd != -1;
7541
7542 if (fd != -1)
7543 close(fd);
7544
7545 if (!has_access)
7546 warnx("Failed to access %s. Some of the counters may not be available\n"
7547 "\tRun as root to enable them or use %s to disable the access explicitly",
7548 "instructions retired perf counter", "--no-perf");
7549
7550 return has_access;
7551 }
7552
add_rapl_perf_counter_(int cpu,struct rapl_counter_info_t * rci,const struct rapl_counter_arch_info * cai,double * scale_,enum rapl_unit * unit_)7553 int add_rapl_perf_counter_(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai,
7554 double *scale_, enum rapl_unit *unit_)
7555 {
7556 if (no_perf)
7557 return -1;
7558
7559 const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name);
7560
7561 if (scale == 0.0)
7562 return -1;
7563
7564 const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name);
7565
7566 if (unit == RAPL_UNIT_INVALID)
7567 return -1;
7568
7569 const unsigned int rapl_type = read_perf_type(cai->perf_subsys);
7570 const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name);
7571
7572 const int fd_counter =
7573 open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP);
7574 if (fd_counter == -1)
7575 return -1;
7576
7577 /* If it's the first counter opened, make it a group descriptor */
7578 if (rci->fd_perf == -1)
7579 rci->fd_perf = fd_counter;
7580
7581 *scale_ = scale;
7582 *unit_ = unit;
7583 return fd_counter;
7584 }
7585
add_rapl_perf_counter(int cpu,struct rapl_counter_info_t * rci,const struct rapl_counter_arch_info * cai,double * scale,enum rapl_unit * unit)7586 int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai,
7587 double *scale, enum rapl_unit *unit)
7588 {
7589 int ret = add_rapl_perf_counter_(cpu, rci, cai, scale, unit);
7590
7591 if (debug >= 2)
7592 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
7593
7594 return ret;
7595 }
7596
7597 /*
7598 * Linux-perf manages the HW instructions-retired counter
7599 * by enabling when requested, and hiding rollover
7600 */
linux_perf_init(void)7601 void linux_perf_init(void)
7602 {
7603 if (access("/proc/sys/kernel/perf_event_paranoid", F_OK))
7604 return;
7605
7606 if (BIC_IS_ENABLED(BIC_IPC) && has_aperf) {
7607 fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
7608 if (fd_instr_count_percpu == NULL)
7609 err(-1, "calloc fd_instr_count_percpu");
7610 }
7611 }
7612
rapl_perf_init(void)7613 void rapl_perf_init(void)
7614 {
7615 const unsigned int num_domains = get_rapl_num_domains();
7616 bool *domain_visited = calloc(num_domains, sizeof(bool));
7617
7618 rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain));
7619 if (rapl_counter_info_perdomain == NULL)
7620 err(-1, "calloc rapl_counter_info_percpu");
7621 rapl_counter_info_perdomain_size = num_domains;
7622
7623 /*
7624 * Initialize rapl_counter_info_percpu
7625 */
7626 for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) {
7627 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id];
7628
7629 rci->fd_perf = -1;
7630 for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) {
7631 rci->data[i] = 0;
7632 rci->source[i] = COUNTER_SOURCE_NONE;
7633 }
7634 }
7635
7636 /*
7637 * Open/probe the counters
7638 * If can't get it via perf, fallback to MSR
7639 */
7640 for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) {
7641
7642 const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i];
7643 bool has_counter = 0;
7644 double scale;
7645 enum rapl_unit unit;
7646 unsigned int next_domain;
7647
7648 memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
7649
7650 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
7651
7652 if (cpu_is_not_allowed(cpu))
7653 continue;
7654
7655 /* Skip already seen and handled RAPL domains */
7656 next_domain = get_rapl_domain_id(cpu);
7657
7658 assert(next_domain < num_domains);
7659
7660 if (domain_visited[next_domain])
7661 continue;
7662
7663 domain_visited[next_domain] = 1;
7664
7665 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain];
7666
7667 /* Check if the counter is enabled and accessible */
7668 if (BIC_IS_ENABLED(cai->bic) && (platform->rapl_msrs & cai->feature_mask)) {
7669
7670 /* Use perf API for this counter */
7671 if (!no_perf && cai->perf_name
7672 && add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) {
7673 rci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
7674 rci->scale[cai->rci_index] = scale * cai->compat_scale;
7675 rci->unit[cai->rci_index] = unit;
7676 rci->flags[cai->rci_index] = cai->flags;
7677
7678 /* Use MSR for this counter */
7679 } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) {
7680 rci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
7681 rci->msr[cai->rci_index] = cai->msr;
7682 rci->msr_mask[cai->rci_index] = cai->msr_mask;
7683 rci->msr_shift[cai->rci_index] = cai->msr_shift;
7684 rci->unit[cai->rci_index] = RAPL_UNIT_JOULES;
7685 rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale;
7686 rci->flags[cai->rci_index] = cai->flags;
7687 }
7688 }
7689
7690 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
7691 has_counter = 1;
7692 }
7693
7694 /* If any CPU has access to the counter, make it present */
7695 if (has_counter)
7696 BIC_PRESENT(cai->bic);
7697 }
7698
7699 free(domain_visited);
7700 }
7701
7702 /* Assumes msr_counter_info is populated */
has_amperf_access(void)7703 static int has_amperf_access(void)
7704 {
7705 return msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present &&
7706 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present;
7707 }
7708
get_cstate_perf_group_fd(struct cstate_counter_info_t * cci,const char * group_name)7709 int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name)
7710 {
7711 if (strcmp(group_name, "cstate_core") == 0)
7712 return &cci->fd_perf_core;
7713
7714 if (strcmp(group_name, "cstate_pkg") == 0)
7715 return &cci->fd_perf_pkg;
7716
7717 return NULL;
7718 }
7719
add_cstate_perf_counter_(int cpu,struct cstate_counter_info_t * cci,const struct cstate_counter_arch_info * cai)7720 int add_cstate_perf_counter_(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
7721 {
7722 if (no_perf)
7723 return -1;
7724
7725 int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys);
7726
7727 if (pfd_group == NULL)
7728 return -1;
7729
7730 const unsigned int type = read_perf_type(cai->perf_subsys);
7731 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
7732
7733 const int fd_counter = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP);
7734
7735 if (fd_counter == -1)
7736 return -1;
7737
7738 /* If it's the first counter opened, make it a group descriptor */
7739 if (*pfd_group == -1)
7740 *pfd_group = fd_counter;
7741
7742 return fd_counter;
7743 }
7744
add_cstate_perf_counter(int cpu,struct cstate_counter_info_t * cci,const struct cstate_counter_arch_info * cai)7745 int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
7746 {
7747 int ret = add_cstate_perf_counter_(cpu, cci, cai);
7748
7749 if (debug >= 2)
7750 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
7751
7752 return ret;
7753 }
7754
add_msr_perf_counter_(int cpu,struct msr_counter_info_t * cci,const struct msr_counter_arch_info * cai)7755 int add_msr_perf_counter_(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
7756 {
7757 if (no_perf)
7758 return -1;
7759
7760 const unsigned int type = read_perf_type(cai->perf_subsys);
7761 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
7762
7763 const int fd_counter = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP);
7764
7765 if (fd_counter == -1)
7766 return -1;
7767
7768 /* If it's the first counter opened, make it a group descriptor */
7769 if (cci->fd_perf == -1)
7770 cci->fd_perf = fd_counter;
7771
7772 return fd_counter;
7773 }
7774
add_msr_perf_counter(int cpu,struct msr_counter_info_t * cci,const struct msr_counter_arch_info * cai)7775 int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
7776 {
7777 int ret = add_msr_perf_counter_(cpu, cci, cai);
7778
7779 if (debug)
7780 fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu);
7781
7782 return ret;
7783 }
7784
msr_perf_init_(void)7785 void msr_perf_init_(void)
7786 {
7787 const int mci_num = topo.max_cpu_num + 1;
7788
7789 msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info));
7790 if (!msr_counter_info)
7791 err(1, "calloc msr_counter_info");
7792 msr_counter_info_size = mci_num;
7793
7794 for (int cpu = 0; cpu < mci_num; ++cpu)
7795 msr_counter_info[cpu].fd_perf = -1;
7796
7797 for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) {
7798
7799 struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx];
7800
7801 cai->present = false;
7802
7803 for (int cpu = 0; cpu < mci_num; ++cpu) {
7804
7805 struct msr_counter_info_t *const cci = &msr_counter_info[cpu];
7806
7807 if (cpu_is_not_allowed(cpu))
7808 continue;
7809
7810 if (cai->needed) {
7811 /* Use perf API for this counter */
7812 if (!no_perf && cai->perf_name && add_msr_perf_counter(cpu, cci, cai) != -1) {
7813 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
7814 cai->present = true;
7815
7816 /* User MSR for this counter */
7817 } else if (!no_msr && cai->msr && probe_msr(cpu, cai->msr) == 0) {
7818 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
7819 cci->msr[cai->rci_index] = cai->msr;
7820 cci->msr_mask[cai->rci_index] = cai->msr_mask;
7821 cai->present = true;
7822 }
7823 }
7824 }
7825 }
7826 }
7827
7828 /* Initialize data for reading perf counters from the MSR group. */
msr_perf_init(void)7829 void msr_perf_init(void)
7830 {
7831 bool need_amperf = false, need_smi = false;
7832 const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1);
7833
7834 need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz)
7835 || BIC_IS_ENABLED(BIC_IPC) || need_soft_c1;
7836
7837 if (BIC_IS_ENABLED(BIC_SMI))
7838 need_smi = true;
7839
7840 /* Enable needed counters */
7841 msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf;
7842 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf;
7843 msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi;
7844
7845 msr_perf_init_();
7846
7847 const bool has_amperf = has_amperf_access();
7848 const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present;
7849
7850 has_aperf_access = has_amperf;
7851
7852 if (has_amperf) {
7853 BIC_PRESENT(BIC_Avg_MHz);
7854 BIC_PRESENT(BIC_Busy);
7855 BIC_PRESENT(BIC_Bzy_MHz);
7856 BIC_PRESENT(BIC_SMI);
7857 }
7858
7859 if (has_smi)
7860 BIC_PRESENT(BIC_SMI);
7861 }
7862
cstate_perf_init_(bool soft_c1)7863 void cstate_perf_init_(bool soft_c1)
7864 {
7865 bool has_counter;
7866 bool *cores_visited = NULL, *pkg_visited = NULL;
7867 const int cores_visited_elems = topo.max_core_id + 1;
7868 const int pkg_visited_elems = topo.max_package_id + 1;
7869 const int cci_num = topo.max_cpu_num + 1;
7870
7871 ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info));
7872 if (!ccstate_counter_info)
7873 err(1, "calloc ccstate_counter_arch_info");
7874 ccstate_counter_info_size = cci_num;
7875
7876 cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited));
7877 if (!cores_visited)
7878 err(1, "calloc cores_visited");
7879
7880 pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited));
7881 if (!pkg_visited)
7882 err(1, "calloc pkg_visited");
7883
7884 /* Initialize cstate_counter_info_percpu */
7885 for (int cpu = 0; cpu < cci_num; ++cpu) {
7886 ccstate_counter_info[cpu].fd_perf_core = -1;
7887 ccstate_counter_info[cpu].fd_perf_pkg = -1;
7888 }
7889
7890 for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) {
7891 has_counter = false;
7892 memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited));
7893 memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited));
7894
7895 const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx];
7896
7897 for (int cpu = 0; cpu < cci_num; ++cpu) {
7898
7899 struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu];
7900
7901 if (cpu_is_not_allowed(cpu))
7902 continue;
7903
7904 const int core_id = cpus[cpu].physical_core_id;
7905 const int pkg_id = cpus[cpu].physical_package_id;
7906
7907 assert(core_id < cores_visited_elems);
7908 assert(pkg_id < pkg_visited_elems);
7909
7910 const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD;
7911 const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE;
7912
7913 if (!per_thread && cores_visited[core_id])
7914 continue;
7915
7916 if (!per_core && pkg_visited[pkg_id])
7917 continue;
7918
7919 const bool counter_needed = BIC_IS_ENABLED(cai->bic) ||
7920 (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY));
7921 const bool counter_supported = (platform->supported_cstates & cai->feature_mask);
7922
7923 if (counter_needed && counter_supported) {
7924 /* Use perf API for this counter */
7925 if (!no_perf && cai->perf_name && add_cstate_perf_counter(cpu, cci, cai) != -1) {
7926
7927 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
7928
7929 /* User MSR for this counter */
7930 } else if (!no_msr && cai->msr && pkg_cstate_limit >= cai->pkg_cstate_limit
7931 && probe_msr(cpu, cai->msr) == 0) {
7932 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
7933 cci->msr[cai->rci_index] = cai->msr;
7934 }
7935 }
7936
7937 if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) {
7938 has_counter = true;
7939 cores_visited[core_id] = true;
7940 pkg_visited[pkg_id] = true;
7941 }
7942 }
7943
7944 /* If any CPU has access to the counter, make it present */
7945 if (has_counter)
7946 BIC_PRESENT(cai->bic);
7947 }
7948
7949 free(cores_visited);
7950 free(pkg_visited);
7951 }
7952
cstate_perf_init(void)7953 void cstate_perf_init(void)
7954 {
7955 /*
7956 * If we don't have a C1 residency MSR, we calculate it "in software",
7957 * but we need APERF, MPERF too.
7958 */
7959 const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access()
7960 && platform->supported_cstates & CC1;
7961
7962 if (soft_c1)
7963 BIC_PRESENT(BIC_CPU_c1);
7964
7965 cstate_perf_init_(soft_c1);
7966 }
7967
probe_cstates(void)7968 void probe_cstates(void)
7969 {
7970 probe_cst_limit();
7971
7972 if (platform->has_msr_module_c6_res_ms)
7973 BIC_PRESENT(BIC_Mod_c6);
7974
7975 if (platform->has_ext_cst_msrs && !no_msr) {
7976 BIC_PRESENT(BIC_Totl_c0);
7977 BIC_PRESENT(BIC_Any_c0);
7978 BIC_PRESENT(BIC_GFX_c0);
7979 BIC_PRESENT(BIC_CPUGFX);
7980 }
7981
7982 if (quiet)
7983 return;
7984
7985 dump_power_ctl();
7986 dump_cst_cfg();
7987 decode_c6_demotion_policy_msr();
7988 print_dev_latency();
7989 dump_sysfs_cstate_config();
7990 print_irtl();
7991 }
7992
probe_lpi(void)7993 void probe_lpi(void)
7994 {
7995 if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK))
7996 BIC_PRESENT(BIC_CPU_LPI);
7997 else
7998 BIC_NOT_PRESENT(BIC_CPU_LPI);
7999
8000 if (!access(sys_lpi_file_sysfs, R_OK)) {
8001 sys_lpi_file = sys_lpi_file_sysfs;
8002 BIC_PRESENT(BIC_SYS_LPI);
8003 } else if (!access(sys_lpi_file_debugfs, R_OK)) {
8004 sys_lpi_file = sys_lpi_file_debugfs;
8005 BIC_PRESENT(BIC_SYS_LPI);
8006 } else {
8007 sys_lpi_file_sysfs = NULL;
8008 BIC_NOT_PRESENT(BIC_SYS_LPI);
8009 }
8010
8011 }
8012
probe_pstates(void)8013 void probe_pstates(void)
8014 {
8015 probe_bclk();
8016
8017 if (quiet)
8018 return;
8019
8020 dump_platform_info();
8021 dump_turbo_ratio_info();
8022 dump_sysfs_pstate_config();
8023 decode_misc_pwr_mgmt_msr();
8024
8025 for_all_cpus(print_hwp, ODD_COUNTERS);
8026 for_all_cpus(print_epb, ODD_COUNTERS);
8027 for_all_cpus(print_perf_limit, ODD_COUNTERS);
8028 }
8029
process_cpuid()8030 void process_cpuid()
8031 {
8032 unsigned int eax, ebx, ecx, edx;
8033 unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
8034 unsigned long long ucode_patch = 0;
8035 bool ucode_patch_valid = false;
8036
8037 eax = ebx = ecx = edx = 0;
8038
8039 __cpuid(0, max_level, ebx, ecx, edx);
8040
8041 if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
8042 genuine_intel = 1;
8043 else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
8044 authentic_amd = 1;
8045 else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)
8046 hygon_genuine = 1;
8047
8048 if (!quiet)
8049 fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n",
8050 (char *)&ebx, (char *)&edx, (char *)&ecx, max_level);
8051
8052 __cpuid(1, fms, ebx, ecx, edx);
8053 family = (fms >> 8) & 0xf;
8054 model = (fms >> 4) & 0xf;
8055 stepping = fms & 0xf;
8056 if (family == 0xf)
8057 family += (fms >> 20) & 0xff;
8058 if (family >= 6)
8059 model += ((fms >> 16) & 0xf) << 4;
8060 ecx_flags = ecx;
8061 edx_flags = edx;
8062
8063 if (!no_msr) {
8064 if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch))
8065 warnx("get_msr(UCODE)");
8066 else
8067 ucode_patch_valid = true;
8068 }
8069
8070 /*
8071 * check max extended function levels of CPUID.
8072 * This is needed to check for invariant TSC.
8073 * This check is valid for both Intel and AMD.
8074 */
8075 ebx = ecx = edx = 0;
8076 __cpuid(0x80000000, max_extended_level, ebx, ecx, edx);
8077
8078 if (!quiet) {
8079 fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)",
8080 family, model, stepping, family, model, stepping);
8081 if (ucode_patch_valid)
8082 fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF));
8083 fputc('\n', outf);
8084
8085 fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
8086 fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n",
8087 ecx_flags & (1 << 0) ? "SSE3" : "-",
8088 ecx_flags & (1 << 3) ? "MONITOR" : "-",
8089 ecx_flags & (1 << 6) ? "SMX" : "-",
8090 ecx_flags & (1 << 7) ? "EIST" : "-",
8091 ecx_flags & (1 << 8) ? "TM2" : "-",
8092 edx_flags & (1 << 4) ? "TSC" : "-",
8093 edx_flags & (1 << 5) ? "MSR" : "-",
8094 edx_flags & (1 << 22) ? "ACPI-TM" : "-",
8095 edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-");
8096 }
8097
8098 probe_platform_features(family, model);
8099
8100 if (!(edx_flags & (1 << 5)))
8101 errx(1, "CPUID: no MSR");
8102
8103 if (max_extended_level >= 0x80000007) {
8104
8105 /*
8106 * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8
8107 * this check is valid for both Intel and AMD
8108 */
8109 __cpuid(0x80000007, eax, ebx, ecx, edx);
8110 has_invariant_tsc = edx & (1 << 8);
8111 }
8112
8113 /*
8114 * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0
8115 * this check is valid for both Intel and AMD
8116 */
8117
8118 __cpuid(0x6, eax, ebx, ecx, edx);
8119 has_aperf = ecx & (1 << 0);
8120 do_dts = eax & (1 << 0);
8121 if (do_dts)
8122 BIC_PRESENT(BIC_CoreTmp);
8123 has_turbo = eax & (1 << 1);
8124 do_ptm = eax & (1 << 6);
8125 if (do_ptm)
8126 BIC_PRESENT(BIC_PkgTmp);
8127 has_hwp = eax & (1 << 7);
8128 has_hwp_notify = eax & (1 << 8);
8129 has_hwp_activity_window = eax & (1 << 9);
8130 has_hwp_epp = eax & (1 << 10);
8131 has_hwp_pkg = eax & (1 << 11);
8132 has_epb = ecx & (1 << 3);
8133
8134 if (!quiet)
8135 fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
8136 "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
8137 has_aperf ? "" : "No-",
8138 has_turbo ? "" : "No-",
8139 do_dts ? "" : "No-",
8140 do_ptm ? "" : "No-",
8141 has_hwp ? "" : "No-",
8142 has_hwp_notify ? "" : "No-",
8143 has_hwp_activity_window ? "" : "No-",
8144 has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-");
8145
8146 if (!quiet)
8147 decode_misc_enable_msr();
8148
8149 if (max_level >= 0x7 && !quiet) {
8150 int has_sgx;
8151
8152 ecx = 0;
8153
8154 __cpuid_count(0x7, 0, eax, ebx, ecx, edx);
8155
8156 has_sgx = ebx & (1 << 2);
8157
8158 is_hybrid = edx & (1 << 15);
8159
8160 fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-");
8161
8162 if (has_sgx)
8163 decode_feature_control_msr();
8164 }
8165
8166 if (max_level >= 0x15) {
8167 unsigned int eax_crystal;
8168 unsigned int ebx_tsc;
8169
8170 /*
8171 * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz
8172 */
8173 eax_crystal = ebx_tsc = crystal_hz = edx = 0;
8174 __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx);
8175
8176 if (ebx_tsc != 0) {
8177 if (!quiet && (ebx != 0))
8178 fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n",
8179 eax_crystal, ebx_tsc, crystal_hz);
8180
8181 if (crystal_hz == 0)
8182 crystal_hz = platform->crystal_freq;
8183
8184 if (crystal_hz) {
8185 tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal;
8186 if (!quiet)
8187 fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n",
8188 tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
8189 }
8190 }
8191 }
8192 if (max_level >= 0x16) {
8193 unsigned int base_mhz, max_mhz, bus_mhz, edx;
8194
8195 /*
8196 * CPUID 16H Base MHz, Max MHz, Bus MHz
8197 */
8198 base_mhz = max_mhz = bus_mhz = edx = 0;
8199
8200 __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
8201
8202 bclk = bus_mhz;
8203
8204 base_hz = base_mhz * 1000000;
8205 has_base_hz = 1;
8206
8207 if (platform->enable_tsc_tweak)
8208 tsc_tweak = base_hz / tsc_hz;
8209
8210 if (!quiet)
8211 fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n",
8212 base_mhz, max_mhz, bus_mhz);
8213 }
8214
8215 if (has_aperf)
8216 aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1;
8217
8218 BIC_PRESENT(BIC_IRQ);
8219 BIC_PRESENT(BIC_TSC_MHz);
8220 }
8221
counter_info_init(void)8222 static void counter_info_init(void)
8223 {
8224 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) {
8225 struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i];
8226
8227 if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY)
8228 cai->msr = MSR_KNL_CORE_C6_RESIDENCY;
8229
8230 if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES)
8231 cai->msr = 0;
8232
8233 if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY)
8234 cai->msr = MSR_ATOM_PKG_C6_RESIDENCY;
8235 }
8236
8237 for (int i = 0; i < NUM_MSR_COUNTERS; ++i) {
8238 msr_counter_arch_infos[i].present = false;
8239 msr_counter_arch_infos[i].needed = false;
8240 }
8241 }
8242
probe_pm_features(void)8243 void probe_pm_features(void)
8244 {
8245 probe_pstates();
8246
8247 probe_cstates();
8248
8249 probe_lpi();
8250
8251 probe_intel_uncore_frequency();
8252
8253 probe_graphics();
8254
8255 probe_rapl();
8256
8257 probe_thermal();
8258
8259 if (platform->has_nhm_msrs && !no_msr)
8260 BIC_PRESENT(BIC_SMI);
8261
8262 if (!quiet)
8263 decode_misc_feature_control();
8264 }
8265
8266 /*
8267 * in /dev/cpu/ return success for names that are numbers
8268 * ie. filter out ".", "..", "microcode".
8269 */
dir_filter(const struct dirent * dirp)8270 int dir_filter(const struct dirent *dirp)
8271 {
8272 if (isdigit(dirp->d_name[0]))
8273 return 1;
8274 else
8275 return 0;
8276 }
8277
8278 char *possible_file = "/sys/devices/system/cpu/possible";
8279 char possible_buf[1024];
8280
initialize_cpu_possible_set(void)8281 int initialize_cpu_possible_set(void)
8282 {
8283 FILE *fp;
8284
8285 fp = fopen(possible_file, "r");
8286 if (!fp) {
8287 warn("open %s", possible_file);
8288 return -1;
8289 }
8290 if (fread(possible_buf, sizeof(char), 1024, fp) == 0) {
8291 warn("read %s", possible_file);
8292 goto err;
8293 }
8294 if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) {
8295 warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str);
8296 goto err;
8297 }
8298 return 0;
8299
8300 err:
8301 fclose(fp);
8302 return -1;
8303 }
8304
topology_probe(bool startup)8305 void topology_probe(bool startup)
8306 {
8307 int i;
8308 int max_core_id = 0;
8309 int max_package_id = 0;
8310 int max_siblings = 0;
8311
8312 /* Initialize num_cpus, max_cpu_num */
8313 set_max_cpu_num();
8314 topo.num_cpus = 0;
8315 for_all_proc_cpus(count_cpus);
8316 if (!summary_only && topo.num_cpus > 1)
8317 BIC_PRESENT(BIC_CPU);
8318
8319 if (debug > 1)
8320 fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
8321
8322 cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology));
8323 if (cpus == NULL)
8324 err(1, "calloc cpus");
8325
8326 /*
8327 * Allocate and initialize cpu_present_set
8328 */
8329 cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
8330 if (cpu_present_set == NULL)
8331 err(3, "CPU_ALLOC");
8332 cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
8333 CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
8334 for_all_proc_cpus(mark_cpu_present);
8335
8336 /*
8337 * Allocate and initialize cpu_possible_set
8338 */
8339 cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
8340 if (cpu_possible_set == NULL)
8341 err(3, "CPU_ALLOC");
8342 cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
8343 CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
8344 initialize_cpu_possible_set();
8345
8346 /*
8347 * Allocate and initialize cpu_effective_set
8348 */
8349 cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
8350 if (cpu_effective_set == NULL)
8351 err(3, "CPU_ALLOC");
8352 cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
8353 CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
8354 update_effective_set(startup);
8355
8356 /*
8357 * Allocate and initialize cpu_allowed_set
8358 */
8359 cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
8360 if (cpu_allowed_set == NULL)
8361 err(3, "CPU_ALLOC");
8362 cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
8363 CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
8364
8365 /*
8366 * Validate and update cpu_allowed_set.
8367 *
8368 * Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
8369 * Give a warning when cpus in cpu_subset become unavailable at runtime.
8370 * Give a warning when cpus are not effective because of cgroup setting.
8371 *
8372 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
8373 */
8374 for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
8375 if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
8376 continue;
8377
8378 if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
8379 if (cpu_subset) {
8380 /* cpus in cpu_subset must be in cpu_present_set during startup */
8381 if (startup)
8382 err(1, "cpu%d not present", i);
8383 else
8384 fprintf(stderr, "cpu%d not present\n", i);
8385 }
8386 continue;
8387 }
8388
8389 if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
8390 if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
8391 fprintf(stderr, "cpu%d not effective\n", i);
8392 continue;
8393 }
8394 }
8395
8396 CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
8397 }
8398
8399 if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
8400 err(-ENODEV, "No valid cpus found");
8401 sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
8402
8403 /*
8404 * Allocate and initialize cpu_affinity_set
8405 */
8406 cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
8407 if (cpu_affinity_set == NULL)
8408 err(3, "CPU_ALLOC");
8409 cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
8410 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
8411
8412 for_all_proc_cpus(init_thread_id);
8413
8414 for_all_proc_cpus(set_cpu_hybrid_type);
8415
8416 /*
8417 * For online cpus
8418 * find max_core_id, max_package_id
8419 */
8420 for (i = 0; i <= topo.max_cpu_num; ++i) {
8421 int siblings;
8422
8423 if (cpu_is_not_present(i)) {
8424 if (debug > 1)
8425 fprintf(outf, "cpu%d NOT PRESENT\n", i);
8426 continue;
8427 }
8428
8429 cpus[i].logical_cpu_id = i;
8430
8431 /* get package information */
8432 cpus[i].physical_package_id = get_physical_package_id(i);
8433 if (cpus[i].physical_package_id > max_package_id)
8434 max_package_id = cpus[i].physical_package_id;
8435
8436 /* get die information */
8437 cpus[i].die_id = get_die_id(i);
8438 if (cpus[i].die_id > topo.max_die_id)
8439 topo.max_die_id = cpus[i].die_id;
8440
8441 /* get numa node information */
8442 cpus[i].physical_node_id = get_physical_node_id(&cpus[i]);
8443 if (cpus[i].physical_node_id > topo.max_node_num)
8444 topo.max_node_num = cpus[i].physical_node_id;
8445
8446 /* get core information */
8447 cpus[i].physical_core_id = get_core_id(i);
8448 if (cpus[i].physical_core_id > max_core_id)
8449 max_core_id = cpus[i].physical_core_id;
8450
8451 /* get thread information */
8452 siblings = get_thread_siblings(&cpus[i]);
8453 if (siblings > max_siblings)
8454 max_siblings = siblings;
8455 if (cpus[i].thread_id == 0)
8456 topo.num_cores++;
8457 }
8458 topo.max_core_id = max_core_id;
8459 topo.max_package_id = max_package_id;
8460
8461 topo.cores_per_node = max_core_id + 1;
8462 if (debug > 1)
8463 fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node);
8464 if (!summary_only && topo.cores_per_node > 1)
8465 BIC_PRESENT(BIC_Core);
8466
8467 topo.num_die = topo.max_die_id + 1;
8468 if (debug > 1)
8469 fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die);
8470 if (!summary_only && topo.num_die > 1)
8471 BIC_PRESENT(BIC_Die);
8472
8473 topo.num_packages = max_package_id + 1;
8474 if (debug > 1)
8475 fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages);
8476 if (!summary_only && topo.num_packages > 1)
8477 BIC_PRESENT(BIC_Package);
8478
8479 set_node_data();
8480 if (debug > 1)
8481 fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg);
8482 if (!summary_only && topo.nodes_per_pkg > 1)
8483 BIC_PRESENT(BIC_Node);
8484
8485 topo.threads_per_core = max_siblings;
8486 if (debug > 1)
8487 fprintf(outf, "max_siblings %d\n", max_siblings);
8488
8489 if (debug < 1)
8490 return;
8491
8492 for (i = 0; i <= topo.max_cpu_num; ++i) {
8493 if (cpu_is_not_present(i))
8494 continue;
8495 fprintf(outf,
8496 "cpu %d pkg %d die %d node %d lnode %d core %d thread %d\n",
8497 i, cpus[i].physical_package_id, cpus[i].die_id,
8498 cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id);
8499 }
8500
8501 }
8502
allocate_counters(struct thread_data ** t,struct core_data ** c,struct pkg_data ** p)8503 void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p)
8504 {
8505 int i;
8506 int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages;
8507 int num_threads = topo.threads_per_core * num_cores;
8508
8509 *t = calloc(num_threads, sizeof(struct thread_data));
8510 if (*t == NULL)
8511 goto error;
8512
8513 for (i = 0; i < num_threads; i++)
8514 (*t)[i].cpu_id = -1;
8515
8516 *c = calloc(num_cores, sizeof(struct core_data));
8517 if (*c == NULL)
8518 goto error;
8519
8520 for (i = 0; i < num_cores; i++) {
8521 (*c)[i].core_id = -1;
8522 (*c)[i].base_cpu = -1;
8523 }
8524
8525 *p = calloc(topo.num_packages, sizeof(struct pkg_data));
8526 if (*p == NULL)
8527 goto error;
8528
8529 for (i = 0; i < topo.num_packages; i++) {
8530 (*p)[i].package_id = i;
8531 (*p)[i].base_cpu = -1;
8532 }
8533
8534 return;
8535 error:
8536 err(1, "calloc counters");
8537 }
8538
8539 /*
8540 * init_counter()
8541 *
8542 * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE
8543 */
init_counter(struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,int cpu_id)8544 void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id)
8545 {
8546 int pkg_id = cpus[cpu_id].physical_package_id;
8547 int node_id = cpus[cpu_id].logical_node_id;
8548 int core_id = cpus[cpu_id].physical_core_id;
8549 int thread_id = cpus[cpu_id].thread_id;
8550 struct thread_data *t;
8551 struct core_data *c;
8552 struct pkg_data *p;
8553
8554 /* Workaround for systems where physical_node_id==-1
8555 * and logical_node_id==(-1 - topo.num_cpus)
8556 */
8557 if (node_id < 0)
8558 node_id = 0;
8559
8560 t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id);
8561 c = GET_CORE(core_base, core_id, node_id, pkg_id);
8562 p = GET_PKG(pkg_base, pkg_id);
8563
8564 t->cpu_id = cpu_id;
8565 if (!cpu_is_not_allowed(cpu_id)) {
8566 if (c->base_cpu < 0)
8567 c->base_cpu = t->cpu_id;
8568 if (p->base_cpu < 0)
8569 p->base_cpu = t->cpu_id;
8570 }
8571
8572 c->core_id = core_id;
8573 p->package_id = pkg_id;
8574 }
8575
initialize_counters(int cpu_id)8576 int initialize_counters(int cpu_id)
8577 {
8578 init_counter(EVEN_COUNTERS, cpu_id);
8579 init_counter(ODD_COUNTERS, cpu_id);
8580 return 0;
8581 }
8582
allocate_output_buffer()8583 void allocate_output_buffer()
8584 {
8585 output_buffer = calloc(1, (1 + topo.num_cpus) * 2048);
8586 outp = output_buffer;
8587 if (outp == NULL)
8588 err(-1, "calloc output buffer");
8589 }
8590
allocate_fd_percpu(void)8591 void allocate_fd_percpu(void)
8592 {
8593 fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8594 if (fd_percpu == NULL)
8595 err(-1, "calloc fd_percpu");
8596 }
8597
allocate_irq_buffers(void)8598 void allocate_irq_buffers(void)
8599 {
8600 irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int));
8601 if (irq_column_2_cpu == NULL)
8602 err(-1, "calloc %d", topo.num_cpus);
8603
8604 irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8605 if (irqs_per_cpu == NULL)
8606 err(-1, "calloc %d", topo.max_cpu_num + 1);
8607 }
8608
update_topo(struct thread_data * t,struct core_data * c,struct pkg_data * p)8609 int update_topo(struct thread_data *t, struct core_data *c, struct pkg_data *p)
8610 {
8611 topo.allowed_cpus++;
8612 if ((int)t->cpu_id == c->base_cpu)
8613 topo.allowed_cores++;
8614 if ((int)t->cpu_id == p->base_cpu)
8615 topo.allowed_packages++;
8616
8617 return 0;
8618 }
8619
topology_update(void)8620 void topology_update(void)
8621 {
8622 topo.allowed_cpus = 0;
8623 topo.allowed_cores = 0;
8624 topo.allowed_packages = 0;
8625 for_all_cpus(update_topo, ODD_COUNTERS);
8626 }
8627
setup_all_buffers(bool startup)8628 void setup_all_buffers(bool startup)
8629 {
8630 topology_probe(startup);
8631 allocate_irq_buffers();
8632 allocate_fd_percpu();
8633 allocate_counters(&thread_even, &core_even, &package_even);
8634 allocate_counters(&thread_odd, &core_odd, &package_odd);
8635 allocate_output_buffer();
8636 for_all_proc_cpus(initialize_counters);
8637 topology_update();
8638 }
8639
set_base_cpu(void)8640 void set_base_cpu(void)
8641 {
8642 int i;
8643
8644 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
8645 if (cpu_is_not_allowed(i))
8646 continue;
8647 base_cpu = i;
8648 if (debug > 1)
8649 fprintf(outf, "base_cpu = %d\n", base_cpu);
8650 return;
8651 }
8652 err(-ENODEV, "No valid cpus found");
8653 }
8654
has_added_counters(void)8655 bool has_added_counters(void)
8656 {
8657 /*
8658 * It only makes sense to call this after the command line is parsed,
8659 * otherwise sys structure is not populated.
8660 */
8661
8662 return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters;
8663 }
8664
check_msr_access(void)8665 void check_msr_access(void)
8666 {
8667 check_dev_msr();
8668 check_msr_permission();
8669
8670 if (no_msr)
8671 bic_disable_msr_access();
8672 }
8673
check_perf_access(void)8674 void check_perf_access(void)
8675 {
8676 if (no_perf || !BIC_IS_ENABLED(BIC_IPC) || !has_instr_count_access())
8677 bic_enabled &= ~BIC_IPC;
8678 }
8679
perf_has_hybrid_devices(void)8680 bool perf_has_hybrid_devices(void)
8681 {
8682 /*
8683 * 0: unknown
8684 * 1: has separate perf device for p and e core
8685 * -1: doesn't have separate perf device for p and e core
8686 */
8687 static int cached;
8688
8689 if (cached > 0)
8690 return true;
8691
8692 if (cached < 0)
8693 return false;
8694
8695 if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) {
8696 cached = -1;
8697 return false;
8698 }
8699
8700 if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) {
8701 cached = -1;
8702 return false;
8703 }
8704
8705 cached = 1;
8706 return true;
8707 }
8708
added_perf_counters_init_(struct perf_counter_info * pinfo)8709 int added_perf_counters_init_(struct perf_counter_info *pinfo)
8710 {
8711 size_t num_domains = 0;
8712 unsigned int next_domain;
8713 bool *domain_visited;
8714 unsigned int perf_type, perf_config;
8715 double perf_scale;
8716 int fd_perf;
8717
8718 if (!pinfo)
8719 return 0;
8720
8721 const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1));
8722
8723 domain_visited = calloc(max_num_domains, sizeof(*domain_visited));
8724
8725 while (pinfo) {
8726 switch (pinfo->scope) {
8727 case SCOPE_CPU:
8728 num_domains = topo.max_cpu_num + 1;
8729 break;
8730
8731 case SCOPE_CORE:
8732 num_domains = topo.max_core_id + 1;
8733 break;
8734
8735 case SCOPE_PACKAGE:
8736 num_domains = topo.max_package_id + 1;
8737 break;
8738 }
8739
8740 /* Allocate buffer for file descriptor for each domain. */
8741 pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain));
8742 if (!pinfo->fd_perf_per_domain)
8743 errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain");
8744
8745 for (size_t i = 0; i < num_domains; ++i)
8746 pinfo->fd_perf_per_domain[i] = -1;
8747
8748 pinfo->num_domains = num_domains;
8749 pinfo->scale = 1.0;
8750
8751 memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
8752
8753 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
8754
8755 next_domain = cpu_to_domain(pinfo, cpu);
8756
8757 assert(next_domain < num_domains);
8758
8759 if (cpu_is_not_allowed(cpu))
8760 continue;
8761
8762 if (domain_visited[next_domain])
8763 continue;
8764
8765 /*
8766 * Intel hybrid platforms expose different perf devices for P and E cores.
8767 * Instead of one, "/sys/bus/event_source/devices/cpu" device, there are
8768 * "/sys/bus/event_source/devices/{cpu_core,cpu_atom}".
8769 *
8770 * This makes it more complicated to the user, because most of the counters
8771 * are available on both and have to be handled manually, otherwise.
8772 *
8773 * Code below, allow user to use the old "cpu" name, which is translated accordingly.
8774 */
8775 const char *perf_device = pinfo->device;
8776
8777 if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) {
8778 switch (cpus[cpu].type) {
8779 case INTEL_PCORE_TYPE:
8780 perf_device = "cpu_core";
8781 break;
8782
8783 case INTEL_ECORE_TYPE:
8784 perf_device = "cpu_atom";
8785 break;
8786
8787 default: /* Don't change, we will probably fail and report a problem soon. */
8788 break;
8789 }
8790 }
8791
8792 perf_type = read_perf_type(perf_device);
8793 if (perf_type == (unsigned int)-1) {
8794 warnx("%s: perf/%s/%s: failed to read %s",
8795 __func__, perf_device, pinfo->event, "type");
8796 continue;
8797 }
8798
8799 perf_config = read_perf_config(perf_device, pinfo->event);
8800 if (perf_config == (unsigned int)-1) {
8801 warnx("%s: perf/%s/%s: failed to read %s",
8802 __func__, perf_device, pinfo->event, "config");
8803 continue;
8804 }
8805
8806 /* Scale is not required, some counters just don't have it. */
8807 perf_scale = read_perf_scale(perf_device, pinfo->event);
8808 if (perf_scale == 0.0)
8809 perf_scale = 1.0;
8810
8811 fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0);
8812 if (fd_perf == -1) {
8813 warnx("%s: perf/%s/%s: failed to open counter on cpu%d",
8814 __func__, perf_device, pinfo->event, cpu);
8815 continue;
8816 }
8817
8818 domain_visited[next_domain] = 1;
8819 pinfo->fd_perf_per_domain[next_domain] = fd_perf;
8820 pinfo->scale = perf_scale;
8821
8822 if (debug)
8823 fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n",
8824 perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]);
8825 }
8826
8827 pinfo = pinfo->next;
8828 }
8829
8830 free(domain_visited);
8831
8832 return 0;
8833 }
8834
added_perf_counters_init(void)8835 void added_perf_counters_init(void)
8836 {
8837 if (added_perf_counters_init_(sys.perf_tp))
8838 errx(1, "%s: %s", __func__, "thread");
8839
8840 if (added_perf_counters_init_(sys.perf_cp))
8841 errx(1, "%s: %s", __func__, "core");
8842
8843 if (added_perf_counters_init_(sys.perf_pp))
8844 errx(1, "%s: %s", __func__, "package");
8845 }
8846
parse_telem_info_file(int fd_dir,const char * info_filename,const char * format,unsigned long * output)8847 int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output)
8848 {
8849 int fd_telem_info;
8850 FILE *file_telem_info;
8851 unsigned long value;
8852
8853 fd_telem_info = openat(fd_dir, info_filename, O_RDONLY);
8854 if (fd_telem_info == -1)
8855 return -1;
8856
8857 file_telem_info = fdopen(fd_telem_info, "r");
8858 if (file_telem_info == NULL) {
8859 close(fd_telem_info);
8860 return -1;
8861 }
8862
8863 if (fscanf(file_telem_info, format, &value) != 1) {
8864 fclose(file_telem_info);
8865 return -1;
8866 }
8867
8868 fclose(file_telem_info);
8869
8870 *output = value;
8871
8872 return 0;
8873 }
8874
pmt_mmio_open(unsigned int target_guid)8875 struct pmt_mmio *pmt_mmio_open(unsigned int target_guid)
8876 {
8877 DIR *dirp;
8878 struct dirent *entry;
8879 struct stat st;
8880 unsigned int telem_idx;
8881 int fd_telem_dir, fd_pmt;
8882 unsigned long guid, size, offset;
8883 size_t mmap_size;
8884 void *mmio;
8885 struct pmt_mmio *ret = NULL;
8886
8887 if (stat(SYSFS_TELEM_PATH, &st) == -1)
8888 return NULL;
8889
8890 dirp = opendir(SYSFS_TELEM_PATH);
8891 if (dirp == NULL)
8892 return NULL;
8893
8894 for (;;) {
8895 entry = readdir(dirp);
8896
8897 if (entry == NULL)
8898 break;
8899
8900 if (strcmp(entry->d_name, ".") == 0)
8901 continue;
8902
8903 if (strcmp(entry->d_name, "..") == 0)
8904 continue;
8905
8906 if (sscanf(entry->d_name, "telem%u", &telem_idx) != 1)
8907 continue;
8908
8909 if (fstatat(dirfd(dirp), entry->d_name, &st, 0) == -1) {
8910 break;
8911 }
8912
8913 if (!S_ISDIR(st.st_mode))
8914 continue;
8915
8916 fd_telem_dir = openat(dirfd(dirp), entry->d_name, O_RDONLY);
8917 if (fd_telem_dir == -1) {
8918 break;
8919 }
8920
8921 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
8922 close(fd_telem_dir);
8923 break;
8924 }
8925
8926 if (parse_telem_info_file(fd_telem_dir, "size", "%lu", &size)) {
8927 close(fd_telem_dir);
8928 break;
8929 }
8930
8931 if (guid != target_guid) {
8932 close(fd_telem_dir);
8933 continue;
8934 }
8935
8936 if (parse_telem_info_file(fd_telem_dir, "offset", "%lu", &offset)) {
8937 close(fd_telem_dir);
8938 break;
8939 }
8940
8941 assert(offset == 0);
8942
8943 fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY);
8944 if (fd_pmt == -1)
8945 goto loop_cleanup_and_break;
8946
8947 mmap_size = ROUND_UP_TO_PAGE_SIZE(size);
8948 mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0);
8949 if (mmio != MAP_FAILED) {
8950
8951 if (debug)
8952 fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio);
8953
8954 ret = calloc(1, sizeof(*ret));
8955
8956 if (!ret) {
8957 fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__);
8958 exit(1);
8959 }
8960
8961 ret->guid = guid;
8962 ret->mmio_base = mmio;
8963 ret->pmt_offset = offset;
8964 ret->size = size;
8965
8966 ret->next = pmt_mmios;
8967 pmt_mmios = ret;
8968 }
8969
8970 loop_cleanup_and_break:
8971 close(fd_pmt);
8972 close(fd_telem_dir);
8973 break;
8974 }
8975
8976 closedir(dirp);
8977
8978 return ret;
8979 }
8980
pmt_mmio_find(unsigned int guid)8981 struct pmt_mmio *pmt_mmio_find(unsigned int guid)
8982 {
8983 struct pmt_mmio *pmmio = pmt_mmios;
8984
8985 while (pmmio) {
8986 if (pmmio->guid == guid)
8987 return pmmio;
8988
8989 pmmio = pmmio->next;
8990 }
8991
8992 return NULL;
8993 }
8994
pmt_get_counter_pointer(struct pmt_mmio * pmmio,unsigned long counter_offset)8995 void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset)
8996 {
8997 char *ret;
8998
8999 /* Get base of mmaped PMT file. */
9000 ret = (char *)pmmio->mmio_base;
9001
9002 /*
9003 * Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data.
9004 * It's not guaranteed that the mmaped memory begins with the telemetry data
9005 * - we might have to apply the offset first.
9006 */
9007 ret += pmmio->pmt_offset;
9008
9009 /* Apply the counter offset to get the address to the mmaped counter. */
9010 ret += counter_offset;
9011
9012 return ret;
9013 }
9014
pmt_add_guid(unsigned int guid)9015 struct pmt_mmio *pmt_add_guid(unsigned int guid)
9016 {
9017 struct pmt_mmio *ret;
9018
9019 ret = pmt_mmio_find(guid);
9020 if (!ret)
9021 ret = pmt_mmio_open(guid);
9022
9023 return ret;
9024 }
9025
9026 enum pmt_open_mode {
9027 PMT_OPEN_TRY, /* Open failure is not an error. */
9028 PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */
9029 };
9030
pmt_find_counter(struct pmt_counter * pcounter,const char * name)9031 struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name)
9032 {
9033 while (pcounter) {
9034 if (strcmp(pcounter->name, name) == 0)
9035 break;
9036
9037 pcounter = pcounter->next;
9038 }
9039
9040 return pcounter;
9041 }
9042
pmt_get_scope_root(enum counter_scope scope)9043 struct pmt_counter **pmt_get_scope_root(enum counter_scope scope)
9044 {
9045 switch (scope) {
9046 case SCOPE_CPU:
9047 return &sys.pmt_tp;
9048 case SCOPE_CORE:
9049 return &sys.pmt_cp;
9050 case SCOPE_PACKAGE:
9051 return &sys.pmt_pp;
9052 }
9053
9054 __builtin_unreachable();
9055 }
9056
pmt_counter_add_domain(struct pmt_counter * pcounter,unsigned long * pmmio,unsigned int domain_id)9057 void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id)
9058 {
9059 /* Make sure the new domain fits. */
9060 if (domain_id >= pcounter->num_domains)
9061 pmt_counter_resize(pcounter, domain_id + 1);
9062
9063 assert(pcounter->domains);
9064 assert(domain_id < pcounter->num_domains);
9065
9066 pcounter->domains[domain_id].pcounter = pmmio;
9067 }
9068
pmt_add_counter(unsigned int guid,const char * name,enum pmt_datatype type,unsigned int lsb,unsigned int msb,unsigned int offset,enum counter_scope scope,enum counter_format format,unsigned int domain_id,enum pmt_open_mode mode)9069 int pmt_add_counter(unsigned int guid, const char *name, enum pmt_datatype type,
9070 unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope,
9071 enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode)
9072 {
9073 struct pmt_mmio *mmio;
9074 struct pmt_counter *pcounter;
9075 struct pmt_counter **const pmt_root = pmt_get_scope_root(scope);
9076 bool new_counter = false;
9077 int conflict = 0;
9078
9079 if (lsb > msb) {
9080 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name);
9081 exit(1);
9082 }
9083
9084 if (msb >= 64) {
9085 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name);
9086 exit(1);
9087 }
9088
9089 mmio = pmt_add_guid(guid);
9090 if (!mmio) {
9091 if (mode != PMT_OPEN_TRY) {
9092 fprintf(stderr, "%s: failed to map PMT MMIO for guid %x\n", __func__, guid);
9093 exit(1);
9094 }
9095
9096 return 1;
9097 }
9098
9099 if (offset >= mmio->size) {
9100 if (mode != PMT_OPEN_TRY) {
9101 fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size);
9102 exit(1);
9103 }
9104
9105 return 1;
9106 }
9107
9108 pcounter = pmt_find_counter(*pmt_root, name);
9109 if (!pcounter) {
9110 pcounter = calloc(1, sizeof(*pcounter));
9111 new_counter = true;
9112 }
9113
9114 if (new_counter) {
9115 strncpy(pcounter->name, name, ARRAY_SIZE(pcounter->name) - 1);
9116 pcounter->type = type;
9117 pcounter->scope = scope;
9118 pcounter->lsb = lsb;
9119 pcounter->msb = msb;
9120 pcounter->format = format;
9121 } else {
9122 conflict += pcounter->type != type;
9123 conflict += pcounter->scope != scope;
9124 conflict += pcounter->lsb != lsb;
9125 conflict += pcounter->msb != msb;
9126 conflict += pcounter->format != format;
9127 }
9128
9129 if (conflict) {
9130 fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n",
9131 __func__, name);
9132 exit(1);
9133 }
9134
9135 pmt_counter_add_domain(pcounter, pmt_get_counter_pointer(mmio, offset), domain_id);
9136
9137 if (new_counter) {
9138 pcounter->next = *pmt_root;
9139 *pmt_root = pcounter;
9140 }
9141
9142 return 0;
9143 }
9144
pmt_init(void)9145 void pmt_init(void)
9146 {
9147 if (BIC_IS_ENABLED(BIC_Diec6)) {
9148 pmt_add_counter(PMT_MTL_DC6_GUID, "Die%c6", PMT_TYPE_XTAL_TIME, PMT_COUNTER_MTL_DC6_LSB,
9149 PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, SCOPE_PACKAGE, FORMAT_DELTA,
9150 0, PMT_OPEN_TRY);
9151 }
9152 }
9153
turbostat_init()9154 void turbostat_init()
9155 {
9156 setup_all_buffers(true);
9157 set_base_cpu();
9158 check_msr_access();
9159 check_perf_access();
9160 process_cpuid();
9161 counter_info_init();
9162 probe_pm_features();
9163 msr_perf_init();
9164 linux_perf_init();
9165 rapl_perf_init();
9166 cstate_perf_init();
9167 added_perf_counters_init();
9168 pmt_init();
9169
9170 for_all_cpus(get_cpu_type, ODD_COUNTERS);
9171 for_all_cpus(get_cpu_type, EVEN_COUNTERS);
9172
9173 if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(base_cpu) != -1)
9174 BIC_PRESENT(BIC_IPC);
9175
9176 /*
9177 * If TSC tweak is needed, but couldn't get it,
9178 * disable more BICs, since it can't be reported accurately.
9179 */
9180 if (platform->enable_tsc_tweak && !has_base_hz) {
9181 bic_enabled &= ~BIC_Busy;
9182 bic_enabled &= ~BIC_Bzy_MHz;
9183 }
9184 }
9185
affinitize_child(void)9186 void affinitize_child(void)
9187 {
9188 /* Prefer cpu_possible_set, if available */
9189 if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
9190 warn("sched_setaffinity cpu_possible_set");
9191
9192 /* Otherwise, allow child to run on same cpu set as turbostat */
9193 if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
9194 warn("sched_setaffinity cpu_allowed_set");
9195 }
9196 }
9197
fork_it(char ** argv)9198 int fork_it(char **argv)
9199 {
9200 pid_t child_pid;
9201 int status;
9202
9203 snapshot_proc_sysfs_files();
9204 status = for_all_cpus(get_counters, EVEN_COUNTERS);
9205 first_counter_read = 0;
9206 if (status)
9207 exit(status);
9208 gettimeofday(&tv_even, (struct timezone *)NULL);
9209
9210 child_pid = fork();
9211 if (!child_pid) {
9212 /* child */
9213 affinitize_child();
9214 execvp(argv[0], argv);
9215 err(errno, "exec %s", argv[0]);
9216 } else {
9217
9218 /* parent */
9219 if (child_pid == -1)
9220 err(1, "fork");
9221
9222 signal(SIGINT, SIG_IGN);
9223 signal(SIGQUIT, SIG_IGN);
9224 if (waitpid(child_pid, &status, 0) == -1)
9225 err(status, "waitpid");
9226
9227 if (WIFEXITED(status))
9228 status = WEXITSTATUS(status);
9229 }
9230 /*
9231 * n.b. fork_it() does not check for errors from for_all_cpus()
9232 * because re-starting is problematic when forking
9233 */
9234 snapshot_proc_sysfs_files();
9235 for_all_cpus(get_counters, ODD_COUNTERS);
9236 gettimeofday(&tv_odd, (struct timezone *)NULL);
9237 timersub(&tv_odd, &tv_even, &tv_delta);
9238 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS))
9239 fprintf(outf, "%s: Counter reset detected\n", progname);
9240 else {
9241 compute_average(EVEN_COUNTERS);
9242 format_all_counters(EVEN_COUNTERS);
9243 }
9244
9245 fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0);
9246
9247 flush_output_stderr();
9248
9249 return status;
9250 }
9251
get_and_dump_counters(void)9252 int get_and_dump_counters(void)
9253 {
9254 int status;
9255
9256 snapshot_proc_sysfs_files();
9257 status = for_all_cpus(get_counters, ODD_COUNTERS);
9258 if (status)
9259 return status;
9260
9261 status = for_all_cpus(dump_counters, ODD_COUNTERS);
9262 if (status)
9263 return status;
9264
9265 flush_output_stdout();
9266
9267 return status;
9268 }
9269
print_version()9270 void print_version()
9271 {
9272 fprintf(outf, "turbostat version 2024.07.26 - Len Brown <lenb@kernel.org>\n");
9273 }
9274
9275 #define COMMAND_LINE_SIZE 2048
9276
print_bootcmd(void)9277 void print_bootcmd(void)
9278 {
9279 char bootcmd[COMMAND_LINE_SIZE];
9280 FILE *fp;
9281 int ret;
9282
9283 memset(bootcmd, 0, COMMAND_LINE_SIZE);
9284 fp = fopen("/proc/cmdline", "r");
9285 if (!fp)
9286 return;
9287
9288 ret = fread(bootcmd, sizeof(char), COMMAND_LINE_SIZE - 1, fp);
9289 if (ret) {
9290 bootcmd[ret] = '\0';
9291 /* the last character is already '\n' */
9292 fprintf(outf, "Kernel command line: %s", bootcmd);
9293 }
9294
9295 fclose(fp);
9296 }
9297
find_msrp_by_name(struct msr_counter * head,char * name)9298 struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
9299 {
9300 struct msr_counter *mp;
9301
9302 for (mp = head; mp; mp = mp->next) {
9303 if (debug)
9304 fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name);
9305 if (!strncmp(name, mp->name, strlen(mp->name)))
9306 return mp;
9307 }
9308 return NULL;
9309 }
9310
add_counter(unsigned int msr_num,char * path,char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format,int flags,int id)9311 int add_counter(unsigned int msr_num, char *path, char *name,
9312 unsigned int width, enum counter_scope scope,
9313 enum counter_type type, enum counter_format format, int flags, int id)
9314 {
9315 struct msr_counter *msrp;
9316
9317 if (no_msr && msr_num)
9318 errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num);
9319
9320 if (debug)
9321 fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n",
9322 __func__, msr_num, path, name, width, scope, type, format, flags, id);
9323
9324 switch (scope) {
9325
9326 case SCOPE_CPU:
9327 msrp = find_msrp_by_name(sys.tp, name);
9328 if (msrp) {
9329 if (debug)
9330 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
9331 break;
9332 }
9333 if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
9334 warnx("ignoring thread counter %s", name);
9335 return -1;
9336 }
9337 break;
9338 case SCOPE_CORE:
9339 msrp = find_msrp_by_name(sys.cp, name);
9340 if (msrp) {
9341 if (debug)
9342 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
9343 break;
9344 }
9345 if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
9346 warnx("ignoring core counter %s", name);
9347 return -1;
9348 }
9349 break;
9350 case SCOPE_PACKAGE:
9351 msrp = find_msrp_by_name(sys.pp, name);
9352 if (msrp) {
9353 if (debug)
9354 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
9355 break;
9356 }
9357 if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
9358 warnx("ignoring package counter %s", name);
9359 return -1;
9360 }
9361 break;
9362 default:
9363 warnx("ignoring counter %s with unknown scope", name);
9364 return -1;
9365 }
9366
9367 if (msrp == NULL) {
9368 msrp = calloc(1, sizeof(struct msr_counter));
9369 if (msrp == NULL)
9370 err(-1, "calloc msr_counter");
9371
9372 msrp->msr_num = msr_num;
9373 strncpy(msrp->name, name, NAME_BYTES - 1);
9374 msrp->width = width;
9375 msrp->type = type;
9376 msrp->format = format;
9377 msrp->flags = flags;
9378
9379 switch (scope) {
9380 case SCOPE_CPU:
9381 msrp->next = sys.tp;
9382 sys.tp = msrp;
9383 break;
9384 case SCOPE_CORE:
9385 msrp->next = sys.cp;
9386 sys.cp = msrp;
9387 break;
9388 case SCOPE_PACKAGE:
9389 msrp->next = sys.pp;
9390 sys.pp = msrp;
9391 break;
9392 }
9393 }
9394
9395 if (path) {
9396 struct sysfs_path *sp;
9397
9398 sp = calloc(1, sizeof(struct sysfs_path));
9399 if (sp == NULL) {
9400 perror("calloc");
9401 exit(1);
9402 }
9403 strncpy(sp->path, path, PATH_BYTES - 1);
9404 sp->id = id;
9405 sp->next = msrp->sp;
9406 msrp->sp = sp;
9407 }
9408
9409 return 0;
9410 }
9411
9412 /*
9413 * Initialize the fields used for identifying and opening the counter.
9414 *
9415 * Defer the initialization of any runtime buffers for actually reading
9416 * the counters for when we initialize all perf counters, so we can later
9417 * easily call re_initialize().
9418 */
make_perf_counter_info(const char * perf_device,const char * perf_event,const char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)9419 struct perf_counter_info *make_perf_counter_info(const char *perf_device,
9420 const char *perf_event,
9421 const char *name,
9422 unsigned int width,
9423 enum counter_scope scope,
9424 enum counter_type type, enum counter_format format)
9425 {
9426 struct perf_counter_info *pinfo;
9427
9428 pinfo = calloc(1, sizeof(*pinfo));
9429 if (!pinfo)
9430 errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event);
9431
9432 strncpy(pinfo->device, perf_device, ARRAY_SIZE(pinfo->device) - 1);
9433 strncpy(pinfo->event, perf_event, ARRAY_SIZE(pinfo->event) - 1);
9434
9435 strncpy(pinfo->name, name, ARRAY_SIZE(pinfo->name) - 1);
9436 pinfo->width = width;
9437 pinfo->scope = scope;
9438 pinfo->type = type;
9439 pinfo->format = format;
9440
9441 return pinfo;
9442 }
9443
add_perf_counter(const char * perf_device,const char * perf_event,const char * name_buffer,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)9444 int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width,
9445 enum counter_scope scope, enum counter_type type, enum counter_format format)
9446 {
9447 struct perf_counter_info *pinfo;
9448
9449 switch (scope) {
9450 case SCOPE_CPU:
9451 if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) {
9452 warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event);
9453 return -1;
9454 }
9455 break;
9456
9457 case SCOPE_CORE:
9458 if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) {
9459 warnx("ignoring core counter perf/%s/%s", perf_device, perf_event);
9460 return -1;
9461 }
9462 break;
9463
9464 case SCOPE_PACKAGE:
9465 if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
9466 warnx("ignoring package counter perf/%s/%s", perf_device, perf_event);
9467 return -1;
9468 }
9469 break;
9470 }
9471
9472 pinfo = make_perf_counter_info(perf_device, perf_event, name_buffer, width, scope, type, format);
9473
9474 if (!pinfo)
9475 return -1;
9476
9477 switch (scope) {
9478 case SCOPE_CPU:
9479 pinfo->next = sys.perf_tp;
9480 sys.perf_tp = pinfo;
9481 ++sys.added_thread_perf_counters;
9482 break;
9483
9484 case SCOPE_CORE:
9485 pinfo->next = sys.perf_cp;
9486 sys.perf_cp = pinfo;
9487 ++sys.added_core_perf_counters;
9488 break;
9489
9490 case SCOPE_PACKAGE:
9491 pinfo->next = sys.perf_pp;
9492 sys.perf_pp = pinfo;
9493 ++sys.added_package_perf_counters;
9494 break;
9495 }
9496
9497 // FIXME: we might not have debug here yet
9498 if (debug)
9499 fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n",
9500 __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope);
9501
9502 return 0;
9503 }
9504
parse_add_command_msr(char * add_command)9505 void parse_add_command_msr(char *add_command)
9506 {
9507 int msr_num = 0;
9508 char *path = NULL;
9509 char perf_device[PERF_DEV_NAME_BYTES] = "";
9510 char perf_event[PERF_EVT_NAME_BYTES] = "";
9511 char name_buffer[PERF_NAME_BYTES] = "";
9512 int width = 64;
9513 int fail = 0;
9514 enum counter_scope scope = SCOPE_CPU;
9515 enum counter_type type = COUNTER_CYCLES;
9516 enum counter_format format = FORMAT_DELTA;
9517
9518 while (add_command) {
9519
9520 if (sscanf(add_command, "msr0x%x", &msr_num) == 1)
9521 goto next;
9522
9523 if (sscanf(add_command, "msr%d", &msr_num) == 1)
9524 goto next;
9525
9526 BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31);
9527 BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31);
9528 if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2)
9529 goto next;
9530
9531 if (*add_command == '/') {
9532 path = add_command;
9533 goto next;
9534 }
9535
9536 if (sscanf(add_command, "u%d", &width) == 1) {
9537 if ((width == 32) || (width == 64))
9538 goto next;
9539 width = 64;
9540 }
9541 if (!strncmp(add_command, "cpu", strlen("cpu"))) {
9542 scope = SCOPE_CPU;
9543 goto next;
9544 }
9545 if (!strncmp(add_command, "core", strlen("core"))) {
9546 scope = SCOPE_CORE;
9547 goto next;
9548 }
9549 if (!strncmp(add_command, "package", strlen("package"))) {
9550 scope = SCOPE_PACKAGE;
9551 goto next;
9552 }
9553 if (!strncmp(add_command, "cycles", strlen("cycles"))) {
9554 type = COUNTER_CYCLES;
9555 goto next;
9556 }
9557 if (!strncmp(add_command, "seconds", strlen("seconds"))) {
9558 type = COUNTER_SECONDS;
9559 goto next;
9560 }
9561 if (!strncmp(add_command, "usec", strlen("usec"))) {
9562 type = COUNTER_USEC;
9563 goto next;
9564 }
9565 if (!strncmp(add_command, "raw", strlen("raw"))) {
9566 format = FORMAT_RAW;
9567 goto next;
9568 }
9569 if (!strncmp(add_command, "delta", strlen("delta"))) {
9570 format = FORMAT_DELTA;
9571 goto next;
9572 }
9573 if (!strncmp(add_command, "percent", strlen("percent"))) {
9574 format = FORMAT_PERCENT;
9575 goto next;
9576 }
9577
9578 BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18);
9579 if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) {
9580 char *eos;
9581
9582 eos = strchr(name_buffer, ',');
9583 if (eos)
9584 *eos = '\0';
9585 goto next;
9586 }
9587
9588 next:
9589 add_command = strchr(add_command, ',');
9590 if (add_command) {
9591 *add_command = '\0';
9592 add_command++;
9593 }
9594
9595 }
9596 if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) {
9597 fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event ) required\n");
9598 fail++;
9599 }
9600
9601 /* Test for non-empty perf_device and perf_event */
9602 const bool is_perf_counter = perf_device[0] && perf_event[0];
9603
9604 /* generate default column header */
9605 if (*name_buffer == '\0') {
9606 if (is_perf_counter) {
9607 snprintf(name_buffer, ARRAY_SIZE(name_buffer), "perf/%s", perf_event);
9608 } else {
9609 if (width == 32)
9610 sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
9611 else
9612 sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
9613 }
9614 }
9615
9616 if (is_perf_counter) {
9617 if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format))
9618 fail++;
9619 } else {
9620 if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0))
9621 fail++;
9622 }
9623
9624 if (fail) {
9625 help();
9626 exit(1);
9627 }
9628 }
9629
starts_with(const char * str,const char * prefix)9630 bool starts_with(const char *str, const char *prefix)
9631 {
9632 return strncmp(prefix, str, strlen(prefix)) == 0;
9633 }
9634
parse_add_command_pmt(char * add_command)9635 void parse_add_command_pmt(char *add_command)
9636 {
9637 char *name = NULL;
9638 char *type_name = NULL;
9639 char *format_name = NULL;
9640 unsigned int offset;
9641 unsigned int lsb;
9642 unsigned int msb;
9643 unsigned int guid;
9644 unsigned int domain_id;
9645 enum counter_scope scope = 0;
9646 enum pmt_datatype type = PMT_TYPE_RAW;
9647 enum counter_format format = FORMAT_RAW;
9648 bool has_offset = false;
9649 bool has_lsb = false;
9650 bool has_msb = false;
9651 bool has_format = true; /* Format has a default value. */
9652 bool has_guid = false;
9653 bool has_scope = false;
9654 bool has_type = true; /* Type has a default value. */
9655
9656 /* Consume the "pmt," prefix. */
9657 add_command = strchr(add_command, ',');
9658 if (!add_command) {
9659 help();
9660 exit(1);
9661 }
9662 ++add_command;
9663
9664 while (add_command) {
9665 if (starts_with(add_command, "name=")) {
9666 name = add_command + strlen("name=");
9667 goto next;
9668 }
9669
9670 if (starts_with(add_command, "type=")) {
9671 type_name = add_command + strlen("type=");
9672 goto next;
9673 }
9674
9675 if (starts_with(add_command, "domain=")) {
9676 const size_t prefix_len = strlen("domain=");
9677
9678 if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) {
9679 scope = SCOPE_CPU;
9680 has_scope = true;
9681 } else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) {
9682 scope = SCOPE_CORE;
9683 has_scope = true;
9684 } else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) {
9685 scope = SCOPE_PACKAGE;
9686 has_scope = true;
9687 }
9688
9689 if (!has_scope) {
9690 printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n",
9691 __func__);
9692 exit(1);
9693 }
9694
9695 goto next;
9696 }
9697
9698 if (starts_with(add_command, "format=")) {
9699 format_name = add_command + strlen("format=");
9700 goto next;
9701 }
9702
9703 if (sscanf(add_command, "offset=%u", &offset) == 1) {
9704 has_offset = true;
9705 goto next;
9706 }
9707
9708 if (sscanf(add_command, "lsb=%u", &lsb) == 1) {
9709 has_lsb = true;
9710 goto next;
9711 }
9712
9713 if (sscanf(add_command, "msb=%u", &msb) == 1) {
9714 has_msb = true;
9715 goto next;
9716 }
9717
9718 if (sscanf(add_command, "guid=%x", &guid) == 1) {
9719 has_guid = true;
9720 goto next;
9721 }
9722
9723 next:
9724 add_command = strchr(add_command, ',');
9725 if (add_command) {
9726 *add_command = '\0';
9727 add_command++;
9728 }
9729 }
9730
9731 if (!name) {
9732 printf("%s: missing %s\n", __func__, "name");
9733 exit(1);
9734 }
9735
9736 if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) {
9737 printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES);
9738 exit(1);
9739 }
9740
9741 if (format_name) {
9742 has_format = false;
9743
9744 if (strcmp("raw", format_name) == 0) {
9745 format = FORMAT_RAW;
9746 has_format = true;
9747 }
9748
9749 if (strcmp("delta", format_name) == 0) {
9750 format = FORMAT_DELTA;
9751 has_format = true;
9752 }
9753
9754 if (!has_format) {
9755 fprintf(stderr, "%s: Invalid format %s. Expected raw or delta\n", __func__, format_name);
9756 exit(1);
9757 }
9758 }
9759
9760 if (type_name) {
9761 has_type = false;
9762
9763 if (strcmp("raw", type_name) == 0) {
9764 type = PMT_TYPE_RAW;
9765 has_type = true;
9766 }
9767
9768 if (strcmp("txtal_time", type_name) == 0) {
9769 type = PMT_TYPE_XTAL_TIME;
9770 has_type = true;
9771 }
9772
9773 if (!has_type) {
9774 printf("%s: invalid %s: %s\n", __func__, "type", type_name);
9775 exit(1);
9776 }
9777 }
9778
9779 if (!has_offset) {
9780 printf("%s : missing %s\n", __func__, "offset");
9781 exit(1);
9782 }
9783
9784 if (!has_lsb) {
9785 printf("%s: missing %s\n", __func__, "lsb");
9786 exit(1);
9787 }
9788
9789 if (!has_msb) {
9790 printf("%s: missing %s\n", __func__, "msb");
9791 exit(1);
9792 }
9793
9794 if (!has_guid) {
9795 printf("%s: missing %s\n", __func__, "guid");
9796 exit(1);
9797 }
9798
9799 if (!has_scope) {
9800 printf("%s: missing %s\n", __func__, "scope");
9801 exit(1);
9802 }
9803
9804 if (lsb > msb) {
9805 printf("%s: lsb > msb doesn't make sense\n", __func__);
9806 exit(1);
9807 }
9808
9809 pmt_add_counter(guid, name, type, lsb, msb, offset, scope, format, domain_id, PMT_OPEN_REQUIRED);
9810 }
9811
parse_add_command(char * add_command)9812 void parse_add_command(char *add_command)
9813 {
9814 if (strncmp(add_command, "pmt", strlen("pmt")) == 0)
9815 return parse_add_command_pmt(add_command);
9816 return parse_add_command_msr(add_command);
9817 }
9818
is_deferred_add(char * name)9819 int is_deferred_add(char *name)
9820 {
9821 int i;
9822
9823 for (i = 0; i < deferred_add_index; ++i)
9824 if (!strcmp(name, deferred_add_names[i]))
9825 return 1;
9826 return 0;
9827 }
9828
is_deferred_skip(char * name)9829 int is_deferred_skip(char *name)
9830 {
9831 int i;
9832
9833 for (i = 0; i < deferred_skip_index; ++i)
9834 if (!strcmp(name, deferred_skip_names[i]))
9835 return 1;
9836 return 0;
9837 }
9838
probe_sysfs(void)9839 void probe_sysfs(void)
9840 {
9841 char path[64];
9842 char name_buf[16];
9843 FILE *input;
9844 int state;
9845 char *sp;
9846
9847 for (state = 10; state >= 0; --state) {
9848
9849 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
9850 input = fopen(path, "r");
9851 if (input == NULL)
9852 continue;
9853 if (!fgets(name_buf, sizeof(name_buf), input))
9854 err(1, "%s: failed to read file", path);
9855
9856 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
9857 sp = strchr(name_buf, '-');
9858 if (!sp)
9859 sp = strchrnul(name_buf, '\n');
9860 *sp = '%';
9861 *(sp + 1) = '\0';
9862
9863 remove_underbar(name_buf);
9864
9865 fclose(input);
9866
9867 sprintf(path, "cpuidle/state%d/time", state);
9868
9869 if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf))
9870 continue;
9871
9872 if (is_deferred_skip(name_buf))
9873 continue;
9874
9875 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0);
9876 }
9877
9878 for (state = 10; state >= 0; --state) {
9879
9880 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
9881 input = fopen(path, "r");
9882 if (input == NULL)
9883 continue;
9884 if (!fgets(name_buf, sizeof(name_buf), input))
9885 err(1, "%s: failed to read file", path);
9886 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
9887 sp = strchr(name_buf, '-');
9888 if (!sp)
9889 sp = strchrnul(name_buf, '\n');
9890 *sp = '\0';
9891 fclose(input);
9892
9893 remove_underbar(name_buf);
9894
9895 sprintf(path, "cpuidle/state%d/usage", state);
9896
9897 if (!DO_BIC(BIC_sysfs) && !is_deferred_add(name_buf))
9898 continue;
9899
9900 if (is_deferred_skip(name_buf))
9901 continue;
9902
9903 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
9904 }
9905
9906 }
9907
9908 /*
9909 * parse cpuset with following syntax
9910 * 1,2,4..6,8-10 and set bits in cpu_subset
9911 */
parse_cpu_command(char * optarg)9912 void parse_cpu_command(char *optarg)
9913 {
9914 if (!strcmp(optarg, "core")) {
9915 if (cpu_subset)
9916 goto error;
9917 show_core_only++;
9918 return;
9919 }
9920 if (!strcmp(optarg, "package")) {
9921 if (cpu_subset)
9922 goto error;
9923 show_pkg_only++;
9924 return;
9925 }
9926 if (show_core_only || show_pkg_only)
9927 goto error;
9928
9929 cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
9930 if (cpu_subset == NULL)
9931 err(3, "CPU_ALLOC");
9932 cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
9933
9934 CPU_ZERO_S(cpu_subset_size, cpu_subset);
9935
9936 if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
9937 goto error;
9938
9939 return;
9940
9941 error:
9942 fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
9943 help();
9944 exit(-1);
9945 }
9946
cmdline(int argc,char ** argv)9947 void cmdline(int argc, char **argv)
9948 {
9949 int opt;
9950 int option_index = 0;
9951 static struct option long_options[] = {
9952 { "add", required_argument, 0, 'a' },
9953 { "cpu", required_argument, 0, 'c' },
9954 { "Dump", no_argument, 0, 'D' },
9955 { "debug", no_argument, 0, 'd' }, /* internal, not documented */
9956 { "enable", required_argument, 0, 'e' },
9957 { "interval", required_argument, 0, 'i' },
9958 { "IPC", no_argument, 0, 'I' },
9959 { "num_iterations", required_argument, 0, 'n' },
9960 { "header_iterations", required_argument, 0, 'N' },
9961 { "help", no_argument, 0, 'h' },
9962 { "hide", required_argument, 0, 'H' }, // meh, -h taken by --help
9963 { "Joules", no_argument, 0, 'J' },
9964 { "list", no_argument, 0, 'l' },
9965 { "out", required_argument, 0, 'o' },
9966 { "quiet", no_argument, 0, 'q' },
9967 { "no-msr", no_argument, 0, 'M' },
9968 { "no-perf", no_argument, 0, 'P' },
9969 { "show", required_argument, 0, 's' },
9970 { "Summary", no_argument, 0, 'S' },
9971 { "TCC", required_argument, 0, 'T' },
9972 { "version", no_argument, 0, 'v' },
9973 { 0, 0, 0, 0 }
9974 };
9975
9976 progname = argv[0];
9977
9978 /*
9979 * Parse some options early, because they may make other options invalid,
9980 * like adding the MSR counter with --add and at the same time using --no-msr.
9981 */
9982 while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) {
9983 switch (opt) {
9984 case 'M':
9985 no_msr = 1;
9986 break;
9987 case 'P':
9988 no_perf = 1;
9989 break;
9990 default:
9991 break;
9992 }
9993 }
9994 optind = 0;
9995
9996 while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qMST:v", long_options, &option_index)) != -1) {
9997 switch (opt) {
9998 case 'a':
9999 parse_add_command(optarg);
10000 break;
10001 case 'c':
10002 parse_cpu_command(optarg);
10003 break;
10004 case 'D':
10005 dump_only++;
10006 break;
10007 case 'e':
10008 /* --enable specified counter */
10009 bic_enabled = bic_enabled | bic_lookup(optarg, SHOW_LIST);
10010 break;
10011 case 'd':
10012 debug++;
10013 ENABLE_BIC(BIC_DISABLED_BY_DEFAULT);
10014 break;
10015 case 'H':
10016 /*
10017 * --hide: do not show those specified
10018 * multiple invocations simply clear more bits in enabled mask
10019 */
10020 bic_enabled &= ~bic_lookup(optarg, HIDE_LIST);
10021 break;
10022 case 'h':
10023 default:
10024 help();
10025 exit(1);
10026 case 'i':
10027 {
10028 double interval = strtod(optarg, NULL);
10029
10030 if (interval < 0.001) {
10031 fprintf(outf, "interval %f seconds is too small\n", interval);
10032 exit(2);
10033 }
10034
10035 interval_tv.tv_sec = interval_ts.tv_sec = interval;
10036 interval_tv.tv_usec = (interval - interval_tv.tv_sec) * 1000000;
10037 interval_ts.tv_nsec = (interval - interval_ts.tv_sec) * 1000000000;
10038 }
10039 break;
10040 case 'J':
10041 rapl_joules++;
10042 break;
10043 case 'l':
10044 ENABLE_BIC(BIC_DISABLED_BY_DEFAULT);
10045 list_header_only++;
10046 quiet++;
10047 break;
10048 case 'o':
10049 outf = fopen_or_die(optarg, "w");
10050 break;
10051 case 'q':
10052 quiet = 1;
10053 break;
10054 case 'M':
10055 case 'P':
10056 /* Parsed earlier */
10057 break;
10058 case 'n':
10059 num_iterations = strtod(optarg, NULL);
10060
10061 if (num_iterations <= 0) {
10062 fprintf(outf, "iterations %d should be positive number\n", num_iterations);
10063 exit(2);
10064 }
10065 break;
10066 case 'N':
10067 header_iterations = strtod(optarg, NULL);
10068
10069 if (header_iterations <= 0) {
10070 fprintf(outf, "iterations %d should be positive number\n", header_iterations);
10071 exit(2);
10072 }
10073 break;
10074 case 's':
10075 /*
10076 * --show: show only those specified
10077 * The 1st invocation will clear and replace the enabled mask
10078 * subsequent invocations can add to it.
10079 */
10080 if (shown == 0)
10081 bic_enabled = bic_lookup(optarg, SHOW_LIST);
10082 else
10083 bic_enabled |= bic_lookup(optarg, SHOW_LIST);
10084 shown = 1;
10085 break;
10086 case 'S':
10087 summary_only++;
10088 break;
10089 case 'T':
10090 tj_max_override = atoi(optarg);
10091 break;
10092 case 'v':
10093 print_version();
10094 exit(0);
10095 break;
10096 }
10097 }
10098 }
10099
set_rlimit(void)10100 void set_rlimit(void)
10101 {
10102 struct rlimit limit;
10103
10104 if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
10105 err(1, "Failed to get rlimit");
10106
10107 if (limit.rlim_max < MAX_NOFILE)
10108 limit.rlim_max = MAX_NOFILE;
10109 if (limit.rlim_cur < MAX_NOFILE)
10110 limit.rlim_cur = MAX_NOFILE;
10111
10112 if (setrlimit(RLIMIT_NOFILE, &limit) < 0)
10113 err(1, "Failed to set rlimit");
10114 }
10115
main(int argc,char ** argv)10116 int main(int argc, char **argv)
10117 {
10118 int fd, ret;
10119
10120 fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY);
10121 if (fd < 0)
10122 goto skip_cgroup_setting;
10123
10124 ret = write(fd, "0\n", 2);
10125 if (ret == -1)
10126 perror("Can't update cgroup\n");
10127
10128 close(fd);
10129
10130 skip_cgroup_setting:
10131 outf = stderr;
10132 cmdline(argc, argv);
10133
10134 if (!quiet) {
10135 print_version();
10136 print_bootcmd();
10137 }
10138
10139 probe_sysfs();
10140
10141 if (!getuid())
10142 set_rlimit();
10143
10144 turbostat_init();
10145
10146 if (!no_msr)
10147 msr_sum_record();
10148
10149 /* dump counters and exit */
10150 if (dump_only)
10151 return get_and_dump_counters();
10152
10153 /* list header and exit */
10154 if (list_header_only) {
10155 print_header(",");
10156 flush_output_stdout();
10157 return 0;
10158 }
10159
10160 /*
10161 * if any params left, it must be a command to fork
10162 */
10163 if (argc - optind)
10164 return fork_it(argv + optind);
10165 else
10166 turbostat_loop();
10167
10168 return 0;
10169 }
10170