1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * (C) 2010,2011 Thomas Renninger <trenn@suse.de>, Novell Inc.
4 */
5
6 #if defined(__i386__) || defined(__x86_64__)
7
8 #include <stdio.h>
9 #include <stdint.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <limits.h>
13
14 #include <cpufreq.h>
15
16 #include "helpers/helpers.h"
17 #include "idle_monitor/cpupower-monitor.h"
18
19 #define MSR_APERF 0xE8
20 #define MSR_MPERF 0xE7
21
22 #define MSR_TSC 0x10
23
24 #define MSR_AMD_HWCR 0xc0010015
25
26 enum mperf_id { C0 = 0, Cx, AVG_FREQ, MPERF_CSTATE_COUNT };
27
28 static int mperf_get_count_percent(unsigned int self_id, double *percent,
29 unsigned int cpu);
30 static int mperf_get_count_freq(unsigned int id, unsigned long long *count,
31 unsigned int cpu);
32 static struct timespec time_start, time_end;
33
34 static cstate_t mperf_cstates[MPERF_CSTATE_COUNT] = {
35 {
36 .name = "C0",
37 .desc = N_("Processor Core not idle"),
38 .id = C0,
39 .range = RANGE_THREAD,
40 .get_count_percent = mperf_get_count_percent,
41 },
42 {
43 .name = "Cx",
44 .desc = N_("Processor Core in an idle state"),
45 .id = Cx,
46 .range = RANGE_THREAD,
47 .get_count_percent = mperf_get_count_percent,
48 },
49
50 {
51 .name = "Freq",
52 .desc = N_("Average Frequency (including boost) in MHz"),
53 .id = AVG_FREQ,
54 .range = RANGE_THREAD,
55 .get_count = mperf_get_count_freq,
56 },
57 };
58
59 enum MAX_FREQ_MODE { MAX_FREQ_SYSFS, MAX_FREQ_TSC_REF };
60 static int max_freq_mode;
61 /*
62 * The max frequency mperf is ticking at (in C0), either retrieved via:
63 * 1) calculated after measurements if we know TSC ticks at mperf/P0 frequency
64 * 2) cpufreq /sys/devices/.../cpu0/cpufreq/cpuinfo_max_freq at init time
65 * 1. Is preferred as it also works without cpufreq subsystem (e.g. on Xen)
66 */
67 static unsigned long max_frequency;
68
69 static unsigned long long *tsc_at_measure_start;
70 static unsigned long long *tsc_at_measure_end;
71 static unsigned long long *mperf_previous_count;
72 static unsigned long long *aperf_previous_count;
73 static unsigned long long *mperf_current_count;
74 static unsigned long long *aperf_current_count;
75
76 /* valid flag for all CPUs. If a MSR read failed it will be zero */
77 static int *is_valid;
78
mperf_get_tsc(unsigned long long * tsc)79 static int mperf_get_tsc(unsigned long long *tsc)
80 {
81 int ret;
82
83 ret = read_msr(base_cpu, MSR_TSC, tsc);
84 if (ret)
85 dprint("Reading TSC MSR failed, returning %llu\n", *tsc);
86 return ret;
87 }
88
mperf_init_stats(unsigned int cpu)89 static int mperf_init_stats(unsigned int cpu)
90 {
91 unsigned long long val;
92 int ret;
93
94 ret = read_msr(cpu, MSR_APERF, &val);
95 aperf_previous_count[cpu] = val;
96 ret |= read_msr(cpu, MSR_MPERF, &val);
97 mperf_previous_count[cpu] = val;
98 is_valid[cpu] = !ret;
99
100 return 0;
101 }
102
mperf_measure_stats(unsigned int cpu)103 static int mperf_measure_stats(unsigned int cpu)
104 {
105 unsigned long long val;
106 int ret;
107
108 ret = read_msr(cpu, MSR_APERF, &val);
109 aperf_current_count[cpu] = val;
110 ret |= read_msr(cpu, MSR_MPERF, &val);
111 mperf_current_count[cpu] = val;
112 is_valid[cpu] = !ret;
113
114 return 0;
115 }
116
mperf_get_count_percent(unsigned int id,double * percent,unsigned int cpu)117 static int mperf_get_count_percent(unsigned int id, double *percent,
118 unsigned int cpu)
119 {
120 unsigned long long aperf_diff, mperf_diff, tsc_diff;
121 unsigned long long timediff;
122
123 if (!is_valid[cpu])
124 return -1;
125
126 if (id != C0 && id != Cx)
127 return -1;
128
129 mperf_diff = mperf_current_count[cpu] - mperf_previous_count[cpu];
130 aperf_diff = aperf_current_count[cpu] - aperf_previous_count[cpu];
131
132 if (max_freq_mode == MAX_FREQ_TSC_REF) {
133 tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu];
134 *percent = 100.0 * mperf_diff / tsc_diff;
135 dprint("%s: TSC Ref - mperf_diff: %llu, tsc_diff: %llu\n",
136 mperf_cstates[id].name, mperf_diff, tsc_diff);
137 } else if (max_freq_mode == MAX_FREQ_SYSFS) {
138 timediff = max_frequency * timespec_diff_us(time_start, time_end);
139 *percent = 100.0 * mperf_diff / timediff;
140 dprint("%s: MAXFREQ - mperf_diff: %llu, time_diff: %llu\n",
141 mperf_cstates[id].name, mperf_diff, timediff);
142 } else
143 return -1;
144
145 if (id == Cx)
146 *percent = 100.0 - *percent;
147
148 dprint("%s: previous: %llu - current: %llu - (%u)\n",
149 mperf_cstates[id].name, mperf_diff, aperf_diff, cpu);
150 dprint("%s: %f\n", mperf_cstates[id].name, *percent);
151 return 0;
152 }
153
mperf_get_count_freq(unsigned int id,unsigned long long * count,unsigned int cpu)154 static int mperf_get_count_freq(unsigned int id, unsigned long long *count,
155 unsigned int cpu)
156 {
157 unsigned long long aperf_diff, mperf_diff, time_diff, tsc_diff;
158
159 if (id != AVG_FREQ)
160 return 1;
161
162 if (!is_valid[cpu])
163 return -1;
164
165 mperf_diff = mperf_current_count[cpu] - mperf_previous_count[cpu];
166 aperf_diff = aperf_current_count[cpu] - aperf_previous_count[cpu];
167
168 if (max_freq_mode == MAX_FREQ_TSC_REF) {
169 /* Calculate max_freq from TSC count */
170 tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu];
171 time_diff = timespec_diff_us(time_start, time_end);
172 max_frequency = tsc_diff / time_diff;
173 }
174
175 *count = max_frequency * ((double)aperf_diff / mperf_diff);
176 dprint("%s: Average freq based on %s maximum frequency:\n",
177 mperf_cstates[id].name,
178 (max_freq_mode == MAX_FREQ_TSC_REF) ? "TSC calculated" : "sysfs read");
179 dprint("max_frequency: %lu\n", max_frequency);
180 dprint("aperf_diff: %llu\n", aperf_diff);
181 dprint("mperf_diff: %llu\n", mperf_diff);
182 dprint("avg freq: %llu\n", *count);
183 return 0;
184 }
185
mperf_start(void)186 static int mperf_start(void)
187 {
188 int cpu;
189
190 clock_gettime(CLOCK_REALTIME, &time_start);
191
192 for (cpu = 0; cpu < cpu_count; cpu++) {
193 mperf_get_tsc(&tsc_at_measure_start[cpu]);
194 mperf_init_stats(cpu);
195 }
196
197 return 0;
198 }
199
mperf_stop(void)200 static int mperf_stop(void)
201 {
202 int cpu;
203
204 for (cpu = 0; cpu < cpu_count; cpu++) {
205 mperf_measure_stats(cpu);
206 mperf_get_tsc(&tsc_at_measure_end[cpu]);
207 }
208
209 clock_gettime(CLOCK_REALTIME, &time_end);
210 return 0;
211 }
212
213 /*
214 * Mperf register is defined to tick at P0 (maximum) frequency
215 *
216 * Instead of reading out P0 which can be tricky to read out from HW,
217 * we use TSC counter if it reliably ticks at P0/mperf frequency.
218 *
219 * Still try to fall back to:
220 * /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq
221 * on older Intel HW without invariant TSC feature.
222 * Or on AMD machines where TSC does not tick at P0 (do not exist yet, but
223 * it's still double checked (MSR_AMD_HWCR)).
224 *
225 * On these machines the user would still get useful mperf
226 * stats when acpi-cpufreq driver is loaded.
227 */
init_maxfreq_mode(void)228 static int init_maxfreq_mode(void)
229 {
230 int ret;
231 unsigned long long hwcr;
232 unsigned long min;
233
234 if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_INV_TSC))
235 goto use_sysfs;
236
237 if (cpupower_cpu_info.vendor == X86_VENDOR_AMD ||
238 cpupower_cpu_info.vendor == X86_VENDOR_HYGON) {
239 /* MSR_AMD_HWCR tells us whether TSC runs at P0/mperf
240 * freq.
241 * A test whether hwcr is accessable/available would be:
242 * (cpupower_cpu_info.family > 0x10 ||
243 * cpupower_cpu_info.family == 0x10 &&
244 * cpupower_cpu_info.model >= 0x2))
245 * This should be the case for all aperf/mperf
246 * capable AMD machines and is therefore safe to test here.
247 * Compare with Linus kernel git commit: acf01734b1747b1ec4
248 */
249 ret = read_msr(0, MSR_AMD_HWCR, &hwcr);
250 /*
251 * If the MSR read failed, assume a Xen system that did
252 * not explicitly provide access to it and assume TSC works
253 */
254 if (ret != 0) {
255 dprint("TSC read 0x%x failed - assume TSC working\n",
256 MSR_AMD_HWCR);
257 return 0;
258 } else if (1 & (hwcr >> 24)) {
259 max_freq_mode = MAX_FREQ_TSC_REF;
260 return 0;
261 } else { /* Use sysfs max frequency if available */ }
262 } else if (cpupower_cpu_info.vendor == X86_VENDOR_INTEL) {
263 /*
264 * On Intel we assume mperf (in C0) is ticking at same
265 * rate than TSC
266 */
267 max_freq_mode = MAX_FREQ_TSC_REF;
268 return 0;
269 }
270 use_sysfs:
271 if (cpufreq_get_hardware_limits(0, &min, &max_frequency)) {
272 dprint("Cannot retrieve max freq from cpufreq kernel "
273 "subsystem\n");
274 return -1;
275 }
276 max_freq_mode = MAX_FREQ_SYSFS;
277 max_frequency /= 1000; /* Default automatically to MHz value */
278 return 0;
279 }
280
281 /*
282 * This monitor provides:
283 *
284 * 1) Average frequency a CPU resided in
285 * This always works if the CPU has aperf/mperf capabilities
286 *
287 * 2) C0 and Cx (any sleep state) time a CPU resided in
288 * Works if mperf timer stops ticking in sleep states which
289 * seem to be the case on all current HW.
290 * Both is directly retrieved from HW registers and is independent
291 * from kernel statistics.
292 */
293 struct cpuidle_monitor mperf_monitor;
mperf_register(void)294 struct cpuidle_monitor *mperf_register(void)
295 {
296 if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF))
297 return NULL;
298
299 if (init_maxfreq_mode())
300 return NULL;
301
302 /* Free this at program termination */
303 is_valid = calloc(cpu_count, sizeof(int));
304 mperf_previous_count = calloc(cpu_count, sizeof(unsigned long long));
305 aperf_previous_count = calloc(cpu_count, sizeof(unsigned long long));
306 mperf_current_count = calloc(cpu_count, sizeof(unsigned long long));
307 aperf_current_count = calloc(cpu_count, sizeof(unsigned long long));
308 tsc_at_measure_start = calloc(cpu_count, sizeof(unsigned long long));
309 tsc_at_measure_end = calloc(cpu_count, sizeof(unsigned long long));
310 mperf_monitor.name_len = strlen(mperf_monitor.name);
311 return &mperf_monitor;
312 }
313
mperf_unregister(void)314 void mperf_unregister(void)
315 {
316 free(mperf_previous_count);
317 free(aperf_previous_count);
318 free(mperf_current_count);
319 free(aperf_current_count);
320 free(tsc_at_measure_start);
321 free(tsc_at_measure_end);
322 free(is_valid);
323 }
324
325 struct cpuidle_monitor mperf_monitor = {
326 .name = "Mperf",
327 .hw_states_num = MPERF_CSTATE_COUNT,
328 .hw_states = mperf_cstates,
329 .start = mperf_start,
330 .stop = mperf_stop,
331 .do_register = mperf_register,
332 .unregister = mperf_unregister,
333 .needs_root = 1,
334 .overflow_s = 922000000 /* 922337203 seconds TSC overflow
335 at 20GHz */
336 };
337 #endif /* #if defined(__i386__) || defined(__x86_64__) */
338