1 /*
2 * @file op_model_amd.c
3 * athlon / K7 / K8 / Family 10h model-specific MSR operations
4 *
5 * @remark Copyright 2002-2009 OProfile authors
6 * @remark Read the file COPYING
7 *
8 * @author John Levon
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf <barry.kasindorf@amd.com>
13 * @author Jason Yeh <jason.yeh@amd.com>
14 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
15 */
16
17 #include <linux/oprofile.h>
18 #include <linux/device.h>
19 #include <linux/pci.h>
20 #include <linux/percpu.h>
21
22 #include <asm/ptrace.h>
23 #include <asm/msr.h>
24 #include <asm/nmi.h>
25 #include <asm/apic.h>
26 #include <asm/processor.h>
27 #include <asm/cpufeature.h>
28
29 #include "op_x86_model.h"
30 #include "op_counter.h"
31
32 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
33 #define NUM_VIRT_COUNTERS 32
34 #else
35 #define NUM_VIRT_COUNTERS 0
36 #endif
37
38 #define OP_EVENT_MASK 0x0FFF
39 #define OP_CTR_OVERFLOW (1ULL<<31)
40
41 #define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
42
43 static int num_counters;
44 static unsigned long reset_value[OP_MAX_COUNTER];
45
46 #define IBS_FETCH_SIZE 6
47 #define IBS_OP_SIZE 12
48
49 static u32 ibs_caps;
50
51 struct ibs_config {
52 unsigned long op_enabled;
53 unsigned long fetch_enabled;
54 unsigned long max_cnt_fetch;
55 unsigned long max_cnt_op;
56 unsigned long rand_en;
57 unsigned long dispatched_ops;
58 unsigned long branch_target;
59 };
60
61 struct ibs_state {
62 u64 ibs_op_ctl;
63 int branch_target;
64 unsigned long sample_size;
65 };
66
67 static struct ibs_config ibs_config;
68 static struct ibs_state ibs_state;
69
70 /*
71 * IBS randomization macros
72 */
73 #define IBS_RANDOM_BITS 12
74 #define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1)
75 #define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5))
76
77 /*
78 * 16-bit Linear Feedback Shift Register (LFSR)
79 *
80 * 16 14 13 11
81 * Feedback polynomial = X + X + X + X + 1
82 */
lfsr_random(void)83 static unsigned int lfsr_random(void)
84 {
85 static unsigned int lfsr_value = 0xF00D;
86 unsigned int bit;
87
88 /* Compute next bit to shift in */
89 bit = ((lfsr_value >> 0) ^
90 (lfsr_value >> 2) ^
91 (lfsr_value >> 3) ^
92 (lfsr_value >> 5)) & 0x0001;
93
94 /* Advance to next register value */
95 lfsr_value = (lfsr_value >> 1) | (bit << 15);
96
97 return lfsr_value;
98 }
99
100 /*
101 * IBS software randomization
102 *
103 * The IBS periodic op counter is randomized in software. The lower 12
104 * bits of the 20 bit counter are randomized. IbsOpCurCnt is
105 * initialized with a 12 bit random value.
106 */
op_amd_randomize_ibs_op(u64 val)107 static inline u64 op_amd_randomize_ibs_op(u64 val)
108 {
109 unsigned int random = lfsr_random();
110
111 if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
112 /*
113 * Work around if the hw can not write to IbsOpCurCnt
114 *
115 * Randomize the lower 8 bits of the 16 bit
116 * IbsOpMaxCnt [15:0] value in the range of -128 to
117 * +127 by adding/subtracting an offset to the
118 * maximum count (IbsOpMaxCnt).
119 *
120 * To avoid over or underflows and protect upper bits
121 * starting at bit 16, the initial value for
122 * IbsOpMaxCnt must fit in the range from 0x0081 to
123 * 0xff80.
124 */
125 val += (s8)(random >> 4);
126 else
127 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
128
129 return val;
130 }
131
132 static inline void
op_amd_handle_ibs(struct pt_regs * const regs,struct op_msrs const * const msrs)133 op_amd_handle_ibs(struct pt_regs * const regs,
134 struct op_msrs const * const msrs)
135 {
136 u64 val, ctl;
137 struct op_entry entry;
138
139 if (!ibs_caps)
140 return;
141
142 if (ibs_config.fetch_enabled) {
143 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
144 if (ctl & IBS_FETCH_VAL) {
145 rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
146 oprofile_write_reserve(&entry, regs, val,
147 IBS_FETCH_CODE, IBS_FETCH_SIZE);
148 oprofile_add_data64(&entry, val);
149 oprofile_add_data64(&entry, ctl);
150 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
151 oprofile_add_data64(&entry, val);
152 oprofile_write_commit(&entry);
153
154 /* reenable the IRQ */
155 ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
156 ctl |= IBS_FETCH_ENABLE;
157 wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
158 }
159 }
160
161 if (ibs_config.op_enabled) {
162 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
163 if (ctl & IBS_OP_VAL) {
164 rdmsrl(MSR_AMD64_IBSOPRIP, val);
165 oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
166 ibs_state.sample_size);
167 oprofile_add_data64(&entry, val);
168 rdmsrl(MSR_AMD64_IBSOPDATA, val);
169 oprofile_add_data64(&entry, val);
170 rdmsrl(MSR_AMD64_IBSOPDATA2, val);
171 oprofile_add_data64(&entry, val);
172 rdmsrl(MSR_AMD64_IBSOPDATA3, val);
173 oprofile_add_data64(&entry, val);
174 rdmsrl(MSR_AMD64_IBSDCLINAD, val);
175 oprofile_add_data64(&entry, val);
176 rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
177 oprofile_add_data64(&entry, val);
178 if (ibs_state.branch_target) {
179 rdmsrl(MSR_AMD64_IBSBRTARGET, val);
180 oprofile_add_data(&entry, (unsigned long)val);
181 }
182 oprofile_write_commit(&entry);
183
184 /* reenable the IRQ */
185 ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
186 wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
187 }
188 }
189 }
190
op_amd_start_ibs(void)191 static inline void op_amd_start_ibs(void)
192 {
193 u64 val;
194
195 if (!ibs_caps)
196 return;
197
198 memset(&ibs_state, 0, sizeof(ibs_state));
199
200 /*
201 * Note: Since the max count settings may out of range we
202 * write back the actual used values so that userland can read
203 * it.
204 */
205
206 if (ibs_config.fetch_enabled) {
207 val = ibs_config.max_cnt_fetch >> 4;
208 val = min(val, IBS_FETCH_MAX_CNT);
209 ibs_config.max_cnt_fetch = val << 4;
210 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
211 val |= IBS_FETCH_ENABLE;
212 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
213 }
214
215 if (ibs_config.op_enabled) {
216 val = ibs_config.max_cnt_op >> 4;
217 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
218 /*
219 * IbsOpCurCnt not supported. See
220 * op_amd_randomize_ibs_op() for details.
221 */
222 val = clamp(val, 0x0081ULL, 0xFF80ULL);
223 ibs_config.max_cnt_op = val << 4;
224 } else {
225 /*
226 * The start value is randomized with a
227 * positive offset, we need to compensate it
228 * with the half of the randomized range. Also
229 * avoid underflows.
230 */
231 val += IBS_RANDOM_MAXCNT_OFFSET;
232 if (ibs_caps & IBS_CAPS_OPCNTEXT)
233 val = min(val, IBS_OP_MAX_CNT_EXT);
234 else
235 val = min(val, IBS_OP_MAX_CNT);
236 ibs_config.max_cnt_op =
237 (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
238 }
239 val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
240 val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
241 val |= IBS_OP_ENABLE;
242 ibs_state.ibs_op_ctl = val;
243 ibs_state.sample_size = IBS_OP_SIZE;
244 if (ibs_config.branch_target) {
245 ibs_state.branch_target = 1;
246 ibs_state.sample_size++;
247 }
248 val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
249 wrmsrl(MSR_AMD64_IBSOPCTL, val);
250 }
251 }
252
op_amd_stop_ibs(void)253 static void op_amd_stop_ibs(void)
254 {
255 if (!ibs_caps)
256 return;
257
258 if (ibs_config.fetch_enabled)
259 /* clear max count and enable */
260 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
261
262 if (ibs_config.op_enabled)
263 /* clear max count and enable */
264 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
265 }
266
267 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
268
op_mux_switch_ctrl(struct op_x86_model_spec const * model,struct op_msrs const * const msrs)269 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
270 struct op_msrs const * const msrs)
271 {
272 u64 val;
273 int i;
274
275 /* enable active counters */
276 for (i = 0; i < num_counters; ++i) {
277 int virt = op_x86_phys_to_virt(i);
278 if (!reset_value[virt])
279 continue;
280 rdmsrl(msrs->controls[i].addr, val);
281 val &= model->reserved;
282 val |= op_x86_get_ctrl(model, &counter_config[virt]);
283 wrmsrl(msrs->controls[i].addr, val);
284 }
285 }
286
287 #endif
288
289 /* functions for op_amd_spec */
290
op_amd_shutdown(struct op_msrs const * const msrs)291 static void op_amd_shutdown(struct op_msrs const * const msrs)
292 {
293 int i;
294
295 for (i = 0; i < num_counters; ++i) {
296 if (!msrs->counters[i].addr)
297 continue;
298 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
299 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
300 }
301 }
302
op_amd_fill_in_addresses(struct op_msrs * const msrs)303 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
304 {
305 int i;
306
307 for (i = 0; i < num_counters; i++) {
308 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
309 goto fail;
310 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
311 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
312 goto fail;
313 }
314 /* both registers must be reserved */
315 if (num_counters == AMD64_NUM_COUNTERS_F15H) {
316 msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
317 msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
318 } else {
319 msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
320 msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
321 }
322 continue;
323 fail:
324 if (!counter_config[i].enabled)
325 continue;
326 op_x86_warn_reserved(i);
327 op_amd_shutdown(msrs);
328 return -EBUSY;
329 }
330
331 return 0;
332 }
333
op_amd_setup_ctrs(struct op_x86_model_spec const * model,struct op_msrs const * const msrs)334 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
335 struct op_msrs const * const msrs)
336 {
337 u64 val;
338 int i;
339
340 /* setup reset_value */
341 for (i = 0; i < OP_MAX_COUNTER; ++i) {
342 if (counter_config[i].enabled
343 && msrs->counters[op_x86_virt_to_phys(i)].addr)
344 reset_value[i] = counter_config[i].count;
345 else
346 reset_value[i] = 0;
347 }
348
349 /* clear all counters */
350 for (i = 0; i < num_counters; ++i) {
351 if (!msrs->controls[i].addr)
352 continue;
353 rdmsrl(msrs->controls[i].addr, val);
354 if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
355 op_x86_warn_in_use(i);
356 val &= model->reserved;
357 wrmsrl(msrs->controls[i].addr, val);
358 /*
359 * avoid a false detection of ctr overflows in NMI
360 * handler
361 */
362 wrmsrl(msrs->counters[i].addr, -1LL);
363 }
364
365 /* enable active counters */
366 for (i = 0; i < num_counters; ++i) {
367 int virt = op_x86_phys_to_virt(i);
368 if (!reset_value[virt])
369 continue;
370
371 /* setup counter registers */
372 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
373
374 /* setup control registers */
375 rdmsrl(msrs->controls[i].addr, val);
376 val &= model->reserved;
377 val |= op_x86_get_ctrl(model, &counter_config[virt]);
378 wrmsrl(msrs->controls[i].addr, val);
379 }
380 }
381
op_amd_check_ctrs(struct pt_regs * const regs,struct op_msrs const * const msrs)382 static int op_amd_check_ctrs(struct pt_regs * const regs,
383 struct op_msrs const * const msrs)
384 {
385 u64 val;
386 int i;
387
388 for (i = 0; i < num_counters; ++i) {
389 int virt = op_x86_phys_to_virt(i);
390 if (!reset_value[virt])
391 continue;
392 rdmsrl(msrs->counters[i].addr, val);
393 /* bit is clear if overflowed: */
394 if (val & OP_CTR_OVERFLOW)
395 continue;
396 oprofile_add_sample(regs, virt);
397 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
398 }
399
400 op_amd_handle_ibs(regs, msrs);
401
402 /* See op_model_ppro.c */
403 return 1;
404 }
405
op_amd_start(struct op_msrs const * const msrs)406 static void op_amd_start(struct op_msrs const * const msrs)
407 {
408 u64 val;
409 int i;
410
411 for (i = 0; i < num_counters; ++i) {
412 if (!reset_value[op_x86_phys_to_virt(i)])
413 continue;
414 rdmsrl(msrs->controls[i].addr, val);
415 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
416 wrmsrl(msrs->controls[i].addr, val);
417 }
418
419 op_amd_start_ibs();
420 }
421
op_amd_stop(struct op_msrs const * const msrs)422 static void op_amd_stop(struct op_msrs const * const msrs)
423 {
424 u64 val;
425 int i;
426
427 /*
428 * Subtle: stop on all counters to avoid race with setting our
429 * pm callback
430 */
431 for (i = 0; i < num_counters; ++i) {
432 if (!reset_value[op_x86_phys_to_virt(i)])
433 continue;
434 rdmsrl(msrs->controls[i].addr, val);
435 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
436 wrmsrl(msrs->controls[i].addr, val);
437 }
438
439 op_amd_stop_ibs();
440 }
441
442 /*
443 * check and reserve APIC extended interrupt LVT offset for IBS if
444 * available
445 */
446
init_ibs(void)447 static void init_ibs(void)
448 {
449 ibs_caps = get_ibs_caps();
450
451 if (!ibs_caps)
452 return;
453
454 printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
455 }
456
457 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
458
setup_ibs_files(struct super_block * sb,struct dentry * root)459 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
460 {
461 struct dentry *dir;
462 int ret = 0;
463
464 /* architecture specific files */
465 if (create_arch_files)
466 ret = create_arch_files(sb, root);
467
468 if (ret)
469 return ret;
470
471 if (!ibs_caps)
472 return ret;
473
474 /* model specific files */
475
476 /* setup some reasonable defaults */
477 memset(&ibs_config, 0, sizeof(ibs_config));
478 ibs_config.max_cnt_fetch = 250000;
479 ibs_config.max_cnt_op = 250000;
480
481 if (ibs_caps & IBS_CAPS_FETCHSAM) {
482 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
483 oprofilefs_create_ulong(sb, dir, "enable",
484 &ibs_config.fetch_enabled);
485 oprofilefs_create_ulong(sb, dir, "max_count",
486 &ibs_config.max_cnt_fetch);
487 oprofilefs_create_ulong(sb, dir, "rand_enable",
488 &ibs_config.rand_en);
489 }
490
491 if (ibs_caps & IBS_CAPS_OPSAM) {
492 dir = oprofilefs_mkdir(sb, root, "ibs_op");
493 oprofilefs_create_ulong(sb, dir, "enable",
494 &ibs_config.op_enabled);
495 oprofilefs_create_ulong(sb, dir, "max_count",
496 &ibs_config.max_cnt_op);
497 if (ibs_caps & IBS_CAPS_OPCNT)
498 oprofilefs_create_ulong(sb, dir, "dispatched_ops",
499 &ibs_config.dispatched_ops);
500 if (ibs_caps & IBS_CAPS_BRNTRGT)
501 oprofilefs_create_ulong(sb, dir, "branch_target",
502 &ibs_config.branch_target);
503 }
504
505 return 0;
506 }
507
508 struct op_x86_model_spec op_amd_spec;
509
op_amd_init(struct oprofile_operations * ops)510 static int op_amd_init(struct oprofile_operations *ops)
511 {
512 init_ibs();
513 create_arch_files = ops->create_files;
514 ops->create_files = setup_ibs_files;
515
516 if (boot_cpu_data.x86 == 0x15) {
517 num_counters = AMD64_NUM_COUNTERS_F15H;
518 } else {
519 num_counters = AMD64_NUM_COUNTERS;
520 }
521
522 op_amd_spec.num_counters = num_counters;
523 op_amd_spec.num_controls = num_counters;
524 op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
525
526 return 0;
527 }
528
529 struct op_x86_model_spec op_amd_spec = {
530 /* num_counters/num_controls filled in at runtime */
531 .reserved = MSR_AMD_EVENTSEL_RESERVED,
532 .event_mask = OP_EVENT_MASK,
533 .init = op_amd_init,
534 .fill_in_addresses = &op_amd_fill_in_addresses,
535 .setup_ctrs = &op_amd_setup_ctrs,
536 .check_ctrs = &op_amd_check_ctrs,
537 .start = &op_amd_start,
538 .stop = &op_amd_stop,
539 .shutdown = &op_amd_shutdown,
540 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
541 .switch_ctrl = &op_mux_switch_ctrl,
542 #endif
543 };
544