1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Advanced Micro Devices, Inc.
4 *
5 * Author: Jacob Shin <jacob.shin@amd.com>
6 */
7
8 #include <linux/perf_event.h>
9 #include <linux/percpu.h>
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/init.h>
13 #include <linux/cpu.h>
14 #include <linux/cpumask.h>
15 #include <linux/cpufeature.h>
16 #include <linux/smp.h>
17
18 #include <asm/perf_event.h>
19 #include <asm/msr.h>
20
21 #define NUM_COUNTERS_NB 4
22 #define NUM_COUNTERS_L2 4
23 #define NUM_COUNTERS_L3 6
24
25 #define RDPMC_BASE_NB 6
26 #define RDPMC_BASE_LLC 10
27
28 #define COUNTER_SHIFT 16
29 #define UNCORE_NAME_LEN 16
30 #define UNCORE_GROUP_MAX 256
31
32 #undef pr_fmt
33 #define pr_fmt(fmt) "amd_uncore: " fmt
34
35 static int pmu_version;
36
37 struct amd_uncore_ctx {
38 int refcnt;
39 int cpu;
40 struct perf_event **events;
41 };
42
43 struct amd_uncore_pmu {
44 char name[UNCORE_NAME_LEN];
45 int num_counters;
46 int rdpmc_base;
47 u32 msr_base;
48 int group;
49 cpumask_t active_mask;
50 struct pmu pmu;
51 struct amd_uncore_ctx * __percpu *ctx;
52 };
53
54 enum {
55 UNCORE_TYPE_DF,
56 UNCORE_TYPE_L3,
57 UNCORE_TYPE_UMC,
58
59 UNCORE_TYPE_MAX
60 };
61
62 union amd_uncore_info {
63 struct {
64 u64 aux_data:32; /* auxiliary data */
65 u64 num_pmcs:8; /* number of counters */
66 u64 gid:8; /* group id */
67 u64 cid:8; /* context id */
68 } split;
69 u64 full;
70 };
71
72 struct amd_uncore {
73 union amd_uncore_info __percpu *info;
74 struct amd_uncore_pmu *pmus;
75 unsigned int num_pmus;
76 bool init_done;
77 void (*scan)(struct amd_uncore *uncore, unsigned int cpu);
78 int (*init)(struct amd_uncore *uncore, unsigned int cpu);
79 void (*move)(struct amd_uncore *uncore, unsigned int cpu);
80 void (*free)(struct amd_uncore *uncore, unsigned int cpu);
81 };
82
83 static struct amd_uncore uncores[UNCORE_TYPE_MAX];
84
event_to_amd_uncore_pmu(struct perf_event * event)85 static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
86 {
87 return container_of(event->pmu, struct amd_uncore_pmu, pmu);
88 }
89
amd_uncore_read(struct perf_event * event)90 static void amd_uncore_read(struct perf_event *event)
91 {
92 struct hw_perf_event *hwc = &event->hw;
93 u64 prev, new;
94 s64 delta;
95
96 /*
97 * since we do not enable counter overflow interrupts,
98 * we do not have to worry about prev_count changing on us
99 */
100
101 prev = local64_read(&hwc->prev_count);
102
103 /*
104 * Some uncore PMUs do not have RDPMC assignments. In such cases,
105 * read counts directly from the corresponding PERF_CTR.
106 */
107 if (hwc->event_base_rdpmc < 0)
108 rdmsrl(hwc->event_base, new);
109 else
110 rdpmcl(hwc->event_base_rdpmc, new);
111
112 local64_set(&hwc->prev_count, new);
113 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
114 delta >>= COUNTER_SHIFT;
115 local64_add(delta, &event->count);
116 }
117
amd_uncore_start(struct perf_event * event,int flags)118 static void amd_uncore_start(struct perf_event *event, int flags)
119 {
120 struct hw_perf_event *hwc = &event->hw;
121
122 if (flags & PERF_EF_RELOAD)
123 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
124
125 hwc->state = 0;
126 wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
127 perf_event_update_userpage(event);
128 }
129
amd_uncore_stop(struct perf_event * event,int flags)130 static void amd_uncore_stop(struct perf_event *event, int flags)
131 {
132 struct hw_perf_event *hwc = &event->hw;
133
134 wrmsrl(hwc->config_base, hwc->config);
135 hwc->state |= PERF_HES_STOPPED;
136
137 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
138 event->pmu->read(event);
139 hwc->state |= PERF_HES_UPTODATE;
140 }
141 }
142
amd_uncore_add(struct perf_event * event,int flags)143 static int amd_uncore_add(struct perf_event *event, int flags)
144 {
145 int i;
146 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
147 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
148 struct hw_perf_event *hwc = &event->hw;
149
150 /* are we already assigned? */
151 if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
152 goto out;
153
154 for (i = 0; i < pmu->num_counters; i++) {
155 if (ctx->events[i] == event) {
156 hwc->idx = i;
157 goto out;
158 }
159 }
160
161 /* if not, take the first available counter */
162 hwc->idx = -1;
163 for (i = 0; i < pmu->num_counters; i++) {
164 struct perf_event *tmp = NULL;
165
166 if (try_cmpxchg(&ctx->events[i], &tmp, event)) {
167 hwc->idx = i;
168 break;
169 }
170 }
171
172 out:
173 if (hwc->idx == -1)
174 return -EBUSY;
175
176 hwc->config_base = pmu->msr_base + (2 * hwc->idx);
177 hwc->event_base = pmu->msr_base + 1 + (2 * hwc->idx);
178 hwc->event_base_rdpmc = pmu->rdpmc_base + hwc->idx;
179 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
180
181 if (pmu->rdpmc_base < 0)
182 hwc->event_base_rdpmc = -1;
183
184 if (flags & PERF_EF_START)
185 event->pmu->start(event, PERF_EF_RELOAD);
186
187 return 0;
188 }
189
amd_uncore_del(struct perf_event * event,int flags)190 static void amd_uncore_del(struct perf_event *event, int flags)
191 {
192 int i;
193 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
194 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
195 struct hw_perf_event *hwc = &event->hw;
196
197 event->pmu->stop(event, PERF_EF_UPDATE);
198
199 for (i = 0; i < pmu->num_counters; i++) {
200 struct perf_event *tmp = event;
201
202 if (try_cmpxchg(&ctx->events[i], &tmp, NULL))
203 break;
204 }
205
206 hwc->idx = -1;
207 }
208
amd_uncore_event_init(struct perf_event * event)209 static int amd_uncore_event_init(struct perf_event *event)
210 {
211 struct amd_uncore_pmu *pmu;
212 struct amd_uncore_ctx *ctx;
213 struct hw_perf_event *hwc = &event->hw;
214
215 if (event->attr.type != event->pmu->type)
216 return -ENOENT;
217
218 if (event->cpu < 0)
219 return -EINVAL;
220
221 pmu = event_to_amd_uncore_pmu(event);
222 ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
223 if (!ctx)
224 return -ENODEV;
225
226 /*
227 * NB and Last level cache counters (MSRs) are shared across all cores
228 * that share the same NB / Last level cache. On family 16h and below,
229 * Interrupts can be directed to a single target core, however, event
230 * counts generated by processes running on other cores cannot be masked
231 * out. So we do not support sampling and per-thread events via
232 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
233 */
234 hwc->config = event->attr.config;
235 hwc->idx = -1;
236
237 /*
238 * since request can come in to any of the shared cores, we will remap
239 * to a single common cpu.
240 */
241 event->cpu = ctx->cpu;
242
243 return 0;
244 }
245
246 static umode_t
amd_f17h_uncore_is_visible(struct kobject * kobj,struct attribute * attr,int i)247 amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
248 {
249 return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ?
250 attr->mode : 0;
251 }
252
253 static umode_t
amd_f19h_uncore_is_visible(struct kobject * kobj,struct attribute * attr,int i)254 amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
255 {
256 return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0;
257 }
258
amd_uncore_attr_show_cpumask(struct device * dev,struct device_attribute * attr,char * buf)259 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
260 struct device_attribute *attr,
261 char *buf)
262 {
263 struct pmu *ptr = dev_get_drvdata(dev);
264 struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu);
265
266 return cpumap_print_to_pagebuf(true, buf, &pmu->active_mask);
267 }
268 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
269
270 static struct attribute *amd_uncore_attrs[] = {
271 &dev_attr_cpumask.attr,
272 NULL,
273 };
274
275 static struct attribute_group amd_uncore_attr_group = {
276 .attrs = amd_uncore_attrs,
277 };
278
279 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
280 static ssize_t __uncore_##_var##_show(struct device *dev, \
281 struct device_attribute *attr, \
282 char *page) \
283 { \
284 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
285 return sprintf(page, _format "\n"); \
286 } \
287 static struct device_attribute format_attr_##_var = \
288 __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
289
290 DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
291 DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
292 DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */
293 DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3, PerfMonV2 UMC */
294 DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15");
295 DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */
296 DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
297 DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */
298 DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */
299 DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */
300 DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
301 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
302 DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
303 DEFINE_UNCORE_FORMAT_ATTR(rdwrmask, rdwrmask, "config:8-9"); /* PerfMonV2 UMC */
304
305 /* Common DF and NB attributes */
306 static struct attribute *amd_uncore_df_format_attr[] = {
307 &format_attr_event12.attr, /* event */
308 &format_attr_umask8.attr, /* umask */
309 NULL,
310 };
311
312 /* Common L2 and L3 attributes */
313 static struct attribute *amd_uncore_l3_format_attr[] = {
314 &format_attr_event12.attr, /* event */
315 &format_attr_umask8.attr, /* umask */
316 NULL, /* threadmask */
317 NULL,
318 };
319
320 /* Common UMC attributes */
321 static struct attribute *amd_uncore_umc_format_attr[] = {
322 &format_attr_event8.attr, /* event */
323 &format_attr_rdwrmask.attr, /* rdwrmask */
324 NULL,
325 };
326
327 /* F17h unique L3 attributes */
328 static struct attribute *amd_f17h_uncore_l3_format_attr[] = {
329 &format_attr_slicemask.attr, /* slicemask */
330 NULL,
331 };
332
333 /* F19h unique L3 attributes */
334 static struct attribute *amd_f19h_uncore_l3_format_attr[] = {
335 &format_attr_coreid.attr, /* coreid */
336 &format_attr_enallslices.attr, /* enallslices */
337 &format_attr_enallcores.attr, /* enallcores */
338 &format_attr_sliceid.attr, /* sliceid */
339 NULL,
340 };
341
342 static struct attribute_group amd_uncore_df_format_group = {
343 .name = "format",
344 .attrs = amd_uncore_df_format_attr,
345 };
346
347 static struct attribute_group amd_uncore_l3_format_group = {
348 .name = "format",
349 .attrs = amd_uncore_l3_format_attr,
350 };
351
352 static struct attribute_group amd_f17h_uncore_l3_format_group = {
353 .name = "format",
354 .attrs = amd_f17h_uncore_l3_format_attr,
355 .is_visible = amd_f17h_uncore_is_visible,
356 };
357
358 static struct attribute_group amd_f19h_uncore_l3_format_group = {
359 .name = "format",
360 .attrs = amd_f19h_uncore_l3_format_attr,
361 .is_visible = amd_f19h_uncore_is_visible,
362 };
363
364 static struct attribute_group amd_uncore_umc_format_group = {
365 .name = "format",
366 .attrs = amd_uncore_umc_format_attr,
367 };
368
369 static const struct attribute_group *amd_uncore_df_attr_groups[] = {
370 &amd_uncore_attr_group,
371 &amd_uncore_df_format_group,
372 NULL,
373 };
374
375 static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
376 &amd_uncore_attr_group,
377 &amd_uncore_l3_format_group,
378 NULL,
379 };
380
381 static const struct attribute_group *amd_uncore_l3_attr_update[] = {
382 &amd_f17h_uncore_l3_format_group,
383 &amd_f19h_uncore_l3_format_group,
384 NULL,
385 };
386
387 static const struct attribute_group *amd_uncore_umc_attr_groups[] = {
388 &amd_uncore_attr_group,
389 &amd_uncore_umc_format_group,
390 NULL,
391 };
392
393 static __always_inline
amd_uncore_ctx_cid(struct amd_uncore * uncore,unsigned int cpu)394 int amd_uncore_ctx_cid(struct amd_uncore *uncore, unsigned int cpu)
395 {
396 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
397 return info->split.cid;
398 }
399
400 static __always_inline
amd_uncore_ctx_gid(struct amd_uncore * uncore,unsigned int cpu)401 int amd_uncore_ctx_gid(struct amd_uncore *uncore, unsigned int cpu)
402 {
403 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
404 return info->split.gid;
405 }
406
407 static __always_inline
amd_uncore_ctx_num_pmcs(struct amd_uncore * uncore,unsigned int cpu)408 int amd_uncore_ctx_num_pmcs(struct amd_uncore *uncore, unsigned int cpu)
409 {
410 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
411 return info->split.num_pmcs;
412 }
413
amd_uncore_ctx_free(struct amd_uncore * uncore,unsigned int cpu)414 static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
415 {
416 struct amd_uncore_pmu *pmu;
417 struct amd_uncore_ctx *ctx;
418 int i;
419
420 if (!uncore->init_done)
421 return;
422
423 for (i = 0; i < uncore->num_pmus; i++) {
424 pmu = &uncore->pmus[i];
425 ctx = *per_cpu_ptr(pmu->ctx, cpu);
426 if (!ctx)
427 continue;
428
429 if (cpu == ctx->cpu)
430 cpumask_clear_cpu(cpu, &pmu->active_mask);
431
432 if (!--ctx->refcnt) {
433 kfree(ctx->events);
434 kfree(ctx);
435 }
436
437 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
438 }
439 }
440
amd_uncore_ctx_init(struct amd_uncore * uncore,unsigned int cpu)441 static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
442 {
443 struct amd_uncore_ctx *curr, *prev;
444 struct amd_uncore_pmu *pmu;
445 int node, cid, gid, i, j;
446
447 if (!uncore->init_done || !uncore->num_pmus)
448 return 0;
449
450 cid = amd_uncore_ctx_cid(uncore, cpu);
451 gid = amd_uncore_ctx_gid(uncore, cpu);
452
453 for (i = 0; i < uncore->num_pmus; i++) {
454 pmu = &uncore->pmus[i];
455 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
456 curr = NULL;
457
458 /* Check for group exclusivity */
459 if (gid != pmu->group)
460 continue;
461
462 /* Find a sibling context */
463 for_each_online_cpu(j) {
464 if (cpu == j)
465 continue;
466
467 prev = *per_cpu_ptr(pmu->ctx, j);
468 if (!prev)
469 continue;
470
471 if (cid == amd_uncore_ctx_cid(uncore, j)) {
472 curr = prev;
473 break;
474 }
475 }
476
477 /* Allocate context if sibling does not exist */
478 if (!curr) {
479 node = cpu_to_node(cpu);
480 curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
481 if (!curr)
482 goto fail;
483
484 curr->cpu = cpu;
485 curr->events = kzalloc_node(sizeof(*curr->events) *
486 pmu->num_counters,
487 GFP_KERNEL, node);
488 if (!curr->events) {
489 kfree(curr);
490 goto fail;
491 }
492
493 cpumask_set_cpu(cpu, &pmu->active_mask);
494 }
495
496 curr->refcnt++;
497 *per_cpu_ptr(pmu->ctx, cpu) = curr;
498 }
499
500 return 0;
501
502 fail:
503 amd_uncore_ctx_free(uncore, cpu);
504
505 return -ENOMEM;
506 }
507
amd_uncore_ctx_move(struct amd_uncore * uncore,unsigned int cpu)508 static void amd_uncore_ctx_move(struct amd_uncore *uncore, unsigned int cpu)
509 {
510 struct amd_uncore_ctx *curr, *next;
511 struct amd_uncore_pmu *pmu;
512 int i, j;
513
514 if (!uncore->init_done)
515 return;
516
517 for (i = 0; i < uncore->num_pmus; i++) {
518 pmu = &uncore->pmus[i];
519 curr = *per_cpu_ptr(pmu->ctx, cpu);
520 if (!curr)
521 continue;
522
523 /* Migrate to a shared sibling if possible */
524 for_each_online_cpu(j) {
525 next = *per_cpu_ptr(pmu->ctx, j);
526 if (!next || cpu == j)
527 continue;
528
529 if (curr == next) {
530 perf_pmu_migrate_context(&pmu->pmu, cpu, j);
531 cpumask_clear_cpu(cpu, &pmu->active_mask);
532 cpumask_set_cpu(j, &pmu->active_mask);
533 next->cpu = j;
534 break;
535 }
536 }
537 }
538 }
539
amd_uncore_cpu_starting(unsigned int cpu)540 static int amd_uncore_cpu_starting(unsigned int cpu)
541 {
542 struct amd_uncore *uncore;
543 int i;
544
545 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
546 uncore = &uncores[i];
547 uncore->scan(uncore, cpu);
548 }
549
550 return 0;
551 }
552
amd_uncore_cpu_online(unsigned int cpu)553 static int amd_uncore_cpu_online(unsigned int cpu)
554 {
555 struct amd_uncore *uncore;
556 int i;
557
558 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
559 uncore = &uncores[i];
560 if (uncore->init(uncore, cpu))
561 break;
562 }
563
564 return 0;
565 }
566
amd_uncore_cpu_down_prepare(unsigned int cpu)567 static int amd_uncore_cpu_down_prepare(unsigned int cpu)
568 {
569 struct amd_uncore *uncore;
570 int i;
571
572 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
573 uncore = &uncores[i];
574 uncore->move(uncore, cpu);
575 }
576
577 return 0;
578 }
579
amd_uncore_cpu_dead(unsigned int cpu)580 static int amd_uncore_cpu_dead(unsigned int cpu)
581 {
582 struct amd_uncore *uncore;
583 int i;
584
585 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
586 uncore = &uncores[i];
587 uncore->free(uncore, cpu);
588 }
589
590 return 0;
591 }
592
amd_uncore_df_event_init(struct perf_event * event)593 static int amd_uncore_df_event_init(struct perf_event *event)
594 {
595 struct hw_perf_event *hwc = &event->hw;
596 int ret = amd_uncore_event_init(event);
597
598 if (ret || pmu_version < 2)
599 return ret;
600
601 hwc->config = event->attr.config &
602 (pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB :
603 AMD64_RAW_EVENT_MASK_NB);
604
605 return 0;
606 }
607
amd_uncore_df_add(struct perf_event * event,int flags)608 static int amd_uncore_df_add(struct perf_event *event, int flags)
609 {
610 int ret = amd_uncore_add(event, flags & ~PERF_EF_START);
611 struct hw_perf_event *hwc = &event->hw;
612
613 if (ret)
614 return ret;
615
616 /*
617 * The first four DF counters are accessible via RDPMC index 6 to 9
618 * followed by the L3 counters from index 10 to 15. For processors
619 * with more than four DF counters, the DF RDPMC assignments become
620 * discontiguous as the additional counters are accessible starting
621 * from index 16.
622 */
623 if (hwc->idx >= NUM_COUNTERS_NB)
624 hwc->event_base_rdpmc += NUM_COUNTERS_L3;
625
626 /* Delayed start after rdpmc base update */
627 if (flags & PERF_EF_START)
628 amd_uncore_start(event, PERF_EF_RELOAD);
629
630 return 0;
631 }
632
633 static
amd_uncore_df_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)634 void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
635 {
636 union cpuid_0x80000022_ebx ebx;
637 union amd_uncore_info info;
638
639 if (!boot_cpu_has(X86_FEATURE_PERFCTR_NB))
640 return;
641
642 info.split.aux_data = 0;
643 info.split.num_pmcs = NUM_COUNTERS_NB;
644 info.split.gid = 0;
645 info.split.cid = topology_logical_package_id(cpu);
646
647 if (pmu_version >= 2) {
648 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
649 info.split.num_pmcs = ebx.split.num_df_pmc;
650 }
651
652 *per_cpu_ptr(uncore->info, cpu) = info;
653 }
654
655 static
amd_uncore_df_ctx_init(struct amd_uncore * uncore,unsigned int cpu)656 int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
657 {
658 struct attribute **df_attr = amd_uncore_df_format_attr;
659 struct amd_uncore_pmu *pmu;
660 int num_counters;
661
662 /* Run just once */
663 if (uncore->init_done)
664 return amd_uncore_ctx_init(uncore, cpu);
665
666 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
667 if (!num_counters)
668 goto done;
669
670 /* No grouping, single instance for a system */
671 uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
672 if (!uncore->pmus)
673 goto done;
674
675 /*
676 * For Family 17h and above, the Northbridge counters are repurposed
677 * as Data Fabric counters. The PMUs are exported based on family as
678 * either NB or DF.
679 */
680 pmu = &uncore->pmus[0];
681 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb",
682 sizeof(pmu->name));
683 pmu->num_counters = num_counters;
684 pmu->msr_base = MSR_F15H_NB_PERF_CTL;
685 pmu->rdpmc_base = RDPMC_BASE_NB;
686 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
687
688 if (pmu_version >= 2) {
689 *df_attr++ = &format_attr_event14v2.attr;
690 *df_attr++ = &format_attr_umask12.attr;
691 } else if (boot_cpu_data.x86 >= 0x17) {
692 *df_attr = &format_attr_event14.attr;
693 }
694
695 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
696 if (!pmu->ctx)
697 goto done;
698
699 pmu->pmu = (struct pmu) {
700 .task_ctx_nr = perf_invalid_context,
701 .attr_groups = amd_uncore_df_attr_groups,
702 .name = pmu->name,
703 .event_init = amd_uncore_df_event_init,
704 .add = amd_uncore_df_add,
705 .del = amd_uncore_del,
706 .start = amd_uncore_start,
707 .stop = amd_uncore_stop,
708 .read = amd_uncore_read,
709 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
710 .module = THIS_MODULE,
711 };
712
713 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
714 free_percpu(pmu->ctx);
715 pmu->ctx = NULL;
716 goto done;
717 }
718
719 pr_info("%d %s%s counters detected\n", pmu->num_counters,
720 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
721 pmu->pmu.name);
722
723 uncore->num_pmus = 1;
724
725 done:
726 uncore->init_done = true;
727
728 return amd_uncore_ctx_init(uncore, cpu);
729 }
730
amd_uncore_l3_event_init(struct perf_event * event)731 static int amd_uncore_l3_event_init(struct perf_event *event)
732 {
733 int ret = amd_uncore_event_init(event);
734 struct hw_perf_event *hwc = &event->hw;
735 u64 config = event->attr.config;
736 u64 mask;
737
738 hwc->config = config & AMD64_RAW_EVENT_MASK_NB;
739
740 /*
741 * SliceMask and ThreadMask need to be set for certain L3 events.
742 * For other events, the two fields do not affect the count.
743 */
744 if (ret || boot_cpu_data.x86 < 0x17)
745 return ret;
746
747 mask = config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
748 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
749 AMD64_L3_COREID_MASK);
750
751 if (boot_cpu_data.x86 <= 0x18)
752 mask = ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
753 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
754
755 /*
756 * If the user doesn't specify a ThreadMask, they're not trying to
757 * count core 0, so we enable all cores & threads.
758 * We'll also assume that they want to count slice 0 if they specify
759 * a ThreadMask and leave SliceId and EnAllSlices unpopulated.
760 */
761 else if (!(config & AMD64_L3_F19H_THREAD_MASK))
762 mask = AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
763 AMD64_L3_EN_ALL_CORES;
764
765 hwc->config |= mask;
766
767 return 0;
768 }
769
770 static
amd_uncore_l3_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)771 void amd_uncore_l3_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
772 {
773 union amd_uncore_info info;
774
775 if (!boot_cpu_has(X86_FEATURE_PERFCTR_LLC))
776 return;
777
778 info.split.aux_data = 0;
779 info.split.num_pmcs = NUM_COUNTERS_L2;
780 info.split.gid = 0;
781 info.split.cid = per_cpu_llc_id(cpu);
782
783 if (boot_cpu_data.x86 >= 0x17)
784 info.split.num_pmcs = NUM_COUNTERS_L3;
785
786 *per_cpu_ptr(uncore->info, cpu) = info;
787 }
788
789 static
amd_uncore_l3_ctx_init(struct amd_uncore * uncore,unsigned int cpu)790 int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
791 {
792 struct attribute **l3_attr = amd_uncore_l3_format_attr;
793 struct amd_uncore_pmu *pmu;
794 int num_counters;
795
796 /* Run just once */
797 if (uncore->init_done)
798 return amd_uncore_ctx_init(uncore, cpu);
799
800 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
801 if (!num_counters)
802 goto done;
803
804 /* No grouping, single instance for a system */
805 uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
806 if (!uncore->pmus)
807 goto done;
808
809 /*
810 * For Family 17h and above, L3 cache counters are available instead
811 * of L2 cache counters. The PMUs are exported based on family as
812 * either L2 or L3.
813 */
814 pmu = &uncore->pmus[0];
815 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2",
816 sizeof(pmu->name));
817 pmu->num_counters = num_counters;
818 pmu->msr_base = MSR_F16H_L2I_PERF_CTL;
819 pmu->rdpmc_base = RDPMC_BASE_LLC;
820 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
821
822 if (boot_cpu_data.x86 >= 0x17) {
823 *l3_attr++ = &format_attr_event8.attr;
824 *l3_attr++ = &format_attr_umask8.attr;
825 *l3_attr++ = boot_cpu_data.x86 >= 0x19 ?
826 &format_attr_threadmask2.attr :
827 &format_attr_threadmask8.attr;
828 }
829
830 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
831 if (!pmu->ctx)
832 goto done;
833
834 pmu->pmu = (struct pmu) {
835 .task_ctx_nr = perf_invalid_context,
836 .attr_groups = amd_uncore_l3_attr_groups,
837 .attr_update = amd_uncore_l3_attr_update,
838 .name = pmu->name,
839 .event_init = amd_uncore_l3_event_init,
840 .add = amd_uncore_add,
841 .del = amd_uncore_del,
842 .start = amd_uncore_start,
843 .stop = amd_uncore_stop,
844 .read = amd_uncore_read,
845 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
846 .module = THIS_MODULE,
847 };
848
849 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
850 free_percpu(pmu->ctx);
851 pmu->ctx = NULL;
852 goto done;
853 }
854
855 pr_info("%d %s%s counters detected\n", pmu->num_counters,
856 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
857 pmu->pmu.name);
858
859 uncore->num_pmus = 1;
860
861 done:
862 uncore->init_done = true;
863
864 return amd_uncore_ctx_init(uncore, cpu);
865 }
866
amd_uncore_umc_event_init(struct perf_event * event)867 static int amd_uncore_umc_event_init(struct perf_event *event)
868 {
869 struct hw_perf_event *hwc = &event->hw;
870 int ret = amd_uncore_event_init(event);
871
872 if (ret)
873 return ret;
874
875 hwc->config = event->attr.config & AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC;
876
877 return 0;
878 }
879
amd_uncore_umc_start(struct perf_event * event,int flags)880 static void amd_uncore_umc_start(struct perf_event *event, int flags)
881 {
882 struct hw_perf_event *hwc = &event->hw;
883
884 if (flags & PERF_EF_RELOAD)
885 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
886
887 hwc->state = 0;
888 wrmsrl(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
889 perf_event_update_userpage(event);
890 }
891
amd_uncore_umc_read(struct perf_event * event)892 static void amd_uncore_umc_read(struct perf_event *event)
893 {
894 struct hw_perf_event *hwc = &event->hw;
895 u64 prev, new, shift;
896 s64 delta;
897
898 shift = COUNTER_SHIFT + 1;
899 prev = local64_read(&hwc->prev_count);
900
901 /*
902 * UMC counters do not have RDPMC assignments. Read counts directly
903 * from the corresponding PERF_CTR.
904 */
905 rdmsrl(hwc->event_base, new);
906
907 /*
908 * Unlike the other uncore counters, UMC counters saturate and set the
909 * Overflow bit (bit 48) on overflow. Since they do not roll over,
910 * proactively reset the corresponding PERF_CTR when bit 47 is set so
911 * that the counter never gets a chance to saturate.
912 */
913 if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
914 wrmsrl(hwc->event_base, 0);
915 local64_set(&hwc->prev_count, 0);
916 } else {
917 local64_set(&hwc->prev_count, new);
918 }
919
920 delta = (new << shift) - (prev << shift);
921 delta >>= shift;
922 local64_add(delta, &event->count);
923 }
924
925 static
amd_uncore_umc_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)926 void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
927 {
928 union cpuid_0x80000022_ebx ebx;
929 union amd_uncore_info info;
930 unsigned int eax, ecx, edx;
931
932 if (pmu_version < 2)
933 return;
934
935 cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx);
936 info.split.aux_data = ecx; /* stash active mask */
937 info.split.num_pmcs = ebx.split.num_umc_pmc;
938 info.split.gid = topology_logical_package_id(cpu);
939 info.split.cid = topology_logical_package_id(cpu);
940 *per_cpu_ptr(uncore->info, cpu) = info;
941 }
942
943 static
amd_uncore_umc_ctx_init(struct amd_uncore * uncore,unsigned int cpu)944 int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
945 {
946 DECLARE_BITMAP(gmask, UNCORE_GROUP_MAX) = { 0 };
947 u8 group_num_pmus[UNCORE_GROUP_MAX] = { 0 };
948 u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 };
949 union amd_uncore_info info;
950 struct amd_uncore_pmu *pmu;
951 int index = 0, gid, i;
952
953 if (pmu_version < 2)
954 return 0;
955
956 /* Run just once */
957 if (uncore->init_done)
958 return amd_uncore_ctx_init(uncore, cpu);
959
960 /* Find unique groups */
961 for_each_online_cpu(i) {
962 info = *per_cpu_ptr(uncore->info, i);
963 gid = info.split.gid;
964 if (test_bit(gid, gmask))
965 continue;
966
967 __set_bit(gid, gmask);
968 group_num_pmus[gid] = hweight32(info.split.aux_data);
969 group_num_pmcs[gid] = info.split.num_pmcs;
970 uncore->num_pmus += group_num_pmus[gid];
971 }
972
973 uncore->pmus = kzalloc(sizeof(*uncore->pmus) * uncore->num_pmus,
974 GFP_KERNEL);
975 if (!uncore->pmus) {
976 uncore->num_pmus = 0;
977 goto done;
978 }
979
980 for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) {
981 for (i = 0; i < group_num_pmus[gid]; i++) {
982 pmu = &uncore->pmus[index];
983 snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%d", index);
984 pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid];
985 pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2;
986 pmu->rdpmc_base = -1;
987 pmu->group = gid;
988
989 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
990 if (!pmu->ctx)
991 goto done;
992
993 pmu->pmu = (struct pmu) {
994 .task_ctx_nr = perf_invalid_context,
995 .attr_groups = amd_uncore_umc_attr_groups,
996 .name = pmu->name,
997 .event_init = amd_uncore_umc_event_init,
998 .add = amd_uncore_add,
999 .del = amd_uncore_del,
1000 .start = amd_uncore_umc_start,
1001 .stop = amd_uncore_stop,
1002 .read = amd_uncore_umc_read,
1003 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
1004 .module = THIS_MODULE,
1005 };
1006
1007 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
1008 free_percpu(pmu->ctx);
1009 pmu->ctx = NULL;
1010 goto done;
1011 }
1012
1013 pr_info("%d %s counters detected\n", pmu->num_counters,
1014 pmu->pmu.name);
1015
1016 index++;
1017 }
1018 }
1019
1020 done:
1021 uncore->num_pmus = index;
1022 uncore->init_done = true;
1023
1024 return amd_uncore_ctx_init(uncore, cpu);
1025 }
1026
1027 static struct amd_uncore uncores[UNCORE_TYPE_MAX] = {
1028 /* UNCORE_TYPE_DF */
1029 {
1030 .scan = amd_uncore_df_ctx_scan,
1031 .init = amd_uncore_df_ctx_init,
1032 .move = amd_uncore_ctx_move,
1033 .free = amd_uncore_ctx_free,
1034 },
1035 /* UNCORE_TYPE_L3 */
1036 {
1037 .scan = amd_uncore_l3_ctx_scan,
1038 .init = amd_uncore_l3_ctx_init,
1039 .move = amd_uncore_ctx_move,
1040 .free = amd_uncore_ctx_free,
1041 },
1042 /* UNCORE_TYPE_UMC */
1043 {
1044 .scan = amd_uncore_umc_ctx_scan,
1045 .init = amd_uncore_umc_ctx_init,
1046 .move = amd_uncore_ctx_move,
1047 .free = amd_uncore_ctx_free,
1048 },
1049 };
1050
amd_uncore_init(void)1051 static int __init amd_uncore_init(void)
1052 {
1053 struct amd_uncore *uncore;
1054 int ret = -ENODEV;
1055 int i;
1056
1057 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1058 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1059 return -ENODEV;
1060
1061 if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
1062 return -ENODEV;
1063
1064 if (boot_cpu_has(X86_FEATURE_PERFMON_V2))
1065 pmu_version = 2;
1066
1067 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1068 uncore = &uncores[i];
1069
1070 BUG_ON(!uncore->scan);
1071 BUG_ON(!uncore->init);
1072 BUG_ON(!uncore->move);
1073 BUG_ON(!uncore->free);
1074
1075 uncore->info = alloc_percpu(union amd_uncore_info);
1076 if (!uncore->info) {
1077 ret = -ENOMEM;
1078 goto fail;
1079 }
1080 };
1081
1082 /*
1083 * Install callbacks. Core will call them for each online cpu.
1084 */
1085 ret = cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
1086 "perf/x86/amd/uncore:prepare",
1087 NULL, amd_uncore_cpu_dead);
1088 if (ret)
1089 goto fail;
1090
1091 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
1092 "perf/x86/amd/uncore:starting",
1093 amd_uncore_cpu_starting, NULL);
1094 if (ret)
1095 goto fail_prep;
1096
1097 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
1098 "perf/x86/amd/uncore:online",
1099 amd_uncore_cpu_online,
1100 amd_uncore_cpu_down_prepare);
1101 if (ret)
1102 goto fail_start;
1103
1104 return 0;
1105
1106 fail_start:
1107 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1108 fail_prep:
1109 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
1110 fail:
1111 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1112 uncore = &uncores[i];
1113 if (uncore->info) {
1114 free_percpu(uncore->info);
1115 uncore->info = NULL;
1116 }
1117 }
1118
1119 return ret;
1120 }
1121
amd_uncore_exit(void)1122 static void __exit amd_uncore_exit(void)
1123 {
1124 struct amd_uncore *uncore;
1125 struct amd_uncore_pmu *pmu;
1126 int i, j;
1127
1128 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE);
1129 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1130 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
1131
1132 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1133 uncore = &uncores[i];
1134 if (!uncore->info)
1135 continue;
1136
1137 free_percpu(uncore->info);
1138 uncore->info = NULL;
1139
1140 for (j = 0; j < uncore->num_pmus; j++) {
1141 pmu = &uncore->pmus[j];
1142 if (!pmu->ctx)
1143 continue;
1144
1145 perf_pmu_unregister(&pmu->pmu);
1146 free_percpu(pmu->ctx);
1147 pmu->ctx = NULL;
1148 }
1149
1150 kfree(uncore->pmus);
1151 uncore->pmus = NULL;
1152 }
1153 }
1154
1155 module_init(amd_uncore_init);
1156 module_exit(amd_uncore_exit);
1157
1158 MODULE_DESCRIPTION("AMD Uncore Driver");
1159 MODULE_LICENSE("GPL v2");
1160