1 /*
2 * Performance event support for s390x - CPU-measurement Counter Facility
3 *
4 * Copyright IBM Corp. 2012, 2017
5 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License (version 2 only)
9 * as published by the Free Software Foundation.
10 */
11 #define KMSG_COMPONENT "cpum_cf"
12 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13
14 #include <linux/kernel.h>
15 #include <linux/kernel_stat.h>
16 #include <linux/perf_event.h>
17 #include <linux/percpu.h>
18 #include <linux/notifier.h>
19 #include <linux/init.h>
20 #include <linux/export.h>
21 #include <asm/ctl_reg.h>
22 #include <asm/irq.h>
23 #include <asm/cpu_mf.h>
24
25 enum cpumf_ctr_set {
26 CPUMF_CTR_SET_BASIC = 0, /* Basic Counter Set */
27 CPUMF_CTR_SET_USER = 1, /* Problem-State Counter Set */
28 CPUMF_CTR_SET_CRYPTO = 2, /* Crypto-Activity Counter Set */
29 CPUMF_CTR_SET_EXT = 3, /* Extended Counter Set */
30 CPUMF_CTR_SET_MT_DIAG = 4, /* MT-diagnostic Counter Set */
31
32 /* Maximum number of counter sets */
33 CPUMF_CTR_SET_MAX,
34 };
35
36 #define CPUMF_LCCTL_ENABLE_SHIFT 16
37 #define CPUMF_LCCTL_ACTCTL_SHIFT 0
38 static const u64 cpumf_state_ctl[CPUMF_CTR_SET_MAX] = {
39 [CPUMF_CTR_SET_BASIC] = 0x02,
40 [CPUMF_CTR_SET_USER] = 0x04,
41 [CPUMF_CTR_SET_CRYPTO] = 0x08,
42 [CPUMF_CTR_SET_EXT] = 0x01,
43 [CPUMF_CTR_SET_MT_DIAG] = 0x20,
44 };
45
ctr_set_enable(u64 * state,int ctr_set)46 static void ctr_set_enable(u64 *state, int ctr_set)
47 {
48 *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT;
49 }
ctr_set_disable(u64 * state,int ctr_set)50 static void ctr_set_disable(u64 *state, int ctr_set)
51 {
52 *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT);
53 }
ctr_set_start(u64 * state,int ctr_set)54 static void ctr_set_start(u64 *state, int ctr_set)
55 {
56 *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT;
57 }
ctr_set_stop(u64 * state,int ctr_set)58 static void ctr_set_stop(u64 *state, int ctr_set)
59 {
60 *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT);
61 }
62
63 /* Local CPUMF event structure */
64 struct cpu_hw_events {
65 struct cpumf_ctr_info info;
66 atomic_t ctr_set[CPUMF_CTR_SET_MAX];
67 u64 state, tx_state;
68 unsigned int flags;
69 unsigned int txn_flags;
70 };
71 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
72 .ctr_set = {
73 [CPUMF_CTR_SET_BASIC] = ATOMIC_INIT(0),
74 [CPUMF_CTR_SET_USER] = ATOMIC_INIT(0),
75 [CPUMF_CTR_SET_CRYPTO] = ATOMIC_INIT(0),
76 [CPUMF_CTR_SET_EXT] = ATOMIC_INIT(0),
77 [CPUMF_CTR_SET_MT_DIAG] = ATOMIC_INIT(0),
78 },
79 .state = 0,
80 .flags = 0,
81 .txn_flags = 0,
82 };
83
get_counter_set(u64 event)84 static enum cpumf_ctr_set get_counter_set(u64 event)
85 {
86 int set = CPUMF_CTR_SET_MAX;
87
88 if (event < 32)
89 set = CPUMF_CTR_SET_BASIC;
90 else if (event < 64)
91 set = CPUMF_CTR_SET_USER;
92 else if (event < 128)
93 set = CPUMF_CTR_SET_CRYPTO;
94 else if (event < 256)
95 set = CPUMF_CTR_SET_EXT;
96 else if (event >= 448 && event < 496)
97 set = CPUMF_CTR_SET_MT_DIAG;
98
99 return set;
100 }
101
validate_ctr_version(const struct hw_perf_event * hwc)102 static int validate_ctr_version(const struct hw_perf_event *hwc)
103 {
104 struct cpu_hw_events *cpuhw;
105 int err = 0;
106 u16 mtdiag_ctl;
107
108 cpuhw = &get_cpu_var(cpu_hw_events);
109
110 /* check required version for counter sets */
111 switch (hwc->config_base) {
112 case CPUMF_CTR_SET_BASIC:
113 case CPUMF_CTR_SET_USER:
114 if (cpuhw->info.cfvn < 1)
115 err = -EOPNOTSUPP;
116 break;
117 case CPUMF_CTR_SET_CRYPTO:
118 case CPUMF_CTR_SET_EXT:
119 if (cpuhw->info.csvn < 1)
120 err = -EOPNOTSUPP;
121 if ((cpuhw->info.csvn == 1 && hwc->config > 159) ||
122 (cpuhw->info.csvn == 2 && hwc->config > 175) ||
123 (cpuhw->info.csvn > 2 && hwc->config > 255))
124 err = -EOPNOTSUPP;
125 break;
126 case CPUMF_CTR_SET_MT_DIAG:
127 if (cpuhw->info.csvn <= 3)
128 err = -EOPNOTSUPP;
129 /*
130 * MT-diagnostic counters are read-only. The counter set
131 * is automatically enabled and activated on all CPUs with
132 * multithreading (SMT). Deactivation of multithreading
133 * also disables the counter set. State changes are ignored
134 * by lcctl(). Because Linux controls SMT enablement through
135 * a kernel parameter only, the counter set is either disabled
136 * or enabled and active.
137 *
138 * Thus, the counters can only be used if SMT is on and the
139 * counter set is enabled and active.
140 */
141 mtdiag_ctl = cpumf_state_ctl[CPUMF_CTR_SET_MT_DIAG];
142 if (!((cpuhw->info.auth_ctl & mtdiag_ctl) &&
143 (cpuhw->info.enable_ctl & mtdiag_ctl) &&
144 (cpuhw->info.act_ctl & mtdiag_ctl)))
145 err = -EOPNOTSUPP;
146 break;
147 }
148
149 put_cpu_var(cpu_hw_events);
150 return err;
151 }
152
validate_ctr_auth(const struct hw_perf_event * hwc)153 static int validate_ctr_auth(const struct hw_perf_event *hwc)
154 {
155 struct cpu_hw_events *cpuhw;
156 u64 ctrs_state;
157 int err = 0;
158
159 cpuhw = &get_cpu_var(cpu_hw_events);
160
161 /* Check authorization for cpu counter sets.
162 * If the particular CPU counter set is not authorized,
163 * return with -ENOENT in order to fall back to other
164 * PMUs that might suffice the event request.
165 */
166 ctrs_state = cpumf_state_ctl[hwc->config_base];
167 if (!(ctrs_state & cpuhw->info.auth_ctl))
168 err = -ENOENT;
169
170 put_cpu_var(cpu_hw_events);
171 return err;
172 }
173
174 /*
175 * Change the CPUMF state to active.
176 * Enable and activate the CPU-counter sets according
177 * to the per-cpu control state.
178 */
cpumf_pmu_enable(struct pmu * pmu)179 static void cpumf_pmu_enable(struct pmu *pmu)
180 {
181 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
182 int err;
183
184 if (cpuhw->flags & PMU_F_ENABLED)
185 return;
186
187 err = lcctl(cpuhw->state);
188 if (err) {
189 pr_err("Enabling the performance measuring unit "
190 "failed with rc=%x\n", err);
191 return;
192 }
193
194 cpuhw->flags |= PMU_F_ENABLED;
195 }
196
197 /*
198 * Change the CPUMF state to inactive.
199 * Disable and enable (inactive) the CPU-counter sets according
200 * to the per-cpu control state.
201 */
cpumf_pmu_disable(struct pmu * pmu)202 static void cpumf_pmu_disable(struct pmu *pmu)
203 {
204 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
205 int err;
206 u64 inactive;
207
208 if (!(cpuhw->flags & PMU_F_ENABLED))
209 return;
210
211 inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
212 err = lcctl(inactive);
213 if (err) {
214 pr_err("Disabling the performance measuring unit "
215 "failed with rc=%x\n", err);
216 return;
217 }
218
219 cpuhw->flags &= ~PMU_F_ENABLED;
220 }
221
222
223 /* Number of perf events counting hardware events */
224 static atomic_t num_events = ATOMIC_INIT(0);
225 /* Used to avoid races in calling reserve/release_cpumf_hardware */
226 static DEFINE_MUTEX(pmc_reserve_mutex);
227
228 /* CPU-measurement alerts for the counter facility */
cpumf_measurement_alert(struct ext_code ext_code,unsigned int alert,unsigned long unused)229 static void cpumf_measurement_alert(struct ext_code ext_code,
230 unsigned int alert, unsigned long unused)
231 {
232 struct cpu_hw_events *cpuhw;
233
234 if (!(alert & CPU_MF_INT_CF_MASK))
235 return;
236
237 inc_irq_stat(IRQEXT_CMC);
238 cpuhw = this_cpu_ptr(&cpu_hw_events);
239
240 /* Measurement alerts are shared and might happen when the PMU
241 * is not reserved. Ignore these alerts in this case. */
242 if (!(cpuhw->flags & PMU_F_RESERVED))
243 return;
244
245 /* counter authorization change alert */
246 if (alert & CPU_MF_INT_CF_CACA)
247 qctri(&cpuhw->info);
248
249 /* loss of counter data alert */
250 if (alert & CPU_MF_INT_CF_LCDA)
251 pr_err("CPU[%i] Counter data was lost\n", smp_processor_id());
252
253 /* loss of MT counter data alert */
254 if (alert & CPU_MF_INT_CF_MTDA)
255 pr_warn("CPU[%i] MT counter data was lost\n",
256 smp_processor_id());
257 }
258
259 #define PMC_INIT 0
260 #define PMC_RELEASE 1
setup_pmc_cpu(void * flags)261 static void setup_pmc_cpu(void *flags)
262 {
263 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
264
265 switch (*((int *) flags)) {
266 case PMC_INIT:
267 memset(&cpuhw->info, 0, sizeof(cpuhw->info));
268 qctri(&cpuhw->info);
269 cpuhw->flags |= PMU_F_RESERVED;
270 break;
271
272 case PMC_RELEASE:
273 cpuhw->flags &= ~PMU_F_RESERVED;
274 break;
275 }
276
277 /* Disable CPU counter sets */
278 lcctl(0);
279 }
280
281 /* Initialize the CPU-measurement facility */
reserve_pmc_hardware(void)282 static int reserve_pmc_hardware(void)
283 {
284 int flags = PMC_INIT;
285
286 on_each_cpu(setup_pmc_cpu, &flags, 1);
287 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
288
289 return 0;
290 }
291
292 /* Release the CPU-measurement facility */
release_pmc_hardware(void)293 static void release_pmc_hardware(void)
294 {
295 int flags = PMC_RELEASE;
296
297 on_each_cpu(setup_pmc_cpu, &flags, 1);
298 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
299 }
300
301 /* Release the PMU if event is the last perf event */
hw_perf_event_destroy(struct perf_event * event)302 static void hw_perf_event_destroy(struct perf_event *event)
303 {
304 if (!atomic_add_unless(&num_events, -1, 1)) {
305 mutex_lock(&pmc_reserve_mutex);
306 if (atomic_dec_return(&num_events) == 0)
307 release_pmc_hardware();
308 mutex_unlock(&pmc_reserve_mutex);
309 }
310 }
311
312 /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */
313 static const int cpumf_generic_events_basic[] = {
314 [PERF_COUNT_HW_CPU_CYCLES] = 0,
315 [PERF_COUNT_HW_INSTRUCTIONS] = 1,
316 [PERF_COUNT_HW_CACHE_REFERENCES] = -1,
317 [PERF_COUNT_HW_CACHE_MISSES] = -1,
318 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
319 [PERF_COUNT_HW_BRANCH_MISSES] = -1,
320 [PERF_COUNT_HW_BUS_CYCLES] = -1,
321 };
322 /* CPUMF <-> perf event mappings for userspace (problem-state set) */
323 static const int cpumf_generic_events_user[] = {
324 [PERF_COUNT_HW_CPU_CYCLES] = 32,
325 [PERF_COUNT_HW_INSTRUCTIONS] = 33,
326 [PERF_COUNT_HW_CACHE_REFERENCES] = -1,
327 [PERF_COUNT_HW_CACHE_MISSES] = -1,
328 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1,
329 [PERF_COUNT_HW_BRANCH_MISSES] = -1,
330 [PERF_COUNT_HW_BUS_CYCLES] = -1,
331 };
332
__hw_perf_event_init(struct perf_event * event)333 static int __hw_perf_event_init(struct perf_event *event)
334 {
335 struct perf_event_attr *attr = &event->attr;
336 struct hw_perf_event *hwc = &event->hw;
337 enum cpumf_ctr_set set;
338 int err;
339 u64 ev;
340
341 switch (attr->type) {
342 case PERF_TYPE_RAW:
343 /* Raw events are used to access counters directly,
344 * hence do not permit excludes */
345 if (attr->exclude_kernel || attr->exclude_user ||
346 attr->exclude_hv)
347 return -EOPNOTSUPP;
348 ev = attr->config;
349 break;
350
351 case PERF_TYPE_HARDWARE:
352 if (is_sampling_event(event)) /* No sampling support */
353 return -ENOENT;
354 ev = attr->config;
355 /* Count user space (problem-state) only */
356 if (!attr->exclude_user && attr->exclude_kernel) {
357 if (ev >= ARRAY_SIZE(cpumf_generic_events_user))
358 return -EOPNOTSUPP;
359 ev = cpumf_generic_events_user[ev];
360
361 /* No support for kernel space counters only */
362 } else if (!attr->exclude_kernel && attr->exclude_user) {
363 return -EOPNOTSUPP;
364
365 /* Count user and kernel space */
366 } else {
367 if (ev >= ARRAY_SIZE(cpumf_generic_events_basic))
368 return -EOPNOTSUPP;
369 ev = cpumf_generic_events_basic[ev];
370 }
371 break;
372
373 default:
374 return -ENOENT;
375 }
376
377 if (ev == -1)
378 return -ENOENT;
379
380 if (ev > PERF_CPUM_CF_MAX_CTR)
381 return -ENOENT;
382
383 /* Obtain the counter set to which the specified counter belongs */
384 set = get_counter_set(ev);
385 switch (set) {
386 case CPUMF_CTR_SET_BASIC:
387 case CPUMF_CTR_SET_USER:
388 case CPUMF_CTR_SET_CRYPTO:
389 case CPUMF_CTR_SET_EXT:
390 case CPUMF_CTR_SET_MT_DIAG:
391 /*
392 * Use the hardware perf event structure to store the
393 * counter number in the 'config' member and the counter
394 * set number in the 'config_base'. The counter set number
395 * is then later used to enable/disable the counter(s).
396 */
397 hwc->config = ev;
398 hwc->config_base = set;
399 break;
400 case CPUMF_CTR_SET_MAX:
401 /* The counter could not be associated to a counter set */
402 return -EINVAL;
403 };
404
405 /* Initialize for using the CPU-measurement counter facility */
406 if (!atomic_inc_not_zero(&num_events)) {
407 mutex_lock(&pmc_reserve_mutex);
408 if (atomic_read(&num_events) == 0 && reserve_pmc_hardware())
409 err = -EBUSY;
410 else
411 atomic_inc(&num_events);
412 mutex_unlock(&pmc_reserve_mutex);
413 }
414 event->destroy = hw_perf_event_destroy;
415
416 /* Finally, validate version and authorization of the counter set */
417 err = validate_ctr_auth(hwc);
418 if (!err)
419 err = validate_ctr_version(hwc);
420
421 return err;
422 }
423
cpumf_pmu_event_init(struct perf_event * event)424 static int cpumf_pmu_event_init(struct perf_event *event)
425 {
426 int err;
427
428 switch (event->attr.type) {
429 case PERF_TYPE_HARDWARE:
430 case PERF_TYPE_HW_CACHE:
431 case PERF_TYPE_RAW:
432 err = __hw_perf_event_init(event);
433 break;
434 default:
435 return -ENOENT;
436 }
437
438 if (unlikely(err) && event->destroy)
439 event->destroy(event);
440
441 return err;
442 }
443
hw_perf_event_reset(struct perf_event * event)444 static int hw_perf_event_reset(struct perf_event *event)
445 {
446 u64 prev, new;
447 int err;
448
449 do {
450 prev = local64_read(&event->hw.prev_count);
451 err = ecctr(event->hw.config, &new);
452 if (err) {
453 if (err != 3)
454 break;
455 /* The counter is not (yet) available. This
456 * might happen if the counter set to which
457 * this counter belongs is in the disabled
458 * state.
459 */
460 new = 0;
461 }
462 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
463
464 return err;
465 }
466
hw_perf_event_update(struct perf_event * event)467 static void hw_perf_event_update(struct perf_event *event)
468 {
469 u64 prev, new, delta;
470 int err;
471
472 do {
473 prev = local64_read(&event->hw.prev_count);
474 err = ecctr(event->hw.config, &new);
475 if (err)
476 return;
477 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev);
478
479 delta = (prev <= new) ? new - prev
480 : (-1ULL - prev) + new + 1; /* overflow */
481 local64_add(delta, &event->count);
482 }
483
cpumf_pmu_read(struct perf_event * event)484 static void cpumf_pmu_read(struct perf_event *event)
485 {
486 if (event->hw.state & PERF_HES_STOPPED)
487 return;
488
489 hw_perf_event_update(event);
490 }
491
cpumf_pmu_start(struct perf_event * event,int flags)492 static void cpumf_pmu_start(struct perf_event *event, int flags)
493 {
494 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
495 struct hw_perf_event *hwc = &event->hw;
496
497 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
498 return;
499
500 if (WARN_ON_ONCE(hwc->config == -1))
501 return;
502
503 if (flags & PERF_EF_RELOAD)
504 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
505
506 hwc->state = 0;
507
508 /* (Re-)enable and activate the counter set */
509 ctr_set_enable(&cpuhw->state, hwc->config_base);
510 ctr_set_start(&cpuhw->state, hwc->config_base);
511
512 /* The counter set to which this counter belongs can be already active.
513 * Because all counters in a set are active, the event->hw.prev_count
514 * needs to be synchronized. At this point, the counter set can be in
515 * the inactive or disabled state.
516 */
517 hw_perf_event_reset(event);
518
519 /* increment refcount for this counter set */
520 atomic_inc(&cpuhw->ctr_set[hwc->config_base]);
521 }
522
cpumf_pmu_stop(struct perf_event * event,int flags)523 static void cpumf_pmu_stop(struct perf_event *event, int flags)
524 {
525 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
526 struct hw_perf_event *hwc = &event->hw;
527
528 if (!(hwc->state & PERF_HES_STOPPED)) {
529 /* Decrement reference count for this counter set and if this
530 * is the last used counter in the set, clear activation
531 * control and set the counter set state to inactive.
532 */
533 if (!atomic_dec_return(&cpuhw->ctr_set[hwc->config_base]))
534 ctr_set_stop(&cpuhw->state, hwc->config_base);
535 event->hw.state |= PERF_HES_STOPPED;
536 }
537
538 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
539 hw_perf_event_update(event);
540 event->hw.state |= PERF_HES_UPTODATE;
541 }
542 }
543
cpumf_pmu_add(struct perf_event * event,int flags)544 static int cpumf_pmu_add(struct perf_event *event, int flags)
545 {
546 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
547
548 /* Check authorization for the counter set to which this
549 * counter belongs.
550 * For group events transaction, the authorization check is
551 * done in cpumf_pmu_commit_txn().
552 */
553 if (!(cpuhw->txn_flags & PERF_PMU_TXN_ADD))
554 if (validate_ctr_auth(&event->hw))
555 return -ENOENT;
556
557 ctr_set_enable(&cpuhw->state, event->hw.config_base);
558 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
559
560 if (flags & PERF_EF_START)
561 cpumf_pmu_start(event, PERF_EF_RELOAD);
562
563 perf_event_update_userpage(event);
564
565 return 0;
566 }
567
cpumf_pmu_del(struct perf_event * event,int flags)568 static void cpumf_pmu_del(struct perf_event *event, int flags)
569 {
570 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
571
572 cpumf_pmu_stop(event, PERF_EF_UPDATE);
573
574 /* Check if any counter in the counter set is still used. If not used,
575 * change the counter set to the disabled state. This also clears the
576 * content of all counters in the set.
577 *
578 * When a new perf event has been added but not yet started, this can
579 * clear enable control and resets all counters in a set. Therefore,
580 * cpumf_pmu_start() always has to reenable a counter set.
581 */
582 if (!atomic_read(&cpuhw->ctr_set[event->hw.config_base]))
583 ctr_set_disable(&cpuhw->state, event->hw.config_base);
584
585 perf_event_update_userpage(event);
586 }
587
588 /*
589 * Start group events scheduling transaction.
590 * Set flags to perform a single test at commit time.
591 *
592 * We only support PERF_PMU_TXN_ADD transactions. Save the
593 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
594 * transactions.
595 */
cpumf_pmu_start_txn(struct pmu * pmu,unsigned int txn_flags)596 static void cpumf_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
597 {
598 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
599
600 WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */
601
602 cpuhw->txn_flags = txn_flags;
603 if (txn_flags & ~PERF_PMU_TXN_ADD)
604 return;
605
606 perf_pmu_disable(pmu);
607 cpuhw->tx_state = cpuhw->state;
608 }
609
610 /*
611 * Stop and cancel a group events scheduling tranctions.
612 * Assumes cpumf_pmu_del() is called for each successful added
613 * cpumf_pmu_add() during the transaction.
614 */
cpumf_pmu_cancel_txn(struct pmu * pmu)615 static void cpumf_pmu_cancel_txn(struct pmu *pmu)
616 {
617 unsigned int txn_flags;
618 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
619
620 WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */
621
622 txn_flags = cpuhw->txn_flags;
623 cpuhw->txn_flags = 0;
624 if (txn_flags & ~PERF_PMU_TXN_ADD)
625 return;
626
627 WARN_ON(cpuhw->tx_state != cpuhw->state);
628
629 perf_pmu_enable(pmu);
630 }
631
632 /*
633 * Commit the group events scheduling transaction. On success, the
634 * transaction is closed. On error, the transaction is kept open
635 * until cpumf_pmu_cancel_txn() is called.
636 */
cpumf_pmu_commit_txn(struct pmu * pmu)637 static int cpumf_pmu_commit_txn(struct pmu *pmu)
638 {
639 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
640 u64 state;
641
642 WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */
643
644 if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) {
645 cpuhw->txn_flags = 0;
646 return 0;
647 }
648
649 /* check if the updated state can be scheduled */
650 state = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1);
651 state >>= CPUMF_LCCTL_ENABLE_SHIFT;
652 if ((state & cpuhw->info.auth_ctl) != state)
653 return -ENOENT;
654
655 cpuhw->txn_flags = 0;
656 perf_pmu_enable(pmu);
657 return 0;
658 }
659
660 /* Performance monitoring unit for s390x */
661 static struct pmu cpumf_pmu = {
662 .task_ctx_nr = perf_sw_context,
663 .capabilities = PERF_PMU_CAP_NO_INTERRUPT,
664 .pmu_enable = cpumf_pmu_enable,
665 .pmu_disable = cpumf_pmu_disable,
666 .event_init = cpumf_pmu_event_init,
667 .add = cpumf_pmu_add,
668 .del = cpumf_pmu_del,
669 .start = cpumf_pmu_start,
670 .stop = cpumf_pmu_stop,
671 .read = cpumf_pmu_read,
672 .start_txn = cpumf_pmu_start_txn,
673 .commit_txn = cpumf_pmu_commit_txn,
674 .cancel_txn = cpumf_pmu_cancel_txn,
675 };
676
cpumf_pmf_setup(unsigned int cpu,int flags)677 static int cpumf_pmf_setup(unsigned int cpu, int flags)
678 {
679 local_irq_disable();
680 setup_pmc_cpu(&flags);
681 local_irq_enable();
682 return 0;
683 }
684
s390_pmu_online_cpu(unsigned int cpu)685 static int s390_pmu_online_cpu(unsigned int cpu)
686 {
687 return cpumf_pmf_setup(cpu, PMC_INIT);
688 }
689
s390_pmu_offline_cpu(unsigned int cpu)690 static int s390_pmu_offline_cpu(unsigned int cpu)
691 {
692 return cpumf_pmf_setup(cpu, PMC_RELEASE);
693 }
694
cpumf_pmu_init(void)695 static int __init cpumf_pmu_init(void)
696 {
697 int rc;
698
699 if (!cpum_cf_avail())
700 return -ENODEV;
701
702 /* clear bit 15 of cr0 to unauthorize problem-state to
703 * extract measurement counters */
704 ctl_clear_bit(0, 48);
705
706 /* register handler for measurement-alert interruptions */
707 rc = register_external_irq(EXT_IRQ_MEASURE_ALERT,
708 cpumf_measurement_alert);
709 if (rc) {
710 pr_err("Registering for CPU-measurement alerts "
711 "failed with rc=%i\n", rc);
712 return rc;
713 }
714
715 cpumf_pmu.attr_groups = cpumf_cf_event_group();
716 rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW);
717 if (rc) {
718 pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc);
719 unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
720 cpumf_measurement_alert);
721 return rc;
722 }
723 return cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE,
724 "perf/s390/cf:online",
725 s390_pmu_online_cpu, s390_pmu_offline_cpu);
726 }
727 early_initcall(cpumf_pmu_init);
728