Lines Matching refs:cpuhw
120 static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw) in ebb_switch_in() argument
122 return cpuhw->mmcr[0]; in ebb_switch_in()
128 static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {} in power_pmu_bhrb_read() argument
351 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_bhrb_enable() local
357 if (event->ctx->task && cpuhw->bhrb_context != event->ctx) { in power_pmu_bhrb_enable()
359 cpuhw->bhrb_context = event->ctx; in power_pmu_bhrb_enable()
361 cpuhw->bhrb_users++; in power_pmu_bhrb_enable()
367 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_bhrb_disable() local
372 WARN_ON_ONCE(!cpuhw->bhrb_users); in power_pmu_bhrb_disable()
373 cpuhw->bhrb_users--; in power_pmu_bhrb_disable()
376 if (!cpuhw->disabled && !cpuhw->bhrb_users) { in power_pmu_bhrb_disable()
382 cpuhw->bhrb_context = NULL; in power_pmu_bhrb_disable()
429 static void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) in power_pmu_bhrb_read() argument
473 cpuhw->bhrb_entries[u_index].to = addr; in power_pmu_bhrb_read()
474 cpuhw->bhrb_entries[u_index].mispred = pred; in power_pmu_bhrb_read()
475 cpuhw->bhrb_entries[u_index].predicted = ~pred; in power_pmu_bhrb_read()
486 cpuhw->bhrb_entries[u_index].from = addr; in power_pmu_bhrb_read()
490 cpuhw->bhrb_entries[u_index].from = addr; in power_pmu_bhrb_read()
491 cpuhw->bhrb_entries[u_index].to = in power_pmu_bhrb_read()
493 cpuhw->bhrb_entries[u_index].mispred = pred; in power_pmu_bhrb_read()
494 cpuhw->bhrb_entries[u_index].predicted = ~pred; in power_pmu_bhrb_read()
500 cpuhw->bhrb_stack.nr = u_index; in power_pmu_bhrb_read()
568 static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw) in ebb_switch_in() argument
570 unsigned long mmcr0 = cpuhw->mmcr[0]; in ebb_switch_in()
604 mtspr(SPRN_MMCR2, cpuhw->mmcr[3] | current->thread.mmcr2); in ebb_switch_in()
832 static int power_check_constraints(struct cpu_hw_events *cpuhw, in power_check_constraints() argument
851 cpuhw->alternatives[i]); in power_check_constraints()
852 event_id[i] = cpuhw->alternatives[i][0]; in power_check_constraints()
854 if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0], in power_check_constraints()
855 &cpuhw->avalues[i][0])) in power_check_constraints()
860 nv = (value | cpuhw->avalues[i][0]) + in power_check_constraints()
861 (value & cpuhw->avalues[i][0] & addf); in power_check_constraints()
863 (((nv + tadd) ^ cpuhw->avalues[i][0]) & in power_check_constraints()
864 cpuhw->amasks[i][0]) != 0) in power_check_constraints()
867 mask |= cpuhw->amasks[i][0]; in power_check_constraints()
878 cpuhw->alternatives[i]); in power_check_constraints()
880 ppmu->get_constraint(cpuhw->alternatives[i][j], in power_check_constraints()
881 &cpuhw->amasks[i][j], in power_check_constraints()
882 &cpuhw->avalues[i][j]); in power_check_constraints()
901 nv = (value | cpuhw->avalues[i][j]) + in power_check_constraints()
902 (value & cpuhw->avalues[i][j] & addf); in power_check_constraints()
904 (((nv + tadd) ^ cpuhw->avalues[i][j]) in power_check_constraints()
905 & cpuhw->amasks[i][j]) == 0) in power_check_constraints()
927 mask |= cpuhw->amasks[i][j]; in power_check_constraints()
935 event_id[i] = cpuhw->alternatives[i][choice[i]]; in power_check_constraints()
1069 static void freeze_limited_counters(struct cpu_hw_events *cpuhw, in freeze_limited_counters() argument
1076 for (i = 0; i < cpuhw->n_limited; ++i) { in freeze_limited_counters()
1077 event = cpuhw->limited_counter[i]; in freeze_limited_counters()
1089 static void thaw_limited_counters(struct cpu_hw_events *cpuhw, in thaw_limited_counters() argument
1096 for (i = 0; i < cpuhw->n_limited; ++i) { in thaw_limited_counters()
1097 event = cpuhw->limited_counter[i]; in thaw_limited_counters()
1098 event->hw.idx = cpuhw->limited_hwidx[i]; in thaw_limited_counters()
1118 static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0) in write_mmcr0() argument
1122 if (!cpuhw->n_limited) { in write_mmcr0()
1141 freeze_limited_counters(cpuhw, pmc5, pmc6); in write_mmcr0()
1143 thaw_limited_counters(cpuhw, pmc5, pmc6); in write_mmcr0()
1159 struct cpu_hw_events *cpuhw; in power_pmu_disable() local
1165 cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_disable()
1167 if (!cpuhw->disabled) { in power_pmu_disable()
1171 if (!cpuhw->pmcs_enabled) { in power_pmu_disable()
1173 cpuhw->pmcs_enabled = 1; in power_pmu_disable()
1189 write_mmcr0(cpuhw, val); in power_pmu_disable()
1195 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) { in power_pmu_disable()
1197 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); in power_pmu_disable()
1201 cpuhw->disabled = 1; in power_pmu_disable()
1202 cpuhw->n_added = 0; in power_pmu_disable()
1218 struct cpu_hw_events *cpuhw; in power_pmu_enable() local
1232 cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_enable()
1233 if (!cpuhw->disabled) in power_pmu_enable()
1236 if (cpuhw->n_events == 0) { in power_pmu_enable()
1241 cpuhw->disabled = 0; in power_pmu_enable()
1248 ebb = is_ebb_event(cpuhw->event[0]); in power_pmu_enable()
1256 if (!cpuhw->n_added) { in power_pmu_enable()
1257 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); in power_pmu_enable()
1258 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]); in power_pmu_enable()
1265 memset(cpuhw->mmcr, 0, sizeof(cpuhw->mmcr)); in power_pmu_enable()
1267 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index, in power_pmu_enable()
1268 cpuhw->mmcr, cpuhw->event)) { in power_pmu_enable()
1280 event = cpuhw->event[0]; in power_pmu_enable()
1282 cpuhw->mmcr[0] |= MMCR0_FCP; in power_pmu_enable()
1284 cpuhw->mmcr[0] |= freeze_events_kernel; in power_pmu_enable()
1286 cpuhw->mmcr[0] |= MMCR0_FCHV; in power_pmu_enable()
1295 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE); in power_pmu_enable()
1296 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]); in power_pmu_enable()
1297 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)) in power_pmu_enable()
1300 mtspr(SPRN_MMCR2, cpuhw->mmcr[3]); in power_pmu_enable()
1306 for (i = 0; i < cpuhw->n_events; ++i) { in power_pmu_enable()
1307 event = cpuhw->event[i]; in power_pmu_enable()
1318 cpuhw->n_limited = n_lim = 0; in power_pmu_enable()
1319 for (i = 0; i < cpuhw->n_events; ++i) { in power_pmu_enable()
1320 event = cpuhw->event[i]; in power_pmu_enable()
1325 cpuhw->limited_counter[n_lim] = event; in power_pmu_enable()
1326 cpuhw->limited_hwidx[n_lim] = idx; in power_pmu_enable()
1350 cpuhw->n_limited = n_lim; in power_pmu_enable()
1351 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE; in power_pmu_enable()
1356 mmcr0 = ebb_switch_in(ebb, cpuhw); in power_pmu_enable()
1359 if (cpuhw->bhrb_users) in power_pmu_enable()
1360 ppmu->config_bhrb(cpuhw->bhrb_filter); in power_pmu_enable()
1362 write_mmcr0(cpuhw, mmcr0); in power_pmu_enable()
1367 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) { in power_pmu_enable()
1369 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]); in power_pmu_enable()
1412 struct cpu_hw_events *cpuhw; in power_pmu_add() local
1424 cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_add()
1425 n0 = cpuhw->n_events; in power_pmu_add()
1428 cpuhw->event[n0] = event; in power_pmu_add()
1429 cpuhw->events[n0] = event->hw.config; in power_pmu_add()
1430 cpuhw->flags[n0] = event->hw.event_base; in power_pmu_add()
1448 if (cpuhw->txn_flags & PERF_PMU_TXN_ADD) in power_pmu_add()
1451 if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1)) in power_pmu_add()
1453 if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1)) in power_pmu_add()
1455 event->hw.config = cpuhw->events[n0]; in power_pmu_add()
1460 ++cpuhw->n_events; in power_pmu_add()
1461 ++cpuhw->n_added; in power_pmu_add()
1467 cpuhw->bhrb_filter = ppmu->bhrb_filter_map( in power_pmu_add()
1481 struct cpu_hw_events *cpuhw; in power_pmu_del() local
1490 cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_del()
1491 for (i = 0; i < cpuhw->n_events; ++i) { in power_pmu_del()
1492 if (event == cpuhw->event[i]) { in power_pmu_del()
1493 while (++i < cpuhw->n_events) { in power_pmu_del()
1494 cpuhw->event[i-1] = cpuhw->event[i]; in power_pmu_del()
1495 cpuhw->events[i-1] = cpuhw->events[i]; in power_pmu_del()
1496 cpuhw->flags[i-1] = cpuhw->flags[i]; in power_pmu_del()
1498 --cpuhw->n_events; in power_pmu_del()
1499 ppmu->disable_pmc(event->hw.idx - 1, cpuhw->mmcr); in power_pmu_del()
1508 for (i = 0; i < cpuhw->n_limited; ++i) in power_pmu_del()
1509 if (event == cpuhw->limited_counter[i]) in power_pmu_del()
1511 if (i < cpuhw->n_limited) { in power_pmu_del()
1512 while (++i < cpuhw->n_limited) { in power_pmu_del()
1513 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i]; in power_pmu_del()
1514 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i]; in power_pmu_del()
1516 --cpuhw->n_limited; in power_pmu_del()
1518 if (cpuhw->n_events == 0) { in power_pmu_del()
1520 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE); in power_pmu_del()
1600 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_start_txn() local
1602 WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ in power_pmu_start_txn()
1604 cpuhw->txn_flags = txn_flags; in power_pmu_start_txn()
1609 cpuhw->n_txn_start = cpuhw->n_events; in power_pmu_start_txn()
1619 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_cancel_txn() local
1622 WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ in power_pmu_cancel_txn()
1624 txn_flags = cpuhw->txn_flags; in power_pmu_cancel_txn()
1625 cpuhw->txn_flags = 0; in power_pmu_cancel_txn()
1639 struct cpu_hw_events *cpuhw; in power_pmu_commit_txn() local
1645 cpuhw = this_cpu_ptr(&cpu_hw_events); in power_pmu_commit_txn()
1646 WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ in power_pmu_commit_txn()
1648 if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) { in power_pmu_commit_txn()
1649 cpuhw->txn_flags = 0; in power_pmu_commit_txn()
1653 n = cpuhw->n_events; in power_pmu_commit_txn()
1654 if (check_excludes(cpuhw->event, cpuhw->flags, 0, n)) in power_pmu_commit_txn()
1656 i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n); in power_pmu_commit_txn()
1660 for (i = cpuhw->n_txn_start; i < n; ++i) in power_pmu_commit_txn()
1661 cpuhw->event[i]->hw.config = cpuhw->events[i]; in power_pmu_commit_txn()
1663 cpuhw->txn_flags = 0; in power_pmu_commit_txn()
1777 struct cpu_hw_events *cpuhw; in power_pmu_event_init() local
1870 cpuhw = &get_cpu_var(cpu_hw_events); in power_pmu_event_init()
1871 err = power_check_constraints(cpuhw, events, cflags, n + 1); in power_pmu_event_init()
1874 cpuhw->bhrb_filter = ppmu->bhrb_filter_map( in power_pmu_event_init()
1877 if (cpuhw->bhrb_filter == -1) { in power_pmu_event_init()
2010 struct cpu_hw_events *cpuhw; in record_and_restart() local
2011 cpuhw = this_cpu_ptr(&cpu_hw_events); in record_and_restart()
2012 power_pmu_bhrb_read(cpuhw); in record_and_restart()
2013 data.br_stack = &cpuhw->bhrb_stack; in record_and_restart()
2084 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); in perf_event_interrupt() local
2090 if (cpuhw->n_limited) in perf_event_interrupt()
2091 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5), in perf_event_interrupt()
2120 for (j = 0; j < cpuhw->n_events; ++j) { in perf_event_interrupt()
2121 event = cpuhw->event[j]; in perf_event_interrupt()
2134 for (i = 0; i < cpuhw->n_events; ++i) { in perf_event_interrupt()
2135 event = cpuhw->event[i]; in perf_event_interrupt()
2157 write_mmcr0(cpuhw, cpuhw->mmcr[0]); in perf_event_interrupt()
2167 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); in power_pmu_prepare_cpu() local
2170 memset(cpuhw, 0, sizeof(*cpuhw)); in power_pmu_prepare_cpu()
2171 cpuhw->mmcr[0] = MMCR0_FC; in power_pmu_prepare_cpu()