1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
4 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
5 * Copyright (C) 2009 Jaswinder Singh Rajput
6 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
7 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
8 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
9 * Copyright (C) 2009 Google, Inc., Stephane Eranian
10 * Copyright 2014 Tilera Corporation. All Rights Reserved.
11 * Copyright (C) 2018 Andes Technology Corporation
12 *
13 * Perf_events support for RISC-V platforms.
14 *
15 * Since the spec. (as of now, Priv-Spec 1.10) does not provide enough
16 * functionality for perf event to fully work, this file provides
17 * the very basic framework only.
18 *
19 * For platform portings, please check Documentations/riscv/pmu.txt.
20 *
21 * The Copyright line includes x86 and tile ones.
22 */
23
24 #include <linux/kprobes.h>
25 #include <linux/kernel.h>
26 #include <linux/kdebug.h>
27 #include <linux/mutex.h>
28 #include <linux/bitmap.h>
29 #include <linux/irq.h>
30 #include <linux/perf_event.h>
31 #include <linux/atomic.h>
32 #include <linux/of.h>
33 #include <asm/perf_event.h>
34
35 static const struct riscv_pmu *riscv_pmu __read_mostly;
36 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
37
38 /*
39 * Hardware & cache maps and their methods
40 */
41
42 static const int riscv_hw_event_map[] = {
43 [PERF_COUNT_HW_CPU_CYCLES] = RISCV_PMU_CYCLE,
44 [PERF_COUNT_HW_INSTRUCTIONS] = RISCV_PMU_INSTRET,
45 [PERF_COUNT_HW_CACHE_REFERENCES] = RISCV_OP_UNSUPP,
46 [PERF_COUNT_HW_CACHE_MISSES] = RISCV_OP_UNSUPP,
47 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = RISCV_OP_UNSUPP,
48 [PERF_COUNT_HW_BRANCH_MISSES] = RISCV_OP_UNSUPP,
49 [PERF_COUNT_HW_BUS_CYCLES] = RISCV_OP_UNSUPP,
50 };
51
52 #define C(x) PERF_COUNT_HW_CACHE_##x
53 static const int riscv_cache_event_map[PERF_COUNT_HW_CACHE_MAX]
54 [PERF_COUNT_HW_CACHE_OP_MAX]
55 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
56 [C(L1D)] = {
57 [C(OP_READ)] = {
58 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
59 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
60 },
61 [C(OP_WRITE)] = {
62 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
63 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
64 },
65 [C(OP_PREFETCH)] = {
66 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
67 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
68 },
69 },
70 [C(L1I)] = {
71 [C(OP_READ)] = {
72 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
73 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
74 },
75 [C(OP_WRITE)] = {
76 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
77 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
78 },
79 [C(OP_PREFETCH)] = {
80 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
81 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
82 },
83 },
84 [C(LL)] = {
85 [C(OP_READ)] = {
86 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
87 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
88 },
89 [C(OP_WRITE)] = {
90 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
91 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
92 },
93 [C(OP_PREFETCH)] = {
94 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
95 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
96 },
97 },
98 [C(DTLB)] = {
99 [C(OP_READ)] = {
100 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
101 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
102 },
103 [C(OP_WRITE)] = {
104 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
105 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
106 },
107 [C(OP_PREFETCH)] = {
108 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
109 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
110 },
111 },
112 [C(ITLB)] = {
113 [C(OP_READ)] = {
114 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
115 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
116 },
117 [C(OP_WRITE)] = {
118 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
119 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
120 },
121 [C(OP_PREFETCH)] = {
122 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
123 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
124 },
125 },
126 [C(BPU)] = {
127 [C(OP_READ)] = {
128 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
129 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
130 },
131 [C(OP_WRITE)] = {
132 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
133 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
134 },
135 [C(OP_PREFETCH)] = {
136 [C(RESULT_ACCESS)] = RISCV_OP_UNSUPP,
137 [C(RESULT_MISS)] = RISCV_OP_UNSUPP,
138 },
139 },
140 };
141
riscv_map_hw_event(u64 config)142 static int riscv_map_hw_event(u64 config)
143 {
144 if (config >= riscv_pmu->max_events)
145 return -EINVAL;
146
147 return riscv_pmu->hw_events[config];
148 }
149
riscv_map_cache_decode(u64 config,unsigned int * type,unsigned int * op,unsigned int * result)150 int riscv_map_cache_decode(u64 config, unsigned int *type,
151 unsigned int *op, unsigned int *result)
152 {
153 return -ENOENT;
154 }
155
riscv_map_cache_event(u64 config)156 static int riscv_map_cache_event(u64 config)
157 {
158 unsigned int type, op, result;
159 int err = -ENOENT;
160 int code;
161
162 err = riscv_map_cache_decode(config, &type, &op, &result);
163 if (!riscv_pmu->cache_events || err)
164 return err;
165
166 if (type >= PERF_COUNT_HW_CACHE_MAX ||
167 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
168 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
169 return -EINVAL;
170
171 code = (*riscv_pmu->cache_events)[type][op][result];
172 if (code == RISCV_OP_UNSUPP)
173 return -EINVAL;
174
175 return code;
176 }
177
178 /*
179 * Low-level functions: reading/writing counters
180 */
181
read_counter(int idx)182 static inline u64 read_counter(int idx)
183 {
184 u64 val = 0;
185
186 switch (idx) {
187 case RISCV_PMU_CYCLE:
188 val = csr_read(CSR_CYCLE);
189 break;
190 case RISCV_PMU_INSTRET:
191 val = csr_read(CSR_INSTRET);
192 break;
193 default:
194 WARN_ON_ONCE(idx < 0 || idx > RISCV_MAX_COUNTERS);
195 return -EINVAL;
196 }
197
198 return val;
199 }
200
write_counter(int idx,u64 value)201 static inline void write_counter(int idx, u64 value)
202 {
203 /* currently not supported */
204 WARN_ON_ONCE(1);
205 }
206
207 /*
208 * pmu->read: read and update the counter
209 *
210 * Other architectures' implementation often have a xxx_perf_event_update
211 * routine, which can return counter values when called in the IRQ, but
212 * return void when being called by the pmu->read method.
213 */
riscv_pmu_read(struct perf_event * event)214 static void riscv_pmu_read(struct perf_event *event)
215 {
216 struct hw_perf_event *hwc = &event->hw;
217 u64 prev_raw_count, new_raw_count;
218 u64 oldval;
219 int idx = hwc->idx;
220 u64 delta;
221
222 do {
223 prev_raw_count = local64_read(&hwc->prev_count);
224 new_raw_count = read_counter(idx);
225
226 oldval = local64_cmpxchg(&hwc->prev_count, prev_raw_count,
227 new_raw_count);
228 } while (oldval != prev_raw_count);
229
230 /*
231 * delta is the value to update the counter we maintain in the kernel.
232 */
233 delta = (new_raw_count - prev_raw_count) &
234 ((1ULL << riscv_pmu->counter_width) - 1);
235 local64_add(delta, &event->count);
236 /*
237 * Something like local64_sub(delta, &hwc->period_left) here is
238 * needed if there is an interrupt for perf.
239 */
240 }
241
242 /*
243 * State transition functions:
244 *
245 * stop()/start() & add()/del()
246 */
247
248 /*
249 * pmu->stop: stop the counter
250 */
riscv_pmu_stop(struct perf_event * event,int flags)251 static void riscv_pmu_stop(struct perf_event *event, int flags)
252 {
253 struct hw_perf_event *hwc = &event->hw;
254
255 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
256 hwc->state |= PERF_HES_STOPPED;
257
258 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
259 riscv_pmu->pmu->read(event);
260 hwc->state |= PERF_HES_UPTODATE;
261 }
262 }
263
264 /*
265 * pmu->start: start the event.
266 */
riscv_pmu_start(struct perf_event * event,int flags)267 static void riscv_pmu_start(struct perf_event *event, int flags)
268 {
269 struct hw_perf_event *hwc = &event->hw;
270
271 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
272 return;
273
274 if (flags & PERF_EF_RELOAD) {
275 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
276
277 /*
278 * Set the counter to the period to the next interrupt here,
279 * if you have any.
280 */
281 }
282
283 hwc->state = 0;
284 perf_event_update_userpage(event);
285
286 /*
287 * Since we cannot write to counters, this serves as an initialization
288 * to the delta-mechanism in pmu->read(); otherwise, the delta would be
289 * wrong when pmu->read is called for the first time.
290 */
291 local64_set(&hwc->prev_count, read_counter(hwc->idx));
292 }
293
294 /*
295 * pmu->add: add the event to PMU.
296 */
riscv_pmu_add(struct perf_event * event,int flags)297 static int riscv_pmu_add(struct perf_event *event, int flags)
298 {
299 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
300 struct hw_perf_event *hwc = &event->hw;
301
302 if (cpuc->n_events == riscv_pmu->num_counters)
303 return -ENOSPC;
304
305 /*
306 * We don't have general conunters, so no binding-event-to-counter
307 * process here.
308 *
309 * Indexing using hwc->config generally not works, since config may
310 * contain extra information, but here the only info we have in
311 * hwc->config is the event index.
312 */
313 hwc->idx = hwc->config;
314 cpuc->events[hwc->idx] = event;
315 cpuc->n_events++;
316
317 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
318
319 if (flags & PERF_EF_START)
320 riscv_pmu->pmu->start(event, PERF_EF_RELOAD);
321
322 return 0;
323 }
324
325 /*
326 * pmu->del: delete the event from PMU.
327 */
riscv_pmu_del(struct perf_event * event,int flags)328 static void riscv_pmu_del(struct perf_event *event, int flags)
329 {
330 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
331 struct hw_perf_event *hwc = &event->hw;
332
333 cpuc->events[hwc->idx] = NULL;
334 cpuc->n_events--;
335 riscv_pmu->pmu->stop(event, PERF_EF_UPDATE);
336 perf_event_update_userpage(event);
337 }
338
339 /*
340 * Interrupt: a skeletion for reference.
341 */
342
343 static DEFINE_MUTEX(pmc_reserve_mutex);
344
riscv_base_pmu_handle_irq(int irq_num,void * dev)345 irqreturn_t riscv_base_pmu_handle_irq(int irq_num, void *dev)
346 {
347 return IRQ_NONE;
348 }
349
reserve_pmc_hardware(void)350 static int reserve_pmc_hardware(void)
351 {
352 int err = 0;
353
354 mutex_lock(&pmc_reserve_mutex);
355 if (riscv_pmu->irq >= 0 && riscv_pmu->handle_irq) {
356 err = request_irq(riscv_pmu->irq, riscv_pmu->handle_irq,
357 IRQF_PERCPU, "riscv-base-perf", NULL);
358 }
359 mutex_unlock(&pmc_reserve_mutex);
360
361 return err;
362 }
363
release_pmc_hardware(void)364 void release_pmc_hardware(void)
365 {
366 mutex_lock(&pmc_reserve_mutex);
367 if (riscv_pmu->irq >= 0)
368 free_irq(riscv_pmu->irq, NULL);
369 mutex_unlock(&pmc_reserve_mutex);
370 }
371
372 /*
373 * Event Initialization/Finalization
374 */
375
376 static atomic_t riscv_active_events = ATOMIC_INIT(0);
377
riscv_event_destroy(struct perf_event * event)378 static void riscv_event_destroy(struct perf_event *event)
379 {
380 if (atomic_dec_return(&riscv_active_events) == 0)
381 release_pmc_hardware();
382 }
383
riscv_event_init(struct perf_event * event)384 static int riscv_event_init(struct perf_event *event)
385 {
386 struct perf_event_attr *attr = &event->attr;
387 struct hw_perf_event *hwc = &event->hw;
388 int err;
389 int code;
390
391 if (atomic_inc_return(&riscv_active_events) == 1) {
392 err = reserve_pmc_hardware();
393
394 if (err) {
395 pr_warn("PMC hardware not available\n");
396 atomic_dec(&riscv_active_events);
397 return -EBUSY;
398 }
399 }
400
401 switch (event->attr.type) {
402 case PERF_TYPE_HARDWARE:
403 code = riscv_pmu->map_hw_event(attr->config);
404 break;
405 case PERF_TYPE_HW_CACHE:
406 code = riscv_pmu->map_cache_event(attr->config);
407 break;
408 case PERF_TYPE_RAW:
409 return -EOPNOTSUPP;
410 default:
411 return -ENOENT;
412 }
413
414 event->destroy = riscv_event_destroy;
415 if (code < 0) {
416 event->destroy(event);
417 return code;
418 }
419
420 /*
421 * idx is set to -1 because the index of a general event should not be
422 * decided until binding to some counter in pmu->add().
423 *
424 * But since we don't have such support, later in pmu->add(), we just
425 * use hwc->config as the index instead.
426 */
427 hwc->config = code;
428 hwc->idx = -1;
429
430 return 0;
431 }
432
433 /*
434 * Initialization
435 */
436
437 static struct pmu min_pmu = {
438 .name = "riscv-base",
439 .event_init = riscv_event_init,
440 .add = riscv_pmu_add,
441 .del = riscv_pmu_del,
442 .start = riscv_pmu_start,
443 .stop = riscv_pmu_stop,
444 .read = riscv_pmu_read,
445 };
446
447 static const struct riscv_pmu riscv_base_pmu = {
448 .pmu = &min_pmu,
449 .max_events = ARRAY_SIZE(riscv_hw_event_map),
450 .map_hw_event = riscv_map_hw_event,
451 .hw_events = riscv_hw_event_map,
452 .map_cache_event = riscv_map_cache_event,
453 .cache_events = &riscv_cache_event_map,
454 .counter_width = 63,
455 .num_counters = RISCV_BASE_COUNTERS + 0,
456 .handle_irq = &riscv_base_pmu_handle_irq,
457
458 /* This means this PMU has no IRQ. */
459 .irq = -1,
460 };
461
462 static const struct of_device_id riscv_pmu_of_ids[] = {
463 {.compatible = "riscv,base-pmu", .data = &riscv_base_pmu},
464 { /* sentinel value */ }
465 };
466
init_hw_perf_events(void)467 int __init init_hw_perf_events(void)
468 {
469 struct device_node *node = of_find_node_by_type(NULL, "pmu");
470 const struct of_device_id *of_id;
471
472 riscv_pmu = &riscv_base_pmu;
473
474 if (node) {
475 of_id = of_match_node(riscv_pmu_of_ids, node);
476
477 if (of_id)
478 riscv_pmu = of_id->data;
479 of_node_put(node);
480 }
481
482 perf_pmu_register(riscv_pmu->pmu, "cpu", PERF_TYPE_RAW);
483 return 0;
484 }
485 arch_initcall(init_hw_perf_events);
486