• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Arm Statistical Profiling Extensions (SPE) support
4  * Copyright (c) 2017-2018, Arm Ltd.
5  */
6 
7 #include <byteswap.h>
8 #include <endian.h>
9 #include <errno.h>
10 #include <inttypes.h>
11 #include <linux/bitops.h>
12 #include <linux/kernel.h>
13 #include <linux/log2.h>
14 #include <linux/types.h>
15 #include <linux/zalloc.h>
16 #include <stdlib.h>
17 #include <unistd.h>
18 
19 #include "auxtrace.h"
20 #include "color.h"
21 #include "debug.h"
22 #include "evlist.h"
23 #include "evsel.h"
24 #include "machine.h"
25 #include "session.h"
26 #include "symbol.h"
27 #include "thread.h"
28 #include "thread-stack.h"
29 #include "tool.h"
30 #include "util/synthetic-events.h"
31 
32 #include "arm-spe.h"
33 #include "arm-spe-decoder/arm-spe-decoder.h"
34 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
35 
36 #define MAX_TIMESTAMP (~0ULL)
37 
38 struct arm_spe {
39 	struct auxtrace			auxtrace;
40 	struct auxtrace_queues		queues;
41 	struct auxtrace_heap		heap;
42 	struct itrace_synth_opts        synth_opts;
43 	u32				auxtrace_type;
44 	struct perf_session		*session;
45 	struct machine			*machine;
46 	u32				pmu_type;
47 
48 	u8				timeless_decoding;
49 	u8				data_queued;
50 
51 	u64				sample_type;
52 	u8				sample_flc;
53 	u8				sample_llc;
54 	u8				sample_tlb;
55 	u8				sample_branch;
56 	u8				sample_remote_access;
57 
58 	u64				l1d_miss_id;
59 	u64				l1d_access_id;
60 	u64				llc_miss_id;
61 	u64				llc_access_id;
62 	u64				tlb_miss_id;
63 	u64				tlb_access_id;
64 	u64				branch_miss_id;
65 	u64				remote_access_id;
66 
67 	u64				kernel_start;
68 
69 	unsigned long			num_events;
70 };
71 
72 struct arm_spe_queue {
73 	struct arm_spe			*spe;
74 	unsigned int			queue_nr;
75 	struct auxtrace_buffer		*buffer;
76 	struct auxtrace_buffer		*old_buffer;
77 	union perf_event		*event_buf;
78 	bool				on_heap;
79 	bool				done;
80 	pid_t				pid;
81 	pid_t				tid;
82 	int				cpu;
83 	struct arm_spe_decoder		*decoder;
84 	u64				time;
85 	u64				timestamp;
86 	struct thread			*thread;
87 };
88 
arm_spe_dump(struct arm_spe * spe __maybe_unused,unsigned char * buf,size_t len)89 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,
90 			 unsigned char *buf, size_t len)
91 {
92 	struct arm_spe_pkt packet;
93 	size_t pos = 0;
94 	int ret, pkt_len, i;
95 	char desc[ARM_SPE_PKT_DESC_MAX];
96 	const char *color = PERF_COLOR_BLUE;
97 
98 	color_fprintf(stdout, color,
99 		      ". ... ARM SPE data: size %zu bytes\n",
100 		      len);
101 
102 	while (len) {
103 		ret = arm_spe_get_packet(buf, len, &packet);
104 		if (ret > 0)
105 			pkt_len = ret;
106 		else
107 			pkt_len = 1;
108 		printf(".");
109 		color_fprintf(stdout, color, "  %08x: ", pos);
110 		for (i = 0; i < pkt_len; i++)
111 			color_fprintf(stdout, color, " %02x", buf[i]);
112 		for (; i < 16; i++)
113 			color_fprintf(stdout, color, "   ");
114 		if (ret > 0) {
115 			ret = arm_spe_pkt_desc(&packet, desc,
116 					       ARM_SPE_PKT_DESC_MAX);
117 			if (ret > 0)
118 				color_fprintf(stdout, color, " %s\n", desc);
119 		} else {
120 			color_fprintf(stdout, color, " Bad packet!\n");
121 		}
122 		pos += pkt_len;
123 		buf += pkt_len;
124 		len -= pkt_len;
125 	}
126 }
127 
arm_spe_dump_event(struct arm_spe * spe,unsigned char * buf,size_t len)128 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,
129 			       size_t len)
130 {
131 	printf(".\n");
132 	arm_spe_dump(spe, buf, len);
133 }
134 
arm_spe_get_trace(struct arm_spe_buffer * b,void * data)135 static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data)
136 {
137 	struct arm_spe_queue *speq = data;
138 	struct auxtrace_buffer *buffer = speq->buffer;
139 	struct auxtrace_buffer *old_buffer = speq->old_buffer;
140 	struct auxtrace_queue *queue;
141 
142 	queue = &speq->spe->queues.queue_array[speq->queue_nr];
143 
144 	buffer = auxtrace_buffer__next(queue, buffer);
145 	/* If no more data, drop the previous auxtrace_buffer and return */
146 	if (!buffer) {
147 		if (old_buffer)
148 			auxtrace_buffer__drop_data(old_buffer);
149 		b->len = 0;
150 		return 0;
151 	}
152 
153 	speq->buffer = buffer;
154 
155 	/* If the aux_buffer doesn't have data associated, try to load it */
156 	if (!buffer->data) {
157 		/* get the file desc associated with the perf data file */
158 		int fd = perf_data__fd(speq->spe->session->data);
159 
160 		buffer->data = auxtrace_buffer__get_data(buffer, fd);
161 		if (!buffer->data)
162 			return -ENOMEM;
163 	}
164 
165 	b->len = buffer->size;
166 	b->buf = buffer->data;
167 
168 	if (b->len) {
169 		if (old_buffer)
170 			auxtrace_buffer__drop_data(old_buffer);
171 		speq->old_buffer = buffer;
172 	} else {
173 		auxtrace_buffer__drop_data(buffer);
174 		return arm_spe_get_trace(b, data);
175 	}
176 
177 	return 0;
178 }
179 
arm_spe__alloc_queue(struct arm_spe * spe,unsigned int queue_nr)180 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,
181 		unsigned int queue_nr)
182 {
183 	struct arm_spe_params params = { .get_trace = 0, };
184 	struct arm_spe_queue *speq;
185 
186 	speq = zalloc(sizeof(*speq));
187 	if (!speq)
188 		return NULL;
189 
190 	speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
191 	if (!speq->event_buf)
192 		goto out_free;
193 
194 	speq->spe = spe;
195 	speq->queue_nr = queue_nr;
196 	speq->pid = -1;
197 	speq->tid = -1;
198 	speq->cpu = -1;
199 
200 	/* params set */
201 	params.get_trace = arm_spe_get_trace;
202 	params.data = speq;
203 
204 	/* create new decoder */
205 	speq->decoder = arm_spe_decoder_new(&params);
206 	if (!speq->decoder)
207 		goto out_free;
208 
209 	return speq;
210 
211 out_free:
212 	zfree(&speq->event_buf);
213 	free(speq);
214 
215 	return NULL;
216 }
217 
arm_spe_cpumode(struct arm_spe * spe,u64 ip)218 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)
219 {
220 	return ip >= spe->kernel_start ?
221 		PERF_RECORD_MISC_KERNEL :
222 		PERF_RECORD_MISC_USER;
223 }
224 
arm_spe_prep_sample(struct arm_spe * spe,struct arm_spe_queue * speq,union perf_event * event,struct perf_sample * sample)225 static void arm_spe_prep_sample(struct arm_spe *spe,
226 				struct arm_spe_queue *speq,
227 				union perf_event *event,
228 				struct perf_sample *sample)
229 {
230 	struct arm_spe_record *record = &speq->decoder->record;
231 
232 	if (!spe->timeless_decoding)
233 		sample->time = speq->timestamp;
234 
235 	sample->ip = record->from_ip;
236 	sample->cpumode = arm_spe_cpumode(spe, sample->ip);
237 	sample->pid = speq->pid;
238 	sample->tid = speq->tid;
239 	sample->addr = record->to_ip;
240 	sample->period = 1;
241 	sample->cpu = speq->cpu;
242 
243 	event->sample.header.type = PERF_RECORD_SAMPLE;
244 	event->sample.header.misc = sample->cpumode;
245 	event->sample.header.size = sizeof(struct perf_event_header);
246 }
247 
arm_spe__inject_event(union perf_event * event,struct perf_sample * sample,u64 type)248 static int arm_spe__inject_event(union perf_event *event, struct perf_sample *sample, u64 type)
249 {
250 	event->header.size = perf_event__sample_event_size(sample, type, 0);
251 	return perf_event__synthesize_sample(event, type, 0, sample);
252 }
253 
254 static inline int
arm_spe_deliver_synth_event(struct arm_spe * spe,struct arm_spe_queue * speq __maybe_unused,union perf_event * event,struct perf_sample * sample)255 arm_spe_deliver_synth_event(struct arm_spe *spe,
256 			    struct arm_spe_queue *speq __maybe_unused,
257 			    union perf_event *event,
258 			    struct perf_sample *sample)
259 {
260 	int ret;
261 
262 	if (spe->synth_opts.inject) {
263 		ret = arm_spe__inject_event(event, sample, spe->sample_type);
264 		if (ret)
265 			return ret;
266 	}
267 
268 	ret = perf_session__deliver_synth_event(spe->session, event, sample);
269 	if (ret)
270 		pr_err("ARM SPE: failed to deliver event, error %d\n", ret);
271 
272 	return ret;
273 }
274 
275 static int
arm_spe_synth_spe_events_sample(struct arm_spe_queue * speq,u64 spe_events_id)276 arm_spe_synth_spe_events_sample(struct arm_spe_queue *speq,
277 				u64 spe_events_id)
278 {
279 	struct arm_spe *spe = speq->spe;
280 	union perf_event *event = speq->event_buf;
281 	struct perf_sample sample = { .ip = 0, };
282 
283 	arm_spe_prep_sample(spe, speq, event, &sample);
284 
285 	sample.id = spe_events_id;
286 	sample.stream_id = spe_events_id;
287 
288 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);
289 }
290 
arm_spe_sample(struct arm_spe_queue * speq)291 static int arm_spe_sample(struct arm_spe_queue *speq)
292 {
293 	const struct arm_spe_record *record = &speq->decoder->record;
294 	struct arm_spe *spe = speq->spe;
295 	int err;
296 
297 	if (spe->sample_flc) {
298 		if (record->type & ARM_SPE_L1D_MISS) {
299 			err = arm_spe_synth_spe_events_sample(
300 					speq, spe->l1d_miss_id);
301 			if (err)
302 				return err;
303 		}
304 
305 		if (record->type & ARM_SPE_L1D_ACCESS) {
306 			err = arm_spe_synth_spe_events_sample(
307 					speq, spe->l1d_access_id);
308 			if (err)
309 				return err;
310 		}
311 	}
312 
313 	if (spe->sample_llc) {
314 		if (record->type & ARM_SPE_LLC_MISS) {
315 			err = arm_spe_synth_spe_events_sample(
316 					speq, spe->llc_miss_id);
317 			if (err)
318 				return err;
319 		}
320 
321 		if (record->type & ARM_SPE_LLC_ACCESS) {
322 			err = arm_spe_synth_spe_events_sample(
323 					speq, spe->llc_access_id);
324 			if (err)
325 				return err;
326 		}
327 	}
328 
329 	if (spe->sample_tlb) {
330 		if (record->type & ARM_SPE_TLB_MISS) {
331 			err = arm_spe_synth_spe_events_sample(
332 					speq, spe->tlb_miss_id);
333 			if (err)
334 				return err;
335 		}
336 
337 		if (record->type & ARM_SPE_TLB_ACCESS) {
338 			err = arm_spe_synth_spe_events_sample(
339 					speq, spe->tlb_access_id);
340 			if (err)
341 				return err;
342 		}
343 	}
344 
345 	if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {
346 		err = arm_spe_synth_spe_events_sample(speq,
347 						      spe->branch_miss_id);
348 		if (err)
349 			return err;
350 	}
351 
352 	if (spe->sample_remote_access &&
353 	    (record->type & ARM_SPE_REMOTE_ACCESS)) {
354 		err = arm_spe_synth_spe_events_sample(speq,
355 						      spe->remote_access_id);
356 		if (err)
357 			return err;
358 	}
359 
360 	return 0;
361 }
362 
arm_spe_run_decoder(struct arm_spe_queue * speq,u64 * timestamp)363 static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp)
364 {
365 	struct arm_spe *spe = speq->spe;
366 	int ret;
367 
368 	if (!spe->kernel_start)
369 		spe->kernel_start = machine__kernel_start(spe->machine);
370 
371 	while (1) {
372 		ret = arm_spe_decode(speq->decoder);
373 		if (!ret) {
374 			pr_debug("No data or all data has been processed.\n");
375 			return 1;
376 		}
377 
378 		/*
379 		 * Error is detected when decode SPE trace data, continue to
380 		 * the next trace data and find out more records.
381 		 */
382 		if (ret < 0)
383 			continue;
384 
385 		ret = arm_spe_sample(speq);
386 		if (ret)
387 			return ret;
388 
389 		if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {
390 			*timestamp = speq->timestamp;
391 			return 0;
392 		}
393 	}
394 
395 	return 0;
396 }
397 
arm_spe__setup_queue(struct arm_spe * spe,struct auxtrace_queue * queue,unsigned int queue_nr)398 static int arm_spe__setup_queue(struct arm_spe *spe,
399 			       struct auxtrace_queue *queue,
400 			       unsigned int queue_nr)
401 {
402 	struct arm_spe_queue *speq = queue->priv;
403 	struct arm_spe_record *record;
404 
405 	if (list_empty(&queue->head) || speq)
406 		return 0;
407 
408 	speq = arm_spe__alloc_queue(spe, queue_nr);
409 
410 	if (!speq)
411 		return -ENOMEM;
412 
413 	queue->priv = speq;
414 
415 	if (queue->cpu != -1)
416 		speq->cpu = queue->cpu;
417 
418 	if (!speq->on_heap) {
419 		int ret;
420 
421 		if (spe->timeless_decoding)
422 			return 0;
423 
424 retry:
425 		ret = arm_spe_decode(speq->decoder);
426 
427 		if (!ret)
428 			return 0;
429 
430 		if (ret < 0)
431 			goto retry;
432 
433 		record = &speq->decoder->record;
434 
435 		speq->timestamp = record->timestamp;
436 		ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);
437 		if (ret)
438 			return ret;
439 		speq->on_heap = true;
440 	}
441 
442 	return 0;
443 }
444 
arm_spe__setup_queues(struct arm_spe * spe)445 static int arm_spe__setup_queues(struct arm_spe *spe)
446 {
447 	unsigned int i;
448 	int ret;
449 
450 	for (i = 0; i < spe->queues.nr_queues; i++) {
451 		ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);
452 		if (ret)
453 			return ret;
454 	}
455 
456 	return 0;
457 }
458 
arm_spe__update_queues(struct arm_spe * spe)459 static int arm_spe__update_queues(struct arm_spe *spe)
460 {
461 	if (spe->queues.new_data) {
462 		spe->queues.new_data = false;
463 		return arm_spe__setup_queues(spe);
464 	}
465 
466 	return 0;
467 }
468 
arm_spe__is_timeless_decoding(struct arm_spe * spe)469 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)
470 {
471 	struct evsel *evsel;
472 	struct evlist *evlist = spe->session->evlist;
473 	bool timeless_decoding = true;
474 
475 	/*
476 	 * Circle through the list of event and complain if we find one
477 	 * with the time bit set.
478 	 */
479 	evlist__for_each_entry(evlist, evsel) {
480 		if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
481 			timeless_decoding = false;
482 	}
483 
484 	return timeless_decoding;
485 }
486 
arm_spe_set_pid_tid_cpu(struct arm_spe * spe,struct auxtrace_queue * queue)487 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,
488 				    struct auxtrace_queue *queue)
489 {
490 	struct arm_spe_queue *speq = queue->priv;
491 	pid_t tid;
492 
493 	tid = machine__get_current_tid(spe->machine, speq->cpu);
494 	if (tid != -1) {
495 		speq->tid = tid;
496 		thread__zput(speq->thread);
497 	} else
498 		speq->tid = queue->tid;
499 
500 	if ((!speq->thread) && (speq->tid != -1)) {
501 		speq->thread = machine__find_thread(spe->machine, -1,
502 						    speq->tid);
503 	}
504 
505 	if (speq->thread) {
506 		speq->pid = speq->thread->pid_;
507 		if (queue->cpu == -1)
508 			speq->cpu = speq->thread->cpu;
509 	}
510 }
511 
arm_spe_process_queues(struct arm_spe * spe,u64 timestamp)512 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)
513 {
514 	unsigned int queue_nr;
515 	u64 ts;
516 	int ret;
517 
518 	while (1) {
519 		struct auxtrace_queue *queue;
520 		struct arm_spe_queue *speq;
521 
522 		if (!spe->heap.heap_cnt)
523 			return 0;
524 
525 		if (spe->heap.heap_array[0].ordinal >= timestamp)
526 			return 0;
527 
528 		queue_nr = spe->heap.heap_array[0].queue_nr;
529 		queue = &spe->queues.queue_array[queue_nr];
530 		speq = queue->priv;
531 
532 		auxtrace_heap__pop(&spe->heap);
533 
534 		if (spe->heap.heap_cnt) {
535 			ts = spe->heap.heap_array[0].ordinal + 1;
536 			if (ts > timestamp)
537 				ts = timestamp;
538 		} else {
539 			ts = timestamp;
540 		}
541 
542 		arm_spe_set_pid_tid_cpu(spe, queue);
543 
544 		ret = arm_spe_run_decoder(speq, &ts);
545 		if (ret < 0) {
546 			auxtrace_heap__add(&spe->heap, queue_nr, ts);
547 			return ret;
548 		}
549 
550 		if (!ret) {
551 			ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);
552 			if (ret < 0)
553 				return ret;
554 		} else {
555 			speq->on_heap = false;
556 		}
557 	}
558 
559 	return 0;
560 }
561 
arm_spe_process_timeless_queues(struct arm_spe * spe,pid_t tid,u64 time_)562 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,
563 					    u64 time_)
564 {
565 	struct auxtrace_queues *queues = &spe->queues;
566 	unsigned int i;
567 	u64 ts = 0;
568 
569 	for (i = 0; i < queues->nr_queues; i++) {
570 		struct auxtrace_queue *queue = &spe->queues.queue_array[i];
571 		struct arm_spe_queue *speq = queue->priv;
572 
573 		if (speq && (tid == -1 || speq->tid == tid)) {
574 			speq->time = time_;
575 			arm_spe_set_pid_tid_cpu(spe, queue);
576 			arm_spe_run_decoder(speq, &ts);
577 		}
578 	}
579 	return 0;
580 }
581 
arm_spe_process_event(struct perf_session * session,union perf_event * event,struct perf_sample * sample,struct perf_tool * tool)582 static int arm_spe_process_event(struct perf_session *session,
583 				 union perf_event *event,
584 				 struct perf_sample *sample,
585 				 struct perf_tool *tool)
586 {
587 	int err = 0;
588 	u64 timestamp;
589 	struct arm_spe *spe = container_of(session->auxtrace,
590 			struct arm_spe, auxtrace);
591 
592 	if (dump_trace)
593 		return 0;
594 
595 	if (!tool->ordered_events) {
596 		pr_err("SPE trace requires ordered events\n");
597 		return -EINVAL;
598 	}
599 
600 	if (sample->time && (sample->time != (u64) -1))
601 		timestamp = sample->time;
602 	else
603 		timestamp = 0;
604 
605 	if (timestamp || spe->timeless_decoding) {
606 		err = arm_spe__update_queues(spe);
607 		if (err)
608 			return err;
609 	}
610 
611 	if (spe->timeless_decoding) {
612 		if (event->header.type == PERF_RECORD_EXIT) {
613 			err = arm_spe_process_timeless_queues(spe,
614 					event->fork.tid,
615 					sample->time);
616 		}
617 	} else if (timestamp) {
618 		if (event->header.type == PERF_RECORD_EXIT) {
619 			err = arm_spe_process_queues(spe, timestamp);
620 			if (err)
621 				return err;
622 		}
623 	}
624 
625 	return err;
626 }
627 
arm_spe_process_auxtrace_event(struct perf_session * session,union perf_event * event,struct perf_tool * tool __maybe_unused)628 static int arm_spe_process_auxtrace_event(struct perf_session *session,
629 					  union perf_event *event,
630 					  struct perf_tool *tool __maybe_unused)
631 {
632 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
633 					     auxtrace);
634 
635 	if (!spe->data_queued) {
636 		struct auxtrace_buffer *buffer;
637 		off_t data_offset;
638 		int fd = perf_data__fd(session->data);
639 		int err;
640 
641 		if (perf_data__is_pipe(session->data)) {
642 			data_offset = 0;
643 		} else {
644 			data_offset = lseek(fd, 0, SEEK_CUR);
645 			if (data_offset == -1)
646 				return -errno;
647 		}
648 
649 		err = auxtrace_queues__add_event(&spe->queues, session, event,
650 				data_offset, &buffer);
651 		if (err)
652 			return err;
653 
654 		/* Dump here now we have copied a piped trace out of the pipe */
655 		if (dump_trace) {
656 			if (auxtrace_buffer__get_data(buffer, fd)) {
657 				arm_spe_dump_event(spe, buffer->data,
658 						buffer->size);
659 				auxtrace_buffer__put_data(buffer);
660 			}
661 		}
662 	}
663 
664 	return 0;
665 }
666 
arm_spe_flush(struct perf_session * session __maybe_unused,struct perf_tool * tool __maybe_unused)667 static int arm_spe_flush(struct perf_session *session __maybe_unused,
668 			 struct perf_tool *tool __maybe_unused)
669 {
670 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
671 			auxtrace);
672 	int ret;
673 
674 	if (dump_trace)
675 		return 0;
676 
677 	if (!tool->ordered_events)
678 		return -EINVAL;
679 
680 	ret = arm_spe__update_queues(spe);
681 	if (ret < 0)
682 		return ret;
683 
684 	if (spe->timeless_decoding)
685 		return arm_spe_process_timeless_queues(spe, -1,
686 				MAX_TIMESTAMP - 1);
687 
688 	return arm_spe_process_queues(spe, MAX_TIMESTAMP);
689 }
690 
arm_spe_free_queue(void * priv)691 static void arm_spe_free_queue(void *priv)
692 {
693 	struct arm_spe_queue *speq = priv;
694 
695 	if (!speq)
696 		return;
697 	thread__zput(speq->thread);
698 	arm_spe_decoder_free(speq->decoder);
699 	zfree(&speq->event_buf);
700 	free(speq);
701 }
702 
arm_spe_free_events(struct perf_session * session)703 static void arm_spe_free_events(struct perf_session *session)
704 {
705 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
706 					     auxtrace);
707 	struct auxtrace_queues *queues = &spe->queues;
708 	unsigned int i;
709 
710 	for (i = 0; i < queues->nr_queues; i++) {
711 		arm_spe_free_queue(queues->queue_array[i].priv);
712 		queues->queue_array[i].priv = NULL;
713 	}
714 	auxtrace_queues__free(queues);
715 }
716 
arm_spe_free(struct perf_session * session)717 static void arm_spe_free(struct perf_session *session)
718 {
719 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,
720 					     auxtrace);
721 
722 	auxtrace_heap__free(&spe->heap);
723 	arm_spe_free_events(session);
724 	session->auxtrace = NULL;
725 	free(spe);
726 }
727 
arm_spe_evsel_is_auxtrace(struct perf_session * session,struct evsel * evsel)728 static bool arm_spe_evsel_is_auxtrace(struct perf_session *session,
729 				      struct evsel *evsel)
730 {
731 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);
732 
733 	return evsel->core.attr.type == spe->pmu_type;
734 }
735 
736 static const char * const arm_spe_info_fmts[] = {
737 	[ARM_SPE_PMU_TYPE]		= "  PMU Type           %"PRId64"\n",
738 };
739 
arm_spe_print_info(__u64 * arr)740 static void arm_spe_print_info(__u64 *arr)
741 {
742 	if (!dump_trace)
743 		return;
744 
745 	fprintf(stdout, arm_spe_info_fmts[ARM_SPE_PMU_TYPE], arr[ARM_SPE_PMU_TYPE]);
746 }
747 
748 struct arm_spe_synth {
749 	struct perf_tool dummy_tool;
750 	struct perf_session *session;
751 };
752 
arm_spe_event_synth(struct perf_tool * tool,union perf_event * event,struct perf_sample * sample __maybe_unused,struct machine * machine __maybe_unused)753 static int arm_spe_event_synth(struct perf_tool *tool,
754 			       union perf_event *event,
755 			       struct perf_sample *sample __maybe_unused,
756 			       struct machine *machine __maybe_unused)
757 {
758 	struct arm_spe_synth *arm_spe_synth =
759 		      container_of(tool, struct arm_spe_synth, dummy_tool);
760 
761 	return perf_session__deliver_synth_event(arm_spe_synth->session,
762 						 event, NULL);
763 }
764 
arm_spe_synth_event(struct perf_session * session,struct perf_event_attr * attr,u64 id)765 static int arm_spe_synth_event(struct perf_session *session,
766 			       struct perf_event_attr *attr, u64 id)
767 {
768 	struct arm_spe_synth arm_spe_synth;
769 
770 	memset(&arm_spe_synth, 0, sizeof(struct arm_spe_synth));
771 	arm_spe_synth.session = session;
772 
773 	return perf_event__synthesize_attr(&arm_spe_synth.dummy_tool, attr, 1,
774 					   &id, arm_spe_event_synth);
775 }
776 
arm_spe_set_event_name(struct evlist * evlist,u64 id,const char * name)777 static void arm_spe_set_event_name(struct evlist *evlist, u64 id,
778 				    const char *name)
779 {
780 	struct evsel *evsel;
781 
782 	evlist__for_each_entry(evlist, evsel) {
783 		if (evsel->core.id && evsel->core.id[0] == id) {
784 			if (evsel->name)
785 				zfree(&evsel->name);
786 			evsel->name = strdup(name);
787 			break;
788 		}
789 	}
790 }
791 
792 static int
arm_spe_synth_events(struct arm_spe * spe,struct perf_session * session)793 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)
794 {
795 	struct evlist *evlist = session->evlist;
796 	struct evsel *evsel;
797 	struct perf_event_attr attr;
798 	bool found = false;
799 	u64 id;
800 	int err;
801 
802 	evlist__for_each_entry(evlist, evsel) {
803 		if (evsel->core.attr.type == spe->pmu_type) {
804 			found = true;
805 			break;
806 		}
807 	}
808 
809 	if (!found) {
810 		pr_debug("No selected events with SPE trace data\n");
811 		return 0;
812 	}
813 
814 	memset(&attr, 0, sizeof(struct perf_event_attr));
815 	attr.size = sizeof(struct perf_event_attr);
816 	attr.type = PERF_TYPE_HARDWARE;
817 	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
818 	attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
819 		PERF_SAMPLE_PERIOD;
820 	if (spe->timeless_decoding)
821 		attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
822 	else
823 		attr.sample_type |= PERF_SAMPLE_TIME;
824 
825 	spe->sample_type = attr.sample_type;
826 
827 	attr.exclude_user = evsel->core.attr.exclude_user;
828 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;
829 	attr.exclude_hv = evsel->core.attr.exclude_hv;
830 	attr.exclude_host = evsel->core.attr.exclude_host;
831 	attr.exclude_guest = evsel->core.attr.exclude_guest;
832 	attr.sample_id_all = evsel->core.attr.sample_id_all;
833 	attr.read_format = evsel->core.attr.read_format;
834 
835 	/* create new id val to be a fixed offset from evsel id */
836 	id = evsel->core.id[0] + 1000000000;
837 
838 	if (!id)
839 		id = 1;
840 
841 	if (spe->synth_opts.flc) {
842 		spe->sample_flc = true;
843 
844 		/* Level 1 data cache miss */
845 		err = arm_spe_synth_event(session, &attr, id);
846 		if (err)
847 			return err;
848 		spe->l1d_miss_id = id;
849 		arm_spe_set_event_name(evlist, id, "l1d-miss");
850 		id += 1;
851 
852 		/* Level 1 data cache access */
853 		err = arm_spe_synth_event(session, &attr, id);
854 		if (err)
855 			return err;
856 		spe->l1d_access_id = id;
857 		arm_spe_set_event_name(evlist, id, "l1d-access");
858 		id += 1;
859 	}
860 
861 	if (spe->synth_opts.llc) {
862 		spe->sample_llc = true;
863 
864 		/* Last level cache miss */
865 		err = arm_spe_synth_event(session, &attr, id);
866 		if (err)
867 			return err;
868 		spe->llc_miss_id = id;
869 		arm_spe_set_event_name(evlist, id, "llc-miss");
870 		id += 1;
871 
872 		/* Last level cache access */
873 		err = arm_spe_synth_event(session, &attr, id);
874 		if (err)
875 			return err;
876 		spe->llc_access_id = id;
877 		arm_spe_set_event_name(evlist, id, "llc-access");
878 		id += 1;
879 	}
880 
881 	if (spe->synth_opts.tlb) {
882 		spe->sample_tlb = true;
883 
884 		/* TLB miss */
885 		err = arm_spe_synth_event(session, &attr, id);
886 		if (err)
887 			return err;
888 		spe->tlb_miss_id = id;
889 		arm_spe_set_event_name(evlist, id, "tlb-miss");
890 		id += 1;
891 
892 		/* TLB access */
893 		err = arm_spe_synth_event(session, &attr, id);
894 		if (err)
895 			return err;
896 		spe->tlb_access_id = id;
897 		arm_spe_set_event_name(evlist, id, "tlb-access");
898 		id += 1;
899 	}
900 
901 	if (spe->synth_opts.branches) {
902 		spe->sample_branch = true;
903 
904 		/* Branch miss */
905 		err = arm_spe_synth_event(session, &attr, id);
906 		if (err)
907 			return err;
908 		spe->branch_miss_id = id;
909 		arm_spe_set_event_name(evlist, id, "branch-miss");
910 		id += 1;
911 	}
912 
913 	if (spe->synth_opts.remote_access) {
914 		spe->sample_remote_access = true;
915 
916 		/* Remote access */
917 		err = arm_spe_synth_event(session, &attr, id);
918 		if (err)
919 			return err;
920 		spe->remote_access_id = id;
921 		arm_spe_set_event_name(evlist, id, "remote-access");
922 		id += 1;
923 	}
924 
925 	return 0;
926 }
927 
arm_spe_process_auxtrace_info(union perf_event * event,struct perf_session * session)928 int arm_spe_process_auxtrace_info(union perf_event *event,
929 				  struct perf_session *session)
930 {
931 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
932 	size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX;
933 	struct arm_spe *spe;
934 	int err;
935 
936 	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
937 					min_sz)
938 		return -EINVAL;
939 
940 	spe = zalloc(sizeof(struct arm_spe));
941 	if (!spe)
942 		return -ENOMEM;
943 
944 	err = auxtrace_queues__init(&spe->queues);
945 	if (err)
946 		goto err_free;
947 
948 	spe->session = session;
949 	spe->machine = &session->machines.host; /* No kvm support */
950 	spe->auxtrace_type = auxtrace_info->type;
951 	spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];
952 
953 	spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);
954 	spe->auxtrace.process_event = arm_spe_process_event;
955 	spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;
956 	spe->auxtrace.flush_events = arm_spe_flush;
957 	spe->auxtrace.free_events = arm_spe_free_events;
958 	spe->auxtrace.free = arm_spe_free;
959 	spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;
960 	session->auxtrace = &spe->auxtrace;
961 
962 	arm_spe_print_info(&auxtrace_info->priv[0]);
963 
964 	if (dump_trace)
965 		return 0;
966 
967 	if (session->itrace_synth_opts && session->itrace_synth_opts->set)
968 		spe->synth_opts = *session->itrace_synth_opts;
969 	else
970 		itrace_synth_opts__set_default(&spe->synth_opts, false);
971 
972 	err = arm_spe_synth_events(spe, session);
973 	if (err)
974 		goto err_free_queues;
975 
976 	err = auxtrace_queues__process_index(&spe->queues, session);
977 	if (err)
978 		goto err_free_queues;
979 
980 	if (spe->queues.populated)
981 		spe->data_queued = true;
982 
983 	return 0;
984 
985 err_free_queues:
986 	auxtrace_queues__free(&spe->queues);
987 	session->auxtrace = NULL;
988 err_free:
989 	free(spe);
990 	return err;
991 }
992