1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * auxtrace.c: AUX area trace support
4 * Copyright (c) 2013-2015, Intel Corporation.
5 */
6
7 #include <inttypes.h>
8 #include <sys/types.h>
9 #include <sys/mman.h>
10 #include <stdbool.h>
11 #include <string.h>
12 #include <limits.h>
13 #include <errno.h>
14
15 #include <linux/kernel.h>
16 #include <linux/perf_event.h>
17 #include <linux/types.h>
18 #include <linux/bitops.h>
19 #include <linux/log2.h>
20 #include <linux/string.h>
21 #include <linux/time64.h>
22
23 #include <sys/param.h>
24 #include <stdlib.h>
25 #include <stdio.h>
26 #include <linux/list.h>
27 #include <linux/zalloc.h>
28
29 #include "evlist.h"
30 #include "dso.h"
31 #include "map.h"
32 #include "pmu.h"
33 #include "evsel.h"
34 #include "evsel_config.h"
35 #include "symbol.h"
36 #include "util/perf_api_probe.h"
37 #include "util/synthetic-events.h"
38 #include "thread_map.h"
39 #include "asm/bug.h"
40 #include "auxtrace.h"
41
42 #include <linux/hash.h>
43
44 #include "event.h"
45 #include "record.h"
46 #include "session.h"
47 #include "debug.h"
48 #include <subcmd/parse-options.h>
49
50 #include "cs-etm.h"
51 #include "intel-pt.h"
52 #include "intel-bts.h"
53 #include "arm-spe.h"
54 #include "s390-cpumsf.h"
55 #include "util/mmap.h"
56
57 #include <linux/ctype.h>
58 #include "symbol/kallsyms.h"
59 #include <internal/lib.h>
60
61 /*
62 * Make a group from 'leader' to 'last', requiring that the events were not
63 * already grouped to a different leader.
64 */
perf_evlist__regroup(struct evlist * evlist,struct evsel * leader,struct evsel * last)65 static int perf_evlist__regroup(struct evlist *evlist,
66 struct evsel *leader,
67 struct evsel *last)
68 {
69 struct evsel *evsel;
70 bool grp;
71
72 if (!evsel__is_group_leader(leader))
73 return -EINVAL;
74
75 grp = false;
76 evlist__for_each_entry(evlist, evsel) {
77 if (grp) {
78 if (!(evsel->leader == leader ||
79 (evsel->leader == evsel &&
80 evsel->core.nr_members <= 1)))
81 return -EINVAL;
82 } else if (evsel == leader) {
83 grp = true;
84 }
85 if (evsel == last)
86 break;
87 }
88
89 grp = false;
90 evlist__for_each_entry(evlist, evsel) {
91 if (grp) {
92 if (evsel->leader != leader) {
93 evsel->leader = leader;
94 if (leader->core.nr_members < 1)
95 leader->core.nr_members = 1;
96 leader->core.nr_members += 1;
97 }
98 } else if (evsel == leader) {
99 grp = true;
100 }
101 if (evsel == last)
102 break;
103 }
104
105 return 0;
106 }
107
auxtrace__dont_decode(struct perf_session * session)108 static bool auxtrace__dont_decode(struct perf_session *session)
109 {
110 return !session->itrace_synth_opts ||
111 session->itrace_synth_opts->dont_decode;
112 }
113
auxtrace_mmap__mmap(struct auxtrace_mmap * mm,struct auxtrace_mmap_params * mp,void * userpg,int fd)114 int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
115 struct auxtrace_mmap_params *mp,
116 void *userpg, int fd)
117 {
118 struct perf_event_mmap_page *pc = userpg;
119
120 WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");
121
122 mm->userpg = userpg;
123 mm->mask = mp->mask;
124 mm->len = mp->len;
125 mm->prev = 0;
126 mm->idx = mp->idx;
127 mm->tid = mp->tid;
128 mm->cpu = mp->cpu;
129
130 if (!mp->len) {
131 mm->base = NULL;
132 return 0;
133 }
134
135 #if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
136 pr_err("Cannot use AUX area tracing mmaps\n");
137 return -1;
138 #endif
139
140 pc->aux_offset = mp->offset;
141 pc->aux_size = mp->len;
142
143 mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);
144 if (mm->base == MAP_FAILED) {
145 pr_debug2("failed to mmap AUX area\n");
146 mm->base = NULL;
147 return -1;
148 }
149
150 return 0;
151 }
152
auxtrace_mmap__munmap(struct auxtrace_mmap * mm)153 void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)
154 {
155 if (mm->base) {
156 munmap(mm->base, mm->len);
157 mm->base = NULL;
158 }
159 }
160
auxtrace_mmap_params__init(struct auxtrace_mmap_params * mp,off_t auxtrace_offset,unsigned int auxtrace_pages,bool auxtrace_overwrite)161 void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
162 off_t auxtrace_offset,
163 unsigned int auxtrace_pages,
164 bool auxtrace_overwrite)
165 {
166 if (auxtrace_pages) {
167 mp->offset = auxtrace_offset;
168 mp->len = auxtrace_pages * (size_t)page_size;
169 mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;
170 mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);
171 pr_debug2("AUX area mmap length %zu\n", mp->len);
172 } else {
173 mp->len = 0;
174 }
175 }
176
auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params * mp,struct evlist * evlist,int idx,bool per_cpu)177 void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
178 struct evlist *evlist, int idx,
179 bool per_cpu)
180 {
181 mp->idx = idx;
182
183 if (per_cpu) {
184 mp->cpu = evlist->core.cpus->map[idx];
185 if (evlist->core.threads)
186 mp->tid = perf_thread_map__pid(evlist->core.threads, 0);
187 else
188 mp->tid = -1;
189 } else {
190 mp->cpu = -1;
191 mp->tid = perf_thread_map__pid(evlist->core.threads, idx);
192 }
193 }
194
195 #define AUXTRACE_INIT_NR_QUEUES 32
196
auxtrace_alloc_queue_array(unsigned int nr_queues)197 static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)
198 {
199 struct auxtrace_queue *queue_array;
200 unsigned int max_nr_queues, i;
201
202 max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);
203 if (nr_queues > max_nr_queues)
204 return NULL;
205
206 queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));
207 if (!queue_array)
208 return NULL;
209
210 for (i = 0; i < nr_queues; i++) {
211 INIT_LIST_HEAD(&queue_array[i].head);
212 queue_array[i].priv = NULL;
213 }
214
215 return queue_array;
216 }
217
auxtrace_queues__init(struct auxtrace_queues * queues)218 int auxtrace_queues__init(struct auxtrace_queues *queues)
219 {
220 queues->nr_queues = AUXTRACE_INIT_NR_QUEUES;
221 queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);
222 if (!queues->queue_array)
223 return -ENOMEM;
224 return 0;
225 }
226
auxtrace_queues__grow(struct auxtrace_queues * queues,unsigned int new_nr_queues)227 static int auxtrace_queues__grow(struct auxtrace_queues *queues,
228 unsigned int new_nr_queues)
229 {
230 unsigned int nr_queues = queues->nr_queues;
231 struct auxtrace_queue *queue_array;
232 unsigned int i;
233
234 if (!nr_queues)
235 nr_queues = AUXTRACE_INIT_NR_QUEUES;
236
237 while (nr_queues && nr_queues < new_nr_queues)
238 nr_queues <<= 1;
239
240 if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)
241 return -EINVAL;
242
243 queue_array = auxtrace_alloc_queue_array(nr_queues);
244 if (!queue_array)
245 return -ENOMEM;
246
247 for (i = 0; i < queues->nr_queues; i++) {
248 list_splice_tail(&queues->queue_array[i].head,
249 &queue_array[i].head);
250 queue_array[i].tid = queues->queue_array[i].tid;
251 queue_array[i].cpu = queues->queue_array[i].cpu;
252 queue_array[i].set = queues->queue_array[i].set;
253 queue_array[i].priv = queues->queue_array[i].priv;
254 }
255
256 queues->nr_queues = nr_queues;
257 queues->queue_array = queue_array;
258
259 return 0;
260 }
261
auxtrace_copy_data(u64 size,struct perf_session * session)262 static void *auxtrace_copy_data(u64 size, struct perf_session *session)
263 {
264 int fd = perf_data__fd(session->data);
265 void *p;
266 ssize_t ret;
267
268 if (size > SSIZE_MAX)
269 return NULL;
270
271 p = malloc(size);
272 if (!p)
273 return NULL;
274
275 ret = readn(fd, p, size);
276 if (ret != (ssize_t)size) {
277 free(p);
278 return NULL;
279 }
280
281 return p;
282 }
283
auxtrace_queues__queue_buffer(struct auxtrace_queues * queues,unsigned int idx,struct auxtrace_buffer * buffer)284 static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues,
285 unsigned int idx,
286 struct auxtrace_buffer *buffer)
287 {
288 struct auxtrace_queue *queue;
289 int err;
290
291 if (idx >= queues->nr_queues) {
292 err = auxtrace_queues__grow(queues, idx + 1);
293 if (err)
294 return err;
295 }
296
297 queue = &queues->queue_array[idx];
298
299 if (!queue->set) {
300 queue->set = true;
301 queue->tid = buffer->tid;
302 queue->cpu = buffer->cpu;
303 }
304
305 buffer->buffer_nr = queues->next_buffer_nr++;
306
307 list_add_tail(&buffer->list, &queue->head);
308
309 queues->new_data = true;
310 queues->populated = true;
311
312 return 0;
313 }
314
315 /* Limit buffers to 32MiB on 32-bit */
316 #define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)
317
auxtrace_queues__split_buffer(struct auxtrace_queues * queues,unsigned int idx,struct auxtrace_buffer * buffer)318 static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,
319 unsigned int idx,
320 struct auxtrace_buffer *buffer)
321 {
322 u64 sz = buffer->size;
323 bool consecutive = false;
324 struct auxtrace_buffer *b;
325 int err;
326
327 while (sz > BUFFER_LIMIT_FOR_32_BIT) {
328 b = memdup(buffer, sizeof(struct auxtrace_buffer));
329 if (!b)
330 return -ENOMEM;
331 b->size = BUFFER_LIMIT_FOR_32_BIT;
332 b->consecutive = consecutive;
333 err = auxtrace_queues__queue_buffer(queues, idx, b);
334 if (err) {
335 auxtrace_buffer__free(b);
336 return err;
337 }
338 buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;
339 sz -= BUFFER_LIMIT_FOR_32_BIT;
340 consecutive = true;
341 }
342
343 buffer->size = sz;
344 buffer->consecutive = consecutive;
345
346 return 0;
347 }
348
filter_cpu(struct perf_session * session,int cpu)349 static bool filter_cpu(struct perf_session *session, int cpu)
350 {
351 unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap;
352
353 return cpu_bitmap && cpu != -1 && !test_bit(cpu, cpu_bitmap);
354 }
355
auxtrace_queues__add_buffer(struct auxtrace_queues * queues,struct perf_session * session,unsigned int idx,struct auxtrace_buffer * buffer,struct auxtrace_buffer ** buffer_ptr)356 static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,
357 struct perf_session *session,
358 unsigned int idx,
359 struct auxtrace_buffer *buffer,
360 struct auxtrace_buffer **buffer_ptr)
361 {
362 int err = -ENOMEM;
363
364 if (filter_cpu(session, buffer->cpu))
365 return 0;
366
367 buffer = memdup(buffer, sizeof(*buffer));
368 if (!buffer)
369 return -ENOMEM;
370
371 if (session->one_mmap) {
372 buffer->data = buffer->data_offset - session->one_mmap_offset +
373 session->one_mmap_addr;
374 } else if (perf_data__is_pipe(session->data)) {
375 buffer->data = auxtrace_copy_data(buffer->size, session);
376 if (!buffer->data)
377 goto out_free;
378 buffer->data_needs_freeing = true;
379 } else if (BITS_PER_LONG == 32 &&
380 buffer->size > BUFFER_LIMIT_FOR_32_BIT) {
381 err = auxtrace_queues__split_buffer(queues, idx, buffer);
382 if (err)
383 goto out_free;
384 }
385
386 err = auxtrace_queues__queue_buffer(queues, idx, buffer);
387 if (err)
388 goto out_free;
389
390 /* FIXME: Doesn't work for split buffer */
391 if (buffer_ptr)
392 *buffer_ptr = buffer;
393
394 return 0;
395
396 out_free:
397 auxtrace_buffer__free(buffer);
398 return err;
399 }
400
auxtrace_queues__add_event(struct auxtrace_queues * queues,struct perf_session * session,union perf_event * event,off_t data_offset,struct auxtrace_buffer ** buffer_ptr)401 int auxtrace_queues__add_event(struct auxtrace_queues *queues,
402 struct perf_session *session,
403 union perf_event *event, off_t data_offset,
404 struct auxtrace_buffer **buffer_ptr)
405 {
406 struct auxtrace_buffer buffer = {
407 .pid = -1,
408 .tid = event->auxtrace.tid,
409 .cpu = event->auxtrace.cpu,
410 .data_offset = data_offset,
411 .offset = event->auxtrace.offset,
412 .reference = event->auxtrace.reference,
413 .size = event->auxtrace.size,
414 };
415 unsigned int idx = event->auxtrace.idx;
416
417 return auxtrace_queues__add_buffer(queues, session, idx, &buffer,
418 buffer_ptr);
419 }
420
auxtrace_queues__add_indexed_event(struct auxtrace_queues * queues,struct perf_session * session,off_t file_offset,size_t sz)421 static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,
422 struct perf_session *session,
423 off_t file_offset, size_t sz)
424 {
425 union perf_event *event;
426 int err;
427 char buf[PERF_SAMPLE_MAX_SIZE];
428
429 err = perf_session__peek_event(session, file_offset, buf,
430 PERF_SAMPLE_MAX_SIZE, &event, NULL);
431 if (err)
432 return err;
433
434 if (event->header.type == PERF_RECORD_AUXTRACE) {
435 if (event->header.size < sizeof(struct perf_record_auxtrace) ||
436 event->header.size != sz) {
437 err = -EINVAL;
438 goto out;
439 }
440 file_offset += event->header.size;
441 err = auxtrace_queues__add_event(queues, session, event,
442 file_offset, NULL);
443 }
444 out:
445 return err;
446 }
447
auxtrace_queues__free(struct auxtrace_queues * queues)448 void auxtrace_queues__free(struct auxtrace_queues *queues)
449 {
450 unsigned int i;
451
452 for (i = 0; i < queues->nr_queues; i++) {
453 while (!list_empty(&queues->queue_array[i].head)) {
454 struct auxtrace_buffer *buffer;
455
456 buffer = list_entry(queues->queue_array[i].head.next,
457 struct auxtrace_buffer, list);
458 list_del_init(&buffer->list);
459 auxtrace_buffer__free(buffer);
460 }
461 }
462
463 zfree(&queues->queue_array);
464 queues->nr_queues = 0;
465 }
466
auxtrace_heapify(struct auxtrace_heap_item * heap_array,unsigned int pos,unsigned int queue_nr,u64 ordinal)467 static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,
468 unsigned int pos, unsigned int queue_nr,
469 u64 ordinal)
470 {
471 unsigned int parent;
472
473 while (pos) {
474 parent = (pos - 1) >> 1;
475 if (heap_array[parent].ordinal <= ordinal)
476 break;
477 heap_array[pos] = heap_array[parent];
478 pos = parent;
479 }
480 heap_array[pos].queue_nr = queue_nr;
481 heap_array[pos].ordinal = ordinal;
482 }
483
auxtrace_heap__add(struct auxtrace_heap * heap,unsigned int queue_nr,u64 ordinal)484 int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
485 u64 ordinal)
486 {
487 struct auxtrace_heap_item *heap_array;
488
489 if (queue_nr >= heap->heap_sz) {
490 unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;
491
492 while (heap_sz <= queue_nr)
493 heap_sz <<= 1;
494 heap_array = realloc(heap->heap_array,
495 heap_sz * sizeof(struct auxtrace_heap_item));
496 if (!heap_array)
497 return -ENOMEM;
498 heap->heap_array = heap_array;
499 heap->heap_sz = heap_sz;
500 }
501
502 auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);
503
504 return 0;
505 }
506
auxtrace_heap__free(struct auxtrace_heap * heap)507 void auxtrace_heap__free(struct auxtrace_heap *heap)
508 {
509 zfree(&heap->heap_array);
510 heap->heap_cnt = 0;
511 heap->heap_sz = 0;
512 }
513
auxtrace_heap__pop(struct auxtrace_heap * heap)514 void auxtrace_heap__pop(struct auxtrace_heap *heap)
515 {
516 unsigned int pos, last, heap_cnt = heap->heap_cnt;
517 struct auxtrace_heap_item *heap_array;
518
519 if (!heap_cnt)
520 return;
521
522 heap->heap_cnt -= 1;
523
524 heap_array = heap->heap_array;
525
526 pos = 0;
527 while (1) {
528 unsigned int left, right;
529
530 left = (pos << 1) + 1;
531 if (left >= heap_cnt)
532 break;
533 right = left + 1;
534 if (right >= heap_cnt) {
535 heap_array[pos] = heap_array[left];
536 return;
537 }
538 if (heap_array[left].ordinal < heap_array[right].ordinal) {
539 heap_array[pos] = heap_array[left];
540 pos = left;
541 } else {
542 heap_array[pos] = heap_array[right];
543 pos = right;
544 }
545 }
546
547 last = heap_cnt - 1;
548 auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,
549 heap_array[last].ordinal);
550 }
551
auxtrace_record__info_priv_size(struct auxtrace_record * itr,struct evlist * evlist)552 size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
553 struct evlist *evlist)
554 {
555 if (itr)
556 return itr->info_priv_size(itr, evlist);
557 return 0;
558 }
559
auxtrace_not_supported(void)560 static int auxtrace_not_supported(void)
561 {
562 pr_err("AUX area tracing is not supported on this architecture\n");
563 return -EINVAL;
564 }
565
auxtrace_record__info_fill(struct auxtrace_record * itr,struct perf_session * session,struct perf_record_auxtrace_info * auxtrace_info,size_t priv_size)566 int auxtrace_record__info_fill(struct auxtrace_record *itr,
567 struct perf_session *session,
568 struct perf_record_auxtrace_info *auxtrace_info,
569 size_t priv_size)
570 {
571 if (itr)
572 return itr->info_fill(itr, session, auxtrace_info, priv_size);
573 return auxtrace_not_supported();
574 }
575
auxtrace_record__free(struct auxtrace_record * itr)576 void auxtrace_record__free(struct auxtrace_record *itr)
577 {
578 if (itr)
579 itr->free(itr);
580 }
581
auxtrace_record__snapshot_start(struct auxtrace_record * itr)582 int auxtrace_record__snapshot_start(struct auxtrace_record *itr)
583 {
584 if (itr && itr->snapshot_start)
585 return itr->snapshot_start(itr);
586 return 0;
587 }
588
auxtrace_record__snapshot_finish(struct auxtrace_record * itr,bool on_exit)589 int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit)
590 {
591 if (!on_exit && itr && itr->snapshot_finish)
592 return itr->snapshot_finish(itr);
593 return 0;
594 }
595
auxtrace_record__find_snapshot(struct auxtrace_record * itr,int idx,struct auxtrace_mmap * mm,unsigned char * data,u64 * head,u64 * old)596 int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
597 struct auxtrace_mmap *mm,
598 unsigned char *data, u64 *head, u64 *old)
599 {
600 if (itr && itr->find_snapshot)
601 return itr->find_snapshot(itr, idx, mm, data, head, old);
602 return 0;
603 }
604
auxtrace_record__options(struct auxtrace_record * itr,struct evlist * evlist,struct record_opts * opts)605 int auxtrace_record__options(struct auxtrace_record *itr,
606 struct evlist *evlist,
607 struct record_opts *opts)
608 {
609 if (itr) {
610 itr->evlist = evlist;
611 return itr->recording_options(itr, evlist, opts);
612 }
613 return 0;
614 }
615
auxtrace_record__reference(struct auxtrace_record * itr)616 u64 auxtrace_record__reference(struct auxtrace_record *itr)
617 {
618 if (itr)
619 return itr->reference(itr);
620 return 0;
621 }
622
auxtrace_parse_snapshot_options(struct auxtrace_record * itr,struct record_opts * opts,const char * str)623 int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
624 struct record_opts *opts, const char *str)
625 {
626 if (!str)
627 return 0;
628
629 /* PMU-agnostic options */
630 switch (*str) {
631 case 'e':
632 opts->auxtrace_snapshot_on_exit = true;
633 str++;
634 break;
635 default:
636 break;
637 }
638
639 if (itr && itr->parse_snapshot_options)
640 return itr->parse_snapshot_options(itr, opts, str);
641
642 pr_err("No AUX area tracing to snapshot\n");
643 return -EINVAL;
644 }
645
auxtrace_record__read_finish(struct auxtrace_record * itr,int idx)646 int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx)
647 {
648 struct evsel *evsel;
649
650 if (!itr->evlist || !itr->pmu)
651 return -EINVAL;
652
653 evlist__for_each_entry(itr->evlist, evsel) {
654 if (evsel->core.attr.type == itr->pmu->type) {
655 if (evsel->disabled)
656 return 0;
657 return perf_evlist__enable_event_idx(itr->evlist, evsel,
658 idx);
659 }
660 }
661 return -EINVAL;
662 }
663
664 /*
665 * Event record size is 16-bit which results in a maximum size of about 64KiB.
666 * Allow about 4KiB for the rest of the sample record, to give a maximum
667 * AUX area sample size of 60KiB.
668 */
669 #define MAX_AUX_SAMPLE_SIZE (60 * 1024)
670
671 /* Arbitrary default size if no other default provided */
672 #define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024)
673
auxtrace_validate_aux_sample_size(struct evlist * evlist,struct record_opts * opts)674 static int auxtrace_validate_aux_sample_size(struct evlist *evlist,
675 struct record_opts *opts)
676 {
677 struct evsel *evsel;
678 bool has_aux_leader = false;
679 u32 sz;
680
681 evlist__for_each_entry(evlist, evsel) {
682 sz = evsel->core.attr.aux_sample_size;
683 if (evsel__is_group_leader(evsel)) {
684 has_aux_leader = evsel__is_aux_event(evsel);
685 if (sz) {
686 if (has_aux_leader)
687 pr_err("Cannot add AUX area sampling to an AUX area event\n");
688 else
689 pr_err("Cannot add AUX area sampling to a group leader\n");
690 return -EINVAL;
691 }
692 }
693 if (sz > MAX_AUX_SAMPLE_SIZE) {
694 pr_err("AUX area sample size %u too big, max. %d\n",
695 sz, MAX_AUX_SAMPLE_SIZE);
696 return -EINVAL;
697 }
698 if (sz) {
699 if (!has_aux_leader) {
700 pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n");
701 return -EINVAL;
702 }
703 evsel__set_sample_bit(evsel, AUX);
704 opts->auxtrace_sample_mode = true;
705 } else {
706 evsel__reset_sample_bit(evsel, AUX);
707 }
708 }
709
710 if (!opts->auxtrace_sample_mode) {
711 pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n");
712 return -EINVAL;
713 }
714
715 if (!perf_can_aux_sample()) {
716 pr_err("AUX area sampling is not supported by kernel\n");
717 return -EINVAL;
718 }
719
720 return 0;
721 }
722
auxtrace_parse_sample_options(struct auxtrace_record * itr,struct evlist * evlist,struct record_opts * opts,const char * str)723 int auxtrace_parse_sample_options(struct auxtrace_record *itr,
724 struct evlist *evlist,
725 struct record_opts *opts, const char *str)
726 {
727 struct evsel_config_term *term;
728 struct evsel *aux_evsel;
729 bool has_aux_sample_size = false;
730 bool has_aux_leader = false;
731 struct evsel *evsel;
732 char *endptr;
733 unsigned long sz;
734
735 if (!str)
736 goto no_opt;
737
738 if (!itr) {
739 pr_err("No AUX area event to sample\n");
740 return -EINVAL;
741 }
742
743 sz = strtoul(str, &endptr, 0);
744 if (*endptr || sz > UINT_MAX) {
745 pr_err("Bad AUX area sampling option: '%s'\n", str);
746 return -EINVAL;
747 }
748
749 if (!sz)
750 sz = itr->default_aux_sample_size;
751
752 if (!sz)
753 sz = DEFAULT_AUX_SAMPLE_SIZE;
754
755 /* Set aux_sample_size based on --aux-sample option */
756 evlist__for_each_entry(evlist, evsel) {
757 if (evsel__is_group_leader(evsel)) {
758 has_aux_leader = evsel__is_aux_event(evsel);
759 } else if (has_aux_leader) {
760 evsel->core.attr.aux_sample_size = sz;
761 }
762 }
763 no_opt:
764 aux_evsel = NULL;
765 /* Override with aux_sample_size from config term */
766 evlist__for_each_entry(evlist, evsel) {
767 if (evsel__is_aux_event(evsel))
768 aux_evsel = evsel;
769 term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE);
770 if (term) {
771 has_aux_sample_size = true;
772 evsel->core.attr.aux_sample_size = term->val.aux_sample_size;
773 /* If possible, group with the AUX event */
774 if (aux_evsel && evsel->core.attr.aux_sample_size)
775 perf_evlist__regroup(evlist, aux_evsel, evsel);
776 }
777 }
778
779 if (!str && !has_aux_sample_size)
780 return 0;
781
782 if (!itr) {
783 pr_err("No AUX area event to sample\n");
784 return -EINVAL;
785 }
786
787 return auxtrace_validate_aux_sample_size(evlist, opts);
788 }
789
790 struct auxtrace_record *__weak
auxtrace_record__init(struct evlist * evlist __maybe_unused,int * err)791 auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err)
792 {
793 *err = 0;
794 return NULL;
795 }
796
auxtrace_index__alloc(struct list_head * head)797 static int auxtrace_index__alloc(struct list_head *head)
798 {
799 struct auxtrace_index *auxtrace_index;
800
801 auxtrace_index = malloc(sizeof(struct auxtrace_index));
802 if (!auxtrace_index)
803 return -ENOMEM;
804
805 auxtrace_index->nr = 0;
806 INIT_LIST_HEAD(&auxtrace_index->list);
807
808 list_add_tail(&auxtrace_index->list, head);
809
810 return 0;
811 }
812
auxtrace_index__free(struct list_head * head)813 void auxtrace_index__free(struct list_head *head)
814 {
815 struct auxtrace_index *auxtrace_index, *n;
816
817 list_for_each_entry_safe(auxtrace_index, n, head, list) {
818 list_del_init(&auxtrace_index->list);
819 free(auxtrace_index);
820 }
821 }
822
auxtrace_index__last(struct list_head * head)823 static struct auxtrace_index *auxtrace_index__last(struct list_head *head)
824 {
825 struct auxtrace_index *auxtrace_index;
826 int err;
827
828 if (list_empty(head)) {
829 err = auxtrace_index__alloc(head);
830 if (err)
831 return NULL;
832 }
833
834 auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);
835
836 if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {
837 err = auxtrace_index__alloc(head);
838 if (err)
839 return NULL;
840 auxtrace_index = list_entry(head->prev, struct auxtrace_index,
841 list);
842 }
843
844 return auxtrace_index;
845 }
846
auxtrace_index__auxtrace_event(struct list_head * head,union perf_event * event,off_t file_offset)847 int auxtrace_index__auxtrace_event(struct list_head *head,
848 union perf_event *event, off_t file_offset)
849 {
850 struct auxtrace_index *auxtrace_index;
851 size_t nr;
852
853 auxtrace_index = auxtrace_index__last(head);
854 if (!auxtrace_index)
855 return -ENOMEM;
856
857 nr = auxtrace_index->nr;
858 auxtrace_index->entries[nr].file_offset = file_offset;
859 auxtrace_index->entries[nr].sz = event->header.size;
860 auxtrace_index->nr += 1;
861
862 return 0;
863 }
864
auxtrace_index__do_write(int fd,struct auxtrace_index * auxtrace_index)865 static int auxtrace_index__do_write(int fd,
866 struct auxtrace_index *auxtrace_index)
867 {
868 struct auxtrace_index_entry ent;
869 size_t i;
870
871 for (i = 0; i < auxtrace_index->nr; i++) {
872 ent.file_offset = auxtrace_index->entries[i].file_offset;
873 ent.sz = auxtrace_index->entries[i].sz;
874 if (writen(fd, &ent, sizeof(ent)) != sizeof(ent))
875 return -errno;
876 }
877 return 0;
878 }
879
auxtrace_index__write(int fd,struct list_head * head)880 int auxtrace_index__write(int fd, struct list_head *head)
881 {
882 struct auxtrace_index *auxtrace_index;
883 u64 total = 0;
884 int err;
885
886 list_for_each_entry(auxtrace_index, head, list)
887 total += auxtrace_index->nr;
888
889 if (writen(fd, &total, sizeof(total)) != sizeof(total))
890 return -errno;
891
892 list_for_each_entry(auxtrace_index, head, list) {
893 err = auxtrace_index__do_write(fd, auxtrace_index);
894 if (err)
895 return err;
896 }
897
898 return 0;
899 }
900
auxtrace_index__process_entry(int fd,struct list_head * head,bool needs_swap)901 static int auxtrace_index__process_entry(int fd, struct list_head *head,
902 bool needs_swap)
903 {
904 struct auxtrace_index *auxtrace_index;
905 struct auxtrace_index_entry ent;
906 size_t nr;
907
908 if (readn(fd, &ent, sizeof(ent)) != sizeof(ent))
909 return -1;
910
911 auxtrace_index = auxtrace_index__last(head);
912 if (!auxtrace_index)
913 return -1;
914
915 nr = auxtrace_index->nr;
916 if (needs_swap) {
917 auxtrace_index->entries[nr].file_offset =
918 bswap_64(ent.file_offset);
919 auxtrace_index->entries[nr].sz = bswap_64(ent.sz);
920 } else {
921 auxtrace_index->entries[nr].file_offset = ent.file_offset;
922 auxtrace_index->entries[nr].sz = ent.sz;
923 }
924
925 auxtrace_index->nr = nr + 1;
926
927 return 0;
928 }
929
auxtrace_index__process(int fd,u64 size,struct perf_session * session,bool needs_swap)930 int auxtrace_index__process(int fd, u64 size, struct perf_session *session,
931 bool needs_swap)
932 {
933 struct list_head *head = &session->auxtrace_index;
934 u64 nr;
935
936 if (readn(fd, &nr, sizeof(u64)) != sizeof(u64))
937 return -1;
938
939 if (needs_swap)
940 nr = bswap_64(nr);
941
942 if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size)
943 return -1;
944
945 while (nr--) {
946 int err;
947
948 err = auxtrace_index__process_entry(fd, head, needs_swap);
949 if (err)
950 return -1;
951 }
952
953 return 0;
954 }
955
auxtrace_queues__process_index_entry(struct auxtrace_queues * queues,struct perf_session * session,struct auxtrace_index_entry * ent)956 static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues,
957 struct perf_session *session,
958 struct auxtrace_index_entry *ent)
959 {
960 return auxtrace_queues__add_indexed_event(queues, session,
961 ent->file_offset, ent->sz);
962 }
963
auxtrace_queues__process_index(struct auxtrace_queues * queues,struct perf_session * session)964 int auxtrace_queues__process_index(struct auxtrace_queues *queues,
965 struct perf_session *session)
966 {
967 struct auxtrace_index *auxtrace_index;
968 struct auxtrace_index_entry *ent;
969 size_t i;
970 int err;
971
972 if (auxtrace__dont_decode(session))
973 return 0;
974
975 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
976 for (i = 0; i < auxtrace_index->nr; i++) {
977 ent = &auxtrace_index->entries[i];
978 err = auxtrace_queues__process_index_entry(queues,
979 session,
980 ent);
981 if (err)
982 return err;
983 }
984 }
985 return 0;
986 }
987
auxtrace_buffer__next(struct auxtrace_queue * queue,struct auxtrace_buffer * buffer)988 struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue,
989 struct auxtrace_buffer *buffer)
990 {
991 if (buffer) {
992 if (list_is_last(&buffer->list, &queue->head))
993 return NULL;
994 return list_entry(buffer->list.next, struct auxtrace_buffer,
995 list);
996 } else {
997 if (list_empty(&queue->head))
998 return NULL;
999 return list_entry(queue->head.next, struct auxtrace_buffer,
1000 list);
1001 }
1002 }
1003
auxtrace_queues__sample_queue(struct auxtrace_queues * queues,struct perf_sample * sample,struct perf_session * session)1004 struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues,
1005 struct perf_sample *sample,
1006 struct perf_session *session)
1007 {
1008 struct perf_sample_id *sid;
1009 unsigned int idx;
1010 u64 id;
1011
1012 id = sample->id;
1013 if (!id)
1014 return NULL;
1015
1016 sid = perf_evlist__id2sid(session->evlist, id);
1017 if (!sid)
1018 return NULL;
1019
1020 idx = sid->idx;
1021
1022 if (idx >= queues->nr_queues)
1023 return NULL;
1024
1025 return &queues->queue_array[idx];
1026 }
1027
auxtrace_queues__add_sample(struct auxtrace_queues * queues,struct perf_session * session,struct perf_sample * sample,u64 data_offset,u64 reference)1028 int auxtrace_queues__add_sample(struct auxtrace_queues *queues,
1029 struct perf_session *session,
1030 struct perf_sample *sample, u64 data_offset,
1031 u64 reference)
1032 {
1033 struct auxtrace_buffer buffer = {
1034 .pid = -1,
1035 .data_offset = data_offset,
1036 .reference = reference,
1037 .size = sample->aux_sample.size,
1038 };
1039 struct perf_sample_id *sid;
1040 u64 id = sample->id;
1041 unsigned int idx;
1042
1043 if (!id)
1044 return -EINVAL;
1045
1046 sid = perf_evlist__id2sid(session->evlist, id);
1047 if (!sid)
1048 return -ENOENT;
1049
1050 idx = sid->idx;
1051 buffer.tid = sid->tid;
1052 buffer.cpu = sid->cpu;
1053
1054 return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL);
1055 }
1056
1057 struct queue_data {
1058 bool samples;
1059 bool events;
1060 };
1061
auxtrace_queue_data_cb(struct perf_session * session,union perf_event * event,u64 offset,void * data)1062 static int auxtrace_queue_data_cb(struct perf_session *session,
1063 union perf_event *event, u64 offset,
1064 void *data)
1065 {
1066 struct queue_data *qd = data;
1067 struct perf_sample sample;
1068 int err;
1069
1070 if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) {
1071 if (event->header.size < sizeof(struct perf_record_auxtrace))
1072 return -EINVAL;
1073 offset += event->header.size;
1074 return session->auxtrace->queue_data(session, NULL, event,
1075 offset);
1076 }
1077
1078 if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE)
1079 return 0;
1080
1081 err = perf_evlist__parse_sample(session->evlist, event, &sample);
1082 if (err)
1083 return err;
1084
1085 if (!sample.aux_sample.size)
1086 return 0;
1087
1088 offset += sample.aux_sample.data - (void *)event;
1089
1090 return session->auxtrace->queue_data(session, &sample, NULL, offset);
1091 }
1092
auxtrace_queue_data(struct perf_session * session,bool samples,bool events)1093 int auxtrace_queue_data(struct perf_session *session, bool samples, bool events)
1094 {
1095 struct queue_data qd = {
1096 .samples = samples,
1097 .events = events,
1098 };
1099
1100 if (auxtrace__dont_decode(session))
1101 return 0;
1102
1103 if (!session->auxtrace || !session->auxtrace->queue_data)
1104 return -EINVAL;
1105
1106 return perf_session__peek_events(session, session->header.data_offset,
1107 session->header.data_size,
1108 auxtrace_queue_data_cb, &qd);
1109 }
1110
auxtrace_buffer__get_data(struct auxtrace_buffer * buffer,int fd)1111 void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd)
1112 {
1113 size_t adj = buffer->data_offset & (page_size - 1);
1114 size_t size = buffer->size + adj;
1115 off_t file_offset = buffer->data_offset - adj;
1116 void *addr;
1117
1118 if (buffer->data)
1119 return buffer->data;
1120
1121 addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, file_offset);
1122 if (addr == MAP_FAILED)
1123 return NULL;
1124
1125 buffer->mmap_addr = addr;
1126 buffer->mmap_size = size;
1127
1128 buffer->data = addr + adj;
1129
1130 return buffer->data;
1131 }
1132
auxtrace_buffer__put_data(struct auxtrace_buffer * buffer)1133 void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer)
1134 {
1135 if (!buffer->data || !buffer->mmap_addr)
1136 return;
1137 munmap(buffer->mmap_addr, buffer->mmap_size);
1138 buffer->mmap_addr = NULL;
1139 buffer->mmap_size = 0;
1140 buffer->data = NULL;
1141 buffer->use_data = NULL;
1142 }
1143
auxtrace_buffer__drop_data(struct auxtrace_buffer * buffer)1144 void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer)
1145 {
1146 auxtrace_buffer__put_data(buffer);
1147 if (buffer->data_needs_freeing) {
1148 buffer->data_needs_freeing = false;
1149 zfree(&buffer->data);
1150 buffer->use_data = NULL;
1151 buffer->size = 0;
1152 }
1153 }
1154
auxtrace_buffer__free(struct auxtrace_buffer * buffer)1155 void auxtrace_buffer__free(struct auxtrace_buffer *buffer)
1156 {
1157 auxtrace_buffer__drop_data(buffer);
1158 free(buffer);
1159 }
1160
auxtrace_synth_error(struct perf_record_auxtrace_error * auxtrace_error,int type,int code,int cpu,pid_t pid,pid_t tid,u64 ip,const char * msg,u64 timestamp)1161 void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type,
1162 int code, int cpu, pid_t pid, pid_t tid, u64 ip,
1163 const char *msg, u64 timestamp)
1164 {
1165 size_t size;
1166
1167 memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error));
1168
1169 auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR;
1170 auxtrace_error->type = type;
1171 auxtrace_error->code = code;
1172 auxtrace_error->cpu = cpu;
1173 auxtrace_error->pid = pid;
1174 auxtrace_error->tid = tid;
1175 auxtrace_error->fmt = 1;
1176 auxtrace_error->ip = ip;
1177 auxtrace_error->time = timestamp;
1178 strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG);
1179
1180 size = (void *)auxtrace_error->msg - (void *)auxtrace_error +
1181 strlen(auxtrace_error->msg) + 1;
1182 auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64));
1183 }
1184
perf_event__synthesize_auxtrace_info(struct auxtrace_record * itr,struct perf_tool * tool,struct perf_session * session,perf_event__handler_t process)1185 int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr,
1186 struct perf_tool *tool,
1187 struct perf_session *session,
1188 perf_event__handler_t process)
1189 {
1190 union perf_event *ev;
1191 size_t priv_size;
1192 int err;
1193
1194 pr_debug2("Synthesizing auxtrace information\n");
1195 priv_size = auxtrace_record__info_priv_size(itr, session->evlist);
1196 ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size);
1197 if (!ev)
1198 return -ENOMEM;
1199
1200 ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO;
1201 ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) +
1202 priv_size;
1203 err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info,
1204 priv_size);
1205 if (err)
1206 goto out_free;
1207
1208 err = process(tool, ev, NULL, NULL);
1209 out_free:
1210 free(ev);
1211 return err;
1212 }
1213
unleader_evsel(struct evlist * evlist,struct evsel * leader)1214 static void unleader_evsel(struct evlist *evlist, struct evsel *leader)
1215 {
1216 struct evsel *new_leader = NULL;
1217 struct evsel *evsel;
1218
1219 /* Find new leader for the group */
1220 evlist__for_each_entry(evlist, evsel) {
1221 if (evsel->leader != leader || evsel == leader)
1222 continue;
1223 if (!new_leader)
1224 new_leader = evsel;
1225 evsel->leader = new_leader;
1226 }
1227
1228 /* Update group information */
1229 if (new_leader) {
1230 zfree(&new_leader->group_name);
1231 new_leader->group_name = leader->group_name;
1232 leader->group_name = NULL;
1233
1234 new_leader->core.nr_members = leader->core.nr_members - 1;
1235 leader->core.nr_members = 1;
1236 }
1237 }
1238
unleader_auxtrace(struct perf_session * session)1239 static void unleader_auxtrace(struct perf_session *session)
1240 {
1241 struct evsel *evsel;
1242
1243 evlist__for_each_entry(session->evlist, evsel) {
1244 if (auxtrace__evsel_is_auxtrace(session, evsel) &&
1245 evsel__is_group_leader(evsel)) {
1246 unleader_evsel(session->evlist, evsel);
1247 }
1248 }
1249 }
1250
perf_event__process_auxtrace_info(struct perf_session * session,union perf_event * event)1251 int perf_event__process_auxtrace_info(struct perf_session *session,
1252 union perf_event *event)
1253 {
1254 enum auxtrace_type type = event->auxtrace_info.type;
1255 int err;
1256
1257 if (dump_trace)
1258 fprintf(stdout, " type: %u\n", type);
1259
1260 switch (type) {
1261 case PERF_AUXTRACE_INTEL_PT:
1262 err = intel_pt_process_auxtrace_info(event, session);
1263 break;
1264 case PERF_AUXTRACE_INTEL_BTS:
1265 err = intel_bts_process_auxtrace_info(event, session);
1266 break;
1267 case PERF_AUXTRACE_ARM_SPE:
1268 err = arm_spe_process_auxtrace_info(event, session);
1269 break;
1270 case PERF_AUXTRACE_CS_ETM:
1271 err = cs_etm__process_auxtrace_info(event, session);
1272 break;
1273 case PERF_AUXTRACE_S390_CPUMSF:
1274 err = s390_cpumsf_process_auxtrace_info(event, session);
1275 break;
1276 case PERF_AUXTRACE_UNKNOWN:
1277 default:
1278 return -EINVAL;
1279 }
1280
1281 if (err)
1282 return err;
1283
1284 unleader_auxtrace(session);
1285
1286 return 0;
1287 }
1288
perf_event__process_auxtrace(struct perf_session * session,union perf_event * event)1289 s64 perf_event__process_auxtrace(struct perf_session *session,
1290 union perf_event *event)
1291 {
1292 s64 err;
1293
1294 if (dump_trace)
1295 fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n",
1296 event->auxtrace.size, event->auxtrace.offset,
1297 event->auxtrace.reference, event->auxtrace.idx,
1298 event->auxtrace.tid, event->auxtrace.cpu);
1299
1300 if (auxtrace__dont_decode(session))
1301 return event->auxtrace.size;
1302
1303 if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE)
1304 return -EINVAL;
1305
1306 err = session->auxtrace->process_auxtrace_event(session, event, session->tool);
1307 if (err < 0)
1308 return err;
1309
1310 return event->auxtrace.size;
1311 }
1312
1313 #define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS
1314 #define PERF_ITRACE_DEFAULT_PERIOD 100000
1315 #define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16
1316 #define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024
1317 #define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64
1318 #define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024
1319
itrace_synth_opts__set_default(struct itrace_synth_opts * synth_opts,bool no_sample)1320 void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts,
1321 bool no_sample)
1322 {
1323 synth_opts->branches = true;
1324 synth_opts->transactions = true;
1325 synth_opts->ptwrites = true;
1326 synth_opts->pwr_events = true;
1327 synth_opts->other_events = true;
1328 synth_opts->errors = true;
1329 synth_opts->flc = true;
1330 synth_opts->llc = true;
1331 synth_opts->tlb = true;
1332 synth_opts->remote_access = true;
1333
1334 if (no_sample) {
1335 synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS;
1336 synth_opts->period = 1;
1337 synth_opts->calls = true;
1338 } else {
1339 synth_opts->instructions = true;
1340 synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE;
1341 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1342 }
1343 synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
1344 synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
1345 synth_opts->initial_skip = 0;
1346 }
1347
get_flag(const char ** ptr,unsigned int * flags)1348 static int get_flag(const char **ptr, unsigned int *flags)
1349 {
1350 while (1) {
1351 char c = **ptr;
1352
1353 if (c >= 'a' && c <= 'z') {
1354 *flags |= 1 << (c - 'a');
1355 ++*ptr;
1356 return 0;
1357 } else if (c == ' ') {
1358 ++*ptr;
1359 continue;
1360 } else {
1361 return -1;
1362 }
1363 }
1364 }
1365
get_flags(const char ** ptr,unsigned int * plus_flags,unsigned int * minus_flags)1366 static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags)
1367 {
1368 while (1) {
1369 switch (**ptr) {
1370 case '+':
1371 ++*ptr;
1372 if (get_flag(ptr, plus_flags))
1373 return -1;
1374 break;
1375 case '-':
1376 ++*ptr;
1377 if (get_flag(ptr, minus_flags))
1378 return -1;
1379 break;
1380 case ' ':
1381 ++*ptr;
1382 break;
1383 default:
1384 return 0;
1385 }
1386 }
1387 }
1388
1389 /*
1390 * Please check tools/perf/Documentation/perf-script.txt for information
1391 * about the options parsed here, which is introduced after this cset,
1392 * when support in 'perf script' for these options is introduced.
1393 */
itrace_parse_synth_opts(const struct option * opt,const char * str,int unset)1394 int itrace_parse_synth_opts(const struct option *opt, const char *str,
1395 int unset)
1396 {
1397 struct itrace_synth_opts *synth_opts = opt->value;
1398 const char *p;
1399 char *endptr;
1400 bool period_type_set = false;
1401 bool period_set = false;
1402
1403 synth_opts->set = true;
1404
1405 if (unset) {
1406 synth_opts->dont_decode = true;
1407 return 0;
1408 }
1409
1410 if (!str) {
1411 itrace_synth_opts__set_default(synth_opts,
1412 synth_opts->default_no_sample);
1413 return 0;
1414 }
1415
1416 for (p = str; *p;) {
1417 switch (*p++) {
1418 case 'i':
1419 synth_opts->instructions = true;
1420 while (*p == ' ' || *p == ',')
1421 p += 1;
1422 if (isdigit(*p)) {
1423 synth_opts->period = strtoull(p, &endptr, 10);
1424 period_set = true;
1425 p = endptr;
1426 while (*p == ' ' || *p == ',')
1427 p += 1;
1428 switch (*p++) {
1429 case 'i':
1430 synth_opts->period_type =
1431 PERF_ITRACE_PERIOD_INSTRUCTIONS;
1432 period_type_set = true;
1433 break;
1434 case 't':
1435 synth_opts->period_type =
1436 PERF_ITRACE_PERIOD_TICKS;
1437 period_type_set = true;
1438 break;
1439 case 'm':
1440 synth_opts->period *= 1000;
1441 /* Fall through */
1442 case 'u':
1443 synth_opts->period *= 1000;
1444 /* Fall through */
1445 case 'n':
1446 if (*p++ != 's')
1447 goto out_err;
1448 synth_opts->period_type =
1449 PERF_ITRACE_PERIOD_NANOSECS;
1450 period_type_set = true;
1451 break;
1452 case '\0':
1453 goto out;
1454 default:
1455 goto out_err;
1456 }
1457 }
1458 break;
1459 case 'b':
1460 synth_opts->branches = true;
1461 break;
1462 case 'x':
1463 synth_opts->transactions = true;
1464 break;
1465 case 'w':
1466 synth_opts->ptwrites = true;
1467 break;
1468 case 'p':
1469 synth_opts->pwr_events = true;
1470 break;
1471 case 'o':
1472 synth_opts->other_events = true;
1473 break;
1474 case 'e':
1475 synth_opts->errors = true;
1476 if (get_flags(&p, &synth_opts->error_plus_flags,
1477 &synth_opts->error_minus_flags))
1478 goto out_err;
1479 break;
1480 case 'd':
1481 synth_opts->log = true;
1482 if (get_flags(&p, &synth_opts->log_plus_flags,
1483 &synth_opts->log_minus_flags))
1484 goto out_err;
1485 break;
1486 case 'c':
1487 synth_opts->branches = true;
1488 synth_opts->calls = true;
1489 break;
1490 case 'r':
1491 synth_opts->branches = true;
1492 synth_opts->returns = true;
1493 break;
1494 case 'G':
1495 case 'g':
1496 if (p[-1] == 'G')
1497 synth_opts->add_callchain = true;
1498 else
1499 synth_opts->callchain = true;
1500 synth_opts->callchain_sz =
1501 PERF_ITRACE_DEFAULT_CALLCHAIN_SZ;
1502 while (*p == ' ' || *p == ',')
1503 p += 1;
1504 if (isdigit(*p)) {
1505 unsigned int val;
1506
1507 val = strtoul(p, &endptr, 10);
1508 p = endptr;
1509 if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ)
1510 goto out_err;
1511 synth_opts->callchain_sz = val;
1512 }
1513 break;
1514 case 'L':
1515 case 'l':
1516 if (p[-1] == 'L')
1517 synth_opts->add_last_branch = true;
1518 else
1519 synth_opts->last_branch = true;
1520 synth_opts->last_branch_sz =
1521 PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ;
1522 while (*p == ' ' || *p == ',')
1523 p += 1;
1524 if (isdigit(*p)) {
1525 unsigned int val;
1526
1527 val = strtoul(p, &endptr, 10);
1528 p = endptr;
1529 if (!val ||
1530 val > PERF_ITRACE_MAX_LAST_BRANCH_SZ)
1531 goto out_err;
1532 synth_opts->last_branch_sz = val;
1533 }
1534 break;
1535 case 's':
1536 synth_opts->initial_skip = strtoul(p, &endptr, 10);
1537 if (p == endptr)
1538 goto out_err;
1539 p = endptr;
1540 break;
1541 case 'f':
1542 synth_opts->flc = true;
1543 break;
1544 case 'm':
1545 synth_opts->llc = true;
1546 break;
1547 case 't':
1548 synth_opts->tlb = true;
1549 break;
1550 case 'a':
1551 synth_opts->remote_access = true;
1552 break;
1553 case 'q':
1554 synth_opts->quick += 1;
1555 break;
1556 case ' ':
1557 case ',':
1558 break;
1559 default:
1560 goto out_err;
1561 }
1562 }
1563 out:
1564 if (synth_opts->instructions) {
1565 if (!period_type_set)
1566 synth_opts->period_type =
1567 PERF_ITRACE_DEFAULT_PERIOD_TYPE;
1568 if (!period_set)
1569 synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD;
1570 }
1571
1572 return 0;
1573
1574 out_err:
1575 pr_err("Bad Instruction Tracing options '%s'\n", str);
1576 return -EINVAL;
1577 }
1578
1579 static const char * const auxtrace_error_type_name[] = {
1580 [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace",
1581 };
1582
auxtrace_error_name(int type)1583 static const char *auxtrace_error_name(int type)
1584 {
1585 const char *error_type_name = NULL;
1586
1587 if (type < PERF_AUXTRACE_ERROR_MAX)
1588 error_type_name = auxtrace_error_type_name[type];
1589 if (!error_type_name)
1590 error_type_name = "unknown AUX";
1591 return error_type_name;
1592 }
1593
perf_event__fprintf_auxtrace_error(union perf_event * event,FILE * fp)1594 size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp)
1595 {
1596 struct perf_record_auxtrace_error *e = &event->auxtrace_error;
1597 unsigned long long nsecs = e->time;
1598 const char *msg = e->msg;
1599 int ret;
1600
1601 ret = fprintf(fp, " %s error type %u",
1602 auxtrace_error_name(e->type), e->type);
1603
1604 if (e->fmt && nsecs) {
1605 unsigned long secs = nsecs / NSEC_PER_SEC;
1606
1607 nsecs -= secs * NSEC_PER_SEC;
1608 ret += fprintf(fp, " time %lu.%09llu", secs, nsecs);
1609 } else {
1610 ret += fprintf(fp, " time 0");
1611 }
1612
1613 if (!e->fmt)
1614 msg = (const char *)&e->time;
1615
1616 ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %s\n",
1617 e->cpu, e->pid, e->tid, e->ip, e->code, msg);
1618 return ret;
1619 }
1620
perf_session__auxtrace_error_inc(struct perf_session * session,union perf_event * event)1621 void perf_session__auxtrace_error_inc(struct perf_session *session,
1622 union perf_event *event)
1623 {
1624 struct perf_record_auxtrace_error *e = &event->auxtrace_error;
1625
1626 if (e->type < PERF_AUXTRACE_ERROR_MAX)
1627 session->evlist->stats.nr_auxtrace_errors[e->type] += 1;
1628 }
1629
events_stats__auxtrace_error_warn(const struct events_stats * stats)1630 void events_stats__auxtrace_error_warn(const struct events_stats *stats)
1631 {
1632 int i;
1633
1634 for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) {
1635 if (!stats->nr_auxtrace_errors[i])
1636 continue;
1637 ui__warning("%u %s errors\n",
1638 stats->nr_auxtrace_errors[i],
1639 auxtrace_error_name(i));
1640 }
1641 }
1642
perf_event__process_auxtrace_error(struct perf_session * session,union perf_event * event)1643 int perf_event__process_auxtrace_error(struct perf_session *session,
1644 union perf_event *event)
1645 {
1646 if (auxtrace__dont_decode(session))
1647 return 0;
1648
1649 perf_event__fprintf_auxtrace_error(event, stdout);
1650 return 0;
1651 }
1652
__auxtrace_mmap__read(struct mmap * map,struct auxtrace_record * itr,struct perf_tool * tool,process_auxtrace_t fn,bool snapshot,size_t snapshot_size)1653 static int __auxtrace_mmap__read(struct mmap *map,
1654 struct auxtrace_record *itr,
1655 struct perf_tool *tool, process_auxtrace_t fn,
1656 bool snapshot, size_t snapshot_size)
1657 {
1658 struct auxtrace_mmap *mm = &map->auxtrace_mmap;
1659 u64 head, old = mm->prev, offset, ref;
1660 unsigned char *data = mm->base;
1661 size_t size, head_off, old_off, len1, len2, padding;
1662 union perf_event ev;
1663 void *data1, *data2;
1664
1665 if (snapshot) {
1666 head = auxtrace_mmap__read_snapshot_head(mm);
1667 if (auxtrace_record__find_snapshot(itr, mm->idx, mm, data,
1668 &head, &old))
1669 return -1;
1670 } else {
1671 head = auxtrace_mmap__read_head(mm);
1672 }
1673
1674 if (old == head)
1675 return 0;
1676
1677 pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n",
1678 mm->idx, old, head, head - old);
1679
1680 if (mm->mask) {
1681 head_off = head & mm->mask;
1682 old_off = old & mm->mask;
1683 } else {
1684 head_off = head % mm->len;
1685 old_off = old % mm->len;
1686 }
1687
1688 if (head_off > old_off)
1689 size = head_off - old_off;
1690 else
1691 size = mm->len - (old_off - head_off);
1692
1693 if (snapshot && size > snapshot_size)
1694 size = snapshot_size;
1695
1696 ref = auxtrace_record__reference(itr);
1697
1698 if (head > old || size <= head || mm->mask) {
1699 offset = head - size;
1700 } else {
1701 /*
1702 * When the buffer size is not a power of 2, 'head' wraps at the
1703 * highest multiple of the buffer size, so we have to subtract
1704 * the remainder here.
1705 */
1706 u64 rem = (0ULL - mm->len) % mm->len;
1707
1708 offset = head - size - rem;
1709 }
1710
1711 if (size > head_off) {
1712 len1 = size - head_off;
1713 data1 = &data[mm->len - len1];
1714 len2 = head_off;
1715 data2 = &data[0];
1716 } else {
1717 len1 = size;
1718 data1 = &data[head_off - len1];
1719 len2 = 0;
1720 data2 = NULL;
1721 }
1722
1723 if (itr->alignment) {
1724 unsigned int unwanted = len1 % itr->alignment;
1725
1726 len1 -= unwanted;
1727 size -= unwanted;
1728 }
1729
1730 /* padding must be written by fn() e.g. record__process_auxtrace() */
1731 padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1);
1732 if (padding)
1733 padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding;
1734
1735 memset(&ev, 0, sizeof(ev));
1736 ev.auxtrace.header.type = PERF_RECORD_AUXTRACE;
1737 ev.auxtrace.header.size = sizeof(ev.auxtrace);
1738 ev.auxtrace.size = size + padding;
1739 ev.auxtrace.offset = offset;
1740 ev.auxtrace.reference = ref;
1741 ev.auxtrace.idx = mm->idx;
1742 ev.auxtrace.tid = mm->tid;
1743 ev.auxtrace.cpu = mm->cpu;
1744
1745 if (fn(tool, map, &ev, data1, len1, data2, len2))
1746 return -1;
1747
1748 mm->prev = head;
1749
1750 if (!snapshot) {
1751 auxtrace_mmap__write_tail(mm, head);
1752 if (itr->read_finish) {
1753 int err;
1754
1755 err = itr->read_finish(itr, mm->idx);
1756 if (err < 0)
1757 return err;
1758 }
1759 }
1760
1761 return 1;
1762 }
1763
auxtrace_mmap__read(struct mmap * map,struct auxtrace_record * itr,struct perf_tool * tool,process_auxtrace_t fn)1764 int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr,
1765 struct perf_tool *tool, process_auxtrace_t fn)
1766 {
1767 return __auxtrace_mmap__read(map, itr, tool, fn, false, 0);
1768 }
1769
auxtrace_mmap__read_snapshot(struct mmap * map,struct auxtrace_record * itr,struct perf_tool * tool,process_auxtrace_t fn,size_t snapshot_size)1770 int auxtrace_mmap__read_snapshot(struct mmap *map,
1771 struct auxtrace_record *itr,
1772 struct perf_tool *tool, process_auxtrace_t fn,
1773 size_t snapshot_size)
1774 {
1775 return __auxtrace_mmap__read(map, itr, tool, fn, true, snapshot_size);
1776 }
1777
1778 /**
1779 * struct auxtrace_cache - hash table to implement a cache
1780 * @hashtable: the hashtable
1781 * @sz: hashtable size (number of hlists)
1782 * @entry_size: size of an entry
1783 * @limit: limit the number of entries to this maximum, when reached the cache
1784 * is dropped and caching begins again with an empty cache
1785 * @cnt: current number of entries
1786 * @bits: hashtable size (@sz = 2^@bits)
1787 */
1788 struct auxtrace_cache {
1789 struct hlist_head *hashtable;
1790 size_t sz;
1791 size_t entry_size;
1792 size_t limit;
1793 size_t cnt;
1794 unsigned int bits;
1795 };
1796
auxtrace_cache__new(unsigned int bits,size_t entry_size,unsigned int limit_percent)1797 struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size,
1798 unsigned int limit_percent)
1799 {
1800 struct auxtrace_cache *c;
1801 struct hlist_head *ht;
1802 size_t sz, i;
1803
1804 c = zalloc(sizeof(struct auxtrace_cache));
1805 if (!c)
1806 return NULL;
1807
1808 sz = 1UL << bits;
1809
1810 ht = calloc(sz, sizeof(struct hlist_head));
1811 if (!ht)
1812 goto out_free;
1813
1814 for (i = 0; i < sz; i++)
1815 INIT_HLIST_HEAD(&ht[i]);
1816
1817 c->hashtable = ht;
1818 c->sz = sz;
1819 c->entry_size = entry_size;
1820 c->limit = (c->sz * limit_percent) / 100;
1821 c->bits = bits;
1822
1823 return c;
1824
1825 out_free:
1826 free(c);
1827 return NULL;
1828 }
1829
auxtrace_cache__drop(struct auxtrace_cache * c)1830 static void auxtrace_cache__drop(struct auxtrace_cache *c)
1831 {
1832 struct auxtrace_cache_entry *entry;
1833 struct hlist_node *tmp;
1834 size_t i;
1835
1836 if (!c)
1837 return;
1838
1839 for (i = 0; i < c->sz; i++) {
1840 hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) {
1841 hlist_del(&entry->hash);
1842 auxtrace_cache__free_entry(c, entry);
1843 }
1844 }
1845
1846 c->cnt = 0;
1847 }
1848
auxtrace_cache__free(struct auxtrace_cache * c)1849 void auxtrace_cache__free(struct auxtrace_cache *c)
1850 {
1851 if (!c)
1852 return;
1853
1854 auxtrace_cache__drop(c);
1855 zfree(&c->hashtable);
1856 free(c);
1857 }
1858
auxtrace_cache__alloc_entry(struct auxtrace_cache * c)1859 void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c)
1860 {
1861 return malloc(c->entry_size);
1862 }
1863
auxtrace_cache__free_entry(struct auxtrace_cache * c __maybe_unused,void * entry)1864 void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused,
1865 void *entry)
1866 {
1867 free(entry);
1868 }
1869
auxtrace_cache__add(struct auxtrace_cache * c,u32 key,struct auxtrace_cache_entry * entry)1870 int auxtrace_cache__add(struct auxtrace_cache *c, u32 key,
1871 struct auxtrace_cache_entry *entry)
1872 {
1873 if (c->limit && ++c->cnt > c->limit)
1874 auxtrace_cache__drop(c);
1875
1876 entry->key = key;
1877 hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]);
1878
1879 return 0;
1880 }
1881
auxtrace_cache__rm(struct auxtrace_cache * c,u32 key)1882 static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c,
1883 u32 key)
1884 {
1885 struct auxtrace_cache_entry *entry;
1886 struct hlist_head *hlist;
1887 struct hlist_node *n;
1888
1889 if (!c)
1890 return NULL;
1891
1892 hlist = &c->hashtable[hash_32(key, c->bits)];
1893 hlist_for_each_entry_safe(entry, n, hlist, hash) {
1894 if (entry->key == key) {
1895 hlist_del(&entry->hash);
1896 return entry;
1897 }
1898 }
1899
1900 return NULL;
1901 }
1902
auxtrace_cache__remove(struct auxtrace_cache * c,u32 key)1903 void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key)
1904 {
1905 struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key);
1906
1907 auxtrace_cache__free_entry(c, entry);
1908 }
1909
auxtrace_cache__lookup(struct auxtrace_cache * c,u32 key)1910 void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key)
1911 {
1912 struct auxtrace_cache_entry *entry;
1913 struct hlist_head *hlist;
1914
1915 if (!c)
1916 return NULL;
1917
1918 hlist = &c->hashtable[hash_32(key, c->bits)];
1919 hlist_for_each_entry(entry, hlist, hash) {
1920 if (entry->key == key)
1921 return entry;
1922 }
1923
1924 return NULL;
1925 }
1926
addr_filter__free_str(struct addr_filter * filt)1927 static void addr_filter__free_str(struct addr_filter *filt)
1928 {
1929 zfree(&filt->str);
1930 filt->action = NULL;
1931 filt->sym_from = NULL;
1932 filt->sym_to = NULL;
1933 filt->filename = NULL;
1934 }
1935
addr_filter__new(void)1936 static struct addr_filter *addr_filter__new(void)
1937 {
1938 struct addr_filter *filt = zalloc(sizeof(*filt));
1939
1940 if (filt)
1941 INIT_LIST_HEAD(&filt->list);
1942
1943 return filt;
1944 }
1945
addr_filter__free(struct addr_filter * filt)1946 static void addr_filter__free(struct addr_filter *filt)
1947 {
1948 if (filt)
1949 addr_filter__free_str(filt);
1950 free(filt);
1951 }
1952
addr_filters__add(struct addr_filters * filts,struct addr_filter * filt)1953 static void addr_filters__add(struct addr_filters *filts,
1954 struct addr_filter *filt)
1955 {
1956 list_add_tail(&filt->list, &filts->head);
1957 filts->cnt += 1;
1958 }
1959
addr_filters__del(struct addr_filters * filts,struct addr_filter * filt)1960 static void addr_filters__del(struct addr_filters *filts,
1961 struct addr_filter *filt)
1962 {
1963 list_del_init(&filt->list);
1964 filts->cnt -= 1;
1965 }
1966
addr_filters__init(struct addr_filters * filts)1967 void addr_filters__init(struct addr_filters *filts)
1968 {
1969 INIT_LIST_HEAD(&filts->head);
1970 filts->cnt = 0;
1971 }
1972
addr_filters__exit(struct addr_filters * filts)1973 void addr_filters__exit(struct addr_filters *filts)
1974 {
1975 struct addr_filter *filt, *n;
1976
1977 list_for_each_entry_safe(filt, n, &filts->head, list) {
1978 addr_filters__del(filts, filt);
1979 addr_filter__free(filt);
1980 }
1981 }
1982
parse_num_or_str(char ** inp,u64 * num,const char ** str,const char * str_delim)1983 static int parse_num_or_str(char **inp, u64 *num, const char **str,
1984 const char *str_delim)
1985 {
1986 *inp += strspn(*inp, " ");
1987
1988 if (isdigit(**inp)) {
1989 char *endptr;
1990
1991 if (!num)
1992 return -EINVAL;
1993 errno = 0;
1994 *num = strtoull(*inp, &endptr, 0);
1995 if (errno)
1996 return -errno;
1997 if (endptr == *inp)
1998 return -EINVAL;
1999 *inp = endptr;
2000 } else {
2001 size_t n;
2002
2003 if (!str)
2004 return -EINVAL;
2005 *inp += strspn(*inp, " ");
2006 *str = *inp;
2007 n = strcspn(*inp, str_delim);
2008 if (!n)
2009 return -EINVAL;
2010 *inp += n;
2011 if (**inp) {
2012 **inp = '\0';
2013 *inp += 1;
2014 }
2015 }
2016 return 0;
2017 }
2018
parse_action(struct addr_filter * filt)2019 static int parse_action(struct addr_filter *filt)
2020 {
2021 if (!strcmp(filt->action, "filter")) {
2022 filt->start = true;
2023 filt->range = true;
2024 } else if (!strcmp(filt->action, "start")) {
2025 filt->start = true;
2026 } else if (!strcmp(filt->action, "stop")) {
2027 filt->start = false;
2028 } else if (!strcmp(filt->action, "tracestop")) {
2029 filt->start = false;
2030 filt->range = true;
2031 filt->action += 5; /* Change 'tracestop' to 'stop' */
2032 } else {
2033 return -EINVAL;
2034 }
2035 return 0;
2036 }
2037
parse_sym_idx(char ** inp,int * idx)2038 static int parse_sym_idx(char **inp, int *idx)
2039 {
2040 *idx = -1;
2041
2042 *inp += strspn(*inp, " ");
2043
2044 if (**inp != '#')
2045 return 0;
2046
2047 *inp += 1;
2048
2049 if (**inp == 'g' || **inp == 'G') {
2050 *inp += 1;
2051 *idx = 0;
2052 } else {
2053 unsigned long num;
2054 char *endptr;
2055
2056 errno = 0;
2057 num = strtoul(*inp, &endptr, 0);
2058 if (errno)
2059 return -errno;
2060 if (endptr == *inp || num > INT_MAX)
2061 return -EINVAL;
2062 *inp = endptr;
2063 *idx = num;
2064 }
2065
2066 return 0;
2067 }
2068
parse_addr_size(char ** inp,u64 * num,const char ** str,int * idx)2069 static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx)
2070 {
2071 int err = parse_num_or_str(inp, num, str, " ");
2072
2073 if (!err && *str)
2074 err = parse_sym_idx(inp, idx);
2075
2076 return err;
2077 }
2078
parse_one_filter(struct addr_filter * filt,const char ** filter_inp)2079 static int parse_one_filter(struct addr_filter *filt, const char **filter_inp)
2080 {
2081 char *fstr;
2082 int err;
2083
2084 filt->str = fstr = strdup(*filter_inp);
2085 if (!fstr)
2086 return -ENOMEM;
2087
2088 err = parse_num_or_str(&fstr, NULL, &filt->action, " ");
2089 if (err)
2090 goto out_err;
2091
2092 err = parse_action(filt);
2093 if (err)
2094 goto out_err;
2095
2096 err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from,
2097 &filt->sym_from_idx);
2098 if (err)
2099 goto out_err;
2100
2101 fstr += strspn(fstr, " ");
2102
2103 if (*fstr == '/') {
2104 fstr += 1;
2105 err = parse_addr_size(&fstr, &filt->size, &filt->sym_to,
2106 &filt->sym_to_idx);
2107 if (err)
2108 goto out_err;
2109 filt->range = true;
2110 }
2111
2112 fstr += strspn(fstr, " ");
2113
2114 if (*fstr == '@') {
2115 fstr += 1;
2116 err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,");
2117 if (err)
2118 goto out_err;
2119 }
2120
2121 fstr += strspn(fstr, " ,");
2122
2123 *filter_inp += fstr - filt->str;
2124
2125 return 0;
2126
2127 out_err:
2128 addr_filter__free_str(filt);
2129
2130 return err;
2131 }
2132
addr_filters__parse_bare_filter(struct addr_filters * filts,const char * filter)2133 int addr_filters__parse_bare_filter(struct addr_filters *filts,
2134 const char *filter)
2135 {
2136 struct addr_filter *filt;
2137 const char *fstr = filter;
2138 int err;
2139
2140 while (*fstr) {
2141 filt = addr_filter__new();
2142 err = parse_one_filter(filt, &fstr);
2143 if (err) {
2144 addr_filter__free(filt);
2145 addr_filters__exit(filts);
2146 return err;
2147 }
2148 addr_filters__add(filts, filt);
2149 }
2150
2151 return 0;
2152 }
2153
2154 struct sym_args {
2155 const char *name;
2156 u64 start;
2157 u64 size;
2158 int idx;
2159 int cnt;
2160 bool started;
2161 bool global;
2162 bool selected;
2163 bool duplicate;
2164 bool near;
2165 };
2166
kern_sym_match(struct sym_args * args,const char * name,char type)2167 static bool kern_sym_match(struct sym_args *args, const char *name, char type)
2168 {
2169 /* A function with the same name, and global or the n'th found or any */
2170 return kallsyms__is_function(type) &&
2171 !strcmp(name, args->name) &&
2172 ((args->global && isupper(type)) ||
2173 (args->selected && ++(args->cnt) == args->idx) ||
2174 (!args->global && !args->selected));
2175 }
2176
find_kern_sym_cb(void * arg,const char * name,char type,u64 start)2177 static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2178 {
2179 struct sym_args *args = arg;
2180
2181 if (args->started) {
2182 if (!args->size)
2183 args->size = start - args->start;
2184 if (args->selected) {
2185 if (args->size)
2186 return 1;
2187 } else if (kern_sym_match(args, name, type)) {
2188 args->duplicate = true;
2189 return 1;
2190 }
2191 } else if (kern_sym_match(args, name, type)) {
2192 args->started = true;
2193 args->start = start;
2194 }
2195
2196 return 0;
2197 }
2198
print_kern_sym_cb(void * arg,const char * name,char type,u64 start)2199 static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start)
2200 {
2201 struct sym_args *args = arg;
2202
2203 if (kern_sym_match(args, name, type)) {
2204 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2205 ++args->cnt, start, type, name);
2206 args->near = true;
2207 } else if (args->near) {
2208 args->near = false;
2209 pr_err("\t\twhich is near\t\t%s\n", name);
2210 }
2211
2212 return 0;
2213 }
2214
sym_not_found_error(const char * sym_name,int idx)2215 static int sym_not_found_error(const char *sym_name, int idx)
2216 {
2217 if (idx > 0) {
2218 pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n",
2219 idx, sym_name);
2220 } else if (!idx) {
2221 pr_err("Global symbol '%s' not found.\n", sym_name);
2222 } else {
2223 pr_err("Symbol '%s' not found.\n", sym_name);
2224 }
2225 pr_err("Note that symbols must be functions.\n");
2226
2227 return -EINVAL;
2228 }
2229
find_kern_sym(const char * sym_name,u64 * start,u64 * size,int idx)2230 static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx)
2231 {
2232 struct sym_args args = {
2233 .name = sym_name,
2234 .idx = idx,
2235 .global = !idx,
2236 .selected = idx > 0,
2237 };
2238 int err;
2239
2240 *start = 0;
2241 *size = 0;
2242
2243 err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb);
2244 if (err < 0) {
2245 pr_err("Failed to parse /proc/kallsyms\n");
2246 return err;
2247 }
2248
2249 if (args.duplicate) {
2250 pr_err("Multiple kernel symbols with name '%s'\n", sym_name);
2251 args.cnt = 0;
2252 kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb);
2253 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2254 sym_name);
2255 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2256 return -EINVAL;
2257 }
2258
2259 if (!args.started) {
2260 pr_err("Kernel symbol lookup: ");
2261 return sym_not_found_error(sym_name, idx);
2262 }
2263
2264 *start = args.start;
2265 *size = args.size;
2266
2267 return 0;
2268 }
2269
find_entire_kern_cb(void * arg,const char * name __maybe_unused,char type,u64 start)2270 static int find_entire_kern_cb(void *arg, const char *name __maybe_unused,
2271 char type, u64 start)
2272 {
2273 struct sym_args *args = arg;
2274
2275 if (!kallsyms__is_function(type))
2276 return 0;
2277
2278 if (!args->started) {
2279 args->started = true;
2280 args->start = start;
2281 }
2282 /* Don't know exactly where the kernel ends, so we add a page */
2283 args->size = round_up(start, page_size) + page_size - args->start;
2284
2285 return 0;
2286 }
2287
addr_filter__entire_kernel(struct addr_filter * filt)2288 static int addr_filter__entire_kernel(struct addr_filter *filt)
2289 {
2290 struct sym_args args = { .started = false };
2291 int err;
2292
2293 err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb);
2294 if (err < 0 || !args.started) {
2295 pr_err("Failed to parse /proc/kallsyms\n");
2296 return err;
2297 }
2298
2299 filt->addr = args.start;
2300 filt->size = args.size;
2301
2302 return 0;
2303 }
2304
check_end_after_start(struct addr_filter * filt,u64 start,u64 size)2305 static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size)
2306 {
2307 if (start + size >= filt->addr)
2308 return 0;
2309
2310 if (filt->sym_from) {
2311 pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n",
2312 filt->sym_to, start, filt->sym_from, filt->addr);
2313 } else {
2314 pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n",
2315 filt->sym_to, start, filt->addr);
2316 }
2317
2318 return -EINVAL;
2319 }
2320
addr_filter__resolve_kernel_syms(struct addr_filter * filt)2321 static int addr_filter__resolve_kernel_syms(struct addr_filter *filt)
2322 {
2323 bool no_size = false;
2324 u64 start, size;
2325 int err;
2326
2327 if (symbol_conf.kptr_restrict) {
2328 pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n");
2329 return -EINVAL;
2330 }
2331
2332 if (filt->sym_from && !strcmp(filt->sym_from, "*"))
2333 return addr_filter__entire_kernel(filt);
2334
2335 if (filt->sym_from) {
2336 err = find_kern_sym(filt->sym_from, &start, &size,
2337 filt->sym_from_idx);
2338 if (err)
2339 return err;
2340 filt->addr = start;
2341 if (filt->range && !filt->size && !filt->sym_to) {
2342 filt->size = size;
2343 no_size = !size;
2344 }
2345 }
2346
2347 if (filt->sym_to) {
2348 err = find_kern_sym(filt->sym_to, &start, &size,
2349 filt->sym_to_idx);
2350 if (err)
2351 return err;
2352
2353 err = check_end_after_start(filt, start, size);
2354 if (err)
2355 return err;
2356 filt->size = start + size - filt->addr;
2357 no_size = !size;
2358 }
2359
2360 /* The very last symbol in kallsyms does not imply a particular size */
2361 if (no_size) {
2362 pr_err("Cannot determine size of symbol '%s'\n",
2363 filt->sym_to ? filt->sym_to : filt->sym_from);
2364 return -EINVAL;
2365 }
2366
2367 return 0;
2368 }
2369
load_dso(const char * name)2370 static struct dso *load_dso(const char *name)
2371 {
2372 struct map *map;
2373 struct dso *dso;
2374
2375 map = dso__new_map(name);
2376 if (!map)
2377 return NULL;
2378
2379 if (map__load(map) < 0)
2380 pr_err("File '%s' not found or has no symbols.\n", name);
2381
2382 dso = dso__get(map->dso);
2383
2384 map__put(map);
2385
2386 return dso;
2387 }
2388
dso_sym_match(struct symbol * sym,const char * name,int * cnt,int idx)2389 static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt,
2390 int idx)
2391 {
2392 /* Same name, and global or the n'th found or any */
2393 return !arch__compare_symbol_names(name, sym->name) &&
2394 ((!idx && sym->binding == STB_GLOBAL) ||
2395 (idx > 0 && ++*cnt == idx) ||
2396 idx < 0);
2397 }
2398
print_duplicate_syms(struct dso * dso,const char * sym_name)2399 static void print_duplicate_syms(struct dso *dso, const char *sym_name)
2400 {
2401 struct symbol *sym;
2402 bool near = false;
2403 int cnt = 0;
2404
2405 pr_err("Multiple symbols with name '%s'\n", sym_name);
2406
2407 sym = dso__first_symbol(dso);
2408 while (sym) {
2409 if (dso_sym_match(sym, sym_name, &cnt, -1)) {
2410 pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n",
2411 ++cnt, sym->start,
2412 sym->binding == STB_GLOBAL ? 'g' :
2413 sym->binding == STB_LOCAL ? 'l' : 'w',
2414 sym->name);
2415 near = true;
2416 } else if (near) {
2417 near = false;
2418 pr_err("\t\twhich is near\t\t%s\n", sym->name);
2419 }
2420 sym = dso__next_symbol(sym);
2421 }
2422
2423 pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n",
2424 sym_name);
2425 pr_err("Or select a global symbol by inserting #0 or #g or #G\n");
2426 }
2427
find_dso_sym(struct dso * dso,const char * sym_name,u64 * start,u64 * size,int idx)2428 static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start,
2429 u64 *size, int idx)
2430 {
2431 struct symbol *sym;
2432 int cnt = 0;
2433
2434 *start = 0;
2435 *size = 0;
2436
2437 sym = dso__first_symbol(dso);
2438 while (sym) {
2439 if (*start) {
2440 if (!*size)
2441 *size = sym->start - *start;
2442 if (idx > 0) {
2443 if (*size)
2444 return 1;
2445 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2446 print_duplicate_syms(dso, sym_name);
2447 return -EINVAL;
2448 }
2449 } else if (dso_sym_match(sym, sym_name, &cnt, idx)) {
2450 *start = sym->start;
2451 *size = sym->end - sym->start;
2452 }
2453 sym = dso__next_symbol(sym);
2454 }
2455
2456 if (!*start)
2457 return sym_not_found_error(sym_name, idx);
2458
2459 return 0;
2460 }
2461
addr_filter__entire_dso(struct addr_filter * filt,struct dso * dso)2462 static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso)
2463 {
2464 if (dso__data_file_size(dso, NULL)) {
2465 pr_err("Failed to determine filter for %s\nCannot determine file size.\n",
2466 filt->filename);
2467 return -EINVAL;
2468 }
2469
2470 filt->addr = 0;
2471 filt->size = dso->data.file_size;
2472
2473 return 0;
2474 }
2475
addr_filter__resolve_syms(struct addr_filter * filt)2476 static int addr_filter__resolve_syms(struct addr_filter *filt)
2477 {
2478 u64 start, size;
2479 struct dso *dso;
2480 int err = 0;
2481
2482 if (!filt->sym_from && !filt->sym_to)
2483 return 0;
2484
2485 if (!filt->filename)
2486 return addr_filter__resolve_kernel_syms(filt);
2487
2488 dso = load_dso(filt->filename);
2489 if (!dso) {
2490 pr_err("Failed to load symbols from: %s\n", filt->filename);
2491 return -EINVAL;
2492 }
2493
2494 if (filt->sym_from && !strcmp(filt->sym_from, "*")) {
2495 err = addr_filter__entire_dso(filt, dso);
2496 goto put_dso;
2497 }
2498
2499 if (filt->sym_from) {
2500 err = find_dso_sym(dso, filt->sym_from, &start, &size,
2501 filt->sym_from_idx);
2502 if (err)
2503 goto put_dso;
2504 filt->addr = start;
2505 if (filt->range && !filt->size && !filt->sym_to)
2506 filt->size = size;
2507 }
2508
2509 if (filt->sym_to) {
2510 err = find_dso_sym(dso, filt->sym_to, &start, &size,
2511 filt->sym_to_idx);
2512 if (err)
2513 goto put_dso;
2514
2515 err = check_end_after_start(filt, start, size);
2516 if (err)
2517 return err;
2518
2519 filt->size = start + size - filt->addr;
2520 }
2521
2522 put_dso:
2523 dso__put(dso);
2524
2525 return err;
2526 }
2527
addr_filter__to_str(struct addr_filter * filt)2528 static char *addr_filter__to_str(struct addr_filter *filt)
2529 {
2530 char filename_buf[PATH_MAX];
2531 const char *at = "";
2532 const char *fn = "";
2533 char *filter;
2534 int err;
2535
2536 if (filt->filename) {
2537 at = "@";
2538 fn = realpath(filt->filename, filename_buf);
2539 if (!fn)
2540 return NULL;
2541 }
2542
2543 if (filt->range) {
2544 err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s",
2545 filt->action, filt->addr, filt->size, at, fn);
2546 } else {
2547 err = asprintf(&filter, "%s 0x%"PRIx64"%s%s",
2548 filt->action, filt->addr, at, fn);
2549 }
2550
2551 return err < 0 ? NULL : filter;
2552 }
2553
parse_addr_filter(struct evsel * evsel,const char * filter,int max_nr)2554 static int parse_addr_filter(struct evsel *evsel, const char *filter,
2555 int max_nr)
2556 {
2557 struct addr_filters filts;
2558 struct addr_filter *filt;
2559 int err;
2560
2561 addr_filters__init(&filts);
2562
2563 err = addr_filters__parse_bare_filter(&filts, filter);
2564 if (err)
2565 goto out_exit;
2566
2567 if (filts.cnt > max_nr) {
2568 pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n",
2569 filts.cnt, max_nr);
2570 err = -EINVAL;
2571 goto out_exit;
2572 }
2573
2574 list_for_each_entry(filt, &filts.head, list) {
2575 char *new_filter;
2576
2577 err = addr_filter__resolve_syms(filt);
2578 if (err)
2579 goto out_exit;
2580
2581 new_filter = addr_filter__to_str(filt);
2582 if (!new_filter) {
2583 err = -ENOMEM;
2584 goto out_exit;
2585 }
2586
2587 if (evsel__append_addr_filter(evsel, new_filter)) {
2588 err = -ENOMEM;
2589 goto out_exit;
2590 }
2591 }
2592
2593 out_exit:
2594 addr_filters__exit(&filts);
2595
2596 if (err) {
2597 pr_err("Failed to parse address filter: '%s'\n", filter);
2598 pr_err("Filter format is: filter|start|stop|tracestop <start symbol or address> [/ <end symbol or size>] [@<file name>]\n");
2599 pr_err("Where multiple filters are separated by space or comma.\n");
2600 }
2601
2602 return err;
2603 }
2604
evsel__nr_addr_filter(struct evsel * evsel)2605 static int evsel__nr_addr_filter(struct evsel *evsel)
2606 {
2607 struct perf_pmu *pmu = evsel__find_pmu(evsel);
2608 int nr_addr_filters = 0;
2609
2610 if (!pmu)
2611 return 0;
2612
2613 perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters);
2614
2615 return nr_addr_filters;
2616 }
2617
auxtrace_parse_filters(struct evlist * evlist)2618 int auxtrace_parse_filters(struct evlist *evlist)
2619 {
2620 struct evsel *evsel;
2621 char *filter;
2622 int err, max_nr;
2623
2624 evlist__for_each_entry(evlist, evsel) {
2625 filter = evsel->filter;
2626 max_nr = evsel__nr_addr_filter(evsel);
2627 if (!filter || !max_nr)
2628 continue;
2629 evsel->filter = NULL;
2630 err = parse_addr_filter(evsel, filter, max_nr);
2631 free(filter);
2632 if (err)
2633 return err;
2634 pr_debug("Address filter: %s\n", evsel->filter);
2635 }
2636
2637 return 0;
2638 }
2639
auxtrace__process_event(struct perf_session * session,union perf_event * event,struct perf_sample * sample,struct perf_tool * tool)2640 int auxtrace__process_event(struct perf_session *session, union perf_event *event,
2641 struct perf_sample *sample, struct perf_tool *tool)
2642 {
2643 if (!session->auxtrace)
2644 return 0;
2645
2646 return session->auxtrace->process_event(session, event, sample, tool);
2647 }
2648
auxtrace__dump_auxtrace_sample(struct perf_session * session,struct perf_sample * sample)2649 void auxtrace__dump_auxtrace_sample(struct perf_session *session,
2650 struct perf_sample *sample)
2651 {
2652 if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample ||
2653 auxtrace__dont_decode(session))
2654 return;
2655
2656 session->auxtrace->dump_auxtrace_sample(session, sample);
2657 }
2658
auxtrace__flush_events(struct perf_session * session,struct perf_tool * tool)2659 int auxtrace__flush_events(struct perf_session *session, struct perf_tool *tool)
2660 {
2661 if (!session->auxtrace)
2662 return 0;
2663
2664 return session->auxtrace->flush_events(session, tool);
2665 }
2666
auxtrace__free_events(struct perf_session * session)2667 void auxtrace__free_events(struct perf_session *session)
2668 {
2669 if (!session->auxtrace)
2670 return;
2671
2672 return session->auxtrace->free_events(session);
2673 }
2674
auxtrace__free(struct perf_session * session)2675 void auxtrace__free(struct perf_session *session)
2676 {
2677 if (!session->auxtrace)
2678 return;
2679
2680 return session->auxtrace->free(session);
2681 }
2682
auxtrace__evsel_is_auxtrace(struct perf_session * session,struct evsel * evsel)2683 bool auxtrace__evsel_is_auxtrace(struct perf_session *session,
2684 struct evsel *evsel)
2685 {
2686 if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace)
2687 return false;
2688
2689 return session->auxtrace->evsel_is_auxtrace(session, evsel);
2690 }
2691