1 // SPDX-License-Identifier: GPL-2.0-only
4  * Copyright (c) 2013-2015, Intel Corporation.
27 #include "thread-stack.h"
34 #include "intel-pt.h"
37 #include "util/synthetic-events.h"
38 #include "time-utils.h"
42 #include "intel-pt-decoder/intel-pt-log.h"
43 #include "intel-pt-decoder/intel-pt-decoder.h"
44 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
45 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
167 	u64 time;  member
221 		len -= pkt_len;  in intel_pt_dump()
243 				 struct perf_sample *sample)  in intel_pt_dump_sample()  argument
245 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_dump_sample()
249 	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);  in intel_pt_dump_sample()
254 	struct perf_time_interval *range = pt->synth_opts.ptime_range;  in intel_pt_log_events()
255 	int n = pt->synth_opts.range_num;  in intel_pt_log_events()
257 	if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)  in intel_pt_log_events()
260 	if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)  in intel_pt_log_events()
263 	/* perf_time__ranges_skip_sample does not work if time is zero */  in intel_pt_log_events()
276 	start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,  in intel_pt_do_fix_overlap()
277 				      pt->have_tsc, &consecutive);  in intel_pt_do_fix_overlap()
279 		return -EINVAL;  in intel_pt_do_fix_overlap()
280 	b->use_size = b->data + b->size - start;  in intel_pt_do_fix_overlap()
281 	b->use_data = start;  in intel_pt_do_fix_overlap()
282 	if (b->use_size && consecutive)  in intel_pt_do_fix_overlap()
283 		b->consecutive = true;  in intel_pt_do_fix_overlap()
294 	if (!buffer->data) {  in intel_pt_get_buffer()
295 		int fd = perf_data__fd(ptq->pt->session->data);  in intel_pt_get_buffer()
297 		buffer->data = auxtrace_buffer__get_data(buffer, fd);  in intel_pt_get_buffer()
298 		if (!buffer->data)  in intel_pt_get_buffer()
299 			return -ENOMEM;  in intel_pt_get_buffer()
302 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;  in intel_pt_get_buffer()
303 	if (might_overlap && !buffer->consecutive && old_buffer &&  in intel_pt_get_buffer()
304 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))  in intel_pt_get_buffer()
305 		return -ENOMEM;  in intel_pt_get_buffer()
307 	if (buffer->use_data) {  in intel_pt_get_buffer()
308 		b->len = buffer->use_size;  in intel_pt_get_buffer()
309 		b->buf = buffer->use_data;  in intel_pt_get_buffer()
311 		b->len = buffer->size;  in intel_pt_get_buffer()
312 		b->buf = buffer->data;  in intel_pt_get_buffer()
314 	b->ref_timestamp = buffer->reference;  in intel_pt_get_buffer()
316 	if (!old_buffer || (might_overlap && !buffer->consecutive)) {  in intel_pt_get_buffer()
317 		b->consecutive = false;  in intel_pt_get_buffer()
318 		b->trace_nr = buffer->buffer_nr + 1;  in intel_pt_get_buffer()
320 		b->consecutive = true;  in intel_pt_get_buffer()
326 /* Do not drop buffers with references - refer intel_pt_get_trace() */
330 	if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)  in intel_pt_lookahead_drop_buffer()
341 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_lookahead()
342 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_lookahead()
346 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_lookahead()
386 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_get_trace()
387 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_get_trace()
391 	if (ptq->stop) {  in intel_pt_get_trace()
392 		b->len = 0;  in intel_pt_get_trace()
396 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_get_trace()
402 		b->len = 0;  in intel_pt_get_trace()
406 	ptq->buffer = buffer;  in intel_pt_get_trace()
412 	if (ptq->step_through_buffers)  in intel_pt_get_trace()
413 		ptq->stop = true;  in intel_pt_get_trace()
415 	if (b->len) {  in intel_pt_get_trace()
418 		ptq->old_buffer = buffer;  in intel_pt_get_trace()
443 	if (!strcmp(var, "intel-pt.cache-divisor")) {  in intel_pt_config_div()
478 	return 32 - __builtin_clz(size);  in intel_pt_cache_size()
487 	if (dso->auxtrace_cache)  in intel_pt_cache()
488 		return dso->auxtrace_cache;  in intel_pt_cache()
495 	dso->auxtrace_cache = c;  in intel_pt_cache()
509 		return -ENOMEM;  in intel_pt_cache_add()
513 		return -ENOMEM;  in intel_pt_cache_add()
515 	e->insn_cnt = insn_cnt;  in intel_pt_cache_add()
516 	e->byte_cnt = byte_cnt;  in intel_pt_cache_add()
517 	e->op = intel_pt_insn->op;  in intel_pt_cache_add()
518 	e->branch = intel_pt_insn->branch;  in intel_pt_cache_add()
519 	e->length = intel_pt_insn->length;  in intel_pt_cache_add()
520 	e->rel = intel_pt_insn->rel;  in intel_pt_cache_add()
521 	memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);  in intel_pt_cache_add()
523 	err = auxtrace_cache__add(c, offset, &e->entry);  in intel_pt_cache_add()
538 	return auxtrace_cache__lookup(dso->auxtrace_cache, offset);  in intel_pt_cache_lookup()
549 	auxtrace_cache__remove(dso->auxtrace_cache, offset);  in intel_pt_cache_invalidate()
554 	return ip >= pt->kernel_start ?  in intel_pt_cpumode()
565 	struct machine *machine = ptq->pt->machine;  in intel_pt_walk_next_insn()
576 	intel_pt_insn->length = 0;  in intel_pt_walk_next_insn()
581 	cpumode = intel_pt_cpumode(ptq->pt, *ip);  in intel_pt_walk_next_insn()
583 	thread = ptq->thread;  in intel_pt_walk_next_insn()
586 			return -EINVAL;  in intel_pt_walk_next_insn()
587 		thread = ptq->pt->unknown_thread;  in intel_pt_walk_next_insn()
591 		if (!thread__find_map(thread, cpumode, *ip, &al) || !al.map->dso)  in intel_pt_walk_next_insn()
592 			return -EINVAL;  in intel_pt_walk_next_insn()
594 		if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&  in intel_pt_walk_next_insn()
595 		    dso__data_status_seen(al.map->dso,  in intel_pt_walk_next_insn()
597 			return -ENOENT;  in intel_pt_walk_next_insn()
599 		offset = al.map->map_ip(al.map, *ip);  in intel_pt_walk_next_insn()
604 			e = intel_pt_cache_lookup(al.map->dso, machine, offset);  in intel_pt_walk_next_insn()
606 			    (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {  in intel_pt_walk_next_insn()
607 				*insn_cnt_ptr = e->insn_cnt;  in intel_pt_walk_next_insn()
608 				*ip += e->byte_cnt;  in intel_pt_walk_next_insn()
609 				intel_pt_insn->op = e->op;  in intel_pt_walk_next_insn()
610 				intel_pt_insn->branch = e->branch;  in intel_pt_walk_next_insn()
611 				intel_pt_insn->length = e->length;  in intel_pt_walk_next_insn()
612 				intel_pt_insn->rel = e->rel;  in intel_pt_walk_next_insn()
613 				memcpy(intel_pt_insn->buf, e->insn,  in intel_pt_walk_next_insn()
623 		/* Load maps to ensure dso->is_64_bit has been updated */  in intel_pt_walk_next_insn()
626 		x86_64 = al.map->dso->is_64_bit;  in intel_pt_walk_next_insn()
629 			len = dso__data_read_offset(al.map->dso, machine,  in intel_pt_walk_next_insn()
633 				return -EINVAL;  in intel_pt_walk_next_insn()
636 				return -EINVAL;  in intel_pt_walk_next_insn()
642 			if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)  in intel_pt_walk_next_insn()
648 			*ip += intel_pt_insn->length;  in intel_pt_walk_next_insn()
651 				intel_pt_insn->length = 0;  in intel_pt_walk_next_insn()
655 			if (*ip >= al.map->end)  in intel_pt_walk_next_insn()
658 			offset += intel_pt_insn->length;  in intel_pt_walk_next_insn()
675 		e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);  in intel_pt_walk_next_insn()
681 	intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,  in intel_pt_walk_next_insn()
682 			   *ip - start_ip, intel_pt_insn);  in intel_pt_walk_next_insn()
699 	list_for_each_entry(filt, &pt->filts.head, list) {  in intel_pt_match_pgd_ip()
700 		if (filt->start)  in intel_pt_match_pgd_ip()
703 		if ((filename && !filt->filename) ||  in intel_pt_match_pgd_ip()
704 		    (!filename && filt->filename) ||  in intel_pt_match_pgd_ip()
705 		    (filename && strcmp(filename, filt->filename)))  in intel_pt_match_pgd_ip()
708 		if (!(offset >= filt->addr && offset < filt->addr + filt->size))  in intel_pt_match_pgd_ip()
713 			     filt->start ? "filter" : "stop",  in intel_pt_match_pgd_ip()
714 			     filt->addr, filt->size);  in intel_pt_match_pgd_ip()
716 		if (filt->start)  in intel_pt_match_pgd_ip()
737 	if (ip >= ptq->pt->kernel_start)  in __intel_pt_pgd_ip()
738 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
742 	thread = ptq->thread;  in __intel_pt_pgd_ip()
744 		return -EINVAL;  in __intel_pt_pgd_ip()
746 	if (!thread__find_map(thread, cpumode, ip, &al) || !al.map->dso)  in __intel_pt_pgd_ip()
747 		return -EINVAL;  in __intel_pt_pgd_ip()
749 	offset = al.map->map_ip(al.map, ip);  in __intel_pt_pgd_ip()
751 	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,  in __intel_pt_pgd_ip()
752 				     al.map->dso->long_name);  in __intel_pt_pgd_ip()
763 	if (attr->type == pt->pmu_type) {  in intel_pt_get_config()
765 			*config = attr->config;  in intel_pt_get_config()
776 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_exclude_kernel()
777 		if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&  in intel_pt_exclude_kernel()
778 		    !evsel->core.attr.exclude_kernel)  in intel_pt_exclude_kernel()
789 	if (!pt->noretcomp_bit)  in intel_pt_return_compression()
792 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_return_compression()
793 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_return_compression()
794 		    (config & pt->noretcomp_bit))  in intel_pt_return_compression()
805 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_branch_enable()
806 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_branch_enable()
819 	if (!pt->mtc_freq_bits)  in intel_pt_mtc_period()
822 	for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)  in intel_pt_mtc_period()
825 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_mtc_period()
826 		if (intel_pt_get_config(pt, &evsel->core.attr, &config))  in intel_pt_mtc_period()
827 			return (config & pt->mtc_freq_bits) >> shift;  in intel_pt_mtc_period()
838 	if (!pt->tsc_bit || !pt->cap_user_time_zero)  in intel_pt_timeless_decoding()
841 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_timeless_decoding()
842 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME))  in intel_pt_timeless_decoding()
844 		if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {  in intel_pt_timeless_decoding()
845 			if (config & pt->tsc_bit)  in intel_pt_timeless_decoding()
858 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_tracing_kernel()
859 		if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&  in intel_pt_tracing_kernel()
860 		    !evsel->core.attr.exclude_kernel)  in intel_pt_tracing_kernel()
872 	if (!pt->tsc_bit)  in intel_pt_have_tsc()
875 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_have_tsc()
876 		if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {  in intel_pt_have_tsc()
877 			if (config & pt->tsc_bit)  in intel_pt_have_tsc()
890 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_sampling_mode()
891 		if ((evsel->core.attr.sample_type & PERF_SAMPLE_AUX) &&  in intel_pt_sampling_mode()
892 		    evsel->core.attr.aux_sample_size)  in intel_pt_sampling_mode()
903 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_ctl()
904 		if (intel_pt_get_config(pt, &evsel->core.attr, &config))  in intel_pt_ctl()
914 	quot = ns / pt->tc.time_mult;  in intel_pt_ns_to_ticks()
915 	rem  = ns % pt->tc.time_mult;  in intel_pt_ns_to_ticks()
916 	return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /  in intel_pt_ns_to_ticks()
917 		pt->tc.time_mult;  in intel_pt_ns_to_ticks()
925 	sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);  in intel_pt_alloc_chain()
933 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_callchain_init()
934 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_CALLCHAIN))  in intel_pt_callchain_init()
935 			evsel->synth_sample_type |= PERF_SAMPLE_CALLCHAIN;  in intel_pt_callchain_init()
938 	pt->chain = intel_pt_alloc_chain(pt);  in intel_pt_callchain_init()
939 	if (!pt->chain)  in intel_pt_callchain_init()
940 		return -ENOMEM;  in intel_pt_callchain_init()
946 				   struct perf_sample *sample)  in intel_pt_add_callchain()  argument
948 	struct thread *thread = machine__findnew_thread(pt->machine,  in intel_pt_add_callchain()
949 							sample->pid,  in intel_pt_add_callchain()
950 							sample->tid);  in intel_pt_add_callchain()
952 	thread_stack__sample_late(thread, sample->cpu, pt->chain,  in intel_pt_add_callchain()
953 				  pt->synth_opts.callchain_sz + 1, sample->ip,  in intel_pt_add_callchain()
954 				  pt->kernel_start);  in intel_pt_add_callchain()
956 	sample->callchain = pt->chain;  in intel_pt_add_callchain()
971 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_br_stack_init()
972 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK))  in intel_pt_br_stack_init()
973 			evsel->synth_sample_type |= PERF_SAMPLE_BRANCH_STACK;  in intel_pt_br_stack_init()
976 	pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz);  in intel_pt_br_stack_init()
977 	if (!pt->br_stack)  in intel_pt_br_stack_init()
978 		return -ENOMEM;  in intel_pt_br_stack_init()
984 				  struct perf_sample *sample)  in intel_pt_add_br_stack()  argument
986 	struct thread *thread = machine__findnew_thread(pt->machine,  in intel_pt_add_br_stack()
987 							sample->pid,  in intel_pt_add_br_stack()
988 							sample->tid);  in intel_pt_add_br_stack()
990 	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,  in intel_pt_add_br_stack()
991 				     pt->br_stack_sz, sample->ip,  in intel_pt_add_br_stack()
992 				     pt->kernel_start);  in intel_pt_add_br_stack()
994 	sample->branch_stack = pt->br_stack;  in intel_pt_add_br_stack()
1004 	struct perf_env *env = pt->machine->env;  in intel_pt_alloc_queue()
1011 	if (pt->synth_opts.callchain) {  in intel_pt_alloc_queue()
1012 		ptq->chain = intel_pt_alloc_chain(pt);  in intel_pt_alloc_queue()
1013 		if (!ptq->chain)  in intel_pt_alloc_queue()
1017 	if (pt->synth_opts.last_branch || pt->synth_opts.other_events) {  in intel_pt_alloc_queue()
1018 		unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz);  in intel_pt_alloc_queue()
1020 		ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);  in intel_pt_alloc_queue()
1021 		if (!ptq->last_branch)  in intel_pt_alloc_queue()
1025 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in intel_pt_alloc_queue()
1026 	if (!ptq->event_buf)  in intel_pt_alloc_queue()
1029 	ptq->pt = pt;  in intel_pt_alloc_queue()
1030 	ptq->queue_nr = queue_nr;  in intel_pt_alloc_queue()
1031 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);  in intel_pt_alloc_queue()
1032 	ptq->pid = -1;  in intel_pt_alloc_queue()
1033 	ptq->tid = -1;  in intel_pt_alloc_queue()
1034 	ptq->cpu = -1;  in intel_pt_alloc_queue()
1035 	ptq->next_tid = -1;  in intel_pt_alloc_queue()
1044 	params.max_non_turbo_ratio = pt->max_non_turbo_ratio;  in intel_pt_alloc_queue()
1046 	params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;  in intel_pt_alloc_queue()
1047 	params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;  in intel_pt_alloc_queue()
1048 	params.quick = pt->synth_opts.quick;  in intel_pt_alloc_queue()
1050 	if (pt->filts.cnt > 0)  in intel_pt_alloc_queue()
1053 	if (pt->synth_opts.instructions) {  in intel_pt_alloc_queue()
1054 		if (pt->synth_opts.period) {  in intel_pt_alloc_queue()
1055 			switch (pt->synth_opts.period_type) {  in intel_pt_alloc_queue()
1059 				params.period = pt->synth_opts.period;  in intel_pt_alloc_queue()
1063 				params.period = pt->synth_opts.period;  in intel_pt_alloc_queue()
1068 							pt->synth_opts.period);  in intel_pt_alloc_queue()
1081 	if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))  in intel_pt_alloc_queue()
1084 	ptq->decoder = intel_pt_decoder_new(&params);  in intel_pt_alloc_queue()
1085 	if (!ptq->decoder)  in intel_pt_alloc_queue()
1091 	zfree(&ptq->event_buf);  in intel_pt_alloc_queue()
1092 	zfree(&ptq->last_branch);  in intel_pt_alloc_queue()
1093 	zfree(&ptq->chain);  in intel_pt_alloc_queue()
1104 	thread__zput(ptq->thread);  in intel_pt_free_queue()
1105 	intel_pt_decoder_free(ptq->decoder);  in intel_pt_free_queue()
1106 	zfree(&ptq->event_buf);  in intel_pt_free_queue()
1107 	zfree(&ptq->last_branch);  in intel_pt_free_queue()
1108 	zfree(&ptq->chain);  in intel_pt_free_queue()
1115 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_set_pid_tid_cpu()
1117 	if (queue->tid == -1 || pt->have_sched_switch) {  in intel_pt_set_pid_tid_cpu()
1118 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);  in intel_pt_set_pid_tid_cpu()
1119 		if (ptq->tid == -1)  in intel_pt_set_pid_tid_cpu()
1120 			ptq->pid = -1;  in intel_pt_set_pid_tid_cpu()
1121 		thread__zput(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1124 	if (!ptq->thread && ptq->tid != -1)  in intel_pt_set_pid_tid_cpu()
1125 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);  in intel_pt_set_pid_tid_cpu()
1127 	if (ptq->thread) {  in intel_pt_set_pid_tid_cpu()
1128 		ptq->pid = ptq->thread->pid_;  in intel_pt_set_pid_tid_cpu()
1129 		if (queue->cpu == -1)  in intel_pt_set_pid_tid_cpu()
1130 			ptq->cpu = ptq->thread->cpu;  in intel_pt_set_pid_tid_cpu()
1136 	ptq->insn_len = 0;  in intel_pt_sample_flags()
1137 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {  in intel_pt_sample_flags()
1138 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;  in intel_pt_sample_flags()
1139 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {  in intel_pt_sample_flags()
1140 		if (ptq->state->to_ip)  in intel_pt_sample_flags()
1141 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1145 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1147 		ptq->insn_len = 0;  in intel_pt_sample_flags()
1149 		if (ptq->state->from_ip)  in intel_pt_sample_flags()
1150 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);  in intel_pt_sample_flags()
1152 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1154 		if (ptq->state->flags & INTEL_PT_IN_TX)  in intel_pt_sample_flags()
1155 			ptq->flags |= PERF_IP_FLAG_IN_TX;  in intel_pt_sample_flags()
1156 		ptq->insn_len = ptq->state->insn_len;  in intel_pt_sample_flags()
1157 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_sample_flags()
1160 	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)  in intel_pt_sample_flags()
1161 		ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;  in intel_pt_sample_flags()
1162 	if (ptq->state->type & INTEL_PT_TRACE_END)  in intel_pt_sample_flags()
1163 		ptq->flags |= PERF_IP_FLAG_TRACE_END;  in intel_pt_sample_flags()
1169 	if (!pt->range_cnt)  in intel_pt_setup_time_range()
1172 	ptq->sel_timestamp = pt->time_ranges[0].start;  in intel_pt_setup_time_range()
1173 	ptq->sel_idx = 0;  in intel_pt_setup_time_range()
1175 	if (ptq->sel_timestamp) {  in intel_pt_setup_time_range()
1176 		ptq->sel_start = true;  in intel_pt_setup_time_range()
1178 		ptq->sel_timestamp = pt->time_ranges[0].end;  in intel_pt_setup_time_range()
1179 		ptq->sel_start = false;  in intel_pt_setup_time_range()
1187 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_setup_queue()
1189 	if (list_empty(&queue->head))  in intel_pt_setup_queue()
1195 			return -ENOMEM;  in intel_pt_setup_queue()
1196 		queue->priv = ptq;  in intel_pt_setup_queue()
1198 		if (queue->cpu != -1)  in intel_pt_setup_queue()
1199 			ptq->cpu = queue->cpu;  in intel_pt_setup_queue()
1200 		ptq->tid = queue->tid;  in intel_pt_setup_queue()
1202 		ptq->cbr_seen = UINT_MAX;  in intel_pt_setup_queue()
1204 		if (pt->sampling_mode && !pt->snapshot_mode &&  in intel_pt_setup_queue()
1205 		    pt->timeless_decoding)  in intel_pt_setup_queue()
1206 			ptq->step_through_buffers = true;  in intel_pt_setup_queue()
1208 		ptq->sync_switch = pt->sync_switch;  in intel_pt_setup_queue()
1213 	if (!ptq->on_heap &&  in intel_pt_setup_queue()
1214 	    (!ptq->sync_switch ||  in intel_pt_setup_queue()
1215 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {  in intel_pt_setup_queue()
1219 		if (pt->timeless_decoding)  in intel_pt_setup_queue()
1224 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_setup_queue()
1226 		if (ptq->sel_start && ptq->sel_timestamp) {  in intel_pt_setup_queue()
1227 			ret = intel_pt_fast_forward(ptq->decoder,  in intel_pt_setup_queue()
1228 						    ptq->sel_timestamp);  in intel_pt_setup_queue()
1234 			state = intel_pt_decode(ptq->decoder);  in intel_pt_setup_queue()
1235 			if (state->err) {  in intel_pt_setup_queue()
1236 				if (state->err == INTEL_PT_ERR_NODATA) {  in intel_pt_setup_queue()
1243 			if (state->timestamp)  in intel_pt_setup_queue()
1247 		ptq->timestamp = state->timestamp;  in intel_pt_setup_queue()
1249 			     queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1250 		ptq->state = state;  in intel_pt_setup_queue()
1251 		ptq->have_sample = true;  in intel_pt_setup_queue()
1252 		if (ptq->sel_start && ptq->sel_timestamp &&  in intel_pt_setup_queue()
1253 		    ptq->timestamp < ptq->sel_timestamp)  in intel_pt_setup_queue()
1254 			ptq->have_sample = false;  in intel_pt_setup_queue()
1256 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1259 		ptq->on_heap = true;  in intel_pt_setup_queue()
1270 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_setup_queues()
1271 		ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);  in intel_pt_setup_queues()
1280 	return pt->synth_opts.initial_skip &&  in intel_pt_skip_event()
1281 	       pt->num_events++ < pt->synth_opts.initial_skip;  in intel_pt_skip_event()
1286  * Also ensure CBR is first non-skipped event by allowing for 4 more samples
1291 	return pt->synth_opts.initial_skip &&  in intel_pt_skip_cbr_event()
1292 	       pt->num_events + 4 < pt->synth_opts.initial_skip;  in intel_pt_skip_cbr_event()
1297 				   struct perf_sample *sample)  in intel_pt_prep_a_sample()  argument
1299 	event->sample.header.type = PERF_RECORD_SAMPLE;  in intel_pt_prep_a_sample()
1300 	event->sample.header.size = sizeof(struct perf_event_header);  in intel_pt_prep_a_sample()
1302 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1303 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1304 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1305 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1306 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1312 				   struct perf_sample *sample)  in intel_pt_prep_b_sample()  argument
1314 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1316 	if (!pt->timeless_decoding)  in intel_pt_prep_b_sample()
1317 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1319 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1320 	sample->cpumode = intel_pt_cpumode(pt, sample->ip);  in intel_pt_prep_b_sample()
1321 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1322 	sample->period = 1;  in intel_pt_prep_b_sample()
1323 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1325 	event->sample.header.misc = sample->cpumode;  in intel_pt_prep_b_sample()
1329 				 struct perf_sample *sample, u64 type)  in intel_pt_inject_event()  argument
1331 	event->header.size = perf_event__sample_event_size(sample, type, 0);  in intel_pt_inject_event()
1332 	return perf_event__synthesize_sample(event, type, 0, sample);  in intel_pt_inject_event()
1337 				      struct perf_sample *sample, u64 type)  in intel_pt_opt_inject()  argument
1339 	if (!pt->synth_opts.inject)  in intel_pt_opt_inject()
1342 	return intel_pt_inject_event(event, sample, type);  in intel_pt_opt_inject()
1347 					struct perf_sample *sample, u64 type)  in intel_pt_deliver_synth_event()  argument
1351 	ret = intel_pt_opt_inject(pt, event, sample, type);  in intel_pt_deliver_synth_event()
1355 	ret = perf_session__deliver_synth_event(pt->session, event, sample);  in intel_pt_deliver_synth_event()
1364 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_branch_sample()
1365 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_branch_sample()
1366 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_branch_sample()  local
1373 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))  in intel_pt_synth_branch_sample()
1379 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1381 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1382 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1388 	if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {  in intel_pt_synth_branch_sample()
1391 			.hw_idx = -1ULL,  in intel_pt_synth_branch_sample()
1393 				.from = sample.ip,  in intel_pt_synth_branch_sample()
1394 				.to = sample.addr,  in intel_pt_synth_branch_sample()
1397 		sample.branch_stack = (struct branch_stack *)&dummy_bs;  in intel_pt_synth_branch_sample()
1400 	if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)  in intel_pt_synth_branch_sample()
1401 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1402 	if (sample.cyc_cnt) {  in intel_pt_synth_branch_sample()
1403 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1404 		ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_branch_sample()
1405 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_branch_sample()
1408 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_branch_sample()
1409 					    pt->branches_sample_type);  in intel_pt_synth_branch_sample()
1415 				 struct perf_sample *sample)  in intel_pt_prep_sample()  argument
1417 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1419 	if (pt->synth_opts.callchain) {  in intel_pt_prep_sample()
1420 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_prep_sample()
1421 				     pt->synth_opts.callchain_sz + 1,  in intel_pt_prep_sample()
1422 				     sample->ip, pt->kernel_start);  in intel_pt_prep_sample()
1423 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1426 	if (pt->synth_opts.last_branch) {  in intel_pt_prep_sample()
1427 		thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,  in intel_pt_prep_sample()
1428 					pt->br_stack_sz);  in intel_pt_prep_sample()
1429 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1435 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_instruction_sample()
1436 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_instruction_sample()
1437 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_instruction_sample()  local
1442 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1444 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1445 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1446 	if (pt->synth_opts.quick)  in intel_pt_synth_instruction_sample()
1447 		sample.period = 1;  in intel_pt_synth_instruction_sample()
1449 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1451 	if (ptq->state->flags & INTEL_PT_SAMPLE_IPC)  in intel_pt_synth_instruction_sample()
1452 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1453 	if (sample.cyc_cnt) {  in intel_pt_synth_instruction_sample()
1454 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1455 		ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_instruction_sample()
1456 		ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_instruction_sample()
1459 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_synth_instruction_sample()
1461 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_instruction_sample()
1462 					    pt->instructions_sample_type);  in intel_pt_synth_instruction_sample()
1467 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_transaction_sample()
1468 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_transaction_sample()
1469 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_transaction_sample()  local
1474 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1476 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1477 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1479 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_transaction_sample()
1480 					    pt->transactions_sample_type);  in intel_pt_synth_transaction_sample()
1486 				   struct perf_sample *sample)  in intel_pt_prep_p_sample()  argument
1488 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1494 	if (!sample->ip)  in intel_pt_prep_p_sample()
1495 		sample->flags = 0;  in intel_pt_prep_p_sample()
1500 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_ptwrite_sample()
1501 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_ptwrite_sample()
1502 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_ptwrite_sample()  local
1508 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1510 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1511 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1514 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_ptwrite_sample()
1515 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);  in intel_pt_synth_ptwrite_sample()
1517 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_ptwrite_sample()
1518 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_ptwrite_sample()
1520 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_ptwrite_sample()
1521 					    pt->ptwrites_sample_type);  in intel_pt_synth_ptwrite_sample()
1526 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cbr_sample()
1527 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cbr_sample()
1528 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cbr_sample()  local
1535 	ptq->cbr_seen = ptq->state->cbr;  in intel_pt_synth_cbr_sample()
1537 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1539 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1540 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1542 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);  in intel_pt_synth_cbr_sample()
1544 	raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);  in intel_pt_synth_cbr_sample()
1547 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_cbr_sample()
1548 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_cbr_sample()
1550 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_cbr_sample()
1551 					    pt->pwr_events_sample_type);  in intel_pt_synth_cbr_sample()
1556 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_mwait_sample()
1557 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_mwait_sample()
1558 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_mwait_sample()  local
1564 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1566 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1567 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1570 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);  in intel_pt_synth_mwait_sample()
1572 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_mwait_sample()
1573 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_mwait_sample()
1575 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_mwait_sample()
1576 					    pt->pwr_events_sample_type);  in intel_pt_synth_mwait_sample()
1581 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwre_sample()
1582 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwre_sample()
1583 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwre_sample()  local
1589 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1591 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1592 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1595 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);  in intel_pt_synth_pwre_sample()
1597 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwre_sample()
1598 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwre_sample()
1600 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwre_sample()
1601 					    pt->pwr_events_sample_type);  in intel_pt_synth_pwre_sample()
1606 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_exstop_sample()
1607 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_exstop_sample()
1608 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_exstop_sample()  local
1614 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
1616 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1617 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1620 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_exstop_sample()
1622 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_exstop_sample()
1623 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_exstop_sample()
1625 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_exstop_sample()
1626 					    pt->pwr_events_sample_type);  in intel_pt_synth_exstop_sample()
1631 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwrx_sample()
1632 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwrx_sample()
1633 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwrx_sample()  local
1639 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
1641 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1642 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1645 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);  in intel_pt_synth_pwrx_sample()
1647 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwrx_sample()
1648 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwrx_sample()
1650 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwrx_sample()
1651 					    pt->pwr_events_sample_type);  in intel_pt_synth_pwrx_sample()
1683 	const u64 *gp_regs = items->val[INTEL_PT_GP_REGS_POS];  in intel_pt_add_gp_regs()
1684 	u32 mask = items->mask[INTEL_PT_GP_REGS_POS];  in intel_pt_add_gp_regs()
1690 		int n = pebs_gp_regs[i] - 1;  in intel_pt_add_gp_regs()
1697 		 * mask (i.e. 'intr_regs->mask') accordingly.  in intel_pt_add_gp_regs()
1700 			intr_regs->mask |= bit;  in intel_pt_add_gp_regs()
1716 	u32 mask = items->has_xmm & (regs_mask >> PERF_REG_X86_XMM0);  in intel_pt_add_xmm()
1717 	const u64 *xmm = items->xmm;  in intel_pt_add_xmm()
1723 	 * and update the resulting mask (i.e. 'intr_regs->mask') accordingly.  in intel_pt_add_xmm()
1725 	intr_regs->mask |= (u64)mask << PERF_REG_X86_XMM0;  in intel_pt_add_xmm()
1762 	br_stack->nr = 0;  in intel_pt_add_lbrs()
1764 	to = &br_stack->entries[0].from;  in intel_pt_add_lbrs()
1767 		u32 mask = items->mask[i];  in intel_pt_add_lbrs()
1768 		const u64 *from = items->val[i];  in intel_pt_add_lbrs()
1775 				br_stack->nr += 1;  in intel_pt_add_lbrs()
1783 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_synth_pebs_sample()
1784 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pebs_sample()  local
1785 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pebs_sample()
1786 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pebs_sample()
1787 	struct evsel *evsel = pt->pebs_evsel;  in intel_pt_synth_pebs_sample()
1788 	u64 sample_type = evsel->core.attr.sample_type;  in intel_pt_synth_pebs_sample()
1789 	u64 id = evsel->core.id[0];  in intel_pt_synth_pebs_sample()
1791 	u64 regs[8 * sizeof(sample.intr_regs.mask)];  in intel_pt_synth_pebs_sample()
1796 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_synth_pebs_sample()
1798 	sample.id = id;  in intel_pt_synth_pebs_sample()
1799 	sample.stream_id = id;  in intel_pt_synth_pebs_sample()
1801 	if (!evsel->core.attr.freq)  in intel_pt_synth_pebs_sample()
1802 		sample.period = evsel->core.attr.sample_period;  in intel_pt_synth_pebs_sample()
1804 	/* No support for non-zero CS base */  in intel_pt_synth_pebs_sample()
1805 	if (items->has_ip)  in intel_pt_synth_pebs_sample()
1806 		sample.ip = items->ip;  in intel_pt_synth_pebs_sample()
1807 	else if (items->has_rip)  in intel_pt_synth_pebs_sample()
1808 		sample.ip = items->rip;  in intel_pt_synth_pebs_sample()
1810 		sample.ip = ptq->state->from_ip;  in intel_pt_synth_pebs_sample()
1812 	/* No support for guest mode at this time */  in intel_pt_synth_pebs_sample()
1813 	cpumode = sample.ip < ptq->pt->kernel_start ?  in intel_pt_synth_pebs_sample()
1817 	event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;  in intel_pt_synth_pebs_sample()
1819 	sample.cpumode = cpumode;  in intel_pt_synth_pebs_sample()
1824 		if (items->has_timestamp)  in intel_pt_synth_pebs_sample()
1825 			timestamp = items->timestamp;  in intel_pt_synth_pebs_sample()
1826 		else if (!pt->timeless_decoding)  in intel_pt_synth_pebs_sample()
1827 			timestamp = ptq->timestamp;  in intel_pt_synth_pebs_sample()
1829 			sample.time = tsc_to_perf_time(timestamp, &pt->tc);  in intel_pt_synth_pebs_sample()
1833 	    pt->synth_opts.callchain) {  in intel_pt_synth_pebs_sample()
1834 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_synth_pebs_sample()
1835 				     pt->synth_opts.callchain_sz, sample.ip,  in intel_pt_synth_pebs_sample()
1836 				     pt->kernel_start);  in intel_pt_synth_pebs_sample()
1837 		sample.callchain = ptq->chain;  in intel_pt_synth_pebs_sample()
1841 	    (items->mask[INTEL_PT_GP_REGS_POS] ||  in intel_pt_synth_pebs_sample()
1842 	     items->mask[INTEL_PT_XMM_POS])) {  in intel_pt_synth_pebs_sample()
1843 		u64 regs_mask = evsel->core.attr.sample_regs_intr;  in intel_pt_synth_pebs_sample()
1846 		sample.intr_regs.abi = items->is_32_bit ?  in intel_pt_synth_pebs_sample()
1849 		sample.intr_regs.regs = regs;  in intel_pt_synth_pebs_sample()
1851 		pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);  in intel_pt_synth_pebs_sample()
1853 		intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);  in intel_pt_synth_pebs_sample()
1857 		if (items->mask[INTEL_PT_LBR_0_POS] ||  in intel_pt_synth_pebs_sample()
1858 		    items->mask[INTEL_PT_LBR_1_POS] ||  in intel_pt_synth_pebs_sample()
1859 		    items->mask[INTEL_PT_LBR_2_POS]) {  in intel_pt_synth_pebs_sample()
1860 			intel_pt_add_lbrs(ptq->last_branch, items);  in intel_pt_synth_pebs_sample()
1861 		} else if (pt->synth_opts.last_branch) {  in intel_pt_synth_pebs_sample()
1862 			thread_stack__br_sample(ptq->thread, ptq->cpu,  in intel_pt_synth_pebs_sample()
1863 						ptq->last_branch,  in intel_pt_synth_pebs_sample()
1864 						pt->br_stack_sz);  in intel_pt_synth_pebs_sample()
1866 			ptq->last_branch->nr = 0;  in intel_pt_synth_pebs_sample()
1868 		sample.branch_stack = ptq->last_branch;  in intel_pt_synth_pebs_sample()
1871 	if (sample_type & PERF_SAMPLE_ADDR && items->has_mem_access_address)  in intel_pt_synth_pebs_sample()
1872 		sample.addr = items->mem_access_address;  in intel_pt_synth_pebs_sample()
1879 		if (items->has_mem_access_latency)  in intel_pt_synth_pebs_sample()
1880 			sample.weight = items->mem_access_latency;  in intel_pt_synth_pebs_sample()
1881 		if (!sample.weight && items->has_tsx_aux_info) {  in intel_pt_synth_pebs_sample()
1883 			sample.weight = (u32)items->tsx_aux_info;  in intel_pt_synth_pebs_sample()
1887 	if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) {  in intel_pt_synth_pebs_sample()
1888 		u64 ax = items->has_rax ? items->rax : 0;  in intel_pt_synth_pebs_sample()
1890 		u64 txn = (u8)(items->tsx_aux_info >> 32);  in intel_pt_synth_pebs_sample()
1895 		sample.transaction = txn;  in intel_pt_synth_pebs_sample()
1898 	return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);  in intel_pt_synth_pebs_sample()
1908 	if (pt->synth_opts.error_minus_flags) {  in intel_pt_synth_error()
1910 		    pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW)  in intel_pt_synth_error()
1913 		    pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST)  in intel_pt_synth_error()
1922 	err = perf_session__deliver_synth_event(pt->session, &event, NULL);  in intel_pt_synth_error()
1933 	struct intel_pt *pt = ptq->pt;  in intel_ptq_synth_error()
1934 	u64 tm = ptq->timestamp;  in intel_ptq_synth_error()
1936 	tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);  in intel_ptq_synth_error()
1938 	return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,  in intel_ptq_synth_error()
1939 				    ptq->tid, state->from_ip, tm);  in intel_ptq_synth_error()
1945 	pid_t tid = ptq->next_tid;  in intel_pt_next_tid()
1948 	if (tid == -1)  in intel_pt_next_tid()
1951 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);  in intel_pt_next_tid()
1953 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);  in intel_pt_next_tid()
1955 	queue = &pt->queues.queue_array[ptq->queue_nr];  in intel_pt_next_tid()
1958 	ptq->next_tid = -1;  in intel_pt_next_tid()
1965 	struct intel_pt *pt = ptq->pt;  in intel_pt_is_switch_ip()
1967 	return ip == pt->switch_ip &&  in intel_pt_is_switch_ip()
1968 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&  in intel_pt_is_switch_ip()
1969 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |  in intel_pt_is_switch_ip()
1978 	const struct intel_pt_state *state = ptq->state;  in intel_pt_sample()
1979 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample()
1982 	if (!ptq->have_sample)  in intel_pt_sample()
1985 	ptq->have_sample = false;  in intel_pt_sample()
1987 	ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
1988 	ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;  in intel_pt_sample()
1994 	if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {  in intel_pt_sample()
2000 	if (pt->sample_pwr_events) {  in intel_pt_sample()
2001 		if (ptq->state->cbr != ptq->cbr_seen) {  in intel_pt_sample()
2006 		if (state->type & INTEL_PT_PWR_EVT) {  in intel_pt_sample()
2007 			if (state->type & INTEL_PT_MWAIT_OP) {  in intel_pt_sample()
2012 			if (state->type & INTEL_PT_PWR_ENTRY) {  in intel_pt_sample()
2017 			if (state->type & INTEL_PT_EX_STOP) {  in intel_pt_sample()
2022 			if (state->type & INTEL_PT_PWR_EXIT) {  in intel_pt_sample()
2030 	if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {  in intel_pt_sample()
2036 	if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {  in intel_pt_sample()
2042 	if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {  in intel_pt_sample()
2048 	if (!(state->type & INTEL_PT_BRANCH))  in intel_pt_sample()
2051 	if (pt->use_thread_stack) {  in intel_pt_sample()
2052 		thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,  in intel_pt_sample()
2053 				    state->from_ip, state->to_ip, ptq->insn_len,  in intel_pt_sample()
2054 				    state->trace_nr, pt->callstack,  in intel_pt_sample()
2055 				    pt->br_stack_sz_plus,  in intel_pt_sample()
2056 				    pt->mispred_all);  in intel_pt_sample()
2058 		thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);  in intel_pt_sample()
2061 	if (pt->sample_branches) {  in intel_pt_sample()
2067 	if (!ptq->sync_switch)  in intel_pt_sample()
2070 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {  in intel_pt_sample()
2071 		switch (ptq->switch_state) {  in intel_pt_sample()
2078 			ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2081 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;  in intel_pt_sample()
2084 	} else if (!state->to_ip) {  in intel_pt_sample()
2085 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_sample()
2086 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {  in intel_pt_sample()
2087 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_sample()
2088 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_sample()
2089 		   state->to_ip == pt->ptss_ip &&  in intel_pt_sample()
2090 		   (ptq->flags & PERF_IP_FLAG_CALL)) {  in intel_pt_sample()
2091 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2099 	struct machine *machine = pt->machine;  in intel_pt_switch_ip()
2115 	start = dso__first_symbol(map->dso);  in intel_pt_switch_ip()
2118 		if (sym->binding == STB_GLOBAL &&  in intel_pt_switch_ip()
2119 		    !strcmp(sym->name, "__switch_to")) {  in intel_pt_switch_ip()
2120 			ip = map->unmap_ip(map, sym->start);  in intel_pt_switch_ip()
2121 			if (ip >= map->start && ip < map->end) {  in intel_pt_switch_ip()
2131 	if (pt->have_sched_switch == 1)  in intel_pt_switch_ip()
2137 		if (!strcmp(sym->name, ptss)) {  in intel_pt_switch_ip()
2138 			ip = map->unmap_ip(map, sym->start);  in intel_pt_switch_ip()
2139 			if (ip >= map->start && ip < map->end) {  in intel_pt_switch_ip()
2153 	pt->sync_switch = true;  in intel_pt_enable_sync_switch()
2155 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_enable_sync_switch()
2156 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_enable_sync_switch()
2157 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_enable_sync_switch()
2160 			ptq->sync_switch = true;  in intel_pt_enable_sync_switch()
2165  * To filter against time ranges, it is only necessary to look at the next start
2166  * or end time.
2170 	struct intel_pt *pt = ptq->pt;  in intel_pt_next_time()
2172 	if (ptq->sel_start) {  in intel_pt_next_time()
2173 		/* Next time is an end time */  in intel_pt_next_time()
2174 		ptq->sel_start = false;  in intel_pt_next_time()
2175 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;  in intel_pt_next_time()
2177 	} else if (ptq->sel_idx + 1 < pt->range_cnt) {  in intel_pt_next_time()
2178 		/* Next time is a start time */  in intel_pt_next_time()
2179 		ptq->sel_start = true;  in intel_pt_next_time()
2180 		ptq->sel_idx += 1;  in intel_pt_next_time()
2181 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;  in intel_pt_next_time()
2185 	/* No next time */  in intel_pt_next_time()
2194 		if (ptq->sel_start) {  in intel_pt_time_filter()
2195 			if (ptq->timestamp >= ptq->sel_timestamp) {  in intel_pt_time_filter()
2196 				/* After start time, so consider next time */  in intel_pt_time_filter()
2198 				if (!ptq->sel_timestamp) {  in intel_pt_time_filter()
2199 					/* No end time */  in intel_pt_time_filter()
2202 				/* Check against end time */  in intel_pt_time_filter()
2205 			/* Before start time, so fast forward */  in intel_pt_time_filter()
2206 			ptq->have_sample = false;  in intel_pt_time_filter()
2207 			if (ptq->sel_timestamp > *ff_timestamp) {  in intel_pt_time_filter()
2208 				if (ptq->sync_switch) {  in intel_pt_time_filter()
2209 					intel_pt_next_tid(ptq->pt, ptq);  in intel_pt_time_filter()
2210 					ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_time_filter()
2212 				*ff_timestamp = ptq->sel_timestamp;  in intel_pt_time_filter()
2213 				err = intel_pt_fast_forward(ptq->decoder,  in intel_pt_time_filter()
2214 							    ptq->sel_timestamp);  in intel_pt_time_filter()
2219 		} else if (ptq->timestamp > ptq->sel_timestamp) {  in intel_pt_time_filter()
2220 			/* After end time, so consider next time */  in intel_pt_time_filter()
2222 				/* No next time range, so stop decoding */  in intel_pt_time_filter()
2223 				ptq->have_sample = false;  in intel_pt_time_filter()
2224 				ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_time_filter()
2227 			/* Check against next start time */  in intel_pt_time_filter()
2230 			/* Before end time */  in intel_pt_time_filter()
2238 	const struct intel_pt_state *state = ptq->state;  in intel_pt_run_decoder()
2239 	struct intel_pt *pt = ptq->pt;  in intel_pt_run_decoder()
2243 	if (!pt->kernel_start) {  in intel_pt_run_decoder()
2244 		pt->kernel_start = machine__kernel_start(pt->machine);  in intel_pt_run_decoder()
2245 		if (pt->per_cpu_mmaps &&  in intel_pt_run_decoder()
2246 		    (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&  in intel_pt_run_decoder()
2247 		    !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&  in intel_pt_run_decoder()
2248 		    !pt->sampling_mode) {  in intel_pt_run_decoder()
2249 			pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);  in intel_pt_run_decoder()
2250 			if (pt->switch_ip) {  in intel_pt_run_decoder()
2252 					     pt->switch_ip, pt->ptss_ip);  in intel_pt_run_decoder()
2259 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_run_decoder()
2265 		state = intel_pt_decode(ptq->decoder);  in intel_pt_run_decoder()
2266 		if (state->err) {  in intel_pt_run_decoder()
2267 			if (state->err == INTEL_PT_ERR_NODATA)  in intel_pt_run_decoder()
2269 			if (ptq->sync_switch &&  in intel_pt_run_decoder()
2270 			    state->from_ip >= pt->kernel_start) {  in intel_pt_run_decoder()
2271 				ptq->sync_switch = false;  in intel_pt_run_decoder()
2274 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2275 			if (pt->synth_opts.errors) {  in intel_pt_run_decoder()
2283 		ptq->state = state;  in intel_pt_run_decoder()
2284 		ptq->have_sample = true;  in intel_pt_run_decoder()
2288 		if (pt->est_tsc &&  in intel_pt_run_decoder()
2289 		    (state->from_ip >= pt->kernel_start || !state->from_ip) &&  in intel_pt_run_decoder()
2290 		    state->to_ip && state->to_ip < pt->kernel_start) {  in intel_pt_run_decoder()
2292 				     state->timestamp, state->est_timestamp);  in intel_pt_run_decoder()
2293 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2295 		} else if (ptq->sync_switch &&  in intel_pt_run_decoder()
2296 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_run_decoder()
2297 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&  in intel_pt_run_decoder()
2298 			   ptq->next_tid == -1) {  in intel_pt_run_decoder()
2300 				     state->timestamp, state->est_timestamp);  in intel_pt_run_decoder()
2301 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
2302 		} else if (state->timestamp > ptq->timestamp) {  in intel_pt_run_decoder()
2303 			ptq->timestamp = state->timestamp;  in intel_pt_run_decoder()
2306 		if (ptq->sel_timestamp) {  in intel_pt_run_decoder()
2312 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {  in intel_pt_run_decoder()
2313 			*timestamp = ptq->timestamp;  in intel_pt_run_decoder()
2322 	if (pt->queues.new_data) {  in intel_pt_update_queues()
2323 		pt->queues.new_data = false;  in intel_pt_update_queues()
2339 		if (!pt->heap.heap_cnt)  in intel_pt_process_queues()
2342 		if (pt->heap.heap_array[0].ordinal >= timestamp)  in intel_pt_process_queues()
2345 		queue_nr = pt->heap.heap_array[0].queue_nr;  in intel_pt_process_queues()
2346 		queue = &pt->queues.queue_array[queue_nr];  in intel_pt_process_queues()
2347 		ptq = queue->priv;  in intel_pt_process_queues()
2350 			     queue_nr, pt->heap.heap_array[0].ordinal,  in intel_pt_process_queues()
2353 		auxtrace_heap__pop(&pt->heap);  in intel_pt_process_queues()
2355 		if (pt->heap.heap_cnt) {  in intel_pt_process_queues()
2356 			ts = pt->heap.heap_array[0].ordinal + 1;  in intel_pt_process_queues()
2368 			auxtrace_heap__add(&pt->heap, queue_nr, ts);  in intel_pt_process_queues()
2373 			ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);  in intel_pt_process_queues()
2377 			ptq->on_heap = false;  in intel_pt_process_queues()
2387 	struct auxtrace_queues *queues = &pt->queues;  in intel_pt_process_timeless_queues()
2391 	for (i = 0; i < queues->nr_queues; i++) {  in intel_pt_process_timeless_queues()
2392 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_process_timeless_queues()
2393 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_process_timeless_queues()
2395 		if (ptq && (tid == -1 || ptq->tid == tid)) {  in intel_pt_process_timeless_queues()
2396 			ptq->time = time_;  in intel_pt_process_timeless_queues()
2406 					    struct perf_sample *sample)  in intel_pt_sample_set_pid_tid_cpu()  argument
2408 	struct machine *m = ptq->pt->machine;  in intel_pt_sample_set_pid_tid_cpu()
2410 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
2411 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
2412 	ptq->cpu = queue->cpu;  in intel_pt_sample_set_pid_tid_cpu()
2415 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2417 	thread__zput(ptq->thread);  in intel_pt_sample_set_pid_tid_cpu()
2419 	if (ptq->tid == -1)  in intel_pt_sample_set_pid_tid_cpu()
2422 	if (ptq->pid == -1) {  in intel_pt_sample_set_pid_tid_cpu()
2423 		ptq->thread = machine__find_thread(m, -1, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2424 		if (ptq->thread)  in intel_pt_sample_set_pid_tid_cpu()
2425 			ptq->pid = ptq->thread->pid_;  in intel_pt_sample_set_pid_tid_cpu()
2429 	ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
2433 					    struct perf_sample *sample)  in intel_pt_process_timeless_sample()  argument
2439 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_timeless_sample()
2441 		return -EINVAL;  in intel_pt_process_timeless_sample()
2443 	ptq = queue->priv;  in intel_pt_process_timeless_sample()
2447 	ptq->stop = false;  in intel_pt_process_timeless_sample()
2448 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
2449 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
2454 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)  in intel_pt_lost()  argument
2456 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,  in intel_pt_lost()
2457 				    sample->pid, sample->tid, 0, sample->time);  in intel_pt_lost()
2464 	if (cpu < 0 || !pt->queues.nr_queues)  in intel_pt_cpu_to_ptq()
2467 	if ((unsigned)cpu >= pt->queues.nr_queues)  in intel_pt_cpu_to_ptq()
2468 		i = pt->queues.nr_queues - 1;  in intel_pt_cpu_to_ptq()
2472 	if (pt->queues.queue_array[i].cpu == cpu)  in intel_pt_cpu_to_ptq()
2473 		return pt->queues.queue_array[i].priv;  in intel_pt_cpu_to_ptq()
2476 		if (pt->queues.queue_array[--i].cpu == cpu)  in intel_pt_cpu_to_ptq()
2477 			return pt->queues.queue_array[i].priv;  in intel_pt_cpu_to_ptq()
2480 	for (; j < pt->queues.nr_queues; j++) {  in intel_pt_cpu_to_ptq()
2481 		if (pt->queues.queue_array[j].cpu == cpu)  in intel_pt_cpu_to_ptq()
2482 			return pt->queues.queue_array[j].priv;  in intel_pt_cpu_to_ptq()
2494 	if (!pt->sync_switch)  in intel_pt_sync_switch()
2498 	if (!ptq || !ptq->sync_switch)  in intel_pt_sync_switch()
2501 	switch (ptq->switch_state) {  in intel_pt_sync_switch()
2506 		ptq->next_tid = tid;  in intel_pt_sync_switch()
2507 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;  in intel_pt_sync_switch()
2510 		if (!ptq->on_heap) {  in intel_pt_sync_switch()
2511 			ptq->timestamp = perf_time_to_tsc(timestamp,  in intel_pt_sync_switch()
2512 							  &pt->tc);  in intel_pt_sync_switch()
2513 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,  in intel_pt_sync_switch()
2514 						 ptq->timestamp);  in intel_pt_sync_switch()
2517 			ptq->on_heap = true;  in intel_pt_sync_switch()
2519 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sync_switch()
2528 	ptq->next_tid = -1;  in intel_pt_sync_switch()
2534 				   struct perf_sample *sample)  in intel_pt_process_switch()  argument
2540 	evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);  in intel_pt_process_switch()
2541 	if (evsel != pt->switch_evsel)  in intel_pt_process_switch()
2544 	tid = evsel__intval(evsel, sample, "next_pid");  in intel_pt_process_switch()
2545 	cpu = sample->cpu;  in intel_pt_process_switch()
2547 	intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",  in intel_pt_process_switch()
2548 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,  in intel_pt_process_switch()
2549 		     &pt->tc));  in intel_pt_process_switch()
2551 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_process_switch()
2555 	return machine__set_current_tid(pt->machine, cpu, -1, tid);  in intel_pt_process_switch()
2559 				      struct perf_sample *sample)  in intel_pt_context_switch_in()  argument
2561 	pid_t pid = sample->pid;  in intel_pt_context_switch_in()
2562 	pid_t tid = sample->tid;  in intel_pt_context_switch_in()
2563 	int cpu = sample->cpu;  in intel_pt_context_switch_in()
2565 	if (pt->sync_switch) {  in intel_pt_context_switch_in()
2569 		if (ptq && ptq->sync_switch) {  in intel_pt_context_switch_in()
2570 			ptq->next_tid = -1;  in intel_pt_context_switch_in()
2571 			switch (ptq->switch_state) {  in intel_pt_context_switch_in()
2578 				ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_context_switch_in()
2590 	if (machine__get_current_tid(pt->machine, cpu) == tid)  in intel_pt_context_switch_in()
2593 	return machine__set_current_tid(pt->machine, cpu, pid, tid);  in intel_pt_context_switch_in()
2597 				   struct perf_sample *sample)  in intel_pt_context_switch()  argument
2599 	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;  in intel_pt_context_switch()
2603 	cpu = sample->cpu;  in intel_pt_context_switch()
2605 	if (pt->have_sched_switch == 3) {  in intel_pt_context_switch()
2607 			return intel_pt_context_switch_in(pt, sample);  in intel_pt_context_switch()
2608 		if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {  in intel_pt_context_switch()
2609 			pr_err("Expecting CPU-wide context switch event\n");  in intel_pt_context_switch()
2610 			return -EINVAL;  in intel_pt_context_switch()
2612 		pid = event->context_switch.next_prev_pid;  in intel_pt_context_switch()
2613 		tid = event->context_switch.next_prev_tid;  in intel_pt_context_switch()
2617 		pid = sample->pid;  in intel_pt_context_switch()
2618 		tid = sample->tid;  in intel_pt_context_switch()
2621 	if (tid == -1)  in intel_pt_context_switch()
2624 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_context_switch()
2628 	return machine__set_current_tid(pt->machine, cpu, pid, tid);  in intel_pt_context_switch()
2633 					 struct perf_sample *sample)  in intel_pt_process_itrace_start()  argument
2635 	if (!pt->per_cpu_mmaps)  in intel_pt_process_itrace_start()
2638 	intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",  in intel_pt_process_itrace_start()
2639 		     sample->cpu, event->itrace_start.pid,  in intel_pt_process_itrace_start()
2640 		     event->itrace_start.tid, sample->time,  in intel_pt_process_itrace_start()
2641 		     perf_time_to_tsc(sample->time, &pt->tc));  in intel_pt_process_itrace_start()
2643 	return machine__set_current_tid(pt->machine, sample->cpu,  in intel_pt_process_itrace_start()
2644 					event->itrace_start.pid,  in intel_pt_process_itrace_start()
2645 					event->itrace_start.tid);  in intel_pt_process_itrace_start()
2651 	if (!al->map || addr < al->map->start || addr >= al->map->end) {  in intel_pt_find_map()
2653 			return -1;  in intel_pt_find_map()
2662 	u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;  in intel_pt_text_poke()
2663 	u64 addr = event->text_poke.addr + event->text_poke.new_len - 1;  in intel_pt_text_poke()
2665 	int cnt = 4096 + event->text_poke.new_len;  in intel_pt_text_poke()
2666 	struct thread *thread = pt->unknown_thread;  in intel_pt_text_poke()
2668 	struct machine *machine = pt->machine;  in intel_pt_text_poke()
2672 	if (!event->text_poke.new_len)  in intel_pt_text_poke()
2675 	for (; cnt; cnt--, addr--) {  in intel_pt_text_poke()
2677 			if (addr < event->text_poke.addr)  in intel_pt_text_poke()
2682 		if (!al.map->dso || !al.map->dso->auxtrace_cache)  in intel_pt_text_poke()
2685 		offset = al.map->map_ip(al.map, addr);  in intel_pt_text_poke()
2687 		e = intel_pt_cache_lookup(al.map->dso, machine, offset);  in intel_pt_text_poke()
2691 		if (addr + e->byte_cnt + e->length <= event->text_poke.addr) {  in intel_pt_text_poke()
2697 			if (e->branch != INTEL_PT_BR_NO_BRANCH)  in intel_pt_text_poke()
2700 			intel_pt_cache_invalidate(al.map->dso, machine, offset);  in intel_pt_text_poke()
2702 				     al.map->dso->long_name, addr);  in intel_pt_text_poke()
2711 				  struct perf_sample *sample,  in intel_pt_process_event()  argument
2714 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_process_event()
2722 	if (!tool->ordered_events) {  in intel_pt_process_event()
2724 		return -EINVAL;  in intel_pt_process_event()
2727 	if (sample->time && sample->time != (u64)-1)  in intel_pt_process_event()
2728 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_process_event()
2732 	if (timestamp || pt->timeless_decoding) {  in intel_pt_process_event()
2738 	if (pt->timeless_decoding) {  in intel_pt_process_event()
2739 		if (pt->sampling_mode) {  in intel_pt_process_event()
2740 			if (sample->aux_sample.size)  in intel_pt_process_event()
2742 								       sample);  in intel_pt_process_event()
2743 		} else if (event->header.type == PERF_RECORD_EXIT) {  in intel_pt_process_event()
2745 							       event->fork.tid,  in intel_pt_process_event()
2746 							       sample->time);  in intel_pt_process_event()
2754 	if (event->header.type == PERF_RECORD_SAMPLE) {  in intel_pt_process_event()
2755 		if (pt->synth_opts.add_callchain && !sample->callchain)  in intel_pt_process_event()
2756 			intel_pt_add_callchain(pt, sample);  in intel_pt_process_event()
2757 		if (pt->synth_opts.add_last_branch && !sample->branch_stack)  in intel_pt_process_event()
2758 			intel_pt_add_br_stack(pt, sample);  in intel_pt_process_event()
2761 	if (event->header.type == PERF_RECORD_AUX &&  in intel_pt_process_event()
2762 	    (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&  in intel_pt_process_event()
2763 	    pt->synth_opts.errors) {  in intel_pt_process_event()
2764 		err = intel_pt_lost(pt, sample);  in intel_pt_process_event()
2769 	if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)  in intel_pt_process_event()
2770 		err = intel_pt_process_switch(pt, sample);  in intel_pt_process_event()
2771 	else if (event->header.type == PERF_RECORD_ITRACE_START)  in intel_pt_process_event()
2772 		err = intel_pt_process_itrace_start(pt, event, sample);  in intel_pt_process_event()
2773 	else if (event->header.type == PERF_RECORD_SWITCH ||  in intel_pt_process_event()
2774 		 event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)  in intel_pt_process_event()
2775 		err = intel_pt_context_switch(pt, event, sample);  in intel_pt_process_event()
2777 	if (!err && event->header.type == PERF_RECORD_TEXT_POKE)  in intel_pt_process_event()
2780 	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {  in intel_pt_process_event()
2781 		intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",  in intel_pt_process_event()
2782 			     event->header.type, sample->cpu, sample->time, timestamp);  in intel_pt_process_event()
2791 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_flush()
2798 	if (!tool->ordered_events)  in intel_pt_flush()
2799 		return -EINVAL;  in intel_pt_flush()
2805 	if (pt->timeless_decoding)  in intel_pt_flush()
2806 		return intel_pt_process_timeless_queues(pt, -1,  in intel_pt_flush()
2807 							MAX_TIMESTAMP - 1);  in intel_pt_flush()
2814 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_free_events()
2816 	struct auxtrace_queues *queues = &pt->queues;  in intel_pt_free_events()
2819 	for (i = 0; i < queues->nr_queues; i++) {  in intel_pt_free_events()
2820 		intel_pt_free_queue(queues->queue_array[i].priv);  in intel_pt_free_events()
2821 		queues->queue_array[i].priv = NULL;  in intel_pt_free_events()
2829 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_free()
2832 	auxtrace_heap__free(&pt->heap);  in intel_pt_free()
2834 	session->auxtrace = NULL;  in intel_pt_free()
2835 	thread__put(pt->unknown_thread);  in intel_pt_free()
2836 	addr_filters__exit(&pt->filts);  in intel_pt_free()
2837 	zfree(&pt->chain);  in intel_pt_free()
2838 	zfree(&pt->filter);  in intel_pt_free()
2839 	zfree(&pt->time_ranges);  in intel_pt_free()
2846 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_evsel_is_auxtrace()
2849 	return evsel->core.attr.type == pt->pmu_type;  in intel_pt_evsel_is_auxtrace()
2856 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_process_auxtrace_event()
2859 	if (!pt->data_queued) {  in intel_pt_process_auxtrace_event()
2862 		int fd = perf_data__fd(session->data);  in intel_pt_process_auxtrace_event()
2865 		if (perf_data__is_pipe(session->data)) {  in intel_pt_process_auxtrace_event()
2869 			if (data_offset == -1)  in intel_pt_process_auxtrace_event()
2870 				return -errno;  in intel_pt_process_auxtrace_event()
2873 		err = auxtrace_queues__add_event(&pt->queues, session, event,  in intel_pt_process_auxtrace_event()
2881 				intel_pt_dump_event(pt, buffer->data,  in intel_pt_process_auxtrace_event()
2882 						    buffer->size);  in intel_pt_process_auxtrace_event()
2892 			       struct perf_sample *sample,  in intel_pt_queue_data()  argument
2895 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_queue_data()
2900 		return auxtrace_queues__add_event(&pt->queues, session, event,  in intel_pt_queue_data()
2904 	if (sample->time && sample->time != (u64)-1)  in intel_pt_queue_data()
2905 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_queue_data()
2909 	return auxtrace_queues__add_sample(&pt->queues, session, sample,  in intel_pt_queue_data()
2920 				struct perf_sample *sample __maybe_unused,  in intel_pt_event_synth()
2926 	return perf_session__deliver_synth_event(intel_pt_synth->session, event,  in intel_pt_event_synth()
2936 	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",  in intel_pt_synth_event()
2937 		 name, id, (u64)attr->sample_type);  in intel_pt_synth_event()
2957 		if (evsel->core.id && evsel->core.id[0] == id) {  in intel_pt_set_event_name()
2958 			if (evsel->name)  in intel_pt_set_event_name()
2959 				zfree(&evsel->name);  in intel_pt_set_event_name()
2960 			evsel->name = strdup(name);  in intel_pt_set_event_name()
2972 		if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)  in intel_pt_evsel()
2982 	struct evlist *evlist = session->evlist;  in intel_pt_synth_events()
2996 	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;  in intel_pt_synth_events()
2999 	if (pt->timeless_decoding)  in intel_pt_synth_events()
3003 	if (!pt->per_cpu_mmaps)  in intel_pt_synth_events()
3005 	attr.exclude_user = evsel->core.attr.exclude_user;  in intel_pt_synth_events()
3006 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;  in intel_pt_synth_events()
3007 	attr.exclude_hv = evsel->core.attr.exclude_hv;  in intel_pt_synth_events()
3008 	attr.exclude_host = evsel->core.attr.exclude_host;  in intel_pt_synth_events()
3009 	attr.exclude_guest = evsel->core.attr.exclude_guest;  in intel_pt_synth_events()
3010 	attr.sample_id_all = evsel->core.attr.sample_id_all;  in intel_pt_synth_events()
3011 	attr.read_format = evsel->core.attr.read_format;  in intel_pt_synth_events()
3013 	id = evsel->core.id[0] + 1000000000;  in intel_pt_synth_events()
3017 	if (pt->synth_opts.branches) {  in intel_pt_synth_events()
3024 		pt->sample_branches = true;  in intel_pt_synth_events()
3025 		pt->branches_sample_type = attr.sample_type;  in intel_pt_synth_events()
3026 		pt->branches_id = id;  in intel_pt_synth_events()
3031 	if (pt->synth_opts.callchain)  in intel_pt_synth_events()
3033 	if (pt->synth_opts.last_branch) {  in intel_pt_synth_events()
3036 		 * We don't use the hardware index, but the sample generation  in intel_pt_synth_events()
3043 	if (pt->synth_opts.instructions) {  in intel_pt_synth_events()
3045 		if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)  in intel_pt_synth_events()
3047 				intel_pt_ns_to_ticks(pt, pt->synth_opts.period);  in intel_pt_synth_events()
3049 			attr.sample_period = pt->synth_opts.period;  in intel_pt_synth_events()
3053 		pt->sample_instructions = true;  in intel_pt_synth_events()
3054 		pt->instructions_sample_type = attr.sample_type;  in intel_pt_synth_events()
3055 		pt->instructions_id = id;  in intel_pt_synth_events()
3062 	if (pt->synth_opts.transactions) {  in intel_pt_synth_events()
3067 		pt->sample_transactions = true;  in intel_pt_synth_events()
3068 		pt->transactions_sample_type = attr.sample_type;  in intel_pt_synth_events()
3069 		pt->transactions_id = id;  in intel_pt_synth_events()
3077 	if (pt->synth_opts.ptwrites) {  in intel_pt_synth_events()
3082 		pt->sample_ptwrites = true;  in intel_pt_synth_events()
3083 		pt->ptwrites_sample_type = attr.sample_type;  in intel_pt_synth_events()
3084 		pt->ptwrites_id = id;  in intel_pt_synth_events()
3089 	if (pt->synth_opts.pwr_events) {  in intel_pt_synth_events()
3090 		pt->sample_pwr_events = true;  in intel_pt_synth_events()
3091 		pt->pwr_events_sample_type = attr.sample_type;  in intel_pt_synth_events()
3097 		pt->cbr_id = id;  in intel_pt_synth_events()
3102 	if (pt->synth_opts.pwr_events && (evsel->core.attr.config & 0x10)) {  in intel_pt_synth_events()
3107 		pt->mwait_id = id;  in intel_pt_synth_events()
3115 		pt->pwre_id = id;  in intel_pt_synth_events()
3123 		pt->exstop_id = id;  in intel_pt_synth_events()
3131 		pt->pwrx_id = id;  in intel_pt_synth_events()
3143 	if (!pt->synth_opts.other_events)  in intel_pt_setup_pebs_events()
3146 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_setup_pebs_events()
3147 		if (evsel->core.attr.aux_output && evsel->core.id) {  in intel_pt_setup_pebs_events()
3148 			pt->sample_pebs = true;  in intel_pt_setup_pebs_events()
3149 			pt->pebs_evsel = evsel;  in intel_pt_setup_pebs_events()
3174 		if (evsel->core.attr.context_switch)  in intel_pt_find_switch()
3185 	if (!strcmp(var, "intel-pt.mispred-all"))  in intel_pt_perf_config()
3186 		pt->mispred_all = perf_config_bool(var, value);  in intel_pt_perf_config()
3196 	tsc = perf_time_to_tsc(ns, &pt->tc);  in intel_pt_tsc_start()
3199 		tm = tsc_to_perf_time(tsc, &pt->tc);  in intel_pt_tsc_start()
3202 		tsc -= 1;  in intel_pt_tsc_start()
3206 		tm = tsc_to_perf_time(++tsc, &pt->tc);  in intel_pt_tsc_start()
3216 	tsc = perf_time_to_tsc(ns, &pt->tc);  in intel_pt_tsc_end()
3219 		tm = tsc_to_perf_time(tsc, &pt->tc);  in intel_pt_tsc_end()
3226 		tm = tsc_to_perf_time(--tsc, &pt->tc);  in intel_pt_tsc_end()
3234 	struct perf_time_interval *p = opts->ptime_range;  in intel_pt_setup_time_ranges()
3235 	int n = opts->range_num;  in intel_pt_setup_time_ranges()
3238 	if (!n || !p || pt->timeless_decoding)  in intel_pt_setup_time_ranges()
3241 	pt->time_ranges = calloc(n, sizeof(struct range));  in intel_pt_setup_time_ranges()
3242 	if (!pt->time_ranges)  in intel_pt_setup_time_ranges()
3243 		return -ENOMEM;  in intel_pt_setup_time_ranges()
3245 	pt->range_cnt = n;  in intel_pt_setup_time_ranges()
3250 		struct range *r = &pt->time_ranges[i];  in intel_pt_setup_time_ranges()
3255 		 * Take care to ensure the TSC range matches the perf-time range  in intel_pt_setup_time_ranges()
3256 		 * when converted back to perf-time.  in intel_pt_setup_time_ranges()
3258 		r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;  in intel_pt_setup_time_ranges()
3259 		r->end   = te ? intel_pt_tsc_end(te, pt) : 0;  in intel_pt_setup_time_ranges()
3261 		intel_pt_log("range %d: perf time interval: %"PRIu64" to %"PRIu64"\n",  in intel_pt_setup_time_ranges()
3263 		intel_pt_log("range %d: TSC time interval: %#"PRIx64" to %#"PRIx64"\n",  in intel_pt_setup_time_ranges()
3264 			     i, r->start, r->end);  in intel_pt_setup_time_ranges()
3272 	[INTEL_PT_TIME_SHIFT]		= "  Time Shift          %"PRIu64"\n",
3273 	[INTEL_PT_TIME_MULT]		= "  Time Muliplier      %"PRIu64"\n",
3274 	[INTEL_PT_TIME_ZERO]		= "  Time Zero           %"PRIu64"\n",
3275 	[INTEL_PT_CAP_USER_TIME_ZERO]	= "  Cap Time Zero       %"PRId64"\n",
3280 	[INTEL_PT_PER_CPU_MMAPS]	= "  Per-cpu maps        %"PRId64"\n",
3286 	[INTEL_PT_MAX_NONTURBO_RATIO]	= "  Max non-turbo ratio %"PRIu64"\n",
3310 	fprintf(stdout, "  %-20s%s\n", name, str ? str : "");  in intel_pt_print_info_str()
3315 	return auxtrace_info->header.size >=  in intel_pt_has()
3322 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;  in intel_pt_process_auxtrace_info()
3329 	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +  in intel_pt_process_auxtrace_info()
3331 		return -EINVAL;  in intel_pt_process_auxtrace_info()
3335 		return -ENOMEM;  in intel_pt_process_auxtrace_info()
3337 	addr_filters__init(&pt->filts);  in intel_pt_process_auxtrace_info()
3343 	err = auxtrace_queues__init(&pt->queues);  in intel_pt_process_auxtrace_info()
3349 	pt->session = session;  in intel_pt_process_auxtrace_info()
3350 	pt->machine = &session->machines.host; /* No kvm support */  in intel_pt_process_auxtrace_info()
3351 	pt->auxtrace_type = auxtrace_info->type;  in intel_pt_process_auxtrace_info()
3352 	pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];  in intel_pt_process_auxtrace_info()
3353 	pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];  in intel_pt_process_auxtrace_info()
3354 	pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];  in intel_pt_process_auxtrace_info()
3355 	pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];  in intel_pt_process_auxtrace_info()
3356 	pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];  in intel_pt_process_auxtrace_info()
3357 	pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];  in intel_pt_process_auxtrace_info()
3358 	pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];  in intel_pt_process_auxtrace_info()
3359 	pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];  in intel_pt_process_auxtrace_info()
3360 	pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];  in intel_pt_process_auxtrace_info()
3361 	pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];  in intel_pt_process_auxtrace_info()
3362 	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,  in intel_pt_process_auxtrace_info()
3366 		pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];  in intel_pt_process_auxtrace_info()
3367 		pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];  in intel_pt_process_auxtrace_info()
3368 		pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];  in intel_pt_process_auxtrace_info()
3369 		pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];  in intel_pt_process_auxtrace_info()
3370 		pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];  in intel_pt_process_auxtrace_info()
3371 		intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,  in intel_pt_process_auxtrace_info()
3376 		pt->max_non_turbo_ratio =  in intel_pt_process_auxtrace_info()
3377 			auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];  in intel_pt_process_auxtrace_info()
3378 		intel_pt_print_info(&auxtrace_info->priv[0],  in intel_pt_process_auxtrace_info()
3383 	info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;  in intel_pt_process_auxtrace_info()
3384 	info_end = (void *)info + auxtrace_info->header.size;  in intel_pt_process_auxtrace_info()
3389 		len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];  in intel_pt_process_auxtrace_info()
3390 		intel_pt_print_info(&auxtrace_info->priv[0],  in intel_pt_process_auxtrace_info()
3400 				err = -EINVAL;  in intel_pt_process_auxtrace_info()
3403 			pt->filter = memdup(filter, len);  in intel_pt_process_auxtrace_info()
3404 			if (!pt->filter) {  in intel_pt_process_auxtrace_info()
3405 				err = -ENOMEM;  in intel_pt_process_auxtrace_info()
3408 			if (session->header.needs_swap)  in intel_pt_process_auxtrace_info()
3409 				mem_bswap_64(pt->filter, len);  in intel_pt_process_auxtrace_info()
3410 			if (pt->filter[len - 1]) {  in intel_pt_process_auxtrace_info()
3412 				err = -EINVAL;  in intel_pt_process_auxtrace_info()
3415 			err = addr_filters__parse_bare_filter(&pt->filts,  in intel_pt_process_auxtrace_info()
3420 		intel_pt_print_info_str("Filter string", pt->filter);  in intel_pt_process_auxtrace_info()
3423 	pt->timeless_decoding = intel_pt_timeless_decoding(pt);  in intel_pt_process_auxtrace_info()
3424 	if (pt->timeless_decoding && !pt->tc.time_mult)  in intel_pt_process_auxtrace_info()
3425 		pt->tc.time_mult = 1;  in intel_pt_process_auxtrace_info()
3426 	pt->have_tsc = intel_pt_have_tsc(pt);  in intel_pt_process_auxtrace_info()
3427 	pt->sampling_mode = intel_pt_sampling_mode(pt);  in intel_pt_process_auxtrace_info()
3428 	pt->est_tsc = !pt->timeless_decoding;  in intel_pt_process_auxtrace_info()
3430 	pt->unknown_thread = thread__new(999999999, 999999999);  in intel_pt_process_auxtrace_info()
3431 	if (!pt->unknown_thread) {  in intel_pt_process_auxtrace_info()
3432 		err = -ENOMEM;  in intel_pt_process_auxtrace_info()
3442 	INIT_LIST_HEAD(&pt->unknown_thread->node);  in intel_pt_process_auxtrace_info()
3444 	err = thread__set_comm(pt->unknown_thread, "unknown", 0);  in intel_pt_process_auxtrace_info()
3447 	if (thread__init_maps(pt->unknown_thread, pt->machine)) {  in intel_pt_process_auxtrace_info()
3448 		err = -ENOMEM;  in intel_pt_process_auxtrace_info()
3452 	pt->auxtrace.process_event = intel_pt_process_event;  in intel_pt_process_auxtrace_info()
3453 	pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;  in intel_pt_process_auxtrace_info()
3454 	pt->auxtrace.queue_data = intel_pt_queue_data;  in intel_pt_process_auxtrace_info()
3455 	pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;  in intel_pt_process_auxtrace_info()
3456 	pt->auxtrace.flush_events = intel_pt_flush;  in intel_pt_process_auxtrace_info()
3457 	pt->auxtrace.free_events = intel_pt_free_events;  in intel_pt_process_auxtrace_info()
3458 	pt->auxtrace.free = intel_pt_free;  in intel_pt_process_auxtrace_info()
3459 	pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace;  in intel_pt_process_auxtrace_info()
3460 	session->auxtrace = &pt->auxtrace;  in intel_pt_process_auxtrace_info()
3465 	if (pt->have_sched_switch == 1) {  in intel_pt_process_auxtrace_info()
3466 		pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);  in intel_pt_process_auxtrace_info()
3467 		if (!pt->switch_evsel) {  in intel_pt_process_auxtrace_info()
3469 			err = -EINVAL;  in intel_pt_process_auxtrace_info()
3472 	} else if (pt->have_sched_switch == 2 &&  in intel_pt_process_auxtrace_info()
3473 		   !intel_pt_find_switch(session->evlist)) {  in intel_pt_process_auxtrace_info()
3475 		err = -EINVAL;  in intel_pt_process_auxtrace_info()
3479 	if (session->itrace_synth_opts->set) {  in intel_pt_process_auxtrace_info()
3480 		pt->synth_opts = *session->itrace_synth_opts;  in intel_pt_process_auxtrace_info()
3482 		itrace_synth_opts__set_default(&pt->synth_opts,  in intel_pt_process_auxtrace_info()
3483 				session->itrace_synth_opts->default_no_sample);  in intel_pt_process_auxtrace_info()
3484 		if (!session->itrace_synth_opts->default_no_sample &&  in intel_pt_process_auxtrace_info()
3485 		    !session->itrace_synth_opts->inject) {  in intel_pt_process_auxtrace_info()
3486 			pt->synth_opts.branches = false;  in intel_pt_process_auxtrace_info()
3487 			pt->synth_opts.callchain = true;  in intel_pt_process_auxtrace_info()
3488 			pt->synth_opts.add_callchain = true;  in intel_pt_process_auxtrace_info()
3490 		pt->synth_opts.thread_stack =  in intel_pt_process_auxtrace_info()
3491 				session->itrace_synth_opts->thread_stack;  in intel_pt_process_auxtrace_info()
3494 	if (pt->synth_opts.log)  in intel_pt_process_auxtrace_info()
3497 	/* Maximum non-turbo ratio is TSC freq / 100 MHz */  in intel_pt_process_auxtrace_info()
3498 	if (pt->tc.time_mult) {  in intel_pt_process_auxtrace_info()
3501 		if (!pt->max_non_turbo_ratio)  in intel_pt_process_auxtrace_info()
3502 			pt->max_non_turbo_ratio =  in intel_pt_process_auxtrace_info()
3505 		intel_pt_log("Maximum non-turbo ratio %u\n",  in intel_pt_process_auxtrace_info()
3506 			     pt->max_non_turbo_ratio);  in intel_pt_process_auxtrace_info()
3507 		pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;  in intel_pt_process_auxtrace_info()
3510 	err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);  in intel_pt_process_auxtrace_info()
3514 	if (pt->synth_opts.calls)  in intel_pt_process_auxtrace_info()
3515 		pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |  in intel_pt_process_auxtrace_info()
3517 	if (pt->synth_opts.returns)  in intel_pt_process_auxtrace_info()
3518 		pt->branches_filter |= PERF_IP_FLAG_RETURN |  in intel_pt_process_auxtrace_info()
3521 	if ((pt->synth_opts.callchain || pt->synth_opts.add_callchain) &&  in intel_pt_process_auxtrace_info()
3526 			pt->synth_opts.callchain = false;  in intel_pt_process_auxtrace_info()
3527 			pt->synth_opts.add_callchain = false;  in intel_pt_process_auxtrace_info()
3531 	if (pt->synth_opts.add_callchain) {  in intel_pt_process_auxtrace_info()
3537 	if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch) {  in intel_pt_process_auxtrace_info()
3538 		pt->br_stack_sz = pt->synth_opts.last_branch_sz;  in intel_pt_process_auxtrace_info()
3539 		pt->br_stack_sz_plus = pt->br_stack_sz;  in intel_pt_process_auxtrace_info()
3542 	if (pt->synth_opts.add_last_branch) {  in intel_pt_process_auxtrace_info()
3548 		 * actual sample ip to where the sample time is recorded.  in intel_pt_process_auxtrace_info()
3554 			pt->br_stack_sz_plus += 1024;  in intel_pt_process_auxtrace_info()
3556 			pt->br_stack_sz_plus += 1;  in intel_pt_process_auxtrace_info()
3559 	pt->use_thread_stack = pt->synth_opts.callchain ||  in intel_pt_process_auxtrace_info()
3560 			       pt->synth_opts.add_callchain ||  in intel_pt_process_auxtrace_info()
3561 			       pt->synth_opts.thread_stack ||  in intel_pt_process_auxtrace_info()
3562 			       pt->synth_opts.last_branch ||  in intel_pt_process_auxtrace_info()
3563 			       pt->synth_opts.add_last_branch;  in intel_pt_process_auxtrace_info()
3565 	pt->callstack = pt->synth_opts.callchain ||  in intel_pt_process_auxtrace_info()
3566 			pt->synth_opts.add_callchain ||  in intel_pt_process_auxtrace_info()
3567 			pt->synth_opts.thread_stack;  in intel_pt_process_auxtrace_info()
3575 	if (pt->sampling_mode || list_empty(&session->auxtrace_index))  in intel_pt_process_auxtrace_info()
3578 		err = auxtrace_queues__process_index(&pt->queues, session);  in intel_pt_process_auxtrace_info()
3582 	if (pt->queues.populated)  in intel_pt_process_auxtrace_info()
3583 		pt->data_queued = true;  in intel_pt_process_auxtrace_info()
3585 	if (pt->timeless_decoding)  in intel_pt_process_auxtrace_info()
3591 	zfree(&pt->chain);  in intel_pt_process_auxtrace_info()
3592 	thread__zput(pt->unknown_thread);  in intel_pt_process_auxtrace_info()
3595 	auxtrace_queues__free(&pt->queues);  in intel_pt_process_auxtrace_info()
3596 	session->auxtrace = NULL;  in intel_pt_process_auxtrace_info()
3598 	addr_filters__exit(&pt->filts);  in intel_pt_process_auxtrace_info()
3599 	zfree(&pt->filter);  in intel_pt_process_auxtrace_info()
3600 	zfree(&pt->time_ranges);  in intel_pt_process_auxtrace_info()