1 // SPDX-License-Identifier: GPL-2.0-only
4  * Copyright (c) 2013-2015, Intel Corporation.
28 #include "thread-stack.h"
35 #include "intel-pt.h"
38 #include "util/synthetic-events.h"
39 #include "time-utils.h"
43 #include "intel-pt-decoder/intel-pt-log.h"
44 #include "intel-pt-decoder/intel-pt-decoder.h"
45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
173 /* applicable_counters is 64-bits */
212 	u64 time;  member
269 		len -= pkt_len;  in intel_pt_dump()
291 				 struct perf_sample *sample)  in intel_pt_dump_sample()  argument
293 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_dump_sample()
297 	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);  in intel_pt_dump_sample()
302 	struct perf_time_interval *range = pt->synth_opts.ptime_range;  in intel_pt_log_events()
303 	int n = pt->synth_opts.range_num;  in intel_pt_log_events()
305 	if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)  in intel_pt_log_events()
308 	if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS)  in intel_pt_log_events()
311 	/* perf_time__ranges_skip_sample does not work if time is zero */  in intel_pt_log_events()
322 	struct rb_node **p = &rb_root->rb_node;  in intel_pt_findnew_vmcs()
330 		if (v->vmcs == vmcs)  in intel_pt_findnew_vmcs()
333 		if (vmcs < v->vmcs)  in intel_pt_findnew_vmcs()
334 			p = &(*p)->rb_left;  in intel_pt_findnew_vmcs()
336 			p = &(*p)->rb_right;  in intel_pt_findnew_vmcs()
341 		v->vmcs = vmcs;  in intel_pt_findnew_vmcs()
342 		v->tsc_offset = dflt_tsc_offset;  in intel_pt_findnew_vmcs()
343 		v->reliable = dflt_tsc_offset;  in intel_pt_findnew_vmcs()
345 		rb_link_node(&v->rb_node, parent, p);  in intel_pt_findnew_vmcs()
346 		rb_insert_color(&v->rb_node, rb_root);  in intel_pt_findnew_vmcs()
355 	struct intel_pt *pt = ptq->pt;  in intel_pt_findnew_vmcs_info()
357 	if (!vmcs && !pt->dflt_tsc_offset)  in intel_pt_findnew_vmcs_info()
360 	return intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, pt->dflt_tsc_offset);  in intel_pt_findnew_vmcs_info()
368 	n = rb_first(&pt->vmcs_info);  in intel_pt_free_vmcs_info()
372 		rb_erase(&v->rb_node, &pt->vmcs_info);  in intel_pt_free_vmcs_info()
383 	start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,  in intel_pt_do_fix_overlap()
384 				      pt->have_tsc, &consecutive,  in intel_pt_do_fix_overlap()
385 				      pt->synth_opts.vm_time_correlation);  in intel_pt_do_fix_overlap()
387 		return -EINVAL;  in intel_pt_do_fix_overlap()
393 	if (pt->synth_opts.vm_time_correlation)  in intel_pt_do_fix_overlap()
394 		memset(b->data, 0, start - b->data);  in intel_pt_do_fix_overlap()
395 	b->use_size = b->data + b->size - start;  in intel_pt_do_fix_overlap()
396 	b->use_data = start;  in intel_pt_do_fix_overlap()
397 	if (b->use_size && consecutive)  in intel_pt_do_fix_overlap()
398 		b->consecutive = true;  in intel_pt_do_fix_overlap()
409 	if (!buffer->data) {  in intel_pt_get_buffer()
410 		int fd = perf_data__fd(ptq->pt->session->data);  in intel_pt_get_buffer()
412 		buffer->data = auxtrace_buffer__get_data(buffer, fd);  in intel_pt_get_buffer()
413 		if (!buffer->data)  in intel_pt_get_buffer()
414 			return -ENOMEM;  in intel_pt_get_buffer()
417 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;  in intel_pt_get_buffer()
418 	if (might_overlap && !buffer->consecutive && old_buffer &&  in intel_pt_get_buffer()
419 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))  in intel_pt_get_buffer()
420 		return -ENOMEM;  in intel_pt_get_buffer()
422 	if (buffer->use_data) {  in intel_pt_get_buffer()
423 		b->len = buffer->use_size;  in intel_pt_get_buffer()
424 		b->buf = buffer->use_data;  in intel_pt_get_buffer()
426 		b->len = buffer->size;  in intel_pt_get_buffer()
427 		b->buf = buffer->data;  in intel_pt_get_buffer()
429 	b->ref_timestamp = buffer->reference;  in intel_pt_get_buffer()
431 	if (!old_buffer || (might_overlap && !buffer->consecutive)) {  in intel_pt_get_buffer()
432 		b->consecutive = false;  in intel_pt_get_buffer()
433 		b->trace_nr = buffer->buffer_nr + 1;  in intel_pt_get_buffer()
435 		b->consecutive = true;  in intel_pt_get_buffer()
441 /* Do not drop buffers with references - refer intel_pt_get_trace() */
445 	if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)  in intel_pt_lookahead_drop_buffer()
456 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_lookahead()
457 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_lookahead()
461 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_lookahead()
501 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_get_trace()
502 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_get_trace()
506 	if (ptq->stop) {  in intel_pt_get_trace()
507 		b->len = 0;  in intel_pt_get_trace()
511 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_get_trace()
517 		b->len = 0;  in intel_pt_get_trace()
521 	ptq->buffer = buffer;  in intel_pt_get_trace()
527 	if (ptq->step_through_buffers)  in intel_pt_get_trace()
528 		ptq->stop = true;  in intel_pt_get_trace()
530 	if (b->len) {  in intel_pt_get_trace()
533 		ptq->old_buffer = buffer;  in intel_pt_get_trace()
559 	if (!strcmp(var, "intel-pt.cache-divisor")) {  in intel_pt_config_div()
594 	return 32 - __builtin_clz(size);  in intel_pt_cache_size()
603 	if (dso->auxtrace_cache)  in intel_pt_cache()
604 		return dso->auxtrace_cache;  in intel_pt_cache()
611 	dso->auxtrace_cache = c;  in intel_pt_cache()
625 		return -ENOMEM;  in intel_pt_cache_add()
629 		return -ENOMEM;  in intel_pt_cache_add()
631 	e->insn_cnt = insn_cnt;  in intel_pt_cache_add()
632 	e->byte_cnt = byte_cnt;  in intel_pt_cache_add()
633 	e->op = intel_pt_insn->op;  in intel_pt_cache_add()
634 	e->branch = intel_pt_insn->branch;  in intel_pt_cache_add()
635 	e->emulated_ptwrite = intel_pt_insn->emulated_ptwrite;  in intel_pt_cache_add()
636 	e->length = intel_pt_insn->length;  in intel_pt_cache_add()
637 	e->rel = intel_pt_insn->rel;  in intel_pt_cache_add()
638 	memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);  in intel_pt_cache_add()
640 	err = auxtrace_cache__add(c, offset, &e->entry);  in intel_pt_cache_add()
655 	return auxtrace_cache__lookup(dso->auxtrace_cache, offset);  in intel_pt_cache_lookup()
666 	auxtrace_cache__remove(dso->auxtrace_cache, offset);  in intel_pt_cache_invalidate()
671 	/* Assumes 64-bit kernel */  in intel_pt_guest_kernel_ip()
683 	return ip >= ptq->pt->kernel_start ?  in intel_pt_nr_cpumode()
690 	/* No support for non-zero CS base */  in intel_pt_cpumode()
692 		return intel_pt_nr_cpumode(ptq, from_ip, ptq->state->from_nr);  in intel_pt_cpumode()
693 	return intel_pt_nr_cpumode(ptq, to_ip, ptq->state->to_nr);  in intel_pt_cpumode()
698 	struct machines *machines = &ptq->pt->session->machines;  in intel_pt_get_guest()
700 	pid_t pid = ptq->pid <= 0 ? DEFAULT_GUEST_KERNEL_ID : ptq->pid;  in intel_pt_get_guest()
702 	if (ptq->guest_machine && pid == ptq->guest_machine->pid)  in intel_pt_get_guest()
705 	ptq->guest_machine = NULL;  in intel_pt_get_guest()
706 	thread__zput(ptq->unknown_guest_thread);  in intel_pt_get_guest()
709 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest()
710 		ptq->guest_thread = machines__findnew_guest_code(machines, pid);  in intel_pt_get_guest()
715 		return -1;  in intel_pt_get_guest()
717 	ptq->unknown_guest_thread = machine__idle_thread(machine);  in intel_pt_get_guest()
718 	if (!ptq->unknown_guest_thread)  in intel_pt_get_guest()
719 		return -1;  in intel_pt_get_guest()
721 	ptq->guest_machine = machine;  in intel_pt_get_guest()
728 	return intel_pt_insn->rel == 16 && intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL;  in intel_pt_jmp_16()
754 	struct machine *machine = ptq->pt->machine;  in intel_pt_walk_next_insn()
768 	intel_pt_insn->length = 0;  in intel_pt_walk_next_insn()
769 	intel_pt_insn->op = INTEL_PT_OP_OTHER;  in intel_pt_walk_next_insn()
774 	nr = ptq->state->to_nr;  in intel_pt_walk_next_insn()
778 		if (ptq->pt->have_guest_sideband) {  in intel_pt_walk_next_insn()
779 			if (!ptq->guest_machine || ptq->guest_machine_pid != ptq->pid) {  in intel_pt_walk_next_insn()
781 				ret = -EINVAL;  in intel_pt_walk_next_insn()
787 			ret = -EINVAL;  in intel_pt_walk_next_insn()
790 		machine = ptq->guest_machine;  in intel_pt_walk_next_insn()
791 		thread = ptq->guest_thread;  in intel_pt_walk_next_insn()
795 				ret = -EINVAL;  in intel_pt_walk_next_insn()
798 			thread = ptq->unknown_guest_thread;  in intel_pt_walk_next_insn()
801 		thread = ptq->thread;  in intel_pt_walk_next_insn()
805 				ret = -EINVAL;  in intel_pt_walk_next_insn()
808 			thread = ptq->pt->unknown_thread;  in intel_pt_walk_next_insn()
821 			ret = -EINVAL;  in intel_pt_walk_next_insn()
826 		if (dso->data.status == DSO_DATA_STATUS_ERROR &&  in intel_pt_walk_next_insn()
828 			ret = -ENOENT;  in intel_pt_walk_next_insn()
839 			    (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {  in intel_pt_walk_next_insn()
840 				*insn_cnt_ptr = e->insn_cnt;  in intel_pt_walk_next_insn()
841 				*ip += e->byte_cnt;  in intel_pt_walk_next_insn()
842 				intel_pt_insn->op = e->op;  in intel_pt_walk_next_insn()
843 				intel_pt_insn->branch = e->branch;  in intel_pt_walk_next_insn()
844 				intel_pt_insn->emulated_ptwrite = e->emulated_ptwrite;  in intel_pt_walk_next_insn()
845 				intel_pt_insn->length = e->length;  in intel_pt_walk_next_insn()
846 				intel_pt_insn->rel = e->rel;  in intel_pt_walk_next_insn()
847 				memcpy(intel_pt_insn->buf, e->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_walk_next_insn()
857 		/* Load maps to ensure dso->is_64_bit has been updated */  in intel_pt_walk_next_insn()
860 		x86_64 = dso->is_64_bit;  in intel_pt_walk_next_insn()
871 				ret = -EINVAL;  in intel_pt_walk_next_insn()
876 				ret = -EINVAL;  in intel_pt_walk_next_insn()
884 			if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH) {  in intel_pt_walk_next_insn()
891 				offs = offset + intel_pt_insn->length;  in intel_pt_walk_next_insn()
893 				intel_pt_insn->emulated_ptwrite = eptw;  in intel_pt_walk_next_insn()
900 			*ip += intel_pt_insn->length;  in intel_pt_walk_next_insn()
903 				intel_pt_insn->length = 0;  in intel_pt_walk_next_insn()
904 				intel_pt_insn->op = INTEL_PT_OP_OTHER;  in intel_pt_walk_next_insn()
911 			offset += intel_pt_insn->length;  in intel_pt_walk_next_insn()
935 			   *ip - start_ip, intel_pt_insn);  in intel_pt_walk_next_insn()
955 	list_for_each_entry(filt, &pt->filts.head, list) {  in intel_pt_match_pgd_ip()
956 		if (filt->start)  in intel_pt_match_pgd_ip()
959 		if ((filename && !filt->filename) ||  in intel_pt_match_pgd_ip()
960 		    (!filename && filt->filename) ||  in intel_pt_match_pgd_ip()
961 		    (filename && strcmp(filename, filt->filename)))  in intel_pt_match_pgd_ip()
964 		if (!(offset >= filt->addr && offset < filt->addr + filt->size))  in intel_pt_match_pgd_ip()
969 			     filt->start ? "filter" : "stop",  in intel_pt_match_pgd_ip()
970 			     filt->addr, filt->size);  in intel_pt_match_pgd_ip()
972 		if (filt->start)  in intel_pt_match_pgd_ip()
994 	if (ptq->state->to_nr) {  in __intel_pt_pgd_ip()
996 			return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
998 		return -EINVAL;  in __intel_pt_pgd_ip()
999 	} else if (ip >= ptq->pt->kernel_start) {  in __intel_pt_pgd_ip()
1000 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
1005 	thread = ptq->thread;  in __intel_pt_pgd_ip()
1007 		return -EINVAL;  in __intel_pt_pgd_ip()
1011 		return -EINVAL;  in __intel_pt_pgd_ip()
1015 	res = intel_pt_match_pgd_ip(ptq->pt, ip, offset, map__dso(al.map)->long_name);  in __intel_pt_pgd_ip()
1028 	if (attr->type == pt->pmu_type) {  in intel_pt_get_config()
1030 			*config = attr->config;  in intel_pt_get_config()
1041 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_exclude_kernel()
1042 		if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&  in intel_pt_exclude_kernel()
1043 		    !evsel->core.attr.exclude_kernel)  in intel_pt_exclude_kernel()
1054 	if (!pt->noretcomp_bit)  in intel_pt_return_compression()
1057 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_return_compression()
1058 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_return_compression()
1059 		    (config & pt->noretcomp_bit))  in intel_pt_return_compression()
1070 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_branch_enable()
1071 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_branch_enable()
1084 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_disabled_tnt()
1085 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_disabled_tnt()
1098 	if (!pt->mtc_freq_bits)  in intel_pt_mtc_period()
1101 	for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)  in intel_pt_mtc_period()
1104 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_mtc_period()
1105 		if (intel_pt_get_config(pt, &evsel->core.attr, &config))  in intel_pt_mtc_period()
1106 			return (config & pt->mtc_freq_bits) >> shift;  in intel_pt_mtc_period()
1117 	if (!pt->tsc_bit || !pt->cap_user_time_zero || pt->synth_opts.timeless_decoding)  in intel_pt_timeless_decoding()
1120 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_timeless_decoding()
1121 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME))  in intel_pt_timeless_decoding()
1123 		if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {  in intel_pt_timeless_decoding()
1124 			if (config & pt->tsc_bit)  in intel_pt_timeless_decoding()
1137 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_tracing_kernel()
1138 		if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&  in intel_pt_tracing_kernel()
1139 		    !evsel->core.attr.exclude_kernel)  in intel_pt_tracing_kernel()
1151 	if (!pt->tsc_bit)  in intel_pt_have_tsc()
1154 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_have_tsc()
1155 		if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {  in intel_pt_have_tsc()
1156 			if (config & pt->tsc_bit)  in intel_pt_have_tsc()
1170 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_have_mtc()
1171 		if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&  in intel_pt_have_mtc()
1172 		    (config & pt->mtc_bit))  in intel_pt_have_mtc()
1182 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_sampling_mode()
1183 		if ((evsel->core.attr.sample_type & PERF_SAMPLE_AUX) &&  in intel_pt_sampling_mode()
1184 		    evsel->core.attr.aux_sample_size)  in intel_pt_sampling_mode()
1195 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_ctl()
1196 		if (intel_pt_get_config(pt, &evsel->core.attr, &config))  in intel_pt_ctl()
1206 	quot = ns / pt->tc.time_mult;  in intel_pt_ns_to_ticks()
1207 	rem  = ns % pt->tc.time_mult;  in intel_pt_ns_to_ticks()
1208 	return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /  in intel_pt_ns_to_ticks()
1209 		pt->tc.time_mult;  in intel_pt_ns_to_ticks()
1217 	sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);  in intel_pt_alloc_chain()
1225 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_callchain_init()
1226 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_CALLCHAIN))  in intel_pt_callchain_init()
1227 			evsel->synth_sample_type |= PERF_SAMPLE_CALLCHAIN;  in intel_pt_callchain_init()
1230 	pt->chain = intel_pt_alloc_chain(pt);  in intel_pt_callchain_init()
1231 	if (!pt->chain)  in intel_pt_callchain_init()
1232 		return -ENOMEM;  in intel_pt_callchain_init()
1238 				   struct perf_sample *sample)  in intel_pt_add_callchain()  argument
1240 	struct thread *thread = machine__findnew_thread(pt->machine,  in intel_pt_add_callchain()
1241 							sample->pid,  in intel_pt_add_callchain()
1242 							sample->tid);  in intel_pt_add_callchain()
1244 	thread_stack__sample_late(thread, sample->cpu, pt->chain,  in intel_pt_add_callchain()
1245 				  pt->synth_opts.callchain_sz + 1, sample->ip,  in intel_pt_add_callchain()
1246 				  pt->kernel_start);  in intel_pt_add_callchain()
1248 	sample->callchain = pt->chain;  in intel_pt_add_callchain()
1263 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_br_stack_init()
1264 		if (!(evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK))  in intel_pt_br_stack_init()
1265 			evsel->synth_sample_type |= PERF_SAMPLE_BRANCH_STACK;  in intel_pt_br_stack_init()
1268 	pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz);  in intel_pt_br_stack_init()
1269 	if (!pt->br_stack)  in intel_pt_br_stack_init()
1270 		return -ENOMEM;  in intel_pt_br_stack_init()
1276 				  struct perf_sample *sample)  in intel_pt_add_br_stack()  argument
1278 	struct thread *thread = machine__findnew_thread(pt->machine,  in intel_pt_add_br_stack()
1279 							sample->pid,  in intel_pt_add_br_stack()
1280 							sample->tid);  in intel_pt_add_br_stack()
1282 	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,  in intel_pt_add_br_stack()
1283 				     pt->br_stack_sz, sample->ip,  in intel_pt_add_br_stack()
1284 				     pt->kernel_start);  in intel_pt_add_br_stack()
1286 	sample->branch_stack = pt->br_stack;  in intel_pt_add_br_stack()
1297 	struct perf_env *env = pt->machine->env;  in intel_pt_alloc_queue()
1304 	if (pt->synth_opts.callchain) {  in intel_pt_alloc_queue()
1305 		ptq->chain = intel_pt_alloc_chain(pt);  in intel_pt_alloc_queue()
1306 		if (!ptq->chain)  in intel_pt_alloc_queue()
1310 	if (pt->synth_opts.last_branch || pt->synth_opts.other_events) {  in intel_pt_alloc_queue()
1311 		unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz);  in intel_pt_alloc_queue()
1313 		ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt);  in intel_pt_alloc_queue()
1314 		if (!ptq->last_branch)  in intel_pt_alloc_queue()
1318 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in intel_pt_alloc_queue()
1319 	if (!ptq->event_buf)  in intel_pt_alloc_queue()
1322 	ptq->pt = pt;  in intel_pt_alloc_queue()
1323 	ptq->queue_nr = queue_nr;  in intel_pt_alloc_queue()
1324 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);  in intel_pt_alloc_queue()
1325 	ptq->pid = -1;  in intel_pt_alloc_queue()
1326 	ptq->tid = -1;  in intel_pt_alloc_queue()
1327 	ptq->cpu = -1;  in intel_pt_alloc_queue()
1328 	ptq->next_tid = -1;  in intel_pt_alloc_queue()
1338 	params.max_non_turbo_ratio = pt->max_non_turbo_ratio;  in intel_pt_alloc_queue()
1340 	params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;  in intel_pt_alloc_queue()
1341 	params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;  in intel_pt_alloc_queue()
1342 	params.quick = pt->synth_opts.quick;  in intel_pt_alloc_queue()
1343 	params.vm_time_correlation = pt->synth_opts.vm_time_correlation;  in intel_pt_alloc_queue()
1344 	params.vm_tm_corr_dry_run = pt->synth_opts.vm_tm_corr_dry_run;  in intel_pt_alloc_queue()
1345 	params.first_timestamp = pt->first_timestamp;  in intel_pt_alloc_queue()
1346 	params.max_loops = pt->max_loops;  in intel_pt_alloc_queue()
1352 	if (pt->filts.cnt > 0)  in intel_pt_alloc_queue()
1355 	if (pt->synth_opts.instructions || pt->synth_opts.cycles) {  in intel_pt_alloc_queue()
1356 		if (pt->synth_opts.period) {  in intel_pt_alloc_queue()
1357 			switch (pt->synth_opts.period_type) {  in intel_pt_alloc_queue()
1361 				params.period = pt->synth_opts.period;  in intel_pt_alloc_queue()
1365 				params.period = pt->synth_opts.period;  in intel_pt_alloc_queue()
1370 							pt->synth_opts.period);  in intel_pt_alloc_queue()
1383 	if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))  in intel_pt_alloc_queue()
1386 	ptq->decoder = intel_pt_decoder_new(&params);  in intel_pt_alloc_queue()
1387 	if (!ptq->decoder)  in intel_pt_alloc_queue()
1393 	zfree(&ptq->event_buf);  in intel_pt_alloc_queue()
1394 	zfree(&ptq->last_branch);  in intel_pt_alloc_queue()
1395 	zfree(&ptq->chain);  in intel_pt_alloc_queue()
1406 	thread__zput(ptq->thread);  in intel_pt_free_queue()
1407 	thread__zput(ptq->guest_thread);  in intel_pt_free_queue()
1408 	thread__zput(ptq->unknown_guest_thread);  in intel_pt_free_queue()
1409 	intel_pt_decoder_free(ptq->decoder);  in intel_pt_free_queue()
1410 	zfree(&ptq->event_buf);  in intel_pt_free_queue()
1411 	zfree(&ptq->last_branch);  in intel_pt_free_queue()
1412 	zfree(&ptq->chain);  in intel_pt_free_queue()
1420 	pt->first_timestamp = timestamp;  in intel_pt_first_timestamp()
1422 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_first_timestamp()
1423 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_first_timestamp()
1424 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_first_timestamp()
1426 		if (ptq && ptq->decoder)  in intel_pt_first_timestamp()
1427 			intel_pt_set_first_timestamp(ptq->decoder, timestamp);  in intel_pt_first_timestamp()
1433 	struct machines *machines = &ptq->pt->session->machines;  in intel_pt_get_guest_from_sideband()
1435 	pid_t machine_pid = ptq->pid;  in intel_pt_get_guest_from_sideband()
1446 	if (ptq->guest_machine != machine) {  in intel_pt_get_guest_from_sideband()
1447 		ptq->guest_machine = NULL;  in intel_pt_get_guest_from_sideband()
1448 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest_from_sideband()
1449 		thread__zput(ptq->unknown_guest_thread);  in intel_pt_get_guest_from_sideband()
1451 		ptq->unknown_guest_thread = machine__find_thread(machine, 0, 0);  in intel_pt_get_guest_from_sideband()
1452 		if (!ptq->unknown_guest_thread)  in intel_pt_get_guest_from_sideband()
1453 			return -1;  in intel_pt_get_guest_from_sideband()
1454 		ptq->guest_machine = machine;  in intel_pt_get_guest_from_sideband()
1457 	vcpu = ptq->thread ? thread__guest_cpu(ptq->thread) : -1;  in intel_pt_get_guest_from_sideband()
1459 		return -1;  in intel_pt_get_guest_from_sideband()
1463 	if (ptq->guest_thread && thread__tid(ptq->guest_thread) != tid)  in intel_pt_get_guest_from_sideband()
1464 		thread__zput(ptq->guest_thread);  in intel_pt_get_guest_from_sideband()
1466 	if (!ptq->guest_thread) {  in intel_pt_get_guest_from_sideband()
1467 		ptq->guest_thread = machine__find_thread(machine, -1, tid);  in intel_pt_get_guest_from_sideband()
1468 		if (!ptq->guest_thread)  in intel_pt_get_guest_from_sideband()
1469 			return -1;  in intel_pt_get_guest_from_sideband()
1472 	ptq->guest_machine_pid = machine_pid;  in intel_pt_get_guest_from_sideband()
1473 	ptq->guest_pid = thread__pid(ptq->guest_thread);  in intel_pt_get_guest_from_sideband()
1474 	ptq->guest_tid = tid;  in intel_pt_get_guest_from_sideband()
1475 	ptq->vcpu = vcpu;  in intel_pt_get_guest_from_sideband()
1483 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_set_pid_tid_cpu()
1485 	if (queue->tid == -1 || pt->have_sched_switch) {  in intel_pt_set_pid_tid_cpu()
1486 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);  in intel_pt_set_pid_tid_cpu()
1487 		if (ptq->tid == -1)  in intel_pt_set_pid_tid_cpu()
1488 			ptq->pid = -1;  in intel_pt_set_pid_tid_cpu()
1489 		thread__zput(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1492 	if (!ptq->thread && ptq->tid != -1)  in intel_pt_set_pid_tid_cpu()
1493 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);  in intel_pt_set_pid_tid_cpu()
1495 	if (ptq->thread) {  in intel_pt_set_pid_tid_cpu()
1496 		ptq->pid = thread__pid(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1497 		if (queue->cpu == -1)  in intel_pt_set_pid_tid_cpu()
1498 			ptq->cpu = thread__cpu(ptq->thread);  in intel_pt_set_pid_tid_cpu()
1501 	if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) {  in intel_pt_set_pid_tid_cpu()
1502 		ptq->guest_machine_pid = 0;  in intel_pt_set_pid_tid_cpu()
1503 		ptq->guest_pid = -1;  in intel_pt_set_pid_tid_cpu()
1504 		ptq->guest_tid = -1;  in intel_pt_set_pid_tid_cpu()
1505 		ptq->vcpu = -1;  in intel_pt_set_pid_tid_cpu()
1511 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample_flags()
1513 	ptq->insn_len = 0;  in intel_pt_sample_flags()
1514 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {  in intel_pt_sample_flags()
1515 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;  in intel_pt_sample_flags()
1516 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {  in intel_pt_sample_flags()
1517 		if (!ptq->state->to_ip)  in intel_pt_sample_flags()
1518 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1521 		else if (ptq->state->from_nr && !ptq->state->to_nr)  in intel_pt_sample_flags()
1522 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1526 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
1530 		if (ptq->state->from_ip)  in intel_pt_sample_flags()
1531 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);  in intel_pt_sample_flags()
1533 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
1535 		if (ptq->state->flags & INTEL_PT_IN_TX)  in intel_pt_sample_flags()
1536 			ptq->flags |= PERF_IP_FLAG_IN_TX;  in intel_pt_sample_flags()
1537 		ptq->insn_len = ptq->state->insn_len;  in intel_pt_sample_flags()
1538 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_sample_flags()
1541 	if (ptq->state->type & INTEL_PT_TRACE_BEGIN)  in intel_pt_sample_flags()
1542 		ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;  in intel_pt_sample_flags()
1543 	if (ptq->state->type & INTEL_PT_TRACE_END)  in intel_pt_sample_flags()
1544 		ptq->flags |= PERF_IP_FLAG_TRACE_END;  in intel_pt_sample_flags()
1546 	if (pt->cap_event_trace) {  in intel_pt_sample_flags()
1547 		if (ptq->state->type & INTEL_PT_IFLAG_CHG) {  in intel_pt_sample_flags()
1548 			if (!ptq->state->from_iflag)  in intel_pt_sample_flags()
1549 				ptq->flags |= PERF_IP_FLAG_INTR_DISABLE;  in intel_pt_sample_flags()
1550 			if (ptq->state->from_iflag != ptq->state->to_iflag)  in intel_pt_sample_flags()
1551 				ptq->flags |= PERF_IP_FLAG_INTR_TOGGLE;  in intel_pt_sample_flags()
1552 		} else if (!ptq->state->to_iflag) {  in intel_pt_sample_flags()
1553 			ptq->flags |= PERF_IP_FLAG_INTR_DISABLE;  in intel_pt_sample_flags()
1561 	if (!pt->range_cnt)  in intel_pt_setup_time_range()
1564 	ptq->sel_timestamp = pt->time_ranges[0].start;  in intel_pt_setup_time_range()
1565 	ptq->sel_idx = 0;  in intel_pt_setup_time_range()
1567 	if (ptq->sel_timestamp) {  in intel_pt_setup_time_range()
1568 		ptq->sel_start = true;  in intel_pt_setup_time_range()
1570 		ptq->sel_timestamp = pt->time_ranges[0].end;  in intel_pt_setup_time_range()
1571 		ptq->sel_start = false;  in intel_pt_setup_time_range()
1579 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_setup_queue()
1581 	if (list_empty(&queue->head))  in intel_pt_setup_queue()
1587 			return -ENOMEM;  in intel_pt_setup_queue()
1588 		queue->priv = ptq;  in intel_pt_setup_queue()
1590 		if (queue->cpu != -1)  in intel_pt_setup_queue()
1591 			ptq->cpu = queue->cpu;  in intel_pt_setup_queue()
1592 		ptq->tid = queue->tid;  in intel_pt_setup_queue()
1594 		ptq->cbr_seen = UINT_MAX;  in intel_pt_setup_queue()
1596 		if (pt->sampling_mode && !pt->snapshot_mode &&  in intel_pt_setup_queue()
1597 		    pt->timeless_decoding)  in intel_pt_setup_queue()
1598 			ptq->step_through_buffers = true;  in intel_pt_setup_queue()
1600 		ptq->sync_switch = pt->sync_switch;  in intel_pt_setup_queue()
1605 	if (!ptq->on_heap &&  in intel_pt_setup_queue()
1606 	    (!ptq->sync_switch ||  in intel_pt_setup_queue()
1607 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {  in intel_pt_setup_queue()
1611 		if (pt->timeless_decoding)  in intel_pt_setup_queue()
1616 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_setup_queue()
1618 		if (ptq->sel_start && ptq->sel_timestamp) {  in intel_pt_setup_queue()
1619 			ret = intel_pt_fast_forward(ptq->decoder,  in intel_pt_setup_queue()
1620 						    ptq->sel_timestamp);  in intel_pt_setup_queue()
1626 			state = intel_pt_decode(ptq->decoder);  in intel_pt_setup_queue()
1627 			if (state->err) {  in intel_pt_setup_queue()
1628 				if (state->err == INTEL_PT_ERR_NODATA) {  in intel_pt_setup_queue()
1635 			if (state->timestamp)  in intel_pt_setup_queue()
1639 		ptq->timestamp = state->timestamp;  in intel_pt_setup_queue()
1641 			     queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1642 		ptq->state = state;  in intel_pt_setup_queue()
1643 		ptq->have_sample = true;  in intel_pt_setup_queue()
1644 		if (ptq->sel_start && ptq->sel_timestamp &&  in intel_pt_setup_queue()
1645 		    ptq->timestamp < ptq->sel_timestamp)  in intel_pt_setup_queue()
1646 			ptq->have_sample = false;  in intel_pt_setup_queue()
1648 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
1651 		ptq->on_heap = true;  in intel_pt_setup_queue()
1662 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_setup_queues()
1663 		ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);  in intel_pt_setup_queues()
1672 	return pt->synth_opts.initial_skip &&  in intel_pt_skip_event()
1673 	       pt->num_events++ < pt->synth_opts.initial_skip;  in intel_pt_skip_event()
1678  * Also ensure CBR is first non-skipped event by allowing for 4 more samples
1683 	return pt->synth_opts.initial_skip &&  in intel_pt_skip_cbr_event()
1684 	       pt->num_events + 4 < pt->synth_opts.initial_skip;  in intel_pt_skip_cbr_event()
1689 				   struct perf_sample *sample)  in intel_pt_prep_a_sample()  argument
1691 	event->sample.header.type = PERF_RECORD_SAMPLE;  in intel_pt_prep_a_sample()
1692 	event->sample.header.size = sizeof(struct perf_event_header);  in intel_pt_prep_a_sample()
1694 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1695 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1697 	if (ptq->pt->have_guest_sideband) {  in intel_pt_prep_a_sample()
1698 		if ((ptq->state->from_ip && ptq->state->from_nr) ||  in intel_pt_prep_a_sample()
1699 		    (ptq->state->to_ip && ptq->state->to_nr)) {  in intel_pt_prep_a_sample()
1700 			sample->pid = ptq->guest_pid;  in intel_pt_prep_a_sample()
1701 			sample->tid = ptq->guest_tid;  in intel_pt_prep_a_sample()
1702 			sample->machine_pid = ptq->guest_machine_pid;  in intel_pt_prep_a_sample()
1703 			sample->vcpu = ptq->vcpu;  in intel_pt_prep_a_sample()
1707 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1708 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1709 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1715 				   struct perf_sample *sample)  in intel_pt_prep_b_sample()  argument
1717 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1719 	if (!pt->timeless_decoding)  in intel_pt_prep_b_sample()
1720 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1722 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1723 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1724 	sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr);  in intel_pt_prep_b_sample()
1725 	sample->period = 1;  in intel_pt_prep_b_sample()
1726 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1728 	event->sample.header.misc = sample->cpumode;  in intel_pt_prep_b_sample()
1732 				 struct perf_sample *sample, u64 type)  in intel_pt_inject_event()  argument
1734 	event->header.size = perf_event__sample_event_size(sample, type, 0);  in intel_pt_inject_event()
1735 	return perf_event__synthesize_sample(event, type, 0, sample);  in intel_pt_inject_event()
1740 				      struct perf_sample *sample, u64 type)  in intel_pt_opt_inject()  argument
1742 	if (!pt->synth_opts.inject)  in intel_pt_opt_inject()
1745 	return intel_pt_inject_event(event, sample, type);  in intel_pt_opt_inject()
1750 					struct perf_sample *sample, u64 type)  in intel_pt_deliver_synth_event()  argument
1754 	ret = intel_pt_opt_inject(pt, event, sample, type);  in intel_pt_deliver_synth_event()
1758 	ret = perf_session__deliver_synth_event(pt->session, event, sample);  in intel_pt_deliver_synth_event()
1767 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_branch_sample()
1768 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_branch_sample()
1769 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_branch_sample()  local
1776 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))  in intel_pt_synth_branch_sample()
1782 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1784 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1785 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1791 	if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {  in intel_pt_synth_branch_sample()
1794 			.hw_idx = -1ULL,  in intel_pt_synth_branch_sample()
1796 				.from = sample.ip,  in intel_pt_synth_branch_sample()
1797 				.to = sample.addr,  in intel_pt_synth_branch_sample()
1800 		sample.branch_stack = (struct branch_stack *)&dummy_bs;  in intel_pt_synth_branch_sample()
1803 	if (ptq->sample_ipc)  in intel_pt_synth_branch_sample()
1804 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1805 	if (sample.cyc_cnt) {  in intel_pt_synth_branch_sample()
1806 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1807 		ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_branch_sample()
1808 		ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_branch_sample()
1811 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_branch_sample()
1812 					    pt->branches_sample_type);  in intel_pt_synth_branch_sample()
1818 				 struct perf_sample *sample)  in intel_pt_prep_sample()  argument
1820 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1822 	if (pt->synth_opts.callchain) {  in intel_pt_prep_sample()
1823 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_prep_sample()
1824 				     pt->synth_opts.callchain_sz + 1,  in intel_pt_prep_sample()
1825 				     sample->ip, pt->kernel_start);  in intel_pt_prep_sample()
1826 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1829 	if (pt->synth_opts.last_branch) {  in intel_pt_prep_sample()
1830 		thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch,  in intel_pt_prep_sample()
1831 					pt->br_stack_sz);  in intel_pt_prep_sample()
1832 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1838 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_instruction_sample()
1839 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_instruction_sample()
1840 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_instruction_sample()  local
1845 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1847 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1848 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1849 	if (pt->synth_opts.quick)  in intel_pt_synth_instruction_sample()
1850 		sample.period = 1;  in intel_pt_synth_instruction_sample()
1852 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1854 	if (ptq->sample_ipc)  in intel_pt_synth_instruction_sample()
1855 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1856 	if (sample.cyc_cnt) {  in intel_pt_synth_instruction_sample()
1857 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1858 		ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_instruction_sample()
1859 		ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_instruction_sample()
1862 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_synth_instruction_sample()
1864 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_instruction_sample()
1865 					    pt->instructions_sample_type);  in intel_pt_synth_instruction_sample()
1870 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cycle_sample()
1871 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cycle_sample()
1872 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cycle_sample()  local
1875 	if (ptq->sample_ipc)  in intel_pt_synth_cycle_sample()
1876 		period = ptq->ipc_cyc_cnt - ptq->last_cy_cyc_cnt;  in intel_pt_synth_cycle_sample()
1881 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_cycle_sample()
1883 	sample.id = ptq->pt->cycles_id;  in intel_pt_synth_cycle_sample()
1884 	sample.stream_id = ptq->pt->cycles_id;  in intel_pt_synth_cycle_sample()
1885 	sample.period = period;  in intel_pt_synth_cycle_sample()
1887 	sample.cyc_cnt = period;  in intel_pt_synth_cycle_sample()
1888 	sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_cy_insn_cnt;  in intel_pt_synth_cycle_sample()
1889 	ptq->last_cy_insn_cnt = ptq->ipc_insn_cnt;  in intel_pt_synth_cycle_sample()
1890 	ptq->last_cy_cyc_cnt = ptq->ipc_cyc_cnt;  in intel_pt_synth_cycle_sample()
1892 	return intel_pt_deliver_synth_event(pt, event, &sample, pt->cycles_sample_type);  in intel_pt_synth_cycle_sample()
1897 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_transaction_sample()
1898 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_transaction_sample()
1899 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_transaction_sample()  local
1904 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1906 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1907 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1909 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_transaction_sample()
1910 					    pt->transactions_sample_type);  in intel_pt_synth_transaction_sample()
1916 				   struct perf_sample *sample)  in intel_pt_prep_p_sample()  argument
1918 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1924 	if (!sample->ip)  in intel_pt_prep_p_sample()
1925 		sample->flags = 0;  in intel_pt_prep_p_sample()
1930 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_ptwrite_sample()
1931 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_ptwrite_sample()
1932 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_ptwrite_sample()  local
1938 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1940 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1941 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1944 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_ptwrite_sample()
1945 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);  in intel_pt_synth_ptwrite_sample()
1947 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_ptwrite_sample()
1948 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_ptwrite_sample()
1950 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_ptwrite_sample()
1951 					    pt->ptwrites_sample_type);  in intel_pt_synth_ptwrite_sample()
1956 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cbr_sample()
1957 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cbr_sample()
1958 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cbr_sample()  local
1965 	ptq->cbr_seen = ptq->state->cbr;  in intel_pt_synth_cbr_sample()
1967 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1969 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1970 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1972 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);  in intel_pt_synth_cbr_sample()
1974 	raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);  in intel_pt_synth_cbr_sample()
1977 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_cbr_sample()
1978 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_cbr_sample()
1980 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_cbr_sample()
1981 					    pt->pwr_events_sample_type);  in intel_pt_synth_cbr_sample()
1986 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_psb_sample()
1987 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_psb_sample()
1988 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_psb_sample()  local
1994 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_psb_sample()
1996 	sample.id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1997 	sample.stream_id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1998 	sample.flags = 0;  in intel_pt_synth_psb_sample()
2001 	raw.offset = ptq->state->psb_offset;  in intel_pt_synth_psb_sample()
2003 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_psb_sample()
2004 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_psb_sample()
2006 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_psb_sample()
2007 					    pt->pwr_events_sample_type);  in intel_pt_synth_psb_sample()
2012 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_mwait_sample()
2013 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_mwait_sample()
2014 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_mwait_sample()  local
2020 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
2022 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
2023 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
2026 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);  in intel_pt_synth_mwait_sample()
2028 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_mwait_sample()
2029 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_mwait_sample()
2031 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_mwait_sample()
2032 					    pt->pwr_events_sample_type);  in intel_pt_synth_mwait_sample()
2037 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwre_sample()
2038 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwre_sample()
2039 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwre_sample()  local
2045 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
2047 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
2048 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
2051 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);  in intel_pt_synth_pwre_sample()
2053 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwre_sample()
2054 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwre_sample()
2056 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwre_sample()
2057 					    pt->pwr_events_sample_type);  in intel_pt_synth_pwre_sample()
2062 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_exstop_sample()
2063 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_exstop_sample()
2064 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_exstop_sample()  local
2070 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
2072 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2073 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2076 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_exstop_sample()
2078 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_exstop_sample()
2079 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_exstop_sample()
2081 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_exstop_sample()
2082 					    pt->pwr_events_sample_type);  in intel_pt_synth_exstop_sample()
2087 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwrx_sample()
2088 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwrx_sample()
2089 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwrx_sample()  local
2095 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
2097 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2098 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2101 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);  in intel_pt_synth_pwrx_sample()
2103 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwrx_sample()
2104 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwrx_sample()
2106 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwrx_sample()
2107 					    pt->pwr_events_sample_type);  in intel_pt_synth_pwrx_sample()
2139 	const u64 *gp_regs = items->val[INTEL_PT_GP_REGS_POS];  in intel_pt_add_gp_regs()
2140 	u32 mask = items->mask[INTEL_PT_GP_REGS_POS];  in intel_pt_add_gp_regs()
2146 		int n = pebs_gp_regs[i] - 1;  in intel_pt_add_gp_regs()
2153 		 * mask (i.e. 'intr_regs->mask') accordingly.  in intel_pt_add_gp_regs()
2156 			intr_regs->mask |= bit;  in intel_pt_add_gp_regs()
2172 	u32 mask = items->has_xmm & (regs_mask >> PERF_REG_X86_XMM0);  in intel_pt_add_xmm()
2173 	const u64 *xmm = items->xmm;  in intel_pt_add_xmm()
2179 	 * and update the resulting mask (i.e. 'intr_regs->mask') accordingly.  in intel_pt_add_xmm()
2181 	intr_regs->mask |= (u64)mask << PERF_REG_X86_XMM0;  in intel_pt_add_xmm()
2218 	br_stack->nr = 0;  in intel_pt_add_lbrs()
2220 	to = &br_stack->entries[0].from;  in intel_pt_add_lbrs()
2223 		u32 mask = items->mask[i];  in intel_pt_add_lbrs()
2224 		const u64 *from = items->val[i];  in intel_pt_add_lbrs()
2231 				br_stack->nr += 1;  in intel_pt_add_lbrs()
2244 #define PERF_PEBS_DATA_SOURCE_GRT_MASK	(PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
2332 	DATA_SRC_FORMAT_ERR  = -1,
2378 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_do_synth_pebs_sample()
2379 	struct perf_sample sample = { .ip = 0, };  in intel_pt_do_synth_pebs_sample()  local
2380 	union perf_event *event = ptq->event_buf;  in intel_pt_do_synth_pebs_sample()
2381 	struct intel_pt *pt = ptq->pt;  in intel_pt_do_synth_pebs_sample()
2382 	u64 sample_type = evsel->core.attr.sample_type;  in intel_pt_do_synth_pebs_sample()
2384 	u64 regs[8 * sizeof(sample.intr_regs.mask)];  in intel_pt_do_synth_pebs_sample()
2389 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_do_synth_pebs_sample()
2391 	sample.id = id;  in intel_pt_do_synth_pebs_sample()
2392 	sample.stream_id = id;  in intel_pt_do_synth_pebs_sample()
2394 	if (!evsel->core.attr.freq)  in intel_pt_do_synth_pebs_sample()
2395 		sample.period = evsel->core.attr.sample_period;  in intel_pt_do_synth_pebs_sample()
2397 	/* No support for non-zero CS base */  in intel_pt_do_synth_pebs_sample()
2398 	if (items->has_ip)  in intel_pt_do_synth_pebs_sample()
2399 		sample.ip = items->ip;  in intel_pt_do_synth_pebs_sample()
2400 	else if (items->has_rip)  in intel_pt_do_synth_pebs_sample()
2401 		sample.ip = items->rip;  in intel_pt_do_synth_pebs_sample()
2403 		sample.ip = ptq->state->from_ip;  in intel_pt_do_synth_pebs_sample()
2405 	cpumode = intel_pt_cpumode(ptq, sample.ip, 0);  in intel_pt_do_synth_pebs_sample()
2407 	event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;  in intel_pt_do_synth_pebs_sample()
2409 	sample.cpumode = cpumode;  in intel_pt_do_synth_pebs_sample()
2414 		if (items->has_timestamp)  in intel_pt_do_synth_pebs_sample()
2415 			timestamp = items->timestamp;  in intel_pt_do_synth_pebs_sample()
2416 		else if (!pt->timeless_decoding)  in intel_pt_do_synth_pebs_sample()
2417 			timestamp = ptq->timestamp;  in intel_pt_do_synth_pebs_sample()
2419 			sample.time = tsc_to_perf_time(timestamp, &pt->tc);  in intel_pt_do_synth_pebs_sample()
2423 	    pt->synth_opts.callchain) {  in intel_pt_do_synth_pebs_sample()
2424 		thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,  in intel_pt_do_synth_pebs_sample()
2425 				     pt->synth_opts.callchain_sz, sample.ip,  in intel_pt_do_synth_pebs_sample()
2426 				     pt->kernel_start);  in intel_pt_do_synth_pebs_sample()
2427 		sample.callchain = ptq->chain;  in intel_pt_do_synth_pebs_sample()
2431 	    (items->mask[INTEL_PT_GP_REGS_POS] ||  in intel_pt_do_synth_pebs_sample()
2432 	     items->mask[INTEL_PT_XMM_POS])) {  in intel_pt_do_synth_pebs_sample()
2433 		u64 regs_mask = evsel->core.attr.sample_regs_intr;  in intel_pt_do_synth_pebs_sample()
2436 		sample.intr_regs.abi = items->is_32_bit ?  in intel_pt_do_synth_pebs_sample()
2439 		sample.intr_regs.regs = regs;  in intel_pt_do_synth_pebs_sample()
2441 		pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);  in intel_pt_do_synth_pebs_sample()
2443 		intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);  in intel_pt_do_synth_pebs_sample()
2447 		if (items->mask[INTEL_PT_LBR_0_POS] ||  in intel_pt_do_synth_pebs_sample()
2448 		    items->mask[INTEL_PT_LBR_1_POS] ||  in intel_pt_do_synth_pebs_sample()
2449 		    items->mask[INTEL_PT_LBR_2_POS]) {  in intel_pt_do_synth_pebs_sample()
2450 			intel_pt_add_lbrs(ptq->last_branch, items);  in intel_pt_do_synth_pebs_sample()
2451 		} else if (pt->synth_opts.last_branch) {  in intel_pt_do_synth_pebs_sample()
2452 			thread_stack__br_sample(ptq->thread, ptq->cpu,  in intel_pt_do_synth_pebs_sample()
2453 						ptq->last_branch,  in intel_pt_do_synth_pebs_sample()
2454 						pt->br_stack_sz);  in intel_pt_do_synth_pebs_sample()
2456 			ptq->last_branch->nr = 0;  in intel_pt_do_synth_pebs_sample()
2458 		sample.branch_stack = ptq->last_branch;  in intel_pt_do_synth_pebs_sample()
2461 	if (sample_type & PERF_SAMPLE_ADDR && items->has_mem_access_address)  in intel_pt_do_synth_pebs_sample()
2462 		sample.addr = items->mem_access_address;  in intel_pt_do_synth_pebs_sample()
2469 		if (items->has_mem_access_latency) {  in intel_pt_do_synth_pebs_sample()
2470 			u64 weight = items->mem_access_latency >> 32;  in intel_pt_do_synth_pebs_sample()
2483 				sample.weight = weight & 0xffff;  in intel_pt_do_synth_pebs_sample()
2484 				sample.ins_lat = items->mem_access_latency & 0xffff;  in intel_pt_do_synth_pebs_sample()
2486 				sample.weight = items->mem_access_latency;  in intel_pt_do_synth_pebs_sample()
2488 		if (!sample.weight && items->has_tsx_aux_info) {  in intel_pt_do_synth_pebs_sample()
2490 			sample.weight = (u32)items->tsx_aux_info;  in intel_pt_do_synth_pebs_sample()
2495 		if (items->has_mem_aux_info && data_src_fmt) {  in intel_pt_do_synth_pebs_sample()
2498 				return -1;  in intel_pt_do_synth_pebs_sample()
2500 			sample.data_src = intel_pt_get_data_src(items->mem_aux_info, data_src_fmt);  in intel_pt_do_synth_pebs_sample()
2502 			sample.data_src = PERF_MEM_NA;  in intel_pt_do_synth_pebs_sample()
2506 	if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) {  in intel_pt_do_synth_pebs_sample()
2507 		u64 ax = items->has_rax ? items->rax : 0;  in intel_pt_do_synth_pebs_sample()
2509 		u64 txn = (u8)(items->tsx_aux_info >> 32);  in intel_pt_do_synth_pebs_sample()
2514 		sample.transaction = txn;  in intel_pt_do_synth_pebs_sample()
2517 	return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);  in intel_pt_do_synth_pebs_sample()
2522 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_single_pebs_sample()
2523 	struct evsel *evsel = pt->pebs_evsel;  in intel_pt_synth_single_pebs_sample()
2524 	int data_src_fmt = pt->pebs_data_src_fmt;  in intel_pt_synth_single_pebs_sample()
2525 	u64 id = evsel->core.id[0];  in intel_pt_synth_single_pebs_sample()
2532 	const struct intel_pt_blk_items *items = &ptq->state->items;  in intel_pt_synth_pebs_sample()
2534 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pebs_sample()
2535 	int err = -EINVAL;  in intel_pt_synth_pebs_sample()
2538 	if (!items->has_applicable_counters || !items->applicable_counters) {  in intel_pt_synth_pebs_sample()
2539 		if (!pt->single_pebs)  in intel_pt_synth_pebs_sample()
2540 			pr_err("PEBS-via-PT record with no applicable_counters\n");  in intel_pt_synth_pebs_sample()
2544 	for_each_set_bit(hw_id, (unsigned long *)&items->applicable_counters, INTEL_PT_MAX_PEBS) {  in intel_pt_synth_pebs_sample()
2545 		pe = &ptq->pebs[hw_id];  in intel_pt_synth_pebs_sample()
2546 		if (!pe->evsel) {  in intel_pt_synth_pebs_sample()
2547 			if (!pt->single_pebs)  in intel_pt_synth_pebs_sample()
2548 				pr_err("PEBS-via-PT record with no matching event, hw_id %d\n",  in intel_pt_synth_pebs_sample()
2552 		err = intel_pt_do_synth_pebs_sample(ptq, pe->evsel, pe->id, pe->data_src_fmt);  in intel_pt_synth_pebs_sample()
2562 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_events_sample()
2563 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_events_sample()
2564 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_events_sample()  local
2574 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_events_sample()
2576 	sample.id        = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2577 	sample.stream_id = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2579 	raw.cfe.type     = ptq->state->cfe_type;  in intel_pt_synth_events_sample()
2581 	raw.cfe.ip       = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_events_sample()
2582 	raw.cfe.vector   = ptq->state->cfe_vector;  in intel_pt_synth_events_sample()
2583 	raw.cfe.evd_cnt  = ptq->state->evd_cnt;  in intel_pt_synth_events_sample()
2585 	for (i = 0; i < ptq->state->evd_cnt; i++) {  in intel_pt_synth_events_sample()
2587 		raw.evd[i].evd_type = ptq->state->evd[i].type;  in intel_pt_synth_events_sample()
2588 		raw.evd[i].payload  = ptq->state->evd[i].payload;  in intel_pt_synth_events_sample()
2591 	sample.raw_size = perf_synth__raw_size(raw) +  in intel_pt_synth_events_sample()
2592 			  ptq->state->evd_cnt * sizeof(struct perf_synth_intel_evd);  in intel_pt_synth_events_sample()
2593 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_events_sample()
2595 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_events_sample()
2596 					    pt->evt_sample_type);  in intel_pt_synth_events_sample()
2601 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_iflag_chg_sample()
2602 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_iflag_chg_sample()
2603 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_iflag_chg_sample()  local
2609 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_iflag_chg_sample()
2611 	sample.id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2612 	sample.stream_id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2615 	raw.iflag = ptq->state->to_iflag;  in intel_pt_synth_iflag_chg_sample()
2617 	if (ptq->state->type & INTEL_PT_BRANCH) {  in intel_pt_synth_iflag_chg_sample()
2619 		raw.branch_ip = ptq->state->to_ip;  in intel_pt_synth_iflag_chg_sample()
2621 		sample.addr = 0;  in intel_pt_synth_iflag_chg_sample()
2623 	sample.flags = ptq->flags;  in intel_pt_synth_iflag_chg_sample()
2625 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_iflag_chg_sample()
2626 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_iflag_chg_sample()
2628 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_iflag_chg_sample()
2629 					    pt->iflag_chg_sample_type);  in intel_pt_synth_iflag_chg_sample()
2636 	bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;  in intel_pt_synth_error()
2637 	bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT;  in intel_pt_synth_error()
2642 	if (pt->synth_opts.error_minus_flags) {  in intel_pt_synth_error()
2644 		    pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW)  in intel_pt_synth_error()
2647 		    pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST)  in intel_pt_synth_error()
2667 	err = perf_session__deliver_synth_event(pt->session, &event, NULL);  in intel_pt_synth_error()
2678 	struct intel_pt *pt = ptq->pt;  in intel_ptq_synth_error()
2679 	u64 tm = ptq->timestamp;  in intel_ptq_synth_error()
2681 	pid_t pid = ptq->pid;  in intel_ptq_synth_error()
2682 	pid_t tid = ptq->tid;  in intel_ptq_synth_error()
2683 	int vcpu = -1;  in intel_ptq_synth_error()
2685 	tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);  in intel_ptq_synth_error()
2687 	if (pt->have_guest_sideband && state->from_nr) {  in intel_ptq_synth_error()
2688 		machine_pid = ptq->guest_machine_pid;  in intel_ptq_synth_error()
2689 		vcpu = ptq->vcpu;  in intel_ptq_synth_error()
2690 		pid = ptq->guest_pid;  in intel_ptq_synth_error()
2691 		tid = ptq->guest_tid;  in intel_ptq_synth_error()
2694 	return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid,  in intel_ptq_synth_error()
2695 				    state->from_ip, tm, machine_pid, vcpu);  in intel_ptq_synth_error()
2701 	pid_t tid = ptq->next_tid;  in intel_pt_next_tid()
2704 	if (tid == -1)  in intel_pt_next_tid()
2707 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);  in intel_pt_next_tid()
2709 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);  in intel_pt_next_tid()
2711 	queue = &pt->queues.queue_array[ptq->queue_nr];  in intel_pt_next_tid()
2714 	ptq->next_tid = -1;  in intel_pt_next_tid()
2721 	struct intel_pt *pt = ptq->pt;  in intel_pt_is_switch_ip()
2723 	return ip == pt->switch_ip &&  in intel_pt_is_switch_ip()
2724 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&  in intel_pt_is_switch_ip()
2725 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |  in intel_pt_is_switch_ip()
2734 	const struct intel_pt_state *state = ptq->state;  in intel_pt_sample()
2735 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample()
2738 	if (!ptq->have_sample)  in intel_pt_sample()
2741 	ptq->have_sample = false;  in intel_pt_sample()
2743 	if (pt->synth_opts.approx_ipc) {  in intel_pt_sample()
2744 		ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
2745 		ptq->ipc_cyc_cnt = ptq->state->cycles;  in intel_pt_sample()
2746 		ptq->sample_ipc = true;  in intel_pt_sample()
2748 		ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_sample()
2749 		ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;  in intel_pt_sample()
2750 		ptq->sample_ipc = ptq->state->flags & INTEL_PT_SAMPLE_IPC;  in intel_pt_sample()
2754 	if (symbol_conf.guest_code && (state->from_nr || state->to_nr))  in intel_pt_sample()
2761 	if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {  in intel_pt_sample()
2767 	if (pt->synth_opts.intr_events) {  in intel_pt_sample()
2768 		if (state->type & INTEL_PT_EVT) {  in intel_pt_sample()
2773 		if (state->type & INTEL_PT_IFLAG_CHG) {  in intel_pt_sample()
2780 	if (pt->sample_pwr_events) {  in intel_pt_sample()
2781 		if (state->type & INTEL_PT_PSB_EVT) {  in intel_pt_sample()
2786 		if (ptq->state->cbr != ptq->cbr_seen) {  in intel_pt_sample()
2791 		if (state->type & INTEL_PT_PWR_EVT) {  in intel_pt_sample()
2792 			if (state->type & INTEL_PT_MWAIT_OP) {  in intel_pt_sample()
2797 			if (state->type & INTEL_PT_PWR_ENTRY) {  in intel_pt_sample()
2802 			if (state->type & INTEL_PT_EX_STOP) {  in intel_pt_sample()
2807 			if (state->type & INTEL_PT_PWR_EXIT) {  in intel_pt_sample()
2815 	if (state->type & INTEL_PT_INSTRUCTION) {  in intel_pt_sample()
2816 		if (pt->sample_instructions) {  in intel_pt_sample()
2821 		if (pt->sample_cycles) {  in intel_pt_sample()
2828 	if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {  in intel_pt_sample()
2834 	if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {  in intel_pt_sample()
2840 	if (!(state->type & INTEL_PT_BRANCH))  in intel_pt_sample()
2843 	if (pt->use_thread_stack) {  in intel_pt_sample()
2844 		thread_stack__event(ptq->thread, ptq->cpu, ptq->flags,  in intel_pt_sample()
2845 				    state->from_ip, state->to_ip, ptq->insn_len,  in intel_pt_sample()
2846 				    state->trace_nr, pt->callstack,  in intel_pt_sample()
2847 				    pt->br_stack_sz_plus,  in intel_pt_sample()
2848 				    pt->mispred_all);  in intel_pt_sample()
2850 		thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);  in intel_pt_sample()
2853 	if (pt->sample_branches) {  in intel_pt_sample()
2854 		if (state->from_nr != state->to_nr &&  in intel_pt_sample()
2855 		    state->from_ip && state->to_ip) {  in intel_pt_sample()
2857 			u64 to_ip = st->to_ip;  in intel_pt_sample()
2858 			u64 from_ip = st->from_ip;  in intel_pt_sample()
2864 			st->to_ip = 0;  in intel_pt_sample()
2868 			st->from_ip = 0;  in intel_pt_sample()
2869 			st->to_ip = to_ip;  in intel_pt_sample()
2871 			st->from_ip = from_ip;  in intel_pt_sample()
2879 	if (!ptq->sync_switch)  in intel_pt_sample()
2882 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {  in intel_pt_sample()
2883 		switch (ptq->switch_state) {  in intel_pt_sample()
2890 			ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2893 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;  in intel_pt_sample()
2896 	} else if (!state->to_ip) {  in intel_pt_sample()
2897 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_sample()
2898 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {  in intel_pt_sample()
2899 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_sample()
2900 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_sample()
2901 		   state->to_ip == pt->ptss_ip &&  in intel_pt_sample()
2902 		   (ptq->flags & PERF_IP_FLAG_CALL)) {  in intel_pt_sample()
2903 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
2911 	struct machine *machine = pt->machine;  in intel_pt_switch_ip()
2930 		if (sym->binding == STB_GLOBAL &&  in intel_pt_switch_ip()
2931 		    !strcmp(sym->name, "__switch_to")) {  in intel_pt_switch_ip()
2932 			ip = map__unmap_ip(map, sym->start);  in intel_pt_switch_ip()
2943 	if (pt->have_sched_switch == 1)  in intel_pt_switch_ip()
2949 		if (!strcmp(sym->name, ptss)) {  in intel_pt_switch_ip()
2950 			ip = map__unmap_ip(map, sym->start);  in intel_pt_switch_ip()
2965 	if (pt->sync_switch_not_supported)  in intel_pt_enable_sync_switch()
2968 	pt->sync_switch = true;  in intel_pt_enable_sync_switch()
2970 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_enable_sync_switch()
2971 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_enable_sync_switch()
2972 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_enable_sync_switch()
2975 			ptq->sync_switch = true;  in intel_pt_enable_sync_switch()
2983 	pt->sync_switch = false;  in intel_pt_disable_sync_switch()
2985 	for (i = 0; i < pt->queues.nr_queues; i++) {  in intel_pt_disable_sync_switch()
2986 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_disable_sync_switch()
2987 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_disable_sync_switch()
2990 			ptq->sync_switch = false;  in intel_pt_disable_sync_switch()
2997  * To filter against time ranges, it is only necessary to look at the next start
2998  * or end time.
3002 	struct intel_pt *pt = ptq->pt;  in intel_pt_next_time()
3004 	if (ptq->sel_start) {  in intel_pt_next_time()
3005 		/* Next time is an end time */  in intel_pt_next_time()
3006 		ptq->sel_start = false;  in intel_pt_next_time()
3007 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;  in intel_pt_next_time()
3009 	} else if (ptq->sel_idx + 1 < pt->range_cnt) {  in intel_pt_next_time()
3010 		/* Next time is a start time */  in intel_pt_next_time()
3011 		ptq->sel_start = true;  in intel_pt_next_time()
3012 		ptq->sel_idx += 1;  in intel_pt_next_time()
3013 		ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;  in intel_pt_next_time()
3017 	/* No next time */  in intel_pt_next_time()
3026 		if (ptq->sel_start) {  in intel_pt_time_filter()
3027 			if (ptq->timestamp >= ptq->sel_timestamp) {  in intel_pt_time_filter()
3028 				/* After start time, so consider next time */  in intel_pt_time_filter()
3030 				if (!ptq->sel_timestamp) {  in intel_pt_time_filter()
3031 					/* No end time */  in intel_pt_time_filter()
3034 				/* Check against end time */  in intel_pt_time_filter()
3037 			/* Before start time, so fast forward */  in intel_pt_time_filter()
3038 			ptq->have_sample = false;  in intel_pt_time_filter()
3039 			if (ptq->sel_timestamp > *ff_timestamp) {  in intel_pt_time_filter()
3040 				if (ptq->sync_switch) {  in intel_pt_time_filter()
3041 					intel_pt_next_tid(ptq->pt, ptq);  in intel_pt_time_filter()
3042 					ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_time_filter()
3044 				*ff_timestamp = ptq->sel_timestamp;  in intel_pt_time_filter()
3045 				err = intel_pt_fast_forward(ptq->decoder,  in intel_pt_time_filter()
3046 							    ptq->sel_timestamp);  in intel_pt_time_filter()
3051 		} else if (ptq->timestamp > ptq->sel_timestamp) {  in intel_pt_time_filter()
3052 			/* After end time, so consider next time */  in intel_pt_time_filter()
3054 				/* No next time range, so stop decoding */  in intel_pt_time_filter()
3055 				ptq->have_sample = false;  in intel_pt_time_filter()
3056 				ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_time_filter()
3059 			/* Check against next start time */  in intel_pt_time_filter()
3062 			/* Before end time */  in intel_pt_time_filter()
3070 	const struct intel_pt_state *state = ptq->state;  in intel_pt_run_decoder()
3071 	struct intel_pt *pt = ptq->pt;  in intel_pt_run_decoder()
3075 	if (!pt->kernel_start) {  in intel_pt_run_decoder()
3076 		pt->kernel_start = machine__kernel_start(pt->machine);  in intel_pt_run_decoder()
3077 		if (pt->per_cpu_mmaps &&  in intel_pt_run_decoder()
3078 		    (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&  in intel_pt_run_decoder()
3079 		    !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&  in intel_pt_run_decoder()
3080 		    !pt->sampling_mode && !pt->synth_opts.vm_time_correlation) {  in intel_pt_run_decoder()
3081 			pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);  in intel_pt_run_decoder()
3082 			if (pt->switch_ip) {  in intel_pt_run_decoder()
3084 					     pt->switch_ip, pt->ptss_ip);  in intel_pt_run_decoder()
3091 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_run_decoder()
3097 		state = intel_pt_decode(ptq->decoder);  in intel_pt_run_decoder()
3098 		if (state->err) {  in intel_pt_run_decoder()
3099 			if (state->err == INTEL_PT_ERR_NODATA)  in intel_pt_run_decoder()
3101 			if (ptq->sync_switch &&  in intel_pt_run_decoder()
3102 			    state->from_ip >= pt->kernel_start) {  in intel_pt_run_decoder()
3103 				ptq->sync_switch = false;  in intel_pt_run_decoder()
3106 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
3107 			if (pt->synth_opts.errors) {  in intel_pt_run_decoder()
3115 		ptq->state = state;  in intel_pt_run_decoder()
3116 		ptq->have_sample = true;  in intel_pt_run_decoder()
3120 		if (pt->est_tsc &&  in intel_pt_run_decoder()
3121 		    (state->from_ip >= pt->kernel_start || !state->from_ip) &&  in intel_pt_run_decoder()
3122 		    state->to_ip && state->to_ip < pt->kernel_start) {  in intel_pt_run_decoder()
3124 				     state->timestamp, state->est_timestamp);  in intel_pt_run_decoder()
3125 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
3127 		} else if (ptq->sync_switch &&  in intel_pt_run_decoder()
3128 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_run_decoder()
3129 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&  in intel_pt_run_decoder()
3130 			   ptq->next_tid == -1) {  in intel_pt_run_decoder()
3132 				     state->timestamp, state->est_timestamp);  in intel_pt_run_decoder()
3133 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
3134 		} else if (state->timestamp > ptq->timestamp) {  in intel_pt_run_decoder()
3135 			ptq->timestamp = state->timestamp;  in intel_pt_run_decoder()
3138 		if (ptq->sel_timestamp) {  in intel_pt_run_decoder()
3144 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {  in intel_pt_run_decoder()
3145 			*timestamp = ptq->timestamp;  in intel_pt_run_decoder()
3154 	if (pt->queues.new_data) {  in intel_pt_update_queues()
3155 		pt->queues.new_data = false;  in intel_pt_update_queues()
3171 		if (!pt->heap.heap_cnt)  in intel_pt_process_queues()
3174 		if (pt->heap.heap_array[0].ordinal >= timestamp)  in intel_pt_process_queues()
3177 		queue_nr = pt->heap.heap_array[0].queue_nr;  in intel_pt_process_queues()
3178 		queue = &pt->queues.queue_array[queue_nr];  in intel_pt_process_queues()
3179 		ptq = queue->priv;  in intel_pt_process_queues()
3182 			     queue_nr, pt->heap.heap_array[0].ordinal,  in intel_pt_process_queues()
3185 		auxtrace_heap__pop(&pt->heap);  in intel_pt_process_queues()
3187 		if (pt->heap.heap_cnt) {  in intel_pt_process_queues()
3188 			ts = pt->heap.heap_array[0].ordinal + 1;  in intel_pt_process_queues()
3200 			auxtrace_heap__add(&pt->heap, queue_nr, ts);  in intel_pt_process_queues()
3205 			ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);  in intel_pt_process_queues()
3209 			ptq->on_heap = false;  in intel_pt_process_queues()
3219 	struct auxtrace_queues *queues = &pt->queues;  in intel_pt_process_timeless_queues()
3223 	for (i = 0; i < queues->nr_queues; i++) {  in intel_pt_process_timeless_queues()
3224 		struct auxtrace_queue *queue = &pt->queues.queue_array[i];  in intel_pt_process_timeless_queues()
3225 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_process_timeless_queues()
3227 		if (ptq && (tid == -1 || ptq->tid == tid)) {  in intel_pt_process_timeless_queues()
3228 			ptq->time = time_;  in intel_pt_process_timeless_queues()
3238 					    struct perf_sample *sample)  in intel_pt_sample_set_pid_tid_cpu()  argument
3240 	struct machine *m = ptq->pt->machine;  in intel_pt_sample_set_pid_tid_cpu()
3242 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
3243 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
3244 	ptq->cpu = queue->cpu;  in intel_pt_sample_set_pid_tid_cpu()
3247 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3249 	thread__zput(ptq->thread);  in intel_pt_sample_set_pid_tid_cpu()
3251 	if (ptq->tid == -1)  in intel_pt_sample_set_pid_tid_cpu()
3254 	if (ptq->pid == -1) {  in intel_pt_sample_set_pid_tid_cpu()
3255 		ptq->thread = machine__find_thread(m, -1, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3256 		if (ptq->thread)  in intel_pt_sample_set_pid_tid_cpu()
3257 			ptq->pid = thread__pid(ptq->thread);  in intel_pt_sample_set_pid_tid_cpu()
3261 	ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);  in intel_pt_sample_set_pid_tid_cpu()
3265 					    struct perf_sample *sample)  in intel_pt_process_timeless_sample()  argument
3271 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_timeless_sample()
3273 		return -EINVAL;  in intel_pt_process_timeless_sample()
3275 	ptq = queue->priv;  in intel_pt_process_timeless_sample()
3279 	ptq->stop = false;  in intel_pt_process_timeless_sample()
3280 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
3281 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
3286 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)  in intel_pt_lost()  argument
3288 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,  in intel_pt_lost()
3289 				    sample->pid, sample->tid, 0, sample->time,  in intel_pt_lost()
3290 				    sample->machine_pid, sample->vcpu);  in intel_pt_lost()
3297 	if (cpu < 0 || !pt->queues.nr_queues)  in intel_pt_cpu_to_ptq()
3300 	if ((unsigned)cpu >= pt->queues.nr_queues)  in intel_pt_cpu_to_ptq()
3301 		i = pt->queues.nr_queues - 1;  in intel_pt_cpu_to_ptq()
3305 	if (pt->queues.queue_array[i].cpu == cpu)  in intel_pt_cpu_to_ptq()
3306 		return pt->queues.queue_array[i].priv;  in intel_pt_cpu_to_ptq()
3309 		if (pt->queues.queue_array[--i].cpu == cpu)  in intel_pt_cpu_to_ptq()
3310 			return pt->queues.queue_array[i].priv;  in intel_pt_cpu_to_ptq()
3313 	for (; j < pt->queues.nr_queues; j++) {  in intel_pt_cpu_to_ptq()
3314 		if (pt->queues.queue_array[j].cpu == cpu)  in intel_pt_cpu_to_ptq()
3315 			return pt->queues.queue_array[j].priv;  in intel_pt_cpu_to_ptq()
3327 	if (!pt->sync_switch)  in intel_pt_sync_switch()
3331 	if (!ptq || !ptq->sync_switch)  in intel_pt_sync_switch()
3334 	switch (ptq->switch_state) {  in intel_pt_sync_switch()
3339 		ptq->next_tid = tid;  in intel_pt_sync_switch()
3340 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;  in intel_pt_sync_switch()
3343 		if (!ptq->on_heap) {  in intel_pt_sync_switch()
3344 			ptq->timestamp = perf_time_to_tsc(timestamp,  in intel_pt_sync_switch()
3345 							  &pt->tc);  in intel_pt_sync_switch()
3346 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,  in intel_pt_sync_switch()
3347 						 ptq->timestamp);  in intel_pt_sync_switch()
3350 			ptq->on_heap = true;  in intel_pt_sync_switch()
3352 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sync_switch()
3361 	ptq->next_tid = -1;  in intel_pt_sync_switch()
3368 				   struct perf_sample *sample)  in intel_pt_process_switch()  argument
3372 	struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id);  in intel_pt_process_switch()
3374 	if (evsel != pt->switch_evsel)  in intel_pt_process_switch()
3377 	tid = evsel__intval(evsel, sample, "next_pid");  in intel_pt_process_switch()
3378 	cpu = sample->cpu;  in intel_pt_process_switch()
3380 	intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",  in intel_pt_process_switch()
3381 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,  in intel_pt_process_switch()
3382 		     &pt->tc));  in intel_pt_process_switch()
3384 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_process_switch()
3388 	return machine__set_current_tid(pt->machine, cpu, -1, tid);  in intel_pt_process_switch()
3393 				      struct perf_sample *sample)  in intel_pt_context_switch_in()  argument
3395 	pid_t pid = sample->pid;  in intel_pt_context_switch_in()
3396 	pid_t tid = sample->tid;  in intel_pt_context_switch_in()
3397 	int cpu = sample->cpu;  in intel_pt_context_switch_in()
3399 	if (pt->sync_switch) {  in intel_pt_context_switch_in()
3403 		if (ptq && ptq->sync_switch) {  in intel_pt_context_switch_in()
3404 			ptq->next_tid = -1;  in intel_pt_context_switch_in()
3405 			switch (ptq->switch_state) {  in intel_pt_context_switch_in()
3412 				ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_context_switch_in()
3424 	if (machine__get_current_tid(pt->machine, cpu) == tid)  in intel_pt_context_switch_in()
3427 	return machine__set_current_tid(pt->machine, cpu, pid, tid);  in intel_pt_context_switch_in()
3432 					 struct perf_sample *sample)  in intel_pt_guest_context_switch()  argument
3434 	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;  in intel_pt_guest_context_switch()
3435 	struct machines *machines = &pt->session->machines;  in intel_pt_guest_context_switch()
3436 	struct machine *machine = machines__find(machines, sample->machine_pid);  in intel_pt_guest_context_switch()
3438 	pt->have_guest_sideband = true;  in intel_pt_guest_context_switch()
3444 	pt->sync_switch_not_supported = true;  in intel_pt_guest_context_switch()
3445 	if (pt->sync_switch)  in intel_pt_guest_context_switch()
3452 		return -EINVAL;  in intel_pt_guest_context_switch()
3454 	return machine__set_current_tid(machine, sample->vcpu, sample->pid, sample->tid);  in intel_pt_guest_context_switch()
3458 				   struct perf_sample *sample)  in intel_pt_context_switch()  argument
3460 	bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;  in intel_pt_context_switch()
3465 		return intel_pt_guest_context_switch(pt, event, sample);  in intel_pt_context_switch()
3467 	cpu = sample->cpu;  in intel_pt_context_switch()
3469 	if (pt->have_sched_switch == 3) {  in intel_pt_context_switch()
3471 			return intel_pt_context_switch_in(pt, sample);  in intel_pt_context_switch()
3472 		if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {  in intel_pt_context_switch()
3473 			pr_err("Expecting CPU-wide context switch event\n");  in intel_pt_context_switch()
3474 			return -EINVAL;  in intel_pt_context_switch()
3476 		pid = event->context_switch.next_prev_pid;  in intel_pt_context_switch()
3477 		tid = event->context_switch.next_prev_tid;  in intel_pt_context_switch()
3481 		pid = sample->pid;  in intel_pt_context_switch()
3482 		tid = sample->tid;  in intel_pt_context_switch()
3485 	if (tid == -1)  in intel_pt_context_switch()
3488 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_context_switch()
3492 	return machine__set_current_tid(pt->machine, cpu, pid, tid);  in intel_pt_context_switch()
3497 					 struct perf_sample *sample)  in intel_pt_process_itrace_start()  argument
3499 	if (!pt->per_cpu_mmaps)  in intel_pt_process_itrace_start()
3502 	intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",  in intel_pt_process_itrace_start()
3503 		     sample->cpu, event->itrace_start.pid,  in intel_pt_process_itrace_start()
3504 		     event->itrace_start.tid, sample->time,  in intel_pt_process_itrace_start()
3505 		     perf_time_to_tsc(sample->time, &pt->tc));  in intel_pt_process_itrace_start()
3507 	return machine__set_current_tid(pt->machine, sample->cpu,  in intel_pt_process_itrace_start()
3508 					event->itrace_start.pid,  in intel_pt_process_itrace_start()
3509 					event->itrace_start.tid);  in intel_pt_process_itrace_start()
3518 	struct perf_env *env = pt->machine->env;  in intel_pt_data_src_fmt()
3521 	if (!env->cpuid)  in intel_pt_data_src_fmt()
3525 	 * PEBS-via-PT is only supported on E-core non-hybrid. Of those only  in intel_pt_data_src_fmt()
3532 	if (!strncmp(env->cpuid, "GenuineIntel,6,190,", 19))  in intel_pt_data_src_fmt()
3535 	if (!strncmp(env->cpuid, "GenuineIntel,6,175,", 19) ||  in intel_pt_data_src_fmt()
3536 	    !strncmp(env->cpuid, "GenuineIntel,6,182,", 19))  in intel_pt_data_src_fmt()
3544 	 *	mem-loads	event=0xd0,umask=0x5  in intel_pt_data_src_fmt()
3545 	 *	mem-stores	event=0xd0,umask=0x6  in intel_pt_data_src_fmt()
3547 	if (evsel->core.attr.type == PERF_TYPE_RAW &&  in intel_pt_data_src_fmt()
3548 	    ((evsel->core.attr.config & 0xffff) == 0x5d0 ||  in intel_pt_data_src_fmt()
3549 	     (evsel->core.attr.config & 0xffff) == 0x6d0))  in intel_pt_data_src_fmt()
3557 					     struct perf_sample *sample)  in intel_pt_process_aux_output_hw_id()  argument
3559 	u64 hw_id = event->aux_output_hw_id.hw_id;  in intel_pt_process_aux_output_hw_id()
3564 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_aux_output_hw_id()
3565 	evsel = evlist__id2evsel_strict(pt->session->evlist, sample->id);  in intel_pt_process_aux_output_hw_id()
3566 	if (!queue || !queue->priv || !evsel || hw_id > INTEL_PT_MAX_PEBS) {  in intel_pt_process_aux_output_hw_id()
3568 		return -EINVAL;  in intel_pt_process_aux_output_hw_id()
3571 	ptq = queue->priv;  in intel_pt_process_aux_output_hw_id()
3573 	ptq->pebs[hw_id].evsel = evsel;  in intel_pt_process_aux_output_hw_id()
3574 	ptq->pebs[hw_id].id = sample->id;  in intel_pt_process_aux_output_hw_id()
3575 	ptq->pebs[hw_id].data_src_fmt = intel_pt_data_src_fmt(pt, evsel);  in intel_pt_process_aux_output_hw_id()
3583 	if (!al->map || addr < map__start(al->map) || addr >= map__end(al->map)) {  in intel_pt_find_map()
3585 			return -1;  in intel_pt_find_map()
3594 	u8 cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;  in intel_pt_text_poke()
3595 	u64 addr = event->text_poke.addr + event->text_poke.new_len - 1;  in intel_pt_text_poke()
3597 	int cnt = 4096 + event->text_poke.new_len;  in intel_pt_text_poke()
3598 	struct thread *thread = pt->unknown_thread;  in intel_pt_text_poke()
3600 	struct machine *machine = pt->machine;  in intel_pt_text_poke()
3606 	if (!event->text_poke.new_len)  in intel_pt_text_poke()
3609 	for (; cnt; cnt--, addr--) {  in intel_pt_text_poke()
3613 			if (addr < event->text_poke.addr)  in intel_pt_text_poke()
3619 		if (!dso || !dso->auxtrace_cache)  in intel_pt_text_poke()
3628 		if (addr + e->byte_cnt + e->length <= event->text_poke.addr) {  in intel_pt_text_poke()
3634 			if (e->branch != INTEL_PT_BR_NO_BRANCH)  in intel_pt_text_poke()
3639 				     dso->long_name, addr);  in intel_pt_text_poke()
3649 				  struct perf_sample *sample,  in intel_pt_process_event()  argument
3652 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_process_event()
3660 	if (!tool->ordered_events) {  in intel_pt_process_event()
3662 		return -EINVAL;  in intel_pt_process_event()
3665 	if (sample->time && sample->time != (u64)-1)  in intel_pt_process_event()
3666 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_process_event()
3670 	if (timestamp || pt->timeless_decoding) {  in intel_pt_process_event()
3676 	if (pt->timeless_decoding) {  in intel_pt_process_event()
3677 		if (pt->sampling_mode) {  in intel_pt_process_event()
3678 			if (sample->aux_sample.size)  in intel_pt_process_event()
3680 								       sample);  in intel_pt_process_event()
3681 		} else if (event->header.type == PERF_RECORD_EXIT) {  in intel_pt_process_event()
3683 							       event->fork.tid,  in intel_pt_process_event()
3684 							       sample->time);  in intel_pt_process_event()
3687 		if (!pt->first_timestamp)  in intel_pt_process_event()
3694 	if (event->header.type == PERF_RECORD_SAMPLE) {  in intel_pt_process_event()
3695 		if (pt->synth_opts.add_callchain && !sample->callchain)  in intel_pt_process_event()
3696 			intel_pt_add_callchain(pt, sample);  in intel_pt_process_event()
3697 		if (pt->synth_opts.add_last_branch && !sample->branch_stack)  in intel_pt_process_event()
3698 			intel_pt_add_br_stack(pt, sample);  in intel_pt_process_event()
3701 	if (event->header.type == PERF_RECORD_AUX &&  in intel_pt_process_event()
3702 	    (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&  in intel_pt_process_event()
3703 	    pt->synth_opts.errors) {  in intel_pt_process_event()
3704 		err = intel_pt_lost(pt, sample);  in intel_pt_process_event()
3710 	if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)  in intel_pt_process_event()
3711 		err = intel_pt_process_switch(pt, sample);  in intel_pt_process_event()
3714 	if (event->header.type == PERF_RECORD_ITRACE_START)  in intel_pt_process_event()
3715 		err = intel_pt_process_itrace_start(pt, event, sample);  in intel_pt_process_event()
3716 	else if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID)  in intel_pt_process_event()
3717 		err = intel_pt_process_aux_output_hw_id(pt, event, sample);  in intel_pt_process_event()
3718 	else if (event->header.type == PERF_RECORD_SWITCH ||  in intel_pt_process_event()
3719 		 event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)  in intel_pt_process_event()
3720 		err = intel_pt_context_switch(pt, event, sample);  in intel_pt_process_event()
3722 	if (!err && event->header.type == PERF_RECORD_TEXT_POKE)  in intel_pt_process_event()
3725 	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {  in intel_pt_process_event()
3726 		intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",  in intel_pt_process_event()
3727 			     event->header.type, sample->cpu, sample->time, timestamp);  in intel_pt_process_event()
3736 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_flush()
3743 	if (!tool->ordered_events)  in intel_pt_flush()
3744 		return -EINVAL;  in intel_pt_flush()
3750 	if (pt->timeless_decoding)  in intel_pt_flush()
3751 		return intel_pt_process_timeless_queues(pt, -1,  in intel_pt_flush()
3752 							MAX_TIMESTAMP - 1);  in intel_pt_flush()
3759 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_free_events()
3761 	struct auxtrace_queues *queues = &pt->queues;  in intel_pt_free_events()
3764 	for (i = 0; i < queues->nr_queues; i++) {  in intel_pt_free_events()
3765 		intel_pt_free_queue(queues->queue_array[i].priv);  in intel_pt_free_events()
3766 		queues->queue_array[i].priv = NULL;  in intel_pt_free_events()
3774 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_free()
3777 	auxtrace_heap__free(&pt->heap);  in intel_pt_free()
3779 	session->auxtrace = NULL;  in intel_pt_free()
3781 	thread__put(pt->unknown_thread);  in intel_pt_free()
3782 	addr_filters__exit(&pt->filts);  in intel_pt_free()
3783 	zfree(&pt->chain);  in intel_pt_free()
3784 	zfree(&pt->filter);  in intel_pt_free()
3785 	zfree(&pt->time_ranges);  in intel_pt_free()
3786 	zfree(&pt->br_stack);  in intel_pt_free()
3793 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_evsel_is_auxtrace()
3796 	return evsel->core.attr.type == pt->pmu_type;  in intel_pt_evsel_is_auxtrace()
3803 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_process_auxtrace_event()
3806 	if (!pt->data_queued) {  in intel_pt_process_auxtrace_event()
3809 		int fd = perf_data__fd(session->data);  in intel_pt_process_auxtrace_event()
3812 		if (perf_data__is_pipe(session->data)) {  in intel_pt_process_auxtrace_event()
3816 			if (data_offset == -1)  in intel_pt_process_auxtrace_event()
3817 				return -errno;  in intel_pt_process_auxtrace_event()
3820 		err = auxtrace_queues__add_event(&pt->queues, session, event,  in intel_pt_process_auxtrace_event()
3828 				intel_pt_dump_event(pt, buffer->data,  in intel_pt_process_auxtrace_event()
3829 						    buffer->size);  in intel_pt_process_auxtrace_event()
3839 			       struct perf_sample *sample,  in intel_pt_queue_data()  argument
3842 	struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,  in intel_pt_queue_data()
3847 		return auxtrace_queues__add_event(&pt->queues, session, event,  in intel_pt_queue_data()
3851 	if (sample->time && sample->time != (u64)-1)  in intel_pt_queue_data()
3852 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_queue_data()
3856 	return auxtrace_queues__add_sample(&pt->queues, session, sample,  in intel_pt_queue_data()
3867 				struct perf_sample *sample __maybe_unused,  in intel_pt_event_synth()
3873 	return perf_session__deliver_synth_event(intel_pt_synth->session, event,  in intel_pt_event_synth()
3883 	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",  in intel_pt_synth_event()
3884 		 name, id, (u64)attr->sample_type);  in intel_pt_synth_event()
3904 		if (evsel->core.id && evsel->core.id[0] == id) {  in intel_pt_set_event_name()
3905 			if (evsel->name)  in intel_pt_set_event_name()
3906 				zfree(&evsel->name);  in intel_pt_set_event_name()
3907 			evsel->name = strdup(name);  in intel_pt_set_event_name()
3919 		if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)  in intel_pt_evsel()
3929 	struct evlist *evlist = session->evlist;  in intel_pt_synth_events()
3943 	attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;  in intel_pt_synth_events()
3946 	if (pt->timeless_decoding)  in intel_pt_synth_events()
3950 	if (!pt->per_cpu_mmaps)  in intel_pt_synth_events()
3952 	attr.exclude_user = evsel->core.attr.exclude_user;  in intel_pt_synth_events()
3953 	attr.exclude_kernel = evsel->core.attr.exclude_kernel;  in intel_pt_synth_events()
3954 	attr.exclude_hv = evsel->core.attr.exclude_hv;  in intel_pt_synth_events()
3955 	attr.exclude_host = evsel->core.attr.exclude_host;  in intel_pt_synth_events()
3956 	attr.exclude_guest = evsel->core.attr.exclude_guest;  in intel_pt_synth_events()
3957 	attr.sample_id_all = evsel->core.attr.sample_id_all;  in intel_pt_synth_events()
3958 	attr.read_format = evsel->core.attr.read_format;  in intel_pt_synth_events()
3960 	id = evsel->core.id[0] + 1000000000;  in intel_pt_synth_events()
3964 	if (pt->synth_opts.branches) {  in intel_pt_synth_events()
3971 		pt->sample_branches = true;  in intel_pt_synth_events()
3972 		pt->branches_sample_type = attr.sample_type;  in intel_pt_synth_events()
3973 		pt->branches_id = id;  in intel_pt_synth_events()
3978 	if (pt->synth_opts.callchain)  in intel_pt_synth_events()
3980 	if (pt->synth_opts.last_branch) {  in intel_pt_synth_events()
3983 		 * We don't use the hardware index, but the sample generation  in intel_pt_synth_events()
3990 	if (pt->synth_opts.instructions) {  in intel_pt_synth_events()
3992 		if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)  in intel_pt_synth_events()
3994 				intel_pt_ns_to_ticks(pt, pt->synth_opts.period);  in intel_pt_synth_events()
3996 			attr.sample_period = pt->synth_opts.period;  in intel_pt_synth_events()
4000 		pt->sample_instructions = true;  in intel_pt_synth_events()
4001 		pt->instructions_sample_type = attr.sample_type;  in intel_pt_synth_events()
4002 		pt->instructions_id = id;  in intel_pt_synth_events()
4006 	if (pt->synth_opts.cycles) {  in intel_pt_synth_events()
4008 		if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)  in intel_pt_synth_events()
4010 				intel_pt_ns_to_ticks(pt, pt->synth_opts.period);  in intel_pt_synth_events()
4012 			attr.sample_period = pt->synth_opts.period;  in intel_pt_synth_events()
4016 		pt->sample_cycles = true;  in intel_pt_synth_events()
4017 		pt->cycles_sample_type = attr.sample_type;  in intel_pt_synth_events()
4018 		pt->cycles_id = id;  in intel_pt_synth_events()
4025 	if (pt->synth_opts.transactions) {  in intel_pt_synth_events()
4030 		pt->sample_transactions = true;  in intel_pt_synth_events()
4031 		pt->transactions_sample_type = attr.sample_type;  in intel_pt_synth_events()
4032 		pt->transactions_id = id;  in intel_pt_synth_events()
4040 	if (pt->synth_opts.ptwrites) {  in intel_pt_synth_events()
4045 		pt->sample_ptwrites = true;  in intel_pt_synth_events()
4046 		pt->ptwrites_sample_type = attr.sample_type;  in intel_pt_synth_events()
4047 		pt->ptwrites_id = id;  in intel_pt_synth_events()
4052 	if (pt->synth_opts.pwr_events) {  in intel_pt_synth_events()
4053 		pt->sample_pwr_events = true;  in intel_pt_synth_events()
4054 		pt->pwr_events_sample_type = attr.sample_type;  in intel_pt_synth_events()
4060 		pt->cbr_id = id;  in intel_pt_synth_events()
4068 		pt->psb_id = id;  in intel_pt_synth_events()
4073 	if (pt->synth_opts.pwr_events && (evsel->core.attr.config & INTEL_PT_CFG_PWR_EVT_EN)) {  in intel_pt_synth_events()
4078 		pt->mwait_id = id;  in intel_pt_synth_events()
4086 		pt->pwre_id = id;  in intel_pt_synth_events()
4094 		pt->exstop_id = id;  in intel_pt_synth_events()
4102 		pt->pwrx_id = id;  in intel_pt_synth_events()
4107 	if (pt->synth_opts.intr_events && (evsel->core.attr.config & INTEL_PT_CFG_EVT_EN)) {  in intel_pt_synth_events()
4112 		pt->evt_sample_type = attr.sample_type;  in intel_pt_synth_events()
4113 		pt->evt_id = id;  in intel_pt_synth_events()
4118 	if (pt->synth_opts.intr_events && pt->cap_event_trace) {  in intel_pt_synth_events()
4123 		pt->iflag_chg_sample_type = attr.sample_type;  in intel_pt_synth_events()
4124 		pt->iflag_chg_id = id;  in intel_pt_synth_events()
4136 	if (!pt->synth_opts.other_events)  in intel_pt_setup_pebs_events()
4139 	evlist__for_each_entry(pt->session->evlist, evsel) {  in intel_pt_setup_pebs_events()
4140 		if (evsel->core.attr.aux_output && evsel->core.id) {  in intel_pt_setup_pebs_events()
4141 			if (pt->single_pebs) {  in intel_pt_setup_pebs_events()
4142 				pt->single_pebs = false;  in intel_pt_setup_pebs_events()
4145 			pt->single_pebs = true;  in intel_pt_setup_pebs_events()
4146 			pt->sample_pebs = true;  in intel_pt_setup_pebs_events()
4147 			pt->pebs_data_src_fmt = intel_pt_data_src_fmt(pt, evsel);  in intel_pt_setup_pebs_events()
4148 			pt->pebs_evsel = evsel;  in intel_pt_setup_pebs_events()
4172 		if (evsel->core.attr.context_switch)  in intel_pt_find_switch()
4183 	if (!strcmp(var, "intel-pt.mispred-all"))  in intel_pt_perf_config()
4184 		pt->mispred_all = perf_config_bool(var, value);  in intel_pt_perf_config()
4186 	if (!strcmp(var, "intel-pt.max-loops"))  in intel_pt_perf_config()
4187 		perf_config_int(&pt->max_loops, var, value);  in intel_pt_perf_config()
4197 	tsc = perf_time_to_tsc(ns, &pt->tc);  in intel_pt_tsc_start()
4200 		tm = tsc_to_perf_time(tsc, &pt->tc);  in intel_pt_tsc_start()
4203 		tsc -= 1;  in intel_pt_tsc_start()
4207 		tm = tsc_to_perf_time(++tsc, &pt->tc);  in intel_pt_tsc_start()
4217 	tsc = perf_time_to_tsc(ns, &pt->tc);  in intel_pt_tsc_end()
4220 		tm = tsc_to_perf_time(tsc, &pt->tc);  in intel_pt_tsc_end()
4227 		tm = tsc_to_perf_time(--tsc, &pt->tc);  in intel_pt_tsc_end()
4235 	struct perf_time_interval *p = opts->ptime_range;  in intel_pt_setup_time_ranges()
4236 	int n = opts->range_num;  in intel_pt_setup_time_ranges()
4239 	if (!n || !p || pt->timeless_decoding)  in intel_pt_setup_time_ranges()
4242 	pt->time_ranges = calloc(n, sizeof(struct range));  in intel_pt_setup_time_ranges()
4243 	if (!pt->time_ranges)  in intel_pt_setup_time_ranges()
4244 		return -ENOMEM;  in intel_pt_setup_time_ranges()
4246 	pt->range_cnt = n;  in intel_pt_setup_time_ranges()
4251 		struct range *r = &pt->time_ranges[i];  in intel_pt_setup_time_ranges()
4256 		 * Take care to ensure the TSC range matches the perf-time range  in intel_pt_setup_time_ranges()
4257 		 * when converted back to perf-time.  in intel_pt_setup_time_ranges()
4259 		r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;  in intel_pt_setup_time_ranges()
4260 		r->end   = te ? intel_pt_tsc_end(te, pt) : 0;  in intel_pt_setup_time_ranges()
4262 		intel_pt_log("range %d: perf time interval: %"PRIu64" to %"PRIu64"\n",  in intel_pt_setup_time_ranges()
4264 		intel_pt_log("range %d: TSC time interval: %#"PRIx64" to %#"PRIx64"\n",  in intel_pt_setup_time_ranges()
4265 			     i, r->start, r->end);  in intel_pt_setup_time_ranges()
4285 		return -errno;  in intel_pt_parse_vm_tm_corr_arg()
4288 		pt->dflt_tsc_offset = tsc_offset;  in intel_pt_parse_vm_tm_corr_arg()
4296 			return -errno;  in intel_pt_parse_vm_tm_corr_arg()
4298 			return -EINVAL;  in intel_pt_parse_vm_tm_corr_arg()
4299 		vmcs_info = intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, tsc_offset);  in intel_pt_parse_vm_tm_corr_arg()
4301 			return -ENOMEM;  in intel_pt_parse_vm_tm_corr_arg()
4313 	char *args = pt->synth_opts.vm_tm_corr_args;  in intel_pt_parse_vm_tm_corr_args()
4324 		pr_err("Failed to parse VM Time Correlation options\n");  in intel_pt_parse_vm_tm_corr_args()
4333 	[INTEL_PT_TIME_SHIFT]		= "  Time Shift          %"PRIu64"\n",
4334 	[INTEL_PT_TIME_MULT]		= "  Time Muliplier      %"PRIu64"\n",
4335 	[INTEL_PT_TIME_ZERO]		= "  Time Zero           %"PRIu64"\n",
4336 	[INTEL_PT_CAP_USER_TIME_ZERO]	= "  Cap Time Zero       %"PRId64"\n",
4341 	[INTEL_PT_PER_CPU_MMAPS]	= "  Per-cpu maps        %"PRId64"\n",
4347 	[INTEL_PT_MAX_NONTURBO_RATIO]	= "  Max non-turbo ratio %"PRIu64"\n",
4371 	fprintf(stdout, "  %-20s%s\n", name, str ? str : "");  in intel_pt_print_info_str()
4376 	return auxtrace_info->header.size >=  in intel_pt_has()
4383 	struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;  in intel_pt_process_auxtrace_info()
4390 	if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +  in intel_pt_process_auxtrace_info()
4392 		return -EINVAL;  in intel_pt_process_auxtrace_info()
4396 		return -ENOMEM;  in intel_pt_process_auxtrace_info()
4398 	pt->vmcs_info = RB_ROOT;  in intel_pt_process_auxtrace_info()
4400 	addr_filters__init(&pt->filts);  in intel_pt_process_auxtrace_info()
4406 	err = auxtrace_queues__init(&pt->queues);  in intel_pt_process_auxtrace_info()
4410 	if (session->itrace_synth_opts->set) {  in intel_pt_process_auxtrace_info()
4411 		pt->synth_opts = *session->itrace_synth_opts;  in intel_pt_process_auxtrace_info()
4413 		struct itrace_synth_opts *opts = session->itrace_synth_opts;  in intel_pt_process_auxtrace_info()
4415 		itrace_synth_opts__set_default(&pt->synth_opts, opts->default_no_sample);  in intel_pt_process_auxtrace_info()
4416 		if (!opts->default_no_sample && !opts->inject) {  in intel_pt_process_auxtrace_info()
4417 			pt->synth_opts.branches = false;  in intel_pt_process_auxtrace_info()
4418 			pt->synth_opts.callchain = true;  in intel_pt_process_auxtrace_info()
4419 			pt->synth_opts.add_callchain = true;  in intel_pt_process_auxtrace_info()
4421 		pt->synth_opts.thread_stack = opts->thread_stack;  in intel_pt_process_auxtrace_info()
4424 	if (!(pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT))  in intel_pt_process_auxtrace_info()
4427 	pt->session = session;  in intel_pt_process_auxtrace_info()
4428 	pt->machine = &session->machines.host; /* No kvm support */  in intel_pt_process_auxtrace_info()
4429 	pt->auxtrace_type = auxtrace_info->type;  in intel_pt_process_auxtrace_info()
4430 	pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];  in intel_pt_process_auxtrace_info()
4431 	pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];  in intel_pt_process_auxtrace_info()
4432 	pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];  in intel_pt_process_auxtrace_info()
4433 	pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];  in intel_pt_process_auxtrace_info()
4434 	pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];  in intel_pt_process_auxtrace_info()
4435 	pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];  in intel_pt_process_auxtrace_info()
4436 	pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];  in intel_pt_process_auxtrace_info()
4437 	pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];  in intel_pt_process_auxtrace_info()
4438 	pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];  in intel_pt_process_auxtrace_info()
4439 	pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];  in intel_pt_process_auxtrace_info()
4440 	intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,  in intel_pt_process_auxtrace_info()
4444 		pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];  in intel_pt_process_auxtrace_info()
4445 		pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];  in intel_pt_process_auxtrace_info()
4446 		pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];  in intel_pt_process_auxtrace_info()
4447 		pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];  in intel_pt_process_auxtrace_info()
4448 		pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];  in intel_pt_process_auxtrace_info()
4449 		intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,  in intel_pt_process_auxtrace_info()
4454 		pt->max_non_turbo_ratio =  in intel_pt_process_auxtrace_info()
4455 			auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];  in intel_pt_process_auxtrace_info()
4456 		intel_pt_print_info(&auxtrace_info->priv[0],  in intel_pt_process_auxtrace_info()
4461 	info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;  in intel_pt_process_auxtrace_info()
4462 	info_end = (void *)auxtrace_info + auxtrace_info->header.size;  in intel_pt_process_auxtrace_info()
4467 		len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];  in intel_pt_process_auxtrace_info()
4468 		intel_pt_print_info(&auxtrace_info->priv[0],  in intel_pt_process_auxtrace_info()
4478 				err = -EINVAL;  in intel_pt_process_auxtrace_info()
4481 			pt->filter = memdup(filter, len);  in intel_pt_process_auxtrace_info()
4482 			if (!pt->filter) {  in intel_pt_process_auxtrace_info()
4483 				err = -ENOMEM;  in intel_pt_process_auxtrace_info()
4486 			if (session->header.needs_swap)  in intel_pt_process_auxtrace_info()
4487 				mem_bswap_64(pt->filter, len);  in intel_pt_process_auxtrace_info()
4488 			if (pt->filter[len - 1]) {  in intel_pt_process_auxtrace_info()
4490 				err = -EINVAL;  in intel_pt_process_auxtrace_info()
4493 			err = addr_filters__parse_bare_filter(&pt->filts,  in intel_pt_process_auxtrace_info()
4498 		intel_pt_print_info_str("Filter string", pt->filter);  in intel_pt_process_auxtrace_info()
4502 		pt->cap_event_trace = *info++;  in intel_pt_process_auxtrace_info()
4505 				pt->cap_event_trace);  in intel_pt_process_auxtrace_info()
4508 	pt->timeless_decoding = intel_pt_timeless_decoding(pt);  in intel_pt_process_auxtrace_info()
4509 	if (pt->timeless_decoding && !pt->tc.time_mult)  in intel_pt_process_auxtrace_info()
4510 		pt->tc.time_mult = 1;  in intel_pt_process_auxtrace_info()
4511 	pt->have_tsc = intel_pt_have_tsc(pt);  in intel_pt_process_auxtrace_info()
4512 	pt->sampling_mode = intel_pt_sampling_mode(pt);  in intel_pt_process_auxtrace_info()
4513 	pt->est_tsc = !pt->timeless_decoding;  in intel_pt_process_auxtrace_info()
4515 	if (pt->synth_opts.vm_time_correlation) {  in intel_pt_process_auxtrace_info()
4516 		if (pt->timeless_decoding) {  in intel_pt_process_auxtrace_info()
4517 			pr_err("Intel PT has no time information for VM Time Correlation\n");  in intel_pt_process_auxtrace_info()
4518 			err = -EINVAL;  in intel_pt_process_auxtrace_info()
4521 		if (session->itrace_synth_opts->ptime_range) {  in intel_pt_process_auxtrace_info()
4522 			pr_err("Time ranges cannot be specified with VM Time Correlation\n");  in intel_pt_process_auxtrace_info()
4523 			err = -EINVAL;  in intel_pt_process_auxtrace_info()
4528 			pr_err("MTC packets must have been enabled for VM Time Correlation\n");  in intel_pt_process_auxtrace_info()
4529 			err = -EINVAL;  in intel_pt_process_auxtrace_info()
4537 	pt->unknown_thread = thread__new(999999999, 999999999);  in intel_pt_process_auxtrace_info()
4538 	if (!pt->unknown_thread) {  in intel_pt_process_auxtrace_info()
4539 		err = -ENOMEM;  in intel_pt_process_auxtrace_info()
4543 	err = thread__set_comm(pt->unknown_thread, "unknown", 0);  in intel_pt_process_auxtrace_info()
4546 	if (thread__init_maps(pt->unknown_thread, pt->machine)) {  in intel_pt_process_auxtrace_info()
4547 		err = -ENOMEM;  in intel_pt_process_auxtrace_info()
4551 	pt->auxtrace.process_event = intel_pt_process_event;  in intel_pt_process_auxtrace_info()
4552 	pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;  in intel_pt_process_auxtrace_info()
4553 	pt->auxtrace.queue_data = intel_pt_queue_data;  in intel_pt_process_auxtrace_info()
4554 	pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;  in intel_pt_process_auxtrace_info()
4555 	pt->auxtrace.flush_events = intel_pt_flush;  in intel_pt_process_auxtrace_info()
4556 	pt->auxtrace.free_events = intel_pt_free_events;  in intel_pt_process_auxtrace_info()
4557 	pt->auxtrace.free = intel_pt_free;  in intel_pt_process_auxtrace_info()
4558 	pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace;  in intel_pt_process_auxtrace_info()
4559 	session->auxtrace = &pt->auxtrace;  in intel_pt_process_auxtrace_info()
4564 	if (pt->have_sched_switch == 1) {  in intel_pt_process_auxtrace_info()
4565 		pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);  in intel_pt_process_auxtrace_info()
4566 		if (!pt->switch_evsel) {  in intel_pt_process_auxtrace_info()
4568 			err = -EINVAL;  in intel_pt_process_auxtrace_info()
4571 	} else if (pt->have_sched_switch == 2 &&  in intel_pt_process_auxtrace_info()
4572 		   !intel_pt_find_switch(session->evlist)) {  in intel_pt_process_auxtrace_info()
4574 		err = -EINVAL;  in intel_pt_process_auxtrace_info()
4578 	if (pt->synth_opts.log) {  in intel_pt_process_auxtrace_info()
4579 		bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR;  in intel_pt_process_auxtrace_info()
4580 		unsigned int log_on_error_size = pt->synth_opts.log_on_error_size;  in intel_pt_process_auxtrace_info()
4585 	/* Maximum non-turbo ratio is TSC freq / 100 MHz */  in intel_pt_process_auxtrace_info()
4586 	if (pt->tc.time_mult) {  in intel_pt_process_auxtrace_info()
4589 		if (!pt->max_non_turbo_ratio)  in intel_pt_process_auxtrace_info()
4590 			pt->max_non_turbo_ratio =  in intel_pt_process_auxtrace_info()
4593 		intel_pt_log("Maximum non-turbo ratio %u\n",  in intel_pt_process_auxtrace_info()
4594 			     pt->max_non_turbo_ratio);  in intel_pt_process_auxtrace_info()
4595 		pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;  in intel_pt_process_auxtrace_info()
4598 	err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);  in intel_pt_process_auxtrace_info()
4602 	if (pt->synth_opts.calls)  in intel_pt_process_auxtrace_info()
4603 		pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |  in intel_pt_process_auxtrace_info()
4605 	if (pt->synth_opts.returns)  in intel_pt_process_auxtrace_info()
4606 		pt->branches_filter |= PERF_IP_FLAG_RETURN |  in intel_pt_process_auxtrace_info()
4609 	if ((pt->synth_opts.callchain || pt->synth_opts.add_callchain) &&  in intel_pt_process_auxtrace_info()
4614 			pt->synth_opts.callchain = false;  in intel_pt_process_auxtrace_info()
4615 			pt->synth_opts.add_callchain = false;  in intel_pt_process_auxtrace_info()
4619 	if (pt->synth_opts.add_callchain) {  in intel_pt_process_auxtrace_info()
4625 	if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch) {  in intel_pt_process_auxtrace_info()
4626 		pt->br_stack_sz = pt->synth_opts.last_branch_sz;  in intel_pt_process_auxtrace_info()
4627 		pt->br_stack_sz_plus = pt->br_stack_sz;  in intel_pt_process_auxtrace_info()
4630 	if (pt->synth_opts.add_last_branch) {  in intel_pt_process_auxtrace_info()
4636 		 * actual sample ip to where the sample time is recorded.  in intel_pt_process_auxtrace_info()
4642 			pt->br_stack_sz_plus += 1024;  in intel_pt_process_auxtrace_info()
4644 			pt->br_stack_sz_plus += 1;  in intel_pt_process_auxtrace_info()
4647 	pt->use_thread_stack = pt->synth_opts.callchain ||  in intel_pt_process_auxtrace_info()
4648 			       pt->synth_opts.add_callchain ||  in intel_pt_process_auxtrace_info()
4649 			       pt->synth_opts.thread_stack ||  in intel_pt_process_auxtrace_info()
4650 			       pt->synth_opts.last_branch ||  in intel_pt_process_auxtrace_info()
4651 			       pt->synth_opts.add_last_branch;  in intel_pt_process_auxtrace_info()
4653 	pt->callstack = pt->synth_opts.callchain ||  in intel_pt_process_auxtrace_info()
4654 			pt->synth_opts.add_callchain ||  in intel_pt_process_auxtrace_info()
4655 			pt->synth_opts.thread_stack;  in intel_pt_process_auxtrace_info()
4663 	if (perf_data__is_pipe(session->data)) {  in intel_pt_process_auxtrace_info()
4669 	if (pt->sampling_mode || list_empty(&session->auxtrace_index))  in intel_pt_process_auxtrace_info()
4672 		err = auxtrace_queues__process_index(&pt->queues, session);  in intel_pt_process_auxtrace_info()
4676 	if (pt->queues.populated)  in intel_pt_process_auxtrace_info()
4677 		pt->data_queued = true;  in intel_pt_process_auxtrace_info()
4679 	if (pt->timeless_decoding)  in intel_pt_process_auxtrace_info()
4685 	zfree(&pt->chain);  in intel_pt_process_auxtrace_info()
4686 	thread__zput(pt->unknown_thread);  in intel_pt_process_auxtrace_info()
4689 	auxtrace_queues__free(&pt->queues);  in intel_pt_process_auxtrace_info()
4690 	session->auxtrace = NULL;  in intel_pt_process_auxtrace_info()
4692 	addr_filters__exit(&pt->filts);  in intel_pt_process_auxtrace_info()
4693 	zfree(&pt->filter);  in intel_pt_process_auxtrace_info()
4694 	zfree(&pt->time_ranges);  in intel_pt_process_auxtrace_info()