28 static DEFINE_PER_CPU(struct pt, pt_ctx);
102 PMU_FORMAT_ATTR(pt,		"config:0"	);
400 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_start()  local
404 	if (READ_ONCE(pt->vmx_on))  in pt_config_start()
405 		perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL);  in pt_config_start()
443 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_filters()  local
465 		if (pt->filters.filter[range].msr_a != filter->msr_a) {  in pt_config_filters()
467 			pt->filters.filter[range].msr_a = filter->msr_a;  in pt_config_filters()
470 		if (pt->filters.filter[range].msr_b != filter->msr_b) {  in pt_config_filters()
472 			pt->filters.filter[range].msr_b = filter->msr_b;  in pt_config_filters()
483 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config()  local
484 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_config()
524 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_stop()  local
532 	if (!READ_ONCE(pt->vmx_on))  in pt_config_stop()
607 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_buffer()  local
620 	if (pt->output_base != reg) {  in pt_config_buffer()
621 		pt->output_base = reg;  in pt_config_buffer()
626 	if (pt->output_mask != reg) {  in pt_config_buffer()
627 		pt->output_mask = reg;  in pt_config_buffer()
827 static void pt_update_head(struct pt *pt)  in pt_update_head()  argument
829 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()
878 static void pt_handle_status(struct pt *pt)  in pt_handle_status()  argument
880 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()
902 			perf_aux_output_flag(&pt->handle,  in pt_handle_status()
938 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_read_offset()  local
942 		rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);  in pt_read_offset()
943 		tp = phys_to_virt(pt->output_base);  in pt_read_offset()
947 	rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);  in pt_read_offset()
949 	buf->output_off = pt->output_mask >> 32;  in pt_read_offset()
952 		buf->cur_idx = (pt->output_mask & 0xffffff80) >> 7;  in pt_read_offset()
1422 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_interrupt()  local
1424 	struct perf_event *event = pt->handle.event;  in intel_pt_interrupt()
1431 	if (!READ_ONCE(pt->handle_nmi))  in intel_pt_interrupt()
1439 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
1445 	pt_handle_status(pt);  in intel_pt_interrupt()
1447 	pt_update_head(pt);  in intel_pt_interrupt()
1449 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in intel_pt_interrupt()
1454 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
1460 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
1462 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
1464 			perf_aux_output_end(&pt->handle, 0);  in intel_pt_interrupt()
1475 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_handle_vmx()  local
1490 	WRITE_ONCE(pt->vmx_on, on);  in intel_pt_handle_vmx()
1496 	event = pt->handle.event;  in intel_pt_handle_vmx()
1498 		perf_aux_output_flag(&pt->handle,  in intel_pt_handle_vmx()
1516 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_start()  local
1519 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_start()
1523 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_start()
1525 		if (pt_buffer_reset_markers(buf, &pt->handle))  in pt_event_start()
1529 	WRITE_ONCE(pt->handle_nmi, 1);  in pt_event_start()
1538 	perf_aux_output_end(&pt->handle, 0);  in pt_event_start()
1545 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_stop()  local
1551 	WRITE_ONCE(pt->handle_nmi, 0);  in pt_event_stop()
1561 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()
1566 		if (WARN_ON_ONCE(pt->handle.event != event))  in pt_event_stop()
1571 		pt_handle_status(pt);  in pt_event_stop()
1573 		pt_update_head(pt);  in pt_event_stop()
1576 			pt->handle.head =  in pt_event_stop()
1579 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in pt_event_stop()
1587 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_snapshot_aux()  local
1588 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_snapshot_aux()
1605 	if (READ_ONCE(pt->handle_nmi))  in pt_event_snapshot_aux()
1609 	pt_update_head(pt);  in pt_event_snapshot_aux()
1616 	ret = perf_output_copy_aux(&pt->handle, handle, from, to);  in pt_event_snapshot_aux()
1623 	if (pt->handle_nmi)  in pt_event_snapshot_aux()
1636 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_add()  local
1640 	if (pt->handle.event)  in pt_event_add()
1691 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in cpu_emergency_stop_pt()  local
1693 	if (pt->handle.event)  in cpu_emergency_stop_pt()
1694 		pt_event_stop(pt->handle.event, PERF_EF_UPDATE);  in cpu_emergency_stop_pt()