3  * Arm Statistical Profiling Extensions (SPE) support
32 #include "arm-spe.h"
33 #include "arm-spe-decoder/arm-spe-decoder.h"
34 #include "arm-spe-decoder/arm-spe-pkt-decoder.h"
73 	struct arm_spe			*spe;  member
89 static void arm_spe_dump(struct arm_spe *spe __maybe_unused,  in arm_spe_dump()
99 		      ". ... ARM SPE data: size %zu bytes\n",  in arm_spe_dump()
128 static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf,  in arm_spe_dump_event()  argument
132 	arm_spe_dump(spe, buf, len);  in arm_spe_dump_event()
142 	queue = &speq->spe->queues.queue_array[speq->queue_nr];  in arm_spe_get_trace()
158 		int fd = perf_data__fd(speq->spe->session->data);  in arm_spe_get_trace()
180 static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe,  in arm_spe__alloc_queue()  argument
194 	speq->spe = spe;  in arm_spe__alloc_queue()
218 static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip)  in arm_spe_cpumode()  argument
220 	return ip >= spe->kernel_start ?  in arm_spe_cpumode()
225 static void arm_spe_prep_sample(struct arm_spe *spe,  in arm_spe_prep_sample()  argument
232 	if (!spe->timeless_decoding)  in arm_spe_prep_sample()
236 	sample->cpumode = arm_spe_cpumode(spe, sample->ip);  in arm_spe_prep_sample()
255 arm_spe_deliver_synth_event(struct arm_spe *spe,  in arm_spe_deliver_synth_event()  argument
262 	if (spe->synth_opts.inject) {  in arm_spe_deliver_synth_event()
263 		ret = arm_spe__inject_event(event, sample, spe->sample_type);  in arm_spe_deliver_synth_event()
268 	ret = perf_session__deliver_synth_event(spe->session, event, sample);  in arm_spe_deliver_synth_event()
270 		pr_err("ARM SPE: failed to deliver event, error %d\n", ret);  in arm_spe_deliver_synth_event()
279 	struct arm_spe *spe = speq->spe;  in arm_spe_synth_spe_events_sample()  local
283 	arm_spe_prep_sample(spe, speq, event, &sample);  in arm_spe_synth_spe_events_sample()
288 	return arm_spe_deliver_synth_event(spe, speq, event, &sample);  in arm_spe_synth_spe_events_sample()
294 	struct arm_spe *spe = speq->spe;  in arm_spe_sample()  local
297 	if (spe->sample_flc) {  in arm_spe_sample()
300 					speq, spe->l1d_miss_id);  in arm_spe_sample()
307 					speq, spe->l1d_access_id);  in arm_spe_sample()
313 	if (spe->sample_llc) {  in arm_spe_sample()
316 					speq, spe->llc_miss_id);  in arm_spe_sample()
323 					speq, spe->llc_access_id);  in arm_spe_sample()
329 	if (spe->sample_tlb) {  in arm_spe_sample()
332 					speq, spe->tlb_miss_id);  in arm_spe_sample()
339 					speq, spe->tlb_access_id);  in arm_spe_sample()
345 	if (spe->sample_branch && (record->type & ARM_SPE_BRANCH_MISS)) {  in arm_spe_sample()
347 						      spe->branch_miss_id);  in arm_spe_sample()
352 	if (spe->sample_remote_access &&  in arm_spe_sample()
355 						      spe->remote_access_id);  in arm_spe_sample()
365 	struct arm_spe *spe = speq->spe;  in arm_spe_run_decoder()  local
368 	if (!spe->kernel_start)  in arm_spe_run_decoder()
369 		spe->kernel_start = machine__kernel_start(spe->machine);  in arm_spe_run_decoder()
379 		 * Error is detected when decode SPE trace data, continue to  in arm_spe_run_decoder()
389 		if (!spe->timeless_decoding && speq->timestamp >= *timestamp) {  in arm_spe_run_decoder()
398 static int arm_spe__setup_queue(struct arm_spe *spe,  in arm_spe__setup_queue()  argument
408 	speq = arm_spe__alloc_queue(spe, queue_nr);  in arm_spe__setup_queue()
421 		if (spe->timeless_decoding)  in arm_spe__setup_queue()
436 		ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp);  in arm_spe__setup_queue()
445 static int arm_spe__setup_queues(struct arm_spe *spe)  in arm_spe__setup_queues()  argument
450 	for (i = 0; i < spe->queues.nr_queues; i++) {  in arm_spe__setup_queues()
451 		ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i);  in arm_spe__setup_queues()
459 static int arm_spe__update_queues(struct arm_spe *spe)  in arm_spe__update_queues()  argument
461 	if (spe->queues.new_data) {  in arm_spe__update_queues()
462 		spe->queues.new_data = false;  in arm_spe__update_queues()
463 		return arm_spe__setup_queues(spe);  in arm_spe__update_queues()
469 static bool arm_spe__is_timeless_decoding(struct arm_spe *spe)  in arm_spe__is_timeless_decoding()  argument
472 	struct evlist *evlist = spe->session->evlist;  in arm_spe__is_timeless_decoding()
487 static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe,  in arm_spe_set_pid_tid_cpu()  argument
493 	tid = machine__get_current_tid(spe->machine, speq->cpu);  in arm_spe_set_pid_tid_cpu()
501 		speq->thread = machine__find_thread(spe->machine, -1,  in arm_spe_set_pid_tid_cpu()
512 static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp)  in arm_spe_process_queues()  argument
522 		if (!spe->heap.heap_cnt)  in arm_spe_process_queues()
525 		if (spe->heap.heap_array[0].ordinal >= timestamp)  in arm_spe_process_queues()
528 		queue_nr = spe->heap.heap_array[0].queue_nr;  in arm_spe_process_queues()
529 		queue = &spe->queues.queue_array[queue_nr];  in arm_spe_process_queues()
532 		auxtrace_heap__pop(&spe->heap);  in arm_spe_process_queues()
534 		if (spe->heap.heap_cnt) {  in arm_spe_process_queues()
535 			ts = spe->heap.heap_array[0].ordinal + 1;  in arm_spe_process_queues()
542 		arm_spe_set_pid_tid_cpu(spe, queue);  in arm_spe_process_queues()
546 			auxtrace_heap__add(&spe->heap, queue_nr, ts);  in arm_spe_process_queues()
551 			ret = auxtrace_heap__add(&spe->heap, queue_nr, ts);  in arm_spe_process_queues()
562 static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid,  in arm_spe_process_timeless_queues()  argument
565 	struct auxtrace_queues *queues = &spe->queues;  in arm_spe_process_timeless_queues()
570 		struct auxtrace_queue *queue = &spe->queues.queue_array[i];  in arm_spe_process_timeless_queues()
575 			arm_spe_set_pid_tid_cpu(spe, queue);  in arm_spe_process_timeless_queues()
589 	struct arm_spe *spe = container_of(session->auxtrace,  in arm_spe_process_event()  local
596 		pr_err("SPE trace requires ordered events\n");  in arm_spe_process_event()
605 	if (timestamp || spe->timeless_decoding) {  in arm_spe_process_event()
606 		err = arm_spe__update_queues(spe);  in arm_spe_process_event()
611 	if (spe->timeless_decoding) {  in arm_spe_process_event()
613 			err = arm_spe_process_timeless_queues(spe,  in arm_spe_process_event()
619 			err = arm_spe_process_queues(spe, timestamp);  in arm_spe_process_event()
632 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_process_auxtrace_event()  local
635 	if (!spe->data_queued) {  in arm_spe_process_auxtrace_event()
649 		err = auxtrace_queues__add_event(&spe->queues, session, event,  in arm_spe_process_auxtrace_event()
657 				arm_spe_dump_event(spe, buffer->data,  in arm_spe_process_auxtrace_event()
670 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_flush()  local
680 	ret = arm_spe__update_queues(spe);  in arm_spe_flush()
684 	if (spe->timeless_decoding)  in arm_spe_flush()
685 		return arm_spe_process_timeless_queues(spe, -1,  in arm_spe_flush()
688 	return arm_spe_process_queues(spe, MAX_TIMESTAMP);  in arm_spe_flush()
705 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_free_events()  local
707 	struct auxtrace_queues *queues = &spe->queues;  in arm_spe_free_events()
719 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe,  in arm_spe_free()  local
722 	auxtrace_heap__free(&spe->heap);  in arm_spe_free()
725 	free(spe);  in arm_spe_free()
731 	struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace);  in arm_spe_evsel_is_auxtrace()  local
733 	return evsel->core.attr.type == spe->pmu_type;  in arm_spe_evsel_is_auxtrace()
793 arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session)  in arm_spe_synth_events()  argument
803 		if (evsel->core.attr.type == spe->pmu_type) {  in arm_spe_synth_events()
810 		pr_debug("No selected events with SPE trace data\n");  in arm_spe_synth_events()
820 	if (spe->timeless_decoding)  in arm_spe_synth_events()
825 	spe->sample_type = attr.sample_type;  in arm_spe_synth_events()
841 	if (spe->synth_opts.flc) {  in arm_spe_synth_events()
842 		spe->sample_flc = true;  in arm_spe_synth_events()
848 		spe->l1d_miss_id = id;  in arm_spe_synth_events()
856 		spe->l1d_access_id = id;  in arm_spe_synth_events()
861 	if (spe->synth_opts.llc) {  in arm_spe_synth_events()
862 		spe->sample_llc = true;  in arm_spe_synth_events()
868 		spe->llc_miss_id = id;  in arm_spe_synth_events()
876 		spe->llc_access_id = id;  in arm_spe_synth_events()
881 	if (spe->synth_opts.tlb) {  in arm_spe_synth_events()
882 		spe->sample_tlb = true;  in arm_spe_synth_events()
888 		spe->tlb_miss_id = id;  in arm_spe_synth_events()
896 		spe->tlb_access_id = id;  in arm_spe_synth_events()
901 	if (spe->synth_opts.branches) {  in arm_spe_synth_events()
902 		spe->sample_branch = true;  in arm_spe_synth_events()
908 		spe->branch_miss_id = id;  in arm_spe_synth_events()
913 	if (spe->synth_opts.remote_access) {  in arm_spe_synth_events()
914 		spe->sample_remote_access = true;  in arm_spe_synth_events()
920 		spe->remote_access_id = id;  in arm_spe_synth_events()
933 	struct arm_spe *spe;  in arm_spe_process_auxtrace_info()  local
940 	spe = zalloc(sizeof(struct arm_spe));  in arm_spe_process_auxtrace_info()
941 	if (!spe)  in arm_spe_process_auxtrace_info()
944 	err = auxtrace_queues__init(&spe->queues);  in arm_spe_process_auxtrace_info()
948 	spe->session = session;  in arm_spe_process_auxtrace_info()
949 	spe->machine = &session->machines.host; /* No kvm support */  in arm_spe_process_auxtrace_info()
950 	spe->auxtrace_type = auxtrace_info->type;  in arm_spe_process_auxtrace_info()
951 	spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE];  in arm_spe_process_auxtrace_info()
953 	spe->timeless_decoding = arm_spe__is_timeless_decoding(spe);  in arm_spe_process_auxtrace_info()
954 	spe->auxtrace.process_event = arm_spe_process_event;  in arm_spe_process_auxtrace_info()
955 	spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event;  in arm_spe_process_auxtrace_info()
956 	spe->auxtrace.flush_events = arm_spe_flush;  in arm_spe_process_auxtrace_info()
957 	spe->auxtrace.free_events = arm_spe_free_events;  in arm_spe_process_auxtrace_info()
958 	spe->auxtrace.free = arm_spe_free;  in arm_spe_process_auxtrace_info()
959 	spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace;  in arm_spe_process_auxtrace_info()
960 	session->auxtrace = &spe->auxtrace;  in arm_spe_process_auxtrace_info()
968 		spe->synth_opts = *session->itrace_synth_opts;  in arm_spe_process_auxtrace_info()
970 		itrace_synth_opts__set_default(&spe->synth_opts, false);  in arm_spe_process_auxtrace_info()
972 	err = arm_spe_synth_events(spe, session);  in arm_spe_process_auxtrace_info()
976 	err = auxtrace_queues__process_index(&spe->queues, session);  in arm_spe_process_auxtrace_info()
980 	if (spe->queues.populated)  in arm_spe_process_auxtrace_info()
981 		spe->data_queued = true;  in arm_spe_process_auxtrace_info()
986 	auxtrace_queues__free(&spe->queues);  in arm_spe_process_auxtrace_info()
989 	free(spe);  in arm_spe_process_auxtrace_info()