Lines Matching full:decoder
31 #include "intel-pt-insn-decoder.h"
32 #include "intel-pt-pkt-decoder.h"
33 #include "intel-pt-decoder.h"
95 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
197 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) in intel_pt_setup_period() argument
199 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { in intel_pt_setup_period()
202 period = intel_pt_lower_power_of_2(decoder->period); in intel_pt_setup_period()
203 decoder->period_mask = ~(period - 1); in intel_pt_setup_period()
204 decoder->period_ticks = period; in intel_pt_setup_period()
217 struct intel_pt_decoder *decoder; in intel_pt_decoder_new() local
222 decoder = zalloc(sizeof(struct intel_pt_decoder)); in intel_pt_decoder_new()
223 if (!decoder) in intel_pt_decoder_new()
226 decoder->get_trace = params->get_trace; in intel_pt_decoder_new()
227 decoder->walk_insn = params->walk_insn; in intel_pt_decoder_new()
228 decoder->pgd_ip = params->pgd_ip; in intel_pt_decoder_new()
229 decoder->data = params->data; in intel_pt_decoder_new()
230 decoder->return_compression = params->return_compression; in intel_pt_decoder_new()
231 decoder->branch_enable = params->branch_enable; in intel_pt_decoder_new()
233 decoder->flags = params->flags; in intel_pt_decoder_new()
235 decoder->period = params->period; in intel_pt_decoder_new()
236 decoder->period_type = params->period_type; in intel_pt_decoder_new()
238 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; in intel_pt_decoder_new()
239 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; in intel_pt_decoder_new()
241 intel_pt_setup_period(decoder); in intel_pt_decoder_new()
243 decoder->mtc_shift = params->mtc_period; in intel_pt_decoder_new()
244 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; in intel_pt_decoder_new()
246 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; in intel_pt_decoder_new()
247 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; in intel_pt_decoder_new()
249 if (!decoder->tsc_ctc_ratio_n) in intel_pt_decoder_new()
250 decoder->tsc_ctc_ratio_d = 0; in intel_pt_decoder_new()
252 if (decoder->tsc_ctc_ratio_d) { in intel_pt_decoder_new()
253 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) in intel_pt_decoder_new()
254 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / in intel_pt_decoder_new()
255 decoder->tsc_ctc_ratio_d; in intel_pt_decoder_new()
264 decoder->tsc_slip = 0x10000; in intel_pt_decoder_new()
266 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); in intel_pt_decoder_new()
267 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); in intel_pt_decoder_new()
268 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); in intel_pt_decoder_new()
269 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); in intel_pt_decoder_new()
270 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); in intel_pt_decoder_new()
272 return decoder; in intel_pt_decoder_new()
346 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) in intel_pt_decoder_free() argument
348 intel_pt_free_stack(&decoder->stack); in intel_pt_decoder_free()
349 free(decoder); in intel_pt_decoder_free()
433 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) in intel_pt_set_last_ip() argument
435 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); in intel_pt_set_last_ip()
436 decoder->have_last_ip = true; in intel_pt_set_last_ip()
439 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) in intel_pt_set_ip() argument
441 intel_pt_set_last_ip(decoder); in intel_pt_set_ip()
442 decoder->ip = decoder->last_ip; in intel_pt_set_ip()
445 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) in intel_pt_decoder_log_packet() argument
447 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, in intel_pt_decoder_log_packet()
448 decoder->buf); in intel_pt_decoder_log_packet()
451 static int intel_pt_bug(struct intel_pt_decoder *decoder) in intel_pt_bug() argument
454 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_bug()
458 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) in intel_pt_clear_tx_flags() argument
460 decoder->tx_flags = 0; in intel_pt_clear_tx_flags()
463 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) in intel_pt_update_in_tx() argument
465 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; in intel_pt_update_in_tx()
468 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) in intel_pt_bad_packet() argument
470 intel_pt_clear_tx_flags(decoder); in intel_pt_bad_packet()
471 decoder->have_tma = false; in intel_pt_bad_packet()
472 decoder->pkt_len = 1; in intel_pt_bad_packet()
473 decoder->pkt_step = 1; in intel_pt_bad_packet()
474 intel_pt_decoder_log_packet(decoder); in intel_pt_bad_packet()
475 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { in intel_pt_bad_packet()
477 decoder->pkt_state = INTEL_PT_STATE_ERR1; in intel_pt_bad_packet()
482 static int intel_pt_get_data(struct intel_pt_decoder *decoder) in intel_pt_get_data() argument
487 decoder->pkt_step = 0; in intel_pt_get_data()
490 ret = decoder->get_trace(&buffer, decoder->data); in intel_pt_get_data()
493 decoder->buf = buffer.buf; in intel_pt_get_data()
494 decoder->len = buffer.len; in intel_pt_get_data()
495 if (!decoder->len) { in intel_pt_get_data()
500 decoder->ip = 0; in intel_pt_get_data()
501 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_get_data()
502 decoder->ref_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
503 decoder->timestamp = 0; in intel_pt_get_data()
504 decoder->have_tma = false; in intel_pt_get_data()
505 decoder->state.trace_nr = buffer.trace_nr; in intel_pt_get_data()
507 decoder->ref_timestamp); in intel_pt_get_data()
514 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder) in intel_pt_get_next_data() argument
516 if (!decoder->next_buf) in intel_pt_get_next_data()
517 return intel_pt_get_data(decoder); in intel_pt_get_next_data()
519 decoder->buf = decoder->next_buf; in intel_pt_get_next_data()
520 decoder->len = decoder->next_len; in intel_pt_get_next_data()
521 decoder->next_buf = 0; in intel_pt_get_next_data()
522 decoder->next_len = 0; in intel_pt_get_next_data()
526 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) in intel_pt_get_split_packet() argument
528 unsigned char *buf = decoder->temp_buf; in intel_pt_get_split_packet()
532 old_len = decoder->len; in intel_pt_get_split_packet()
533 len = decoder->len; in intel_pt_get_split_packet()
534 memcpy(buf, decoder->buf, len); in intel_pt_get_split_packet()
536 ret = intel_pt_get_data(decoder); in intel_pt_get_split_packet()
538 decoder->pos += old_len; in intel_pt_get_split_packet()
543 if (n > decoder->len) in intel_pt_get_split_packet()
544 n = decoder->len; in intel_pt_get_split_packet()
545 memcpy(buf + len, decoder->buf, n); in intel_pt_get_split_packet()
548 ret = intel_pt_get_packet(buf, len, &decoder->packet); in intel_pt_get_split_packet()
550 decoder->next_buf = decoder->buf; in intel_pt_get_split_packet()
551 decoder->next_len = decoder->len; in intel_pt_get_split_packet()
552 decoder->buf = buf; in intel_pt_get_split_packet()
553 decoder->len = old_len; in intel_pt_get_split_packet()
554 return intel_pt_bad_packet(decoder); in intel_pt_get_split_packet()
557 decoder->next_buf = decoder->buf + (ret - old_len); in intel_pt_get_split_packet()
558 decoder->next_len = decoder->len - (ret - old_len); in intel_pt_get_split_packet()
560 decoder->buf = buf; in intel_pt_get_split_packet()
561 decoder->len = ret; in intel_pt_get_split_packet()
567 struct intel_pt_decoder *decoder; member
578 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, in intel_pt_pkt_lookahead() argument
582 const unsigned char *buf = decoder->buf; in intel_pt_pkt_lookahead()
583 size_t len = decoder->len; in intel_pt_pkt_lookahead()
586 pkt_info.decoder = decoder; in intel_pt_pkt_lookahead()
587 pkt_info.pos = decoder->pos; in intel_pt_pkt_lookahead()
588 pkt_info.pkt_len = decoder->pkt_step; in intel_pt_pkt_lookahead()
589 pkt_info.last_packet_type = decoder->last_packet_type; in intel_pt_pkt_lookahead()
654 struct intel_pt_decoder *decoder = pkt_info->decoder; in intel_pt_calc_cyc_cb() local
683 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { in intel_pt_calc_cyc_cb()
685 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_cyc_cb()
692 data->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_cyc_cb()
695 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
697 data->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
701 decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
702 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
725 data->timestamp - timestamp < decoder->tsc_slip) in intel_pt_calc_cyc_cb()
742 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_cyc_cb()
747 ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_cyc_cb()
749 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_cyc_cb()
752 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
753 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
756 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
757 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
775 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cyc_cb()
791 if (!data->cbr && decoder->cbr) { in intel_pt_calc_cyc_cb()
792 data->cbr = decoder->cbr; in intel_pt_calc_cyc_cb()
793 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_cb()
799 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; in intel_pt_calc_cyc_cb()
808 decoder->calc_cyc_to_tsc = cyc_to_tsc; in intel_pt_calc_cyc_cb()
809 decoder->have_calc_cyc_to_tsc = true; in intel_pt_calc_cyc_cb()
822 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, in intel_pt_calc_cyc_to_tsc() argument
828 .last_mtc = decoder->last_mtc, in intel_pt_calc_cyc_to_tsc()
829 .ctc_timestamp = decoder->ctc_timestamp, in intel_pt_calc_cyc_to_tsc()
830 .ctc_delta = decoder->ctc_delta, in intel_pt_calc_cyc_to_tsc()
831 .tsc_timestamp = decoder->tsc_timestamp, in intel_pt_calc_cyc_to_tsc()
832 .timestamp = decoder->timestamp, in intel_pt_calc_cyc_to_tsc()
833 .have_tma = decoder->have_tma, in intel_pt_calc_cyc_to_tsc()
834 .fixup_last_mtc = decoder->fixup_last_mtc, in intel_pt_calc_cyc_to_tsc()
847 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); in intel_pt_calc_cyc_to_tsc()
850 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) in intel_pt_get_next_packet() argument
854 decoder->last_packet_type = decoder->packet.type; in intel_pt_get_next_packet()
857 decoder->pos += decoder->pkt_step; in intel_pt_get_next_packet()
858 decoder->buf += decoder->pkt_step; in intel_pt_get_next_packet()
859 decoder->len -= decoder->pkt_step; in intel_pt_get_next_packet()
861 if (!decoder->len) { in intel_pt_get_next_packet()
862 ret = intel_pt_get_next_data(decoder); in intel_pt_get_next_packet()
867 ret = intel_pt_get_packet(decoder->buf, decoder->len, in intel_pt_get_next_packet()
868 &decoder->packet); in intel_pt_get_next_packet()
870 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { in intel_pt_get_next_packet()
871 ret = intel_pt_get_split_packet(decoder); in intel_pt_get_next_packet()
876 return intel_pt_bad_packet(decoder); in intel_pt_get_next_packet()
878 decoder->pkt_len = ret; in intel_pt_get_next_packet()
879 decoder->pkt_step = ret; in intel_pt_get_next_packet()
880 intel_pt_decoder_log_packet(decoder); in intel_pt_get_next_packet()
881 } while (decoder->packet.type == INTEL_PT_PAD); in intel_pt_get_next_packet()
886 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) in intel_pt_next_period() argument
890 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_next_period()
891 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
892 if (decoder->continuous_period) { in intel_pt_next_period()
893 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_next_period()
897 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
898 if (masked_timestamp > decoder->last_masked_timestamp) { in intel_pt_next_period()
899 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_next_period()
900 decoder->continuous_period = true; in intel_pt_next_period()
904 if (masked_timestamp < decoder->last_masked_timestamp) in intel_pt_next_period()
905 return decoder->period_ticks; in intel_pt_next_period()
907 return decoder->period_ticks - (timestamp - masked_timestamp); in intel_pt_next_period()
910 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) in intel_pt_next_sample() argument
912 switch (decoder->period_type) { in intel_pt_next_sample()
914 return decoder->period - decoder->period_insn_cnt; in intel_pt_next_sample()
916 return intel_pt_next_period(decoder); in intel_pt_next_sample()
924 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) in intel_pt_sample_insn() argument
928 switch (decoder->period_type) { in intel_pt_sample_insn()
930 decoder->period_insn_cnt = 0; in intel_pt_sample_insn()
933 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_sample_insn()
934 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_sample_insn()
935 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_sample_insn()
936 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_sample_insn()
938 decoder->last_masked_timestamp += decoder->period_ticks; in intel_pt_sample_insn()
946 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_sample_insn()
949 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, in intel_pt_walk_insn() argument
955 if (!decoder->mtc_insn) in intel_pt_walk_insn()
956 decoder->mtc_insn = true; in intel_pt_walk_insn()
958 max_insn_cnt = intel_pt_next_sample(decoder); in intel_pt_walk_insn()
960 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, in intel_pt_walk_insn()
961 max_insn_cnt, decoder->data); in intel_pt_walk_insn()
963 decoder->tot_insn_cnt += insn_cnt; in intel_pt_walk_insn()
964 decoder->timestamp_insn_cnt += insn_cnt; in intel_pt_walk_insn()
965 decoder->sample_insn_cnt += insn_cnt; in intel_pt_walk_insn()
966 decoder->period_insn_cnt += insn_cnt; in intel_pt_walk_insn()
969 decoder->no_progress = 0; in intel_pt_walk_insn()
970 decoder->pkt_state = INTEL_PT_STATE_ERR2; in intel_pt_walk_insn()
972 decoder->ip); in intel_pt_walk_insn()
978 if (ip && decoder->ip == ip) { in intel_pt_walk_insn()
984 intel_pt_sample_insn(decoder); in intel_pt_walk_insn()
987 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_insn()
988 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
989 decoder->state.to_ip = 0; in intel_pt_walk_insn()
990 decoder->ip += intel_pt_insn->length; in intel_pt_walk_insn()
999 err = intel_pt_push(&decoder->stack, decoder->ip + in intel_pt_walk_insn()
1005 decoder->ret_addr = intel_pt_pop(&decoder->stack); in intel_pt_walk_insn()
1009 int cnt = decoder->no_progress++; in intel_pt_walk_insn()
1011 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1012 decoder->ip += intel_pt_insn->length + in intel_pt_walk_insn()
1014 decoder->state.to_ip = decoder->ip; in intel_pt_walk_insn()
1019 * decoder error results in the decoder erroneously setting the in intel_pt_walk_insn()
1026 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1027 decoder->stuck_ip_prd = 1; in intel_pt_walk_insn()
1028 decoder->stuck_ip_cnt = 1; in intel_pt_walk_insn()
1030 decoder->state.to_ip == decoder->stuck_ip) { in intel_pt_walk_insn()
1032 decoder->state.to_ip); in intel_pt_walk_insn()
1033 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_insn()
1036 } else if (!--decoder->stuck_ip_cnt) { in intel_pt_walk_insn()
1037 decoder->stuck_ip_prd += 1; in intel_pt_walk_insn()
1038 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; in intel_pt_walk_insn()
1039 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1045 decoder->no_progress = 0; in intel_pt_walk_insn()
1047 decoder->state.insn_op = intel_pt_insn->op; in intel_pt_walk_insn()
1048 decoder->state.insn_len = intel_pt_insn->length; in intel_pt_walk_insn()
1049 memcpy(decoder->state.insn, intel_pt_insn->buf, in intel_pt_walk_insn()
1052 if (decoder->tx_flags & INTEL_PT_IN_TX) in intel_pt_walk_insn()
1053 decoder->state.flags |= INTEL_PT_IN_TX; in intel_pt_walk_insn()
1058 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) in intel_pt_fup_event() argument
1062 if (decoder->set_fup_tx_flags) { in intel_pt_fup_event()
1063 decoder->set_fup_tx_flags = false; in intel_pt_fup_event()
1064 decoder->tx_flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1065 decoder->state.type = INTEL_PT_TRANSACTION; in intel_pt_fup_event()
1066 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1067 decoder->state.to_ip = 0; in intel_pt_fup_event()
1068 decoder->state.flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1071 if (decoder->set_fup_ptw) { in intel_pt_fup_event()
1072 decoder->set_fup_ptw = false; in intel_pt_fup_event()
1073 decoder->state.type = INTEL_PT_PTW; in intel_pt_fup_event()
1074 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1075 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1076 decoder->state.to_ip = 0; in intel_pt_fup_event()
1077 decoder->state.ptw_payload = decoder->fup_ptw_payload; in intel_pt_fup_event()
1080 if (decoder->set_fup_mwait) { in intel_pt_fup_event()
1081 decoder->set_fup_mwait = false; in intel_pt_fup_event()
1082 decoder->state.type = INTEL_PT_MWAIT_OP; in intel_pt_fup_event()
1083 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1084 decoder->state.to_ip = 0; in intel_pt_fup_event()
1085 decoder->state.mwait_payload = decoder->fup_mwait_payload; in intel_pt_fup_event()
1088 if (decoder->set_fup_pwre) { in intel_pt_fup_event()
1089 decoder->set_fup_pwre = false; in intel_pt_fup_event()
1090 decoder->state.type |= INTEL_PT_PWR_ENTRY; in intel_pt_fup_event()
1091 decoder->state.type &= ~INTEL_PT_BRANCH; in intel_pt_fup_event()
1092 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1093 decoder->state.to_ip = 0; in intel_pt_fup_event()
1094 decoder->state.pwre_payload = decoder->fup_pwre_payload; in intel_pt_fup_event()
1097 if (decoder->set_fup_exstop) { in intel_pt_fup_event()
1098 decoder->set_fup_exstop = false; in intel_pt_fup_event()
1099 decoder->state.type |= INTEL_PT_EX_STOP; in intel_pt_fup_event()
1100 decoder->state.type &= ~INTEL_PT_BRANCH; in intel_pt_fup_event()
1101 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1102 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1103 decoder->state.to_ip = 0; in intel_pt_fup_event()
1109 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, in intel_pt_fup_with_nlip() argument
1113 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && in intel_pt_fup_with_nlip()
1115 ip == decoder->ip + intel_pt_insn->length; in intel_pt_fup_with_nlip()
1118 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) in intel_pt_walk_fup() argument
1124 ip = decoder->last_ip; in intel_pt_walk_fup()
1127 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); in intel_pt_walk_fup()
1131 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { in intel_pt_walk_fup()
1132 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_fup()
1133 if (intel_pt_fup_event(decoder)) in intel_pt_walk_fup()
1137 decoder->set_fup_tx_flags = false; in intel_pt_walk_fup()
1143 decoder->ip); in intel_pt_walk_fup()
1144 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1150 decoder->ip); in intel_pt_walk_fup()
1151 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1155 intel_pt_bug(decoder); in intel_pt_walk_fup()
1159 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_tip() argument
1164 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tip()
1166 decoder->pgd_ip && in intel_pt_walk_tip()
1167 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1168 (decoder->state.type & INTEL_PT_BRANCH) && in intel_pt_walk_tip()
1169 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { in intel_pt_walk_tip()
1171 decoder->no_progress = 0; in intel_pt_walk_tip()
1172 decoder->pge = false; in intel_pt_walk_tip()
1173 decoder->continuous_period = false; in intel_pt_walk_tip()
1174 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1175 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1184 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { in intel_pt_walk_tip()
1185 decoder->pge = false; in intel_pt_walk_tip()
1186 decoder->continuous_period = false; in intel_pt_walk_tip()
1187 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1188 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1189 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1190 if (decoder->packet.count != 0) in intel_pt_walk_tip()
1191 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1193 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1194 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1195 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1196 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1198 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1199 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1206 uint64_t to_ip = decoder->ip + intel_pt_insn.length + in intel_pt_walk_tip()
1209 if (decoder->pgd_ip && in intel_pt_walk_tip()
1210 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1211 decoder->pgd_ip(to_ip, decoder->data)) { in intel_pt_walk_tip()
1213 decoder->pge = false; in intel_pt_walk_tip()
1214 decoder->continuous_period = false; in intel_pt_walk_tip()
1215 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1216 decoder->ip = to_ip; in intel_pt_walk_tip()
1217 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1218 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1222 decoder->ip); in intel_pt_walk_tip()
1223 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_tip()
1227 return intel_pt_bug(decoder); in intel_pt_walk_tip()
1230 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) in intel_pt_walk_tnt() argument
1236 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tnt()
1243 if (!decoder->return_compression) { in intel_pt_walk_tnt()
1245 decoder->ip); in intel_pt_walk_tnt()
1246 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1249 if (!decoder->ret_addr) { in intel_pt_walk_tnt()
1251 decoder->ip); in intel_pt_walk_tnt()
1252 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1255 if (!(decoder->tnt.payload & BIT63)) { in intel_pt_walk_tnt()
1257 decoder->ip); in intel_pt_walk_tnt()
1258 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1261 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1262 if (decoder->tnt.count) in intel_pt_walk_tnt()
1263 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1265 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1266 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1267 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1268 decoder->ip = decoder->ret_addr; in intel_pt_walk_tnt()
1269 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1275 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_tnt()
1278 if (decoder->packet.type != INTEL_PT_TIP || in intel_pt_walk_tnt()
1279 decoder->packet.count == 0) { in intel_pt_walk_tnt()
1281 decoder->ip); in intel_pt_walk_tnt()
1282 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1283 decoder->pkt_step = 0; in intel_pt_walk_tnt()
1286 intel_pt_set_last_ip(decoder); in intel_pt_walk_tnt()
1287 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1288 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tnt()
1289 decoder->ip = decoder->last_ip; in intel_pt_walk_tnt()
1294 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1295 if (decoder->tnt.count) in intel_pt_walk_tnt()
1296 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1298 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1299 if (decoder->tnt.payload & BIT63) { in intel_pt_walk_tnt()
1300 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1301 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1302 decoder->ip += intel_pt_insn.length + in intel_pt_walk_tnt()
1304 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1308 if (decoder->state.type & INTEL_PT_INSTRUCTION) { in intel_pt_walk_tnt()
1309 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1310 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_tnt()
1311 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1312 decoder->state.to_ip = 0; in intel_pt_walk_tnt()
1313 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1316 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1317 if (!decoder->tnt.count) { in intel_pt_walk_tnt()
1318 decoder->sample_timestamp = decoder->timestamp; in intel_pt_walk_tnt()
1319 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_walk_tnt()
1322 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1326 return intel_pt_bug(decoder); in intel_pt_walk_tnt()
1330 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) in intel_pt_mode_tsx() argument
1335 fup_tx_flags = decoder->packet.payload & in intel_pt_mode_tsx()
1337 err = intel_pt_get_next_packet(decoder); in intel_pt_mode_tsx()
1340 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_mode_tsx()
1341 decoder->fup_tx_flags = fup_tx_flags; in intel_pt_mode_tsx()
1342 decoder->set_fup_tx_flags = true; in intel_pt_mode_tsx()
1343 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) in intel_pt_mode_tsx()
1347 decoder->pos); in intel_pt_mode_tsx()
1348 intel_pt_update_in_tx(decoder); in intel_pt_mode_tsx()
1353 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_tsc_timestamp() argument
1357 decoder->have_tma = false; in intel_pt_calc_tsc_timestamp()
1359 if (decoder->ref_timestamp) { in intel_pt_calc_tsc_timestamp()
1360 timestamp = decoder->packet.payload | in intel_pt_calc_tsc_timestamp()
1361 (decoder->ref_timestamp & (0xffULL << 56)); in intel_pt_calc_tsc_timestamp()
1362 if (timestamp < decoder->ref_timestamp) { in intel_pt_calc_tsc_timestamp()
1363 if (decoder->ref_timestamp - timestamp > (1ULL << 55)) in intel_pt_calc_tsc_timestamp()
1366 if (timestamp - decoder->ref_timestamp > (1ULL << 55)) in intel_pt_calc_tsc_timestamp()
1369 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1370 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1371 decoder->ref_timestamp = 0; in intel_pt_calc_tsc_timestamp()
1372 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1373 } else if (decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1374 timestamp = decoder->packet.payload | in intel_pt_calc_tsc_timestamp()
1375 (decoder->timestamp & (0xffULL << 56)); in intel_pt_calc_tsc_timestamp()
1376 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1377 if (timestamp < decoder->timestamp && in intel_pt_calc_tsc_timestamp()
1378 decoder->timestamp - timestamp < decoder->tsc_slip) { in intel_pt_calc_tsc_timestamp()
1381 timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1383 if (timestamp < decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1386 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1388 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1389 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1392 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_tsc_timestamp()
1393 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1394 decoder->cycle_cnt = 0; in intel_pt_calc_tsc_timestamp()
1395 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_tsc_timestamp()
1396 intel_pt_calc_cyc_to_tsc(decoder, false); in intel_pt_calc_tsc_timestamp()
1399 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_tsc_timestamp()
1402 static int intel_pt_overflow(struct intel_pt_decoder *decoder) in intel_pt_overflow() argument
1405 intel_pt_clear_tx_flags(decoder); in intel_pt_overflow()
1406 decoder->timestamp_insn_cnt = 0; in intel_pt_overflow()
1407 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_overflow()
1408 decoder->overflow = true; in intel_pt_overflow()
1412 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) in intel_pt_calc_tma() argument
1414 uint32_t ctc = decoder->packet.payload; in intel_pt_calc_tma()
1415 uint32_t fc = decoder->packet.count; in intel_pt_calc_tma()
1416 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_tma()
1418 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_tma()
1421 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_tma()
1422 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; in intel_pt_calc_tma()
1423 if (decoder->tsc_ctc_mult) { in intel_pt_calc_tma()
1424 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_tma()
1426 decoder->ctc_timestamp -= multdiv(ctc_rem, in intel_pt_calc_tma()
1427 decoder->tsc_ctc_ratio_n, in intel_pt_calc_tma()
1428 decoder->tsc_ctc_ratio_d); in intel_pt_calc_tma()
1430 decoder->ctc_delta = 0; in intel_pt_calc_tma()
1431 decoder->have_tma = true; in intel_pt_calc_tma()
1432 decoder->fixup_last_mtc = true; in intel_pt_calc_tma()
1434 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); in intel_pt_calc_tma()
1437 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_mtc_timestamp() argument
1442 if (!decoder->have_tma) in intel_pt_calc_mtc_timestamp()
1445 mtc = decoder->packet.payload; in intel_pt_calc_mtc_timestamp()
1447 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { in intel_pt_calc_mtc_timestamp()
1448 decoder->fixup_last_mtc = false; in intel_pt_calc_mtc_timestamp()
1449 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_mtc_timestamp()
1450 &decoder->last_mtc); in intel_pt_calc_mtc_timestamp()
1453 if (mtc > decoder->last_mtc) in intel_pt_calc_mtc_timestamp()
1454 mtc_delta = mtc - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1456 mtc_delta = mtc + 256 - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1458 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_mtc_timestamp()
1460 if (decoder->tsc_ctc_mult) { in intel_pt_calc_mtc_timestamp()
1461 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1462 decoder->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_mtc_timestamp()
1464 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1465 multdiv(decoder->ctc_delta, in intel_pt_calc_mtc_timestamp()
1466 decoder->tsc_ctc_ratio_n, in intel_pt_calc_mtc_timestamp()
1467 decoder->tsc_ctc_ratio_d); in intel_pt_calc_mtc_timestamp()
1470 if (timestamp < decoder->timestamp) in intel_pt_calc_mtc_timestamp()
1472 timestamp, decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1474 decoder->timestamp = timestamp; in intel_pt_calc_mtc_timestamp()
1476 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_mtc_timestamp()
1477 decoder->last_mtc = mtc; in intel_pt_calc_mtc_timestamp()
1479 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_mtc_timestamp()
1480 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_mtc_timestamp()
1481 decoder->cycle_cnt = 0; in intel_pt_calc_mtc_timestamp()
1482 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_mtc_timestamp()
1483 intel_pt_calc_cyc_to_tsc(decoder, true); in intel_pt_calc_mtc_timestamp()
1487 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) in intel_pt_calc_cbr() argument
1489 unsigned int cbr = decoder->packet.payload & 0xff; in intel_pt_calc_cbr()
1491 decoder->cbr_payload = decoder->packet.payload; in intel_pt_calc_cbr()
1493 if (decoder->cbr == cbr) in intel_pt_calc_cbr()
1496 decoder->cbr = cbr; in intel_pt_calc_cbr()
1497 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cbr()
1500 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_cyc_timestamp() argument
1502 uint64_t timestamp = decoder->cyc_ref_timestamp; in intel_pt_calc_cyc_timestamp()
1504 decoder->have_cyc = true; in intel_pt_calc_cyc_timestamp()
1506 decoder->cycle_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
1508 if (!decoder->cyc_ref_timestamp) in intel_pt_calc_cyc_timestamp()
1511 if (decoder->have_calc_cyc_to_tsc) in intel_pt_calc_cyc_timestamp()
1512 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1513 else if (decoder->cbr) in intel_pt_calc_cyc_timestamp()
1514 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1518 if (timestamp < decoder->timestamp) in intel_pt_calc_cyc_timestamp()
1520 timestamp, decoder->timestamp); in intel_pt_calc_cyc_timestamp()
1522 decoder->timestamp = timestamp; in intel_pt_calc_cyc_timestamp()
1524 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_cyc_timestamp()
1528 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) in intel_pt_walk_psbend() argument
1533 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psbend()
1537 switch (decoder->packet.type) { in intel_pt_walk_psbend()
1555 decoder->have_tma = false; in intel_pt_walk_psbend()
1560 return intel_pt_overflow(decoder); in intel_pt_walk_psbend()
1563 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psbend()
1567 intel_pt_calc_tma(decoder); in intel_pt_walk_psbend()
1571 intel_pt_calc_cbr(decoder); in intel_pt_walk_psbend()
1575 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psbend()
1579 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psbend()
1583 decoder->pge = true; in intel_pt_walk_psbend()
1584 if (decoder->packet.count) in intel_pt_walk_psbend()
1585 intel_pt_set_last_ip(decoder); in intel_pt_walk_psbend()
1589 intel_pt_update_in_tx(decoder); in intel_pt_walk_psbend()
1593 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psbend()
1594 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_psbend()
1595 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_psbend()
1608 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_fup_tip() argument
1612 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { in intel_pt_walk_fup_tip()
1613 decoder->tx_flags = 0; in intel_pt_walk_fup_tip()
1614 decoder->state.flags &= ~INTEL_PT_IN_TX; in intel_pt_walk_fup_tip()
1615 decoder->state.flags |= INTEL_PT_ABORT_TX; in intel_pt_walk_fup_tip()
1617 decoder->state.flags |= INTEL_PT_ASYNC; in intel_pt_walk_fup_tip()
1621 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_fup_tip()
1625 switch (decoder->packet.type) { in intel_pt_walk_fup_tip()
1643 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_fup_tip()
1644 decoder->pkt_step = 0; in intel_pt_walk_fup_tip()
1648 intel_pt_calc_cbr(decoder); in intel_pt_walk_fup_tip()
1652 return intel_pt_overflow(decoder); in intel_pt_walk_fup_tip()
1655 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1656 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1657 if (decoder->packet.count != 0) { in intel_pt_walk_fup_tip()
1658 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1660 decoder->ip); in intel_pt_walk_fup_tip()
1662 decoder->pge = false; in intel_pt_walk_fup_tip()
1663 decoder->continuous_period = false; in intel_pt_walk_fup_tip()
1667 decoder->pge = true; in intel_pt_walk_fup_tip()
1669 decoder->ip); in intel_pt_walk_fup_tip()
1670 decoder->state.from_ip = 0; in intel_pt_walk_fup_tip()
1671 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1672 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1674 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1675 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1680 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1681 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1682 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1684 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1685 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1690 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_fup_tip()
1694 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_fup_tip()
1695 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_fup_tip()
1696 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_fup_tip()
1700 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_fup_tip()
1704 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_fup_tip()
1713 return intel_pt_bug(decoder); in intel_pt_walk_fup_tip()
1718 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) in intel_pt_walk_trace() argument
1724 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
1728 switch (decoder->packet.type) { in intel_pt_walk_trace()
1730 if (!decoder->packet.count) in intel_pt_walk_trace()
1732 decoder->tnt = decoder->packet; in intel_pt_walk_trace()
1733 decoder->pkt_state = INTEL_PT_STATE_TNT; in intel_pt_walk_trace()
1734 err = intel_pt_walk_tnt(decoder); in intel_pt_walk_trace()
1740 if (decoder->packet.count != 0) in intel_pt_walk_trace()
1741 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
1742 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; in intel_pt_walk_trace()
1743 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
1746 decoder->pge = true; in intel_pt_walk_trace()
1747 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
1749 decoder->pos); in intel_pt_walk_trace()
1752 intel_pt_set_ip(decoder); in intel_pt_walk_trace()
1753 decoder->state.from_ip = 0; in intel_pt_walk_trace()
1754 decoder->state.to_ip = decoder->ip; in intel_pt_walk_trace()
1759 return intel_pt_overflow(decoder); in intel_pt_walk_trace()
1762 if (decoder->packet.count != 0) in intel_pt_walk_trace()
1763 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
1764 decoder->pkt_state = INTEL_PT_STATE_TIP; in intel_pt_walk_trace()
1765 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
1768 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
1770 decoder->pos); in intel_pt_walk_trace()
1774 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
1775 if (!decoder->branch_enable) { in intel_pt_walk_trace()
1776 decoder->ip = decoder->last_ip; in intel_pt_walk_trace()
1777 if (intel_pt_fup_event(decoder)) in intel_pt_walk_trace()
1782 if (decoder->set_fup_mwait) in intel_pt_walk_trace()
1785 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; in intel_pt_walk_trace()
1787 decoder->pkt_state = INTEL_PT_STATE_FUP; in intel_pt_walk_trace()
1788 err = intel_pt_walk_fup(decoder); in intel_pt_walk_trace()
1795 return intel_pt_walk_fup_tip(decoder); in intel_pt_walk_trace()
1798 decoder->pge = false; in intel_pt_walk_trace()
1799 decoder->continuous_period = false; in intel_pt_walk_trace()
1800 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_trace()
1801 decoder->have_tma = false; in intel_pt_walk_trace()
1805 decoder->last_ip = 0; in intel_pt_walk_trace()
1806 decoder->have_last_ip = true; in intel_pt_walk_trace()
1807 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_trace()
1808 err = intel_pt_walk_psbend(decoder); in intel_pt_walk_trace()
1816 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_trace()
1820 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_trace()
1821 if (decoder->period_type != INTEL_PT_PERIOD_MTC) in intel_pt_walk_trace()
1827 if (!decoder->mtc_insn) in intel_pt_walk_trace()
1829 decoder->mtc_insn = false; in intel_pt_walk_trace()
1831 if (!decoder->timestamp) in intel_pt_walk_trace()
1833 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_trace()
1834 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1835 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1836 decoder->mtc_insn = false; in intel_pt_walk_trace()
1840 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_trace()
1844 intel_pt_calc_tma(decoder); in intel_pt_walk_trace()
1848 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_trace()
1852 intel_pt_calc_cbr(decoder); in intel_pt_walk_trace()
1853 if (!decoder->branch_enable && in intel_pt_walk_trace()
1854 decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
1855 decoder->cbr_seen = decoder->cbr; in intel_pt_walk_trace()
1856 decoder->state.type = INTEL_PT_CBR_CHG; in intel_pt_walk_trace()
1857 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1858 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1859 decoder->state.cbr_payload = in intel_pt_walk_trace()
1860 decoder->packet.payload; in intel_pt_walk_trace()
1866 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_trace()
1871 if (!decoder->pge) { in intel_pt_walk_trace()
1872 intel_pt_update_in_tx(decoder); in intel_pt_walk_trace()
1875 err = intel_pt_mode_tsx(decoder, &no_tip); in intel_pt_walk_trace()
1881 return intel_pt_bug(decoder); in intel_pt_walk_trace()
1890 decoder->fup_ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
1891 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
1894 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
1895 decoder->set_fup_ptw = true; in intel_pt_walk_trace()
1899 decoder->pos); in intel_pt_walk_trace()
1904 decoder->state.type = INTEL_PT_PTW; in intel_pt_walk_trace()
1905 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1906 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1907 decoder->state.ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
1911 decoder->fup_mwait_payload = decoder->packet.payload; in intel_pt_walk_trace()
1912 decoder->set_fup_mwait = true; in intel_pt_walk_trace()
1916 if (decoder->set_fup_mwait) { in intel_pt_walk_trace()
1917 decoder->fup_pwre_payload = in intel_pt_walk_trace()
1918 decoder->packet.payload; in intel_pt_walk_trace()
1919 decoder->set_fup_pwre = true; in intel_pt_walk_trace()
1922 decoder->state.type = INTEL_PT_PWR_ENTRY; in intel_pt_walk_trace()
1923 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1924 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1925 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
1929 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
1932 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
1933 decoder->set_fup_exstop = true; in intel_pt_walk_trace()
1937 decoder->pos); in intel_pt_walk_trace()
1942 decoder->state.type = INTEL_PT_EX_STOP; in intel_pt_walk_trace()
1943 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1944 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1948 decoder->state.type = INTEL_PT_PWR_EXIT; in intel_pt_walk_trace()
1949 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
1950 decoder->state.to_ip = 0; in intel_pt_walk_trace()
1951 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
1955 return intel_pt_bug(decoder); in intel_pt_walk_trace()
1960 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) in intel_pt_have_ip() argument
1962 return decoder->packet.count && in intel_pt_have_ip()
1963 (decoder->have_last_ip || decoder->packet.count == 3 || in intel_pt_have_ip()
1964 decoder->packet.count == 6); in intel_pt_have_ip()
1968 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) in intel_pt_walk_psb() argument
1973 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psb()
1977 switch (decoder->packet.type) { in intel_pt_walk_psb()
1979 decoder->continuous_period = false; in intel_pt_walk_psb()
1994 decoder->pge = true; in intel_pt_walk_psb()
1995 if (intel_pt_have_ip(decoder)) { in intel_pt_walk_psb()
1996 uint64_t current_ip = decoder->ip; in intel_pt_walk_psb()
1998 intel_pt_set_ip(decoder); in intel_pt_walk_psb()
2001 decoder->ip); in intel_pt_walk_psb()
2006 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psb()
2010 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psb()
2014 intel_pt_calc_tma(decoder); in intel_pt_walk_psb()
2018 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_psb()
2022 intel_pt_calc_cbr(decoder); in intel_pt_walk_psb()
2026 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psb()
2030 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psb()
2034 intel_pt_update_in_tx(decoder); in intel_pt_walk_psb()
2038 decoder->pge = false; in intel_pt_walk_psb()
2039 decoder->continuous_period = false; in intel_pt_walk_psb()
2040 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_psb()
2044 decoder->have_tma = false; in intel_pt_walk_psb()
2046 if (decoder->ip) in intel_pt_walk_psb()
2047 decoder->pkt_state = INTEL_PT_STATE_ERR4; in intel_pt_walk_psb()
2049 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_psb()
2053 return intel_pt_bug(decoder); in intel_pt_walk_psb()
2056 return intel_pt_overflow(decoder); in intel_pt_walk_psb()
2071 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) in intel_pt_walk_to_ip() argument
2076 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_to_ip()
2080 switch (decoder->packet.type) { in intel_pt_walk_to_ip()
2082 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2086 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD; in intel_pt_walk_to_ip()
2087 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2088 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2089 if (decoder->ip) in intel_pt_walk_to_ip()
2094 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2095 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2096 if (decoder->ip) in intel_pt_walk_to_ip()
2101 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_to_ip()
2105 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_to_ip()
2109 intel_pt_calc_tma(decoder); in intel_pt_walk_to_ip()
2113 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_to_ip()
2117 intel_pt_calc_cbr(decoder); in intel_pt_walk_to_ip()
2121 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_to_ip()
2125 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_to_ip()
2129 intel_pt_update_in_tx(decoder); in intel_pt_walk_to_ip()
2133 return intel_pt_overflow(decoder); in intel_pt_walk_to_ip()
2136 return intel_pt_bug(decoder); in intel_pt_walk_to_ip()
2139 decoder->pge = false; in intel_pt_walk_to_ip()
2140 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2141 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_to_ip()
2142 decoder->have_tma = false; in intel_pt_walk_to_ip()
2146 decoder->last_ip = 0; in intel_pt_walk_to_ip()
2147 decoder->have_last_ip = true; in intel_pt_walk_to_ip()
2148 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_to_ip()
2149 err = intel_pt_walk_psb(decoder); in intel_pt_walk_to_ip()
2152 if (decoder->ip) { in intel_pt_walk_to_ip()
2154 decoder->state.type = 0; in intel_pt_walk_to_ip()
2177 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) in intel_pt_sync_ip() argument
2181 decoder->set_fup_tx_flags = false; in intel_pt_sync_ip()
2182 decoder->set_fup_ptw = false; in intel_pt_sync_ip()
2183 decoder->set_fup_mwait = false; in intel_pt_sync_ip()
2184 decoder->set_fup_pwre = false; in intel_pt_sync_ip()
2185 decoder->set_fup_exstop = false; in intel_pt_sync_ip()
2187 if (!decoder->branch_enable) { in intel_pt_sync_ip()
2188 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2189 decoder->overflow = false; in intel_pt_sync_ip()
2190 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync_ip()
2195 err = intel_pt_walk_to_ip(decoder); in intel_pt_sync_ip()
2199 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2200 decoder->overflow = false; in intel_pt_sync_ip()
2202 decoder->state.from_ip = 0; in intel_pt_sync_ip()
2203 decoder->state.to_ip = decoder->ip; in intel_pt_sync_ip()
2204 intel_pt_log_to("Setting IP", decoder->ip); in intel_pt_sync_ip()
2209 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) in intel_pt_part_psb() argument
2211 const unsigned char *end = decoder->buf + decoder->len; in intel_pt_part_psb()
2215 if (i > decoder->len) in intel_pt_part_psb()
2223 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) in intel_pt_rest_psb() argument
2228 if (rest_psb > decoder->len || in intel_pt_rest_psb()
2229 memcmp(decoder->buf, psb + part_psb, rest_psb)) in intel_pt_rest_psb()
2235 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, in intel_pt_get_split_psb() argument
2240 decoder->pos += decoder->len; in intel_pt_get_split_psb()
2241 decoder->len = 0; in intel_pt_get_split_psb()
2243 ret = intel_pt_get_next_data(decoder); in intel_pt_get_split_psb()
2247 rest_psb = intel_pt_rest_psb(decoder, part_psb); in intel_pt_get_split_psb()
2251 decoder->pos -= part_psb; in intel_pt_get_split_psb()
2252 decoder->next_buf = decoder->buf + rest_psb; in intel_pt_get_split_psb()
2253 decoder->next_len = decoder->len - rest_psb; in intel_pt_get_split_psb()
2254 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); in intel_pt_get_split_psb()
2255 decoder->buf = decoder->temp_buf; in intel_pt_get_split_psb()
2256 decoder->len = INTEL_PT_PSB_LEN; in intel_pt_get_split_psb()
2261 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) in intel_pt_scan_for_psb() argument
2268 if (!decoder->len) { in intel_pt_scan_for_psb()
2269 ret = intel_pt_get_next_data(decoder); in intel_pt_scan_for_psb()
2274 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, in intel_pt_scan_for_psb()
2279 part_psb = intel_pt_part_psb(decoder); in intel_pt_scan_for_psb()
2281 ret = intel_pt_get_split_psb(decoder, part_psb); in intel_pt_scan_for_psb()
2285 decoder->pos += decoder->len; in intel_pt_scan_for_psb()
2286 decoder->len = 0; in intel_pt_scan_for_psb()
2291 decoder->pkt_step = next - decoder->buf; in intel_pt_scan_for_psb()
2292 return intel_pt_get_next_packet(decoder); in intel_pt_scan_for_psb()
2296 static int intel_pt_sync(struct intel_pt_decoder *decoder) in intel_pt_sync() argument
2300 decoder->pge = false; in intel_pt_sync()
2301 decoder->continuous_period = false; in intel_pt_sync()
2302 decoder->have_last_ip = false; in intel_pt_sync()
2303 decoder->last_ip = 0; in intel_pt_sync()
2304 decoder->ip = 0; in intel_pt_sync()
2305 intel_pt_clear_stack(&decoder->stack); in intel_pt_sync()
2307 err = intel_pt_scan_for_psb(decoder); in intel_pt_sync()
2311 decoder->have_last_ip = true; in intel_pt_sync()
2312 decoder->pkt_state = INTEL_PT_STATE_NO_IP; in intel_pt_sync()
2314 err = intel_pt_walk_psb(decoder); in intel_pt_sync()
2318 if (decoder->ip) { in intel_pt_sync()
2319 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync()
2320 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
2322 return intel_pt_sync_ip(decoder); in intel_pt_sync()
2328 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) in intel_pt_est_timestamp() argument
2330 uint64_t est = decoder->sample_insn_cnt << 1; in intel_pt_est_timestamp()
2332 if (!decoder->cbr || !decoder->max_non_turbo_ratio) in intel_pt_est_timestamp()
2335 est *= decoder->max_non_turbo_ratio; in intel_pt_est_timestamp()
2336 est /= decoder->cbr; in intel_pt_est_timestamp()
2338 return decoder->sample_timestamp + est; in intel_pt_est_timestamp()
2341 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) in intel_pt_decode() argument
2346 decoder->state.type = INTEL_PT_BRANCH; in intel_pt_decode()
2347 decoder->state.flags = 0; in intel_pt_decode()
2349 switch (decoder->pkt_state) { in intel_pt_decode()
2351 err = intel_pt_sync(decoder); in intel_pt_decode()
2354 decoder->have_last_ip = false; in intel_pt_decode()
2355 decoder->last_ip = 0; in intel_pt_decode()
2356 decoder->ip = 0; in intel_pt_decode()
2359 err = intel_pt_sync_ip(decoder); in intel_pt_decode()
2362 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2366 err = intel_pt_walk_tnt(decoder); in intel_pt_decode()
2368 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2372 err = intel_pt_walk_tip(decoder); in intel_pt_decode()
2375 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2377 err = intel_pt_walk_fup_tip(decoder); in intel_pt_decode()
2380 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2382 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2385 err = intel_pt_bug(decoder); in intel_pt_decode()
2391 decoder->state.err = intel_pt_ext_err(err); in intel_pt_decode()
2392 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
2393 decoder->sample_timestamp = decoder->timestamp; in intel_pt_decode()
2394 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_decode()
2396 decoder->state.err = 0; in intel_pt_decode()
2397 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) { in intel_pt_decode()
2398 decoder->cbr_seen = decoder->cbr; in intel_pt_decode()
2399 decoder->state.type |= INTEL_PT_CBR_CHG; in intel_pt_decode()
2400 decoder->state.cbr_payload = decoder->cbr_payload; in intel_pt_decode()
2402 if (intel_pt_sample_time(decoder->pkt_state)) { in intel_pt_decode()
2403 decoder->sample_timestamp = decoder->timestamp; in intel_pt_decode()
2404 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_decode()
2408 decoder->state.timestamp = decoder->sample_timestamp; in intel_pt_decode()
2409 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); in intel_pt_decode()
2410 decoder->state.cr3 = decoder->cr3; in intel_pt_decode()
2411 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; in intel_pt_decode()
2413 return &decoder->state; in intel_pt_decode()