Lines Matching full:decoder
22 #include "intel-pt-insn-decoder.h"
23 #include "intel-pt-pkt-decoder.h"
24 #include "intel-pt-decoder.h"
95 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
224 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) in intel_pt_setup_period() argument
226 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { in intel_pt_setup_period()
229 period = intel_pt_lower_power_of_2(decoder->period); in intel_pt_setup_period()
230 decoder->period_mask = ~(period - 1); in intel_pt_setup_period()
231 decoder->period_ticks = period; in intel_pt_setup_period()
244 struct intel_pt_decoder *decoder; in intel_pt_decoder_new() local
249 decoder = zalloc(sizeof(struct intel_pt_decoder)); in intel_pt_decoder_new()
250 if (!decoder) in intel_pt_decoder_new()
253 decoder->get_trace = params->get_trace; in intel_pt_decoder_new()
254 decoder->walk_insn = params->walk_insn; in intel_pt_decoder_new()
255 decoder->pgd_ip = params->pgd_ip; in intel_pt_decoder_new()
256 decoder->lookahead = params->lookahead; in intel_pt_decoder_new()
257 decoder->data = params->data; in intel_pt_decoder_new()
258 decoder->return_compression = params->return_compression; in intel_pt_decoder_new()
259 decoder->branch_enable = params->branch_enable; in intel_pt_decoder_new()
260 decoder->hop = params->quick >= 1; in intel_pt_decoder_new()
261 decoder->leap = params->quick >= 2; in intel_pt_decoder_new()
263 decoder->flags = params->flags; in intel_pt_decoder_new()
265 decoder->ctl = params->ctl; in intel_pt_decoder_new()
266 decoder->period = params->period; in intel_pt_decoder_new()
267 decoder->period_type = params->period_type; in intel_pt_decoder_new()
269 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; in intel_pt_decoder_new()
270 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; in intel_pt_decoder_new()
272 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); in intel_pt_decoder_new()
274 intel_pt_setup_period(decoder); in intel_pt_decoder_new()
276 decoder->mtc_shift = params->mtc_period; in intel_pt_decoder_new()
277 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; in intel_pt_decoder_new()
279 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; in intel_pt_decoder_new()
280 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; in intel_pt_decoder_new()
282 if (!decoder->tsc_ctc_ratio_n) in intel_pt_decoder_new()
283 decoder->tsc_ctc_ratio_d = 0; in intel_pt_decoder_new()
285 if (decoder->tsc_ctc_ratio_d) { in intel_pt_decoder_new()
286 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) in intel_pt_decoder_new()
287 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / in intel_pt_decoder_new()
288 decoder->tsc_ctc_ratio_d; in intel_pt_decoder_new()
297 decoder->tsc_slip = 0x10000; in intel_pt_decoder_new()
299 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); in intel_pt_decoder_new()
300 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); in intel_pt_decoder_new()
301 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); in intel_pt_decoder_new()
302 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); in intel_pt_decoder_new()
303 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); in intel_pt_decoder_new()
305 if (decoder->hop) in intel_pt_decoder_new()
308 return decoder; in intel_pt_decoder_new()
382 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) in intel_pt_decoder_free() argument
384 intel_pt_free_stack(&decoder->stack); in intel_pt_decoder_free()
385 free(decoder); in intel_pt_decoder_free()
469 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) in intel_pt_set_last_ip() argument
471 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); in intel_pt_set_last_ip()
472 decoder->have_last_ip = true; in intel_pt_set_last_ip()
475 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) in intel_pt_set_ip() argument
477 intel_pt_set_last_ip(decoder); in intel_pt_set_ip()
478 decoder->ip = decoder->last_ip; in intel_pt_set_ip()
481 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) in intel_pt_decoder_log_packet() argument
483 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, in intel_pt_decoder_log_packet()
484 decoder->buf); in intel_pt_decoder_log_packet()
487 static int intel_pt_bug(struct intel_pt_decoder *decoder) in intel_pt_bug() argument
490 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_bug()
494 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) in intel_pt_clear_tx_flags() argument
496 decoder->tx_flags = 0; in intel_pt_clear_tx_flags()
499 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) in intel_pt_update_in_tx() argument
501 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; in intel_pt_update_in_tx()
504 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) in intel_pt_bad_packet() argument
506 intel_pt_clear_tx_flags(decoder); in intel_pt_bad_packet()
507 decoder->have_tma = false; in intel_pt_bad_packet()
508 decoder->pkt_len = 1; in intel_pt_bad_packet()
509 decoder->pkt_step = 1; in intel_pt_bad_packet()
510 intel_pt_decoder_log_packet(decoder); in intel_pt_bad_packet()
511 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { in intel_pt_bad_packet()
513 decoder->pkt_state = INTEL_PT_STATE_ERR1; in intel_pt_bad_packet()
518 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) in intel_pt_update_sample_time() argument
520 decoder->sample_timestamp = decoder->timestamp; in intel_pt_update_sample_time()
521 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_update_sample_time()
524 static void intel_pt_reposition(struct intel_pt_decoder *decoder) in intel_pt_reposition() argument
526 decoder->ip = 0; in intel_pt_reposition()
527 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_reposition()
528 decoder->timestamp = 0; in intel_pt_reposition()
529 decoder->have_tma = false; in intel_pt_reposition()
532 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) in intel_pt_get_data() argument
537 decoder->pkt_step = 0; in intel_pt_get_data()
540 ret = decoder->get_trace(&buffer, decoder->data); in intel_pt_get_data()
543 decoder->buf = buffer.buf; in intel_pt_get_data()
544 decoder->len = buffer.len; in intel_pt_get_data()
545 if (!decoder->len) { in intel_pt_get_data()
549 decoder->buf_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
551 intel_pt_reposition(decoder); in intel_pt_get_data()
552 decoder->ref_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
553 decoder->state.trace_nr = buffer.trace_nr; in intel_pt_get_data()
555 decoder->ref_timestamp); in intel_pt_get_data()
562 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, in intel_pt_get_next_data() argument
565 if (!decoder->next_buf) in intel_pt_get_next_data()
566 return intel_pt_get_data(decoder, reposition); in intel_pt_get_next_data()
568 decoder->buf = decoder->next_buf; in intel_pt_get_next_data()
569 decoder->len = decoder->next_len; in intel_pt_get_next_data()
570 decoder->next_buf = 0; in intel_pt_get_next_data()
571 decoder->next_len = 0; in intel_pt_get_next_data()
575 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) in intel_pt_get_split_packet() argument
577 unsigned char *buf = decoder->temp_buf; in intel_pt_get_split_packet()
581 old_len = decoder->len; in intel_pt_get_split_packet()
582 len = decoder->len; in intel_pt_get_split_packet()
583 memcpy(buf, decoder->buf, len); in intel_pt_get_split_packet()
585 ret = intel_pt_get_data(decoder, false); in intel_pt_get_split_packet()
587 decoder->pos += old_len; in intel_pt_get_split_packet()
592 if (n > decoder->len) in intel_pt_get_split_packet()
593 n = decoder->len; in intel_pt_get_split_packet()
594 memcpy(buf + len, decoder->buf, n); in intel_pt_get_split_packet()
597 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_split_packet()
598 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_split_packet()
600 decoder->next_buf = decoder->buf; in intel_pt_get_split_packet()
601 decoder->next_len = decoder->len; in intel_pt_get_split_packet()
602 decoder->buf = buf; in intel_pt_get_split_packet()
603 decoder->len = old_len; in intel_pt_get_split_packet()
604 return intel_pt_bad_packet(decoder); in intel_pt_get_split_packet()
607 decoder->next_buf = decoder->buf + (ret - old_len); in intel_pt_get_split_packet()
608 decoder->next_len = decoder->len - (ret - old_len); in intel_pt_get_split_packet()
610 decoder->buf = buf; in intel_pt_get_split_packet()
611 decoder->len = ret; in intel_pt_get_split_packet()
617 struct intel_pt_decoder *decoder; member
628 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, in intel_pt_pkt_lookahead() argument
632 const unsigned char *buf = decoder->buf; in intel_pt_pkt_lookahead()
633 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; in intel_pt_pkt_lookahead()
634 size_t len = decoder->len; in intel_pt_pkt_lookahead()
637 pkt_info.decoder = decoder; in intel_pt_pkt_lookahead()
638 pkt_info.pos = decoder->pos; in intel_pt_pkt_lookahead()
639 pkt_info.pkt_len = decoder->pkt_step; in intel_pt_pkt_lookahead()
640 pkt_info.last_packet_type = decoder->last_packet_type; in intel_pt_pkt_lookahead()
706 struct intel_pt_decoder *decoder = pkt_info->decoder; in intel_pt_calc_cyc_cb() local
739 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { in intel_pt_calc_cyc_cb()
741 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_cyc_cb()
748 data->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_cyc_cb()
751 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
753 data->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
757 decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
758 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
781 data->timestamp - timestamp < decoder->tsc_slip) in intel_pt_calc_cyc_cb()
798 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_cyc_cb()
803 ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_cyc_cb()
805 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_cyc_cb()
808 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
809 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
812 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
813 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
831 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cyc_cb()
847 if (!data->cbr && decoder->cbr) { in intel_pt_calc_cyc_cb()
848 data->cbr = decoder->cbr; in intel_pt_calc_cyc_cb()
849 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_cb()
855 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; in intel_pt_calc_cyc_cb()
864 decoder->calc_cyc_to_tsc = cyc_to_tsc; in intel_pt_calc_cyc_cb()
865 decoder->have_calc_cyc_to_tsc = true; in intel_pt_calc_cyc_cb()
878 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, in intel_pt_calc_cyc_to_tsc() argument
884 .last_mtc = decoder->last_mtc, in intel_pt_calc_cyc_to_tsc()
885 .ctc_timestamp = decoder->ctc_timestamp, in intel_pt_calc_cyc_to_tsc()
886 .ctc_delta = decoder->ctc_delta, in intel_pt_calc_cyc_to_tsc()
887 .tsc_timestamp = decoder->tsc_timestamp, in intel_pt_calc_cyc_to_tsc()
888 .timestamp = decoder->timestamp, in intel_pt_calc_cyc_to_tsc()
889 .have_tma = decoder->have_tma, in intel_pt_calc_cyc_to_tsc()
890 .fixup_last_mtc = decoder->fixup_last_mtc, in intel_pt_calc_cyc_to_tsc()
903 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); in intel_pt_calc_cyc_to_tsc()
906 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) in intel_pt_get_next_packet() argument
910 decoder->last_packet_type = decoder->packet.type; in intel_pt_get_next_packet()
913 decoder->pos += decoder->pkt_step; in intel_pt_get_next_packet()
914 decoder->buf += decoder->pkt_step; in intel_pt_get_next_packet()
915 decoder->len -= decoder->pkt_step; in intel_pt_get_next_packet()
917 if (!decoder->len) { in intel_pt_get_next_packet()
918 ret = intel_pt_get_next_data(decoder, false); in intel_pt_get_next_packet()
923 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_next_packet()
924 ret = intel_pt_get_packet(decoder->buf, decoder->len, in intel_pt_get_next_packet()
925 &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_next_packet()
927 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { in intel_pt_get_next_packet()
928 ret = intel_pt_get_split_packet(decoder); in intel_pt_get_next_packet()
933 return intel_pt_bad_packet(decoder); in intel_pt_get_next_packet()
935 decoder->pkt_len = ret; in intel_pt_get_next_packet()
936 decoder->pkt_step = ret; in intel_pt_get_next_packet()
937 intel_pt_decoder_log_packet(decoder); in intel_pt_get_next_packet()
938 } while (decoder->packet.type == INTEL_PT_PAD); in intel_pt_get_next_packet()
943 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) in intel_pt_next_period() argument
947 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_next_period()
948 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
949 if (decoder->continuous_period) { in intel_pt_next_period()
950 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_next_period()
954 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
955 if (masked_timestamp > decoder->last_masked_timestamp) { in intel_pt_next_period()
956 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_next_period()
957 decoder->continuous_period = true; in intel_pt_next_period()
961 if (masked_timestamp < decoder->last_masked_timestamp) in intel_pt_next_period()
962 return decoder->period_ticks; in intel_pt_next_period()
964 return decoder->period_ticks - (timestamp - masked_timestamp); in intel_pt_next_period()
967 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) in intel_pt_next_sample() argument
969 switch (decoder->period_type) { in intel_pt_next_sample()
971 return decoder->period - decoder->period_insn_cnt; in intel_pt_next_sample()
973 return intel_pt_next_period(decoder); in intel_pt_next_sample()
981 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) in intel_pt_sample_insn() argument
985 switch (decoder->period_type) { in intel_pt_sample_insn()
987 decoder->period_insn_cnt = 0; in intel_pt_sample_insn()
990 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_sample_insn()
991 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_sample_insn()
992 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_sample_insn()
993 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_sample_insn()
995 decoder->last_masked_timestamp += decoder->period_ticks; in intel_pt_sample_insn()
1003 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_sample_insn()
1006 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, in intel_pt_walk_insn() argument
1012 if (!decoder->mtc_insn) in intel_pt_walk_insn()
1013 decoder->mtc_insn = true; in intel_pt_walk_insn()
1015 max_insn_cnt = intel_pt_next_sample(decoder); in intel_pt_walk_insn()
1017 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, in intel_pt_walk_insn()
1018 max_insn_cnt, decoder->data); in intel_pt_walk_insn()
1020 decoder->tot_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1021 decoder->timestamp_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1022 decoder->sample_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1023 decoder->period_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1026 decoder->no_progress = 0; in intel_pt_walk_insn()
1027 decoder->pkt_state = INTEL_PT_STATE_ERR2; in intel_pt_walk_insn()
1029 decoder->ip); in intel_pt_walk_insn()
1035 if (ip && decoder->ip == ip) { in intel_pt_walk_insn()
1041 intel_pt_sample_insn(decoder); in intel_pt_walk_insn()
1044 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_insn()
1045 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1046 decoder->state.to_ip = 0; in intel_pt_walk_insn()
1047 decoder->ip += intel_pt_insn->length; in intel_pt_walk_insn()
1056 err = intel_pt_push(&decoder->stack, decoder->ip + in intel_pt_walk_insn()
1062 decoder->ret_addr = intel_pt_pop(&decoder->stack); in intel_pt_walk_insn()
1066 int cnt = decoder->no_progress++; in intel_pt_walk_insn()
1068 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1069 decoder->ip += intel_pt_insn->length + in intel_pt_walk_insn()
1071 decoder->state.to_ip = decoder->ip; in intel_pt_walk_insn()
1076 * decoder error results in the decoder erroneously setting the in intel_pt_walk_insn()
1083 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1084 decoder->stuck_ip_prd = 1; in intel_pt_walk_insn()
1085 decoder->stuck_ip_cnt = 1; in intel_pt_walk_insn()
1087 decoder->state.to_ip == decoder->stuck_ip) { in intel_pt_walk_insn()
1089 decoder->state.to_ip); in intel_pt_walk_insn()
1090 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_insn()
1093 } else if (!--decoder->stuck_ip_cnt) { in intel_pt_walk_insn()
1094 decoder->stuck_ip_prd += 1; in intel_pt_walk_insn()
1095 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; in intel_pt_walk_insn()
1096 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1102 decoder->no_progress = 0; in intel_pt_walk_insn()
1104 decoder->state.insn_op = intel_pt_insn->op; in intel_pt_walk_insn()
1105 decoder->state.insn_len = intel_pt_insn->length; in intel_pt_walk_insn()
1106 memcpy(decoder->state.insn, intel_pt_insn->buf, in intel_pt_walk_insn()
1109 if (decoder->tx_flags & INTEL_PT_IN_TX) in intel_pt_walk_insn()
1110 decoder->state.flags |= INTEL_PT_IN_TX; in intel_pt_walk_insn()
1115 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) in intel_pt_fup_event() argument
1117 enum intel_pt_sample_type type = decoder->state.type; in intel_pt_fup_event()
1120 decoder->state.type &= ~INTEL_PT_BRANCH; in intel_pt_fup_event()
1122 if (decoder->set_fup_tx_flags) { in intel_pt_fup_event()
1123 decoder->set_fup_tx_flags = false; in intel_pt_fup_event()
1124 decoder->tx_flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1125 decoder->state.type |= INTEL_PT_TRANSACTION; in intel_pt_fup_event()
1126 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) in intel_pt_fup_event()
1127 decoder->state.type |= INTEL_PT_BRANCH; in intel_pt_fup_event()
1128 decoder->state.flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1131 if (decoder->set_fup_ptw) { in intel_pt_fup_event()
1132 decoder->set_fup_ptw = false; in intel_pt_fup_event()
1133 decoder->state.type |= INTEL_PT_PTW; in intel_pt_fup_event()
1134 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1135 decoder->state.ptw_payload = decoder->fup_ptw_payload; in intel_pt_fup_event()
1138 if (decoder->set_fup_mwait) { in intel_pt_fup_event()
1139 decoder->set_fup_mwait = false; in intel_pt_fup_event()
1140 decoder->state.type |= INTEL_PT_MWAIT_OP; in intel_pt_fup_event()
1141 decoder->state.mwait_payload = decoder->fup_mwait_payload; in intel_pt_fup_event()
1144 if (decoder->set_fup_pwre) { in intel_pt_fup_event()
1145 decoder->set_fup_pwre = false; in intel_pt_fup_event()
1146 decoder->state.type |= INTEL_PT_PWR_ENTRY; in intel_pt_fup_event()
1147 decoder->state.pwre_payload = decoder->fup_pwre_payload; in intel_pt_fup_event()
1150 if (decoder->set_fup_exstop) { in intel_pt_fup_event()
1151 decoder->set_fup_exstop = false; in intel_pt_fup_event()
1152 decoder->state.type |= INTEL_PT_EX_STOP; in intel_pt_fup_event()
1153 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1156 if (decoder->set_fup_bep) { in intel_pt_fup_event()
1157 decoder->set_fup_bep = false; in intel_pt_fup_event()
1158 decoder->state.type |= INTEL_PT_BLK_ITEMS; in intel_pt_fup_event()
1161 if (decoder->overflow) { in intel_pt_fup_event()
1162 decoder->overflow = false; in intel_pt_fup_event()
1163 if (!ret && !decoder->pge) { in intel_pt_fup_event()
1164 if (decoder->hop) { in intel_pt_fup_event()
1165 decoder->state.type = 0; in intel_pt_fup_event()
1166 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_fup_event()
1168 decoder->pge = true; in intel_pt_fup_event()
1169 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; in intel_pt_fup_event()
1170 decoder->state.from_ip = 0; in intel_pt_fup_event()
1171 decoder->state.to_ip = decoder->ip; in intel_pt_fup_event()
1176 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1177 decoder->state.to_ip = 0; in intel_pt_fup_event()
1179 decoder->state.type = type; in intel_pt_fup_event()
1184 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, in intel_pt_fup_with_nlip() argument
1188 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && in intel_pt_fup_with_nlip()
1190 ip == decoder->ip + intel_pt_insn->length; in intel_pt_fup_with_nlip()
1193 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) in intel_pt_walk_fup() argument
1199 ip = decoder->last_ip; in intel_pt_walk_fup()
1202 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); in intel_pt_walk_fup()
1206 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { in intel_pt_walk_fup()
1207 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; in intel_pt_walk_fup()
1209 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_fup()
1210 if (intel_pt_fup_event(decoder) && no_tip) in intel_pt_walk_fup()
1214 decoder->set_fup_tx_flags = false; in intel_pt_walk_fup()
1220 decoder->ip); in intel_pt_walk_fup()
1221 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1227 decoder->ip); in intel_pt_walk_fup()
1228 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1232 intel_pt_bug(decoder); in intel_pt_walk_fup()
1236 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_tip() argument
1241 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tip()
1243 decoder->pgd_ip && in intel_pt_walk_tip()
1244 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1245 (decoder->state.type & INTEL_PT_BRANCH) && in intel_pt_walk_tip()
1246 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { in intel_pt_walk_tip()
1248 decoder->no_progress = 0; in intel_pt_walk_tip()
1249 decoder->pge = false; in intel_pt_walk_tip()
1250 decoder->continuous_period = false; in intel_pt_walk_tip()
1251 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1252 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1261 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { in intel_pt_walk_tip()
1262 decoder->pge = false; in intel_pt_walk_tip()
1263 decoder->continuous_period = false; in intel_pt_walk_tip()
1264 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1265 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1266 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1267 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1269 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1270 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1272 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1274 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1275 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1276 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1277 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1279 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1280 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1287 uint64_t to_ip = decoder->ip + intel_pt_insn.length + in intel_pt_walk_tip()
1290 if (decoder->pgd_ip && in intel_pt_walk_tip()
1291 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1292 decoder->pgd_ip(to_ip, decoder->data)) { in intel_pt_walk_tip()
1294 decoder->pge = false; in intel_pt_walk_tip()
1295 decoder->continuous_period = false; in intel_pt_walk_tip()
1296 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1297 decoder->ip = to_ip; in intel_pt_walk_tip()
1298 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1299 decoder->state.to_ip = to_ip; in intel_pt_walk_tip()
1300 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1304 decoder->ip); in intel_pt_walk_tip()
1305 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_tip()
1309 return intel_pt_bug(decoder); in intel_pt_walk_tip()
1312 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) in intel_pt_walk_tnt() argument
1318 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tnt()
1325 if (!decoder->return_compression) { in intel_pt_walk_tnt()
1327 decoder->ip); in intel_pt_walk_tnt()
1328 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1331 if (!decoder->ret_addr) { in intel_pt_walk_tnt()
1333 decoder->ip); in intel_pt_walk_tnt()
1334 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1337 if (!(decoder->tnt.payload & BIT63)) { in intel_pt_walk_tnt()
1339 decoder->ip); in intel_pt_walk_tnt()
1340 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1343 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1344 if (decoder->tnt.count) in intel_pt_walk_tnt()
1345 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1347 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1348 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1349 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1350 decoder->ip = decoder->ret_addr; in intel_pt_walk_tnt()
1351 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1357 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_tnt()
1360 if (decoder->packet.type != INTEL_PT_TIP || in intel_pt_walk_tnt()
1361 decoder->packet.count == 0) { in intel_pt_walk_tnt()
1363 decoder->ip); in intel_pt_walk_tnt()
1364 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1365 decoder->pkt_step = 0; in intel_pt_walk_tnt()
1368 intel_pt_set_last_ip(decoder); in intel_pt_walk_tnt()
1369 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1370 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tnt()
1371 decoder->ip = decoder->last_ip; in intel_pt_walk_tnt()
1376 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1377 if (decoder->tnt.count) in intel_pt_walk_tnt()
1378 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1380 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1381 if (decoder->tnt.payload & BIT63) { in intel_pt_walk_tnt()
1382 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1383 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1384 decoder->ip += intel_pt_insn.length + in intel_pt_walk_tnt()
1386 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1390 if (decoder->state.type & INTEL_PT_INSTRUCTION) { in intel_pt_walk_tnt()
1391 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1392 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_tnt()
1393 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1394 decoder->state.to_ip = 0; in intel_pt_walk_tnt()
1395 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1398 decoder->sample_cyc = false; in intel_pt_walk_tnt()
1399 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1400 if (!decoder->tnt.count) { in intel_pt_walk_tnt()
1401 intel_pt_update_sample_time(decoder); in intel_pt_walk_tnt()
1404 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1408 return intel_pt_bug(decoder); in intel_pt_walk_tnt()
1412 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) in intel_pt_mode_tsx() argument
1417 fup_tx_flags = decoder->packet.payload & in intel_pt_mode_tsx()
1419 err = intel_pt_get_next_packet(decoder); in intel_pt_mode_tsx()
1422 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_mode_tsx()
1423 decoder->fup_tx_flags = fup_tx_flags; in intel_pt_mode_tsx()
1424 decoder->set_fup_tx_flags = true; in intel_pt_mode_tsx()
1425 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) in intel_pt_mode_tsx()
1429 decoder->pos); in intel_pt_mode_tsx()
1430 intel_pt_update_in_tx(decoder); in intel_pt_mode_tsx()
1450 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_tsc_timestamp() argument
1454 decoder->have_tma = false; in intel_pt_calc_tsc_timestamp()
1456 if (decoder->ref_timestamp) { in intel_pt_calc_tsc_timestamp()
1457 timestamp = intel_pt_8b_tsc(decoder->packet.payload, in intel_pt_calc_tsc_timestamp()
1458 decoder->ref_timestamp); in intel_pt_calc_tsc_timestamp()
1459 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1460 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1461 decoder->ref_timestamp = 0; in intel_pt_calc_tsc_timestamp()
1462 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1463 } else if (decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1464 timestamp = decoder->packet.payload | in intel_pt_calc_tsc_timestamp()
1465 (decoder->timestamp & (0xffULL << 56)); in intel_pt_calc_tsc_timestamp()
1466 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1467 if (timestamp < decoder->timestamp && in intel_pt_calc_tsc_timestamp()
1468 decoder->timestamp - timestamp < decoder->tsc_slip) { in intel_pt_calc_tsc_timestamp()
1471 timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1473 if (timestamp < decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1476 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1478 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1479 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1482 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_tsc_timestamp()
1483 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1484 decoder->cycle_cnt = 0; in intel_pt_calc_tsc_timestamp()
1485 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_tsc_timestamp()
1486 intel_pt_calc_cyc_to_tsc(decoder, false); in intel_pt_calc_tsc_timestamp()
1489 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_tsc_timestamp()
1492 static int intel_pt_overflow(struct intel_pt_decoder *decoder) in intel_pt_overflow() argument
1495 intel_pt_clear_tx_flags(decoder); in intel_pt_overflow()
1496 decoder->timestamp_insn_cnt = 0; in intel_pt_overflow()
1497 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_overflow()
1498 decoder->state.from_ip = decoder->ip; in intel_pt_overflow()
1499 decoder->ip = 0; in intel_pt_overflow()
1500 decoder->pge = false; in intel_pt_overflow()
1501 decoder->set_fup_tx_flags = false; in intel_pt_overflow()
1502 decoder->set_fup_ptw = false; in intel_pt_overflow()
1503 decoder->set_fup_mwait = false; in intel_pt_overflow()
1504 decoder->set_fup_pwre = false; in intel_pt_overflow()
1505 decoder->set_fup_exstop = false; in intel_pt_overflow()
1506 decoder->set_fup_bep = false; in intel_pt_overflow()
1507 decoder->overflow = true; in intel_pt_overflow()
1511 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_pge() argument
1513 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_pge()
1516 decoder->cyc_cnt_timestamp = decoder->timestamp; in intel_pt_mtc_cyc_cnt_pge()
1517 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_pge()
1520 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_cbr() argument
1522 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; in intel_pt_mtc_cyc_cnt_cbr()
1524 if (decoder->pge) in intel_pt_mtc_cyc_cnt_cbr()
1525 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_mtc_cyc_cnt_cbr()
1528 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_upd() argument
1532 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_upd()
1535 decoder->sample_cyc = true; in intel_pt_mtc_cyc_cnt_upd()
1537 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) in intel_pt_mtc_cyc_cnt_upd()
1540 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; in intel_pt_mtc_cyc_cnt_upd()
1541 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1543 if (tot_cyc_cnt > decoder->tot_cyc_cnt) in intel_pt_mtc_cyc_cnt_upd()
1544 decoder->tot_cyc_cnt = tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1547 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) in intel_pt_calc_tma() argument
1549 uint32_t ctc = decoder->packet.payload; in intel_pt_calc_tma()
1550 uint32_t fc = decoder->packet.count; in intel_pt_calc_tma()
1551 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_tma()
1553 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_tma()
1556 if (decoder->pge && !decoder->in_psb) in intel_pt_calc_tma()
1557 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_calc_tma()
1559 intel_pt_mtc_cyc_cnt_upd(decoder); in intel_pt_calc_tma()
1561 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_tma()
1562 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; in intel_pt_calc_tma()
1563 if (decoder->tsc_ctc_mult) { in intel_pt_calc_tma()
1564 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_tma()
1566 decoder->ctc_timestamp -= multdiv(ctc_rem, in intel_pt_calc_tma()
1567 decoder->tsc_ctc_ratio_n, in intel_pt_calc_tma()
1568 decoder->tsc_ctc_ratio_d); in intel_pt_calc_tma()
1570 decoder->ctc_delta = 0; in intel_pt_calc_tma()
1571 decoder->have_tma = true; in intel_pt_calc_tma()
1572 decoder->fixup_last_mtc = true; in intel_pt_calc_tma()
1574 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); in intel_pt_calc_tma()
1577 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_mtc_timestamp() argument
1582 if (!decoder->have_tma) in intel_pt_calc_mtc_timestamp()
1585 mtc = decoder->packet.payload; in intel_pt_calc_mtc_timestamp()
1587 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { in intel_pt_calc_mtc_timestamp()
1588 decoder->fixup_last_mtc = false; in intel_pt_calc_mtc_timestamp()
1589 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_mtc_timestamp()
1590 &decoder->last_mtc); in intel_pt_calc_mtc_timestamp()
1593 if (mtc > decoder->last_mtc) in intel_pt_calc_mtc_timestamp()
1594 mtc_delta = mtc - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1596 mtc_delta = mtc + 256 - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1598 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_mtc_timestamp()
1600 if (decoder->tsc_ctc_mult) { in intel_pt_calc_mtc_timestamp()
1601 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1602 decoder->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_mtc_timestamp()
1604 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1605 multdiv(decoder->ctc_delta, in intel_pt_calc_mtc_timestamp()
1606 decoder->tsc_ctc_ratio_n, in intel_pt_calc_mtc_timestamp()
1607 decoder->tsc_ctc_ratio_d); in intel_pt_calc_mtc_timestamp()
1610 if (timestamp < decoder->timestamp) in intel_pt_calc_mtc_timestamp()
1612 timestamp, decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1614 decoder->timestamp = timestamp; in intel_pt_calc_mtc_timestamp()
1616 intel_pt_mtc_cyc_cnt_upd(decoder); in intel_pt_calc_mtc_timestamp()
1618 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_mtc_timestamp()
1619 decoder->last_mtc = mtc; in intel_pt_calc_mtc_timestamp()
1621 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_mtc_timestamp()
1622 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_mtc_timestamp()
1623 decoder->cycle_cnt = 0; in intel_pt_calc_mtc_timestamp()
1624 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_mtc_timestamp()
1625 intel_pt_calc_cyc_to_tsc(decoder, true); in intel_pt_calc_mtc_timestamp()
1628 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1631 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) in intel_pt_calc_cbr() argument
1633 unsigned int cbr = decoder->packet.payload & 0xff; in intel_pt_calc_cbr()
1635 decoder->cbr_payload = decoder->packet.payload; in intel_pt_calc_cbr()
1637 if (decoder->cbr == cbr) in intel_pt_calc_cbr()
1640 decoder->cbr = cbr; in intel_pt_calc_cbr()
1641 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cbr()
1642 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_cbr()
1643 decoder->cycle_cnt = 0; in intel_pt_calc_cbr()
1645 intel_pt_mtc_cyc_cnt_cbr(decoder); in intel_pt_calc_cbr()
1648 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_cyc_timestamp() argument
1650 uint64_t timestamp = decoder->cyc_ref_timestamp; in intel_pt_calc_cyc_timestamp()
1652 decoder->have_cyc = true; in intel_pt_calc_cyc_timestamp()
1654 decoder->cycle_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
1655 if (decoder->pge) in intel_pt_calc_cyc_timestamp()
1656 decoder->tot_cyc_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
1657 decoder->sample_cyc = true; in intel_pt_calc_cyc_timestamp()
1659 if (!decoder->cyc_ref_timestamp) in intel_pt_calc_cyc_timestamp()
1662 if (decoder->have_calc_cyc_to_tsc) in intel_pt_calc_cyc_timestamp()
1663 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1664 else if (decoder->cbr) in intel_pt_calc_cyc_timestamp()
1665 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1669 if (timestamp < decoder->timestamp) in intel_pt_calc_cyc_timestamp()
1671 timestamp, decoder->timestamp); in intel_pt_calc_cyc_timestamp()
1673 decoder->timestamp = timestamp; in intel_pt_calc_cyc_timestamp()
1675 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_cyc_timestamp()
1677 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_cyc_timestamp()
1680 static void intel_pt_bbp(struct intel_pt_decoder *decoder) in intel_pt_bbp() argument
1682 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { in intel_pt_bbp()
1683 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); in intel_pt_bbp()
1684 decoder->state.items.is_32_bit = false; in intel_pt_bbp()
1686 decoder->blk_type = decoder->packet.payload; in intel_pt_bbp()
1687 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); in intel_pt_bbp()
1688 if (decoder->blk_type == INTEL_PT_GP_REGS) in intel_pt_bbp()
1689 decoder->state.items.is_32_bit = decoder->packet.count; in intel_pt_bbp()
1690 if (decoder->blk_type_pos < 0) { in intel_pt_bbp()
1692 decoder->blk_type); in intel_pt_bbp()
1693 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { in intel_pt_bbp()
1695 decoder->blk_type); in intel_pt_bbp()
1699 static void intel_pt_bip(struct intel_pt_decoder *decoder) in intel_pt_bip() argument
1701 uint32_t id = decoder->packet.count; in intel_pt_bip()
1703 int pos = decoder->blk_type_pos; in intel_pt_bip()
1707 id, decoder->blk_type); in intel_pt_bip()
1711 if (decoder->state.items.mask[pos] & bit) { in intel_pt_bip()
1713 id, decoder->blk_type); in intel_pt_bip()
1716 decoder->state.items.mask[pos] |= bit; in intel_pt_bip()
1717 decoder->state.items.val[pos][id] = decoder->packet.payload; in intel_pt_bip()
1721 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) in intel_pt_walk_psbend() argument
1725 decoder->in_psb = true; in intel_pt_walk_psbend()
1728 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psbend()
1732 switch (decoder->packet.type) { in intel_pt_walk_psbend()
1755 decoder->have_tma = false; in intel_pt_walk_psbend()
1761 err = intel_pt_overflow(decoder); in intel_pt_walk_psbend()
1765 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psbend()
1769 intel_pt_calc_tma(decoder); in intel_pt_walk_psbend()
1773 intel_pt_calc_cbr(decoder); in intel_pt_walk_psbend()
1777 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psbend()
1781 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psbend()
1785 decoder->pge = true; in intel_pt_walk_psbend()
1786 if (decoder->packet.count) { in intel_pt_walk_psbend()
1787 intel_pt_set_last_ip(decoder); in intel_pt_walk_psbend()
1788 if (decoder->hop) { in intel_pt_walk_psbend()
1790 decoder->ip = decoder->last_ip; in intel_pt_walk_psbend()
1791 decoder->hop_psb_fup = true; in intel_pt_walk_psbend()
1797 intel_pt_update_in_tx(decoder); in intel_pt_walk_psbend()
1801 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psbend()
1802 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_psbend()
1803 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_psbend()
1807 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_psbend()
1818 decoder->in_psb = false; in intel_pt_walk_psbend()
1823 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_fup_tip() argument
1827 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { in intel_pt_walk_fup_tip()
1828 decoder->tx_flags = 0; in intel_pt_walk_fup_tip()
1829 decoder->state.flags &= ~INTEL_PT_IN_TX; in intel_pt_walk_fup_tip()
1830 decoder->state.flags |= INTEL_PT_ABORT_TX; in intel_pt_walk_fup_tip()
1832 decoder->state.flags |= INTEL_PT_ASYNC; in intel_pt_walk_fup_tip()
1836 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_fup_tip()
1840 switch (decoder->packet.type) { in intel_pt_walk_fup_tip()
1862 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_fup_tip()
1863 decoder->pkt_step = 0; in intel_pt_walk_fup_tip()
1867 intel_pt_calc_cbr(decoder); in intel_pt_walk_fup_tip()
1871 return intel_pt_overflow(decoder); in intel_pt_walk_fup_tip()
1874 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1875 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1876 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1878 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1879 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1881 decoder->pge = false; in intel_pt_walk_fup_tip()
1882 decoder->continuous_period = false; in intel_pt_walk_fup_tip()
1883 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_fup_tip()
1887 decoder->pge = true; in intel_pt_walk_fup_tip()
1889 decoder->ip); in intel_pt_walk_fup_tip()
1890 decoder->state.from_ip = 0; in intel_pt_walk_fup_tip()
1891 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1892 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1894 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1895 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1897 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_fup_tip()
1898 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_fup_tip()
1902 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1903 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1904 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1906 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1907 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1912 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_fup_tip()
1916 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_fup_tip()
1917 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_fup_tip()
1918 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_fup_tip()
1922 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_fup_tip()
1926 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_fup_tip()
1935 return intel_pt_bug(decoder); in intel_pt_walk_fup_tip()
1940 static int intel_pt_resample(struct intel_pt_decoder *decoder) in intel_pt_resample() argument
1942 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_resample()
1943 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_resample()
1944 decoder->state.from_ip = decoder->ip; in intel_pt_resample()
1945 decoder->state.to_ip = 0; in intel_pt_resample()
1954 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
1957 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) in intel_pt_hop_trace() argument
1962 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { in intel_pt_hop_trace()
1963 *err = intel_pt_scan_for_psb(decoder); in intel_pt_hop_trace()
1968 switch (decoder->packet.type) { in intel_pt_hop_trace()
1973 decoder->pge = false; in intel_pt_hop_trace()
1974 if (!decoder->packet.count) in intel_pt_hop_trace()
1976 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1977 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_hop_trace()
1978 decoder->state.from_ip = 0; in intel_pt_hop_trace()
1979 decoder->state.to_ip = decoder->ip; in intel_pt_hop_trace()
1983 if (!decoder->packet.count) in intel_pt_hop_trace()
1985 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1986 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
1987 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
1988 decoder->state.to_ip = 0; in intel_pt_hop_trace()
1992 if (!decoder->packet.count) in intel_pt_hop_trace()
1994 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1995 if (decoder->set_fup_mwait || decoder->set_fup_pwre) in intel_pt_hop_trace()
1997 if (!decoder->branch_enable || !decoder->pge) in intel_pt_hop_trace()
2000 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
2001 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2002 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2003 intel_pt_fup_event(decoder); in intel_pt_hop_trace()
2006 intel_pt_fup_event(decoder); in intel_pt_hop_trace()
2007 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; in intel_pt_hop_trace()
2008 *err = intel_pt_walk_fup_tip(decoder); in intel_pt_hop_trace()
2009 if (!*err && decoder->state.to_ip) in intel_pt_hop_trace()
2010 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_hop_trace()
2014 decoder->last_ip = 0; in intel_pt_hop_trace()
2015 decoder->have_last_ip = true; in intel_pt_hop_trace()
2016 decoder->hop_psb_fup = false; in intel_pt_hop_trace()
2017 *err = intel_pt_walk_psbend(decoder); in intel_pt_hop_trace()
2022 if (decoder->hop_psb_fup) { in intel_pt_hop_trace()
2023 decoder->hop_psb_fup = false; in intel_pt_hop_trace()
2024 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
2025 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2026 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2029 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_hop_trace()
2030 decoder->state.type = 0; in intel_pt_hop_trace()
2067 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) in intel_pt_walk_trace() argument
2074 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2079 if (decoder->cyc_threshold) { in intel_pt_walk_trace()
2080 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) in intel_pt_walk_trace()
2081 decoder->sample_cyc = false; in intel_pt_walk_trace()
2082 last_packet_type = decoder->packet.type; in intel_pt_walk_trace()
2085 if (decoder->hop) { in intel_pt_walk_trace()
2086 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { in intel_pt_walk_trace()
2098 switch (decoder->packet.type) { in intel_pt_walk_trace()
2100 if (!decoder->packet.count) in intel_pt_walk_trace()
2102 decoder->tnt = decoder->packet; in intel_pt_walk_trace()
2103 decoder->pkt_state = INTEL_PT_STATE_TNT; in intel_pt_walk_trace()
2104 err = intel_pt_walk_tnt(decoder); in intel_pt_walk_trace()
2110 if (decoder->packet.count != 0) in intel_pt_walk_trace()
2111 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2112 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; in intel_pt_walk_trace()
2113 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
2116 decoder->pge = true; in intel_pt_walk_trace()
2117 decoder->overflow = false; in intel_pt_walk_trace()
2118 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_trace()
2119 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
2121 decoder->pos); in intel_pt_walk_trace()
2124 intel_pt_set_ip(decoder); in intel_pt_walk_trace()
2125 decoder->state.from_ip = 0; in intel_pt_walk_trace()
2126 decoder->state.to_ip = decoder->ip; in intel_pt_walk_trace()
2127 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_trace()
2132 if (decoder->hop) in intel_pt_walk_trace()
2133 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_walk_trace()
2138 return intel_pt_overflow(decoder); in intel_pt_walk_trace()
2141 if (decoder->packet.count != 0) in intel_pt_walk_trace()
2142 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2143 decoder->pkt_state = INTEL_PT_STATE_TIP; in intel_pt_walk_trace()
2144 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
2147 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
2149 decoder->pos); in intel_pt_walk_trace()
2153 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2154 if (!decoder->branch_enable || !decoder->pge) { in intel_pt_walk_trace()
2155 decoder->ip = decoder->last_ip; in intel_pt_walk_trace()
2156 if (intel_pt_fup_event(decoder)) in intel_pt_walk_trace()
2161 if (decoder->set_fup_mwait) in intel_pt_walk_trace()
2164 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; in intel_pt_walk_trace()
2166 decoder->pkt_state = INTEL_PT_STATE_FUP; in intel_pt_walk_trace()
2167 err = intel_pt_walk_fup(decoder); in intel_pt_walk_trace()
2174 return intel_pt_walk_fup_tip(decoder); in intel_pt_walk_trace()
2177 decoder->pge = false; in intel_pt_walk_trace()
2178 decoder->continuous_period = false; in intel_pt_walk_trace()
2179 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_trace()
2180 decoder->have_tma = false; in intel_pt_walk_trace()
2184 decoder->last_ip = 0; in intel_pt_walk_trace()
2185 decoder->have_last_ip = true; in intel_pt_walk_trace()
2186 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_trace()
2187 err = intel_pt_walk_psbend(decoder); in intel_pt_walk_trace()
2197 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
2198 decoder->state.type = 0; in intel_pt_walk_trace()
2204 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_trace()
2208 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_trace()
2209 if (decoder->period_type != INTEL_PT_PERIOD_MTC) in intel_pt_walk_trace()
2215 if (!decoder->mtc_insn) in intel_pt_walk_trace()
2217 decoder->mtc_insn = false; in intel_pt_walk_trace()
2219 if (!decoder->timestamp) in intel_pt_walk_trace()
2221 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_trace()
2222 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2223 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2224 decoder->mtc_insn = false; in intel_pt_walk_trace()
2228 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_trace()
2232 intel_pt_calc_tma(decoder); in intel_pt_walk_trace()
2236 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_trace()
2240 intel_pt_calc_cbr(decoder); in intel_pt_walk_trace()
2241 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
2242 decoder->state.type = 0; in intel_pt_walk_trace()
2248 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_trace()
2253 if (!decoder->pge || decoder->in_psb) { in intel_pt_walk_trace()
2254 intel_pt_update_in_tx(decoder); in intel_pt_walk_trace()
2257 err = intel_pt_mode_tsx(decoder, &no_tip); in intel_pt_walk_trace()
2263 return intel_pt_bug(decoder); in intel_pt_walk_trace()
2272 decoder->fup_ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
2273 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2276 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2277 decoder->set_fup_ptw = true; in intel_pt_walk_trace()
2281 decoder->pos); in intel_pt_walk_trace()
2286 decoder->state.type = INTEL_PT_PTW; in intel_pt_walk_trace()
2287 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2288 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2289 decoder->state.ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
2293 decoder->fup_mwait_payload = decoder->packet.payload; in intel_pt_walk_trace()
2294 decoder->set_fup_mwait = true; in intel_pt_walk_trace()
2298 if (decoder->set_fup_mwait) { in intel_pt_walk_trace()
2299 decoder->fup_pwre_payload = in intel_pt_walk_trace()
2300 decoder->packet.payload; in intel_pt_walk_trace()
2301 decoder->set_fup_pwre = true; in intel_pt_walk_trace()
2304 decoder->state.type = INTEL_PT_PWR_ENTRY; in intel_pt_walk_trace()
2305 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2306 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2307 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
2311 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2314 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2315 decoder->set_fup_exstop = true; in intel_pt_walk_trace()
2319 decoder->pos); in intel_pt_walk_trace()
2324 decoder->state.type = INTEL_PT_EX_STOP; in intel_pt_walk_trace()
2325 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2326 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2330 decoder->state.type = INTEL_PT_PWR_EXIT; in intel_pt_walk_trace()
2331 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2332 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2333 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
2337 intel_pt_bbp(decoder); in intel_pt_walk_trace()
2341 intel_pt_bip(decoder); in intel_pt_walk_trace()
2345 decoder->state.type = INTEL_PT_BLK_ITEMS; in intel_pt_walk_trace()
2346 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2347 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2351 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2354 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2355 decoder->set_fup_bep = true; in intel_pt_walk_trace()
2359 decoder->pos); in intel_pt_walk_trace()
2364 return intel_pt_bug(decoder); in intel_pt_walk_trace()
2369 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) in intel_pt_have_ip() argument
2371 return decoder->packet.count && in intel_pt_have_ip()
2372 (decoder->have_last_ip || decoder->packet.count == 3 || in intel_pt_have_ip()
2373 decoder->packet.count == 6); in intel_pt_have_ip()
2377 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) in intel_pt_walk_psb() argument
2381 decoder->in_psb = true; in intel_pt_walk_psb()
2384 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psb()
2388 switch (decoder->packet.type) { in intel_pt_walk_psb()
2390 decoder->continuous_period = false; in intel_pt_walk_psb()
2410 decoder->pge = true; in intel_pt_walk_psb()
2411 if (intel_pt_have_ip(decoder)) { in intel_pt_walk_psb()
2412 uint64_t current_ip = decoder->ip; in intel_pt_walk_psb()
2414 intel_pt_set_ip(decoder); in intel_pt_walk_psb()
2417 decoder->ip); in intel_pt_walk_psb()
2422 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psb()
2426 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psb()
2430 intel_pt_calc_tma(decoder); in intel_pt_walk_psb()
2434 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_psb()
2438 intel_pt_calc_cbr(decoder); in intel_pt_walk_psb()
2442 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psb()
2446 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psb()
2450 intel_pt_update_in_tx(decoder); in intel_pt_walk_psb()
2454 decoder->pge = false; in intel_pt_walk_psb()
2455 decoder->continuous_period = false; in intel_pt_walk_psb()
2456 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_psb()
2460 decoder->have_tma = false; in intel_pt_walk_psb()
2462 if (decoder->ip) in intel_pt_walk_psb()
2463 decoder->pkt_state = INTEL_PT_STATE_ERR4; in intel_pt_walk_psb()
2465 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_psb()
2470 err = intel_pt_bug(decoder); in intel_pt_walk_psb()
2474 err = intel_pt_overflow(decoder); in intel_pt_walk_psb()
2490 decoder->in_psb = false; in intel_pt_walk_psb()
2495 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) in intel_pt_walk_to_ip() argument
2500 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_to_ip()
2504 switch (decoder->packet.type) { in intel_pt_walk_to_ip()
2506 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2507 decoder->pge = false; in intel_pt_walk_to_ip()
2508 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2509 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2510 if (!decoder->ip) in intel_pt_walk_to_ip()
2512 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_to_ip()
2516 decoder->pge = true; in intel_pt_walk_to_ip()
2517 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_to_ip()
2518 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2519 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2520 if (!decoder->ip) in intel_pt_walk_to_ip()
2522 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_to_ip()
2526 decoder->pge = true; in intel_pt_walk_to_ip()
2527 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2528 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2529 if (!decoder->ip) in intel_pt_walk_to_ip()
2534 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2535 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2536 if (decoder->ip) in intel_pt_walk_to_ip()
2541 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_to_ip()
2545 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_to_ip()
2549 intel_pt_calc_tma(decoder); in intel_pt_walk_to_ip()
2553 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_to_ip()
2557 intel_pt_calc_cbr(decoder); in intel_pt_walk_to_ip()
2561 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_to_ip()
2565 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_to_ip()
2569 intel_pt_update_in_tx(decoder); in intel_pt_walk_to_ip()
2573 return intel_pt_overflow(decoder); in intel_pt_walk_to_ip()
2576 return intel_pt_bug(decoder); in intel_pt_walk_to_ip()
2579 decoder->pge = false; in intel_pt_walk_to_ip()
2580 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2581 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_to_ip()
2582 decoder->have_tma = false; in intel_pt_walk_to_ip()
2586 decoder->last_ip = 0; in intel_pt_walk_to_ip()
2587 decoder->have_last_ip = true; in intel_pt_walk_to_ip()
2588 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_to_ip()
2589 err = intel_pt_walk_psb(decoder); in intel_pt_walk_to_ip()
2592 if (decoder->ip) { in intel_pt_walk_to_ip()
2594 decoder->state.type = 0; in intel_pt_walk_to_ip()
2621 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) in intel_pt_sync_ip() argument
2625 decoder->set_fup_tx_flags = false; in intel_pt_sync_ip()
2626 decoder->set_fup_ptw = false; in intel_pt_sync_ip()
2627 decoder->set_fup_mwait = false; in intel_pt_sync_ip()
2628 decoder->set_fup_pwre = false; in intel_pt_sync_ip()
2629 decoder->set_fup_exstop = false; in intel_pt_sync_ip()
2630 decoder->set_fup_bep = false; in intel_pt_sync_ip()
2631 decoder->overflow = false; in intel_pt_sync_ip()
2633 if (!decoder->branch_enable) { in intel_pt_sync_ip()
2634 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2635 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync_ip()
2640 err = intel_pt_walk_to_ip(decoder); in intel_pt_sync_ip()
2645 if (decoder->hop) in intel_pt_sync_ip()
2646 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync_ip()
2648 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2650 decoder->state.from_ip = 0; in intel_pt_sync_ip()
2651 decoder->state.to_ip = decoder->ip; in intel_pt_sync_ip()
2652 intel_pt_log_to("Setting IP", decoder->ip); in intel_pt_sync_ip()
2657 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) in intel_pt_part_psb() argument
2659 const unsigned char *end = decoder->buf + decoder->len; in intel_pt_part_psb()
2663 if (i > decoder->len) in intel_pt_part_psb()
2671 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) in intel_pt_rest_psb() argument
2676 if (rest_psb > decoder->len || in intel_pt_rest_psb()
2677 memcmp(decoder->buf, psb + part_psb, rest_psb)) in intel_pt_rest_psb()
2683 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, in intel_pt_get_split_psb() argument
2688 decoder->pos += decoder->len; in intel_pt_get_split_psb()
2689 decoder->len = 0; in intel_pt_get_split_psb()
2691 ret = intel_pt_get_next_data(decoder, false); in intel_pt_get_split_psb()
2695 rest_psb = intel_pt_rest_psb(decoder, part_psb); in intel_pt_get_split_psb()
2699 decoder->pos -= part_psb; in intel_pt_get_split_psb()
2700 decoder->next_buf = decoder->buf + rest_psb; in intel_pt_get_split_psb()
2701 decoder->next_len = decoder->len - rest_psb; in intel_pt_get_split_psb()
2702 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); in intel_pt_get_split_psb()
2703 decoder->buf = decoder->temp_buf; in intel_pt_get_split_psb()
2704 decoder->len = INTEL_PT_PSB_LEN; in intel_pt_get_split_psb()
2709 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) in intel_pt_scan_for_psb() argument
2716 if (!decoder->len) { in intel_pt_scan_for_psb()
2717 ret = intel_pt_get_next_data(decoder, false); in intel_pt_scan_for_psb()
2722 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, in intel_pt_scan_for_psb()
2727 part_psb = intel_pt_part_psb(decoder); in intel_pt_scan_for_psb()
2729 ret = intel_pt_get_split_psb(decoder, part_psb); in intel_pt_scan_for_psb()
2733 decoder->pos += decoder->len; in intel_pt_scan_for_psb()
2734 decoder->len = 0; in intel_pt_scan_for_psb()
2739 decoder->pkt_step = next - decoder->buf; in intel_pt_scan_for_psb()
2740 return intel_pt_get_next_packet(decoder); in intel_pt_scan_for_psb()
2744 static int intel_pt_sync(struct intel_pt_decoder *decoder) in intel_pt_sync() argument
2748 decoder->pge = false; in intel_pt_sync()
2749 decoder->continuous_period = false; in intel_pt_sync()
2750 decoder->have_last_ip = false; in intel_pt_sync()
2751 decoder->last_ip = 0; in intel_pt_sync()
2752 decoder->ip = 0; in intel_pt_sync()
2753 intel_pt_clear_stack(&decoder->stack); in intel_pt_sync()
2756 err = intel_pt_scan_for_psb(decoder); in intel_pt_sync()
2760 decoder->have_last_ip = true; in intel_pt_sync()
2761 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
2763 err = intel_pt_walk_psb(decoder); in intel_pt_sync()
2767 if (decoder->ip) { in intel_pt_sync()
2768 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync()
2773 if (decoder->hop) in intel_pt_sync()
2774 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync()
2776 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
2777 } else if (decoder->leap) { in intel_pt_sync()
2784 return intel_pt_sync_ip(decoder); in intel_pt_sync()
2790 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) in intel_pt_est_timestamp() argument
2792 uint64_t est = decoder->sample_insn_cnt << 1; in intel_pt_est_timestamp()
2794 if (!decoder->cbr || !decoder->max_non_turbo_ratio) in intel_pt_est_timestamp()
2797 est *= decoder->max_non_turbo_ratio; in intel_pt_est_timestamp()
2798 est /= decoder->cbr; in intel_pt_est_timestamp()
2800 return decoder->sample_timestamp + est; in intel_pt_est_timestamp()
2803 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) in intel_pt_decode() argument
2808 decoder->state.type = INTEL_PT_BRANCH; in intel_pt_decode()
2809 decoder->state.flags = 0; in intel_pt_decode()
2811 switch (decoder->pkt_state) { in intel_pt_decode()
2813 err = intel_pt_sync(decoder); in intel_pt_decode()
2816 decoder->have_last_ip = false; in intel_pt_decode()
2817 decoder->last_ip = 0; in intel_pt_decode()
2818 decoder->ip = 0; in intel_pt_decode()
2821 err = intel_pt_sync_ip(decoder); in intel_pt_decode()
2824 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2828 err = intel_pt_walk_tnt(decoder); in intel_pt_decode()
2830 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2834 err = intel_pt_walk_tip(decoder); in intel_pt_decode()
2837 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2839 err = intel_pt_walk_fup_tip(decoder); in intel_pt_decode()
2842 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2844 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2847 err = intel_pt_resample(decoder); in intel_pt_decode()
2850 err = intel_pt_bug(decoder); in intel_pt_decode()
2856 decoder->state.err = intel_pt_ext_err(err); in intel_pt_decode()
2858 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
2859 intel_pt_update_sample_time(decoder); in intel_pt_decode()
2860 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
2862 decoder->state.err = 0; in intel_pt_decode()
2863 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_decode()
2864 decoder->cbr_seen = decoder->cbr; in intel_pt_decode()
2865 if (!decoder->state.type) { in intel_pt_decode()
2866 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
2867 decoder->state.to_ip = 0; in intel_pt_decode()
2869 decoder->state.type |= INTEL_PT_CBR_CHG; in intel_pt_decode()
2870 decoder->state.cbr_payload = decoder->cbr_payload; in intel_pt_decode()
2871 decoder->state.cbr = decoder->cbr; in intel_pt_decode()
2873 if (intel_pt_sample_time(decoder->pkt_state)) { in intel_pt_decode()
2874 intel_pt_update_sample_time(decoder); in intel_pt_decode()
2875 if (decoder->sample_cyc) { in intel_pt_decode()
2876 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
2877 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
2878 decoder->sample_cyc = false; in intel_pt_decode()
2885 if (!decoder->have_cyc) in intel_pt_decode()
2886 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
2889 decoder->state.timestamp = decoder->sample_timestamp; in intel_pt_decode()
2890 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); in intel_pt_decode()
2891 decoder->state.cr3 = decoder->cr3; in intel_pt_decode()
2892 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; in intel_pt_decode()
2893 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; in intel_pt_decode()
2895 return &decoder->state; in intel_pt_decode()
3259 * intel_pt_fast_forward - reposition decoder forwards.
3260 * @decoder: Intel PT decoder
3263 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
3267 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) in intel_pt_fast_forward() argument
3277 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); in intel_pt_fast_forward()
3284 decoder->pos += decoder->len; in intel_pt_fast_forward()
3285 decoder->len = 0; in intel_pt_fast_forward()
3286 err = intel_pt_get_next_data(decoder, true); in intel_pt_fast_forward()
3290 } while (decoder->buf_timestamp != d.buf_timestamp); in intel_pt_fast_forward()
3293 if (!decoder->buf) in intel_pt_fast_forward()
3296 buf = (unsigned char *)decoder->buf; in intel_pt_fast_forward()
3297 len = decoder->len; in intel_pt_fast_forward()
3312 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); in intel_pt_fast_forward()
3321 decoder->pos += decoder->len - len; in intel_pt_fast_forward()
3322 decoder->buf = buf; in intel_pt_fast_forward()
3323 decoder->len = len; in intel_pt_fast_forward()
3324 intel_pt_reposition(decoder); in intel_pt_fast_forward()