Lines Matching refs:decoder
95 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
224 static void intel_pt_setup_period(struct intel_pt_decoder *decoder) in intel_pt_setup_period() argument
226 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) { in intel_pt_setup_period()
229 period = intel_pt_lower_power_of_2(decoder->period); in intel_pt_setup_period()
230 decoder->period_mask = ~(period - 1); in intel_pt_setup_period()
231 decoder->period_ticks = period; in intel_pt_setup_period()
244 struct intel_pt_decoder *decoder; in intel_pt_decoder_new() local
249 decoder = zalloc(sizeof(struct intel_pt_decoder)); in intel_pt_decoder_new()
250 if (!decoder) in intel_pt_decoder_new()
253 decoder->get_trace = params->get_trace; in intel_pt_decoder_new()
254 decoder->walk_insn = params->walk_insn; in intel_pt_decoder_new()
255 decoder->pgd_ip = params->pgd_ip; in intel_pt_decoder_new()
256 decoder->lookahead = params->lookahead; in intel_pt_decoder_new()
257 decoder->data = params->data; in intel_pt_decoder_new()
258 decoder->return_compression = params->return_compression; in intel_pt_decoder_new()
259 decoder->branch_enable = params->branch_enable; in intel_pt_decoder_new()
260 decoder->hop = params->quick >= 1; in intel_pt_decoder_new()
261 decoder->leap = params->quick >= 2; in intel_pt_decoder_new()
263 decoder->flags = params->flags; in intel_pt_decoder_new()
265 decoder->ctl = params->ctl; in intel_pt_decoder_new()
266 decoder->period = params->period; in intel_pt_decoder_new()
267 decoder->period_type = params->period_type; in intel_pt_decoder_new()
269 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio; in intel_pt_decoder_new()
270 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio; in intel_pt_decoder_new()
272 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl); in intel_pt_decoder_new()
274 intel_pt_setup_period(decoder); in intel_pt_decoder_new()
276 decoder->mtc_shift = params->mtc_period; in intel_pt_decoder_new()
277 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1; in intel_pt_decoder_new()
279 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n; in intel_pt_decoder_new()
280 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d; in intel_pt_decoder_new()
282 if (!decoder->tsc_ctc_ratio_n) in intel_pt_decoder_new()
283 decoder->tsc_ctc_ratio_d = 0; in intel_pt_decoder_new()
285 if (decoder->tsc_ctc_ratio_d) { in intel_pt_decoder_new()
286 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d)) in intel_pt_decoder_new()
287 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n / in intel_pt_decoder_new()
288 decoder->tsc_ctc_ratio_d; in intel_pt_decoder_new()
297 decoder->tsc_slip = 0x10000; in intel_pt_decoder_new()
299 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift); in intel_pt_decoder_new()
300 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n); in intel_pt_decoder_new()
301 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d); in intel_pt_decoder_new()
302 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult); in intel_pt_decoder_new()
303 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip); in intel_pt_decoder_new()
305 if (decoder->hop) in intel_pt_decoder_new()
308 return decoder; in intel_pt_decoder_new()
382 void intel_pt_decoder_free(struct intel_pt_decoder *decoder) in intel_pt_decoder_free() argument
384 intel_pt_free_stack(&decoder->stack); in intel_pt_decoder_free()
385 free(decoder); in intel_pt_decoder_free()
469 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder) in intel_pt_set_last_ip() argument
471 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip); in intel_pt_set_last_ip()
472 decoder->have_last_ip = true; in intel_pt_set_last_ip()
475 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder) in intel_pt_set_ip() argument
477 intel_pt_set_last_ip(decoder); in intel_pt_set_ip()
478 decoder->ip = decoder->last_ip; in intel_pt_set_ip()
481 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder) in intel_pt_decoder_log_packet() argument
483 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos, in intel_pt_decoder_log_packet()
484 decoder->buf); in intel_pt_decoder_log_packet()
487 static int intel_pt_bug(struct intel_pt_decoder *decoder) in intel_pt_bug() argument
490 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_bug()
494 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder) in intel_pt_clear_tx_flags() argument
496 decoder->tx_flags = 0; in intel_pt_clear_tx_flags()
499 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder) in intel_pt_update_in_tx() argument
501 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX; in intel_pt_update_in_tx()
504 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder) in intel_pt_bad_packet() argument
506 intel_pt_clear_tx_flags(decoder); in intel_pt_bad_packet()
507 decoder->have_tma = false; in intel_pt_bad_packet()
508 decoder->pkt_len = 1; in intel_pt_bad_packet()
509 decoder->pkt_step = 1; in intel_pt_bad_packet()
510 intel_pt_decoder_log_packet(decoder); in intel_pt_bad_packet()
511 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) { in intel_pt_bad_packet()
513 decoder->pkt_state = INTEL_PT_STATE_ERR1; in intel_pt_bad_packet()
518 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder) in intel_pt_update_sample_time() argument
520 decoder->sample_timestamp = decoder->timestamp; in intel_pt_update_sample_time()
521 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt; in intel_pt_update_sample_time()
524 static void intel_pt_reposition(struct intel_pt_decoder *decoder) in intel_pt_reposition() argument
526 decoder->ip = 0; in intel_pt_reposition()
527 decoder->pkt_state = INTEL_PT_STATE_NO_PSB; in intel_pt_reposition()
528 decoder->timestamp = 0; in intel_pt_reposition()
529 decoder->have_tma = false; in intel_pt_reposition()
532 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) in intel_pt_get_data() argument
537 decoder->pkt_step = 0; in intel_pt_get_data()
540 ret = decoder->get_trace(&buffer, decoder->data); in intel_pt_get_data()
543 decoder->buf = buffer.buf; in intel_pt_get_data()
544 decoder->len = buffer.len; in intel_pt_get_data()
545 if (!decoder->len) { in intel_pt_get_data()
549 decoder->buf_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
551 intel_pt_reposition(decoder); in intel_pt_get_data()
552 decoder->ref_timestamp = buffer.ref_timestamp; in intel_pt_get_data()
553 decoder->state.trace_nr = buffer.trace_nr; in intel_pt_get_data()
555 decoder->ref_timestamp); in intel_pt_get_data()
562 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder, in intel_pt_get_next_data() argument
565 if (!decoder->next_buf) in intel_pt_get_next_data()
566 return intel_pt_get_data(decoder, reposition); in intel_pt_get_next_data()
568 decoder->buf = decoder->next_buf; in intel_pt_get_next_data()
569 decoder->len = decoder->next_len; in intel_pt_get_next_data()
570 decoder->next_buf = 0; in intel_pt_get_next_data()
571 decoder->next_len = 0; in intel_pt_get_next_data()
575 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder) in intel_pt_get_split_packet() argument
577 unsigned char *buf = decoder->temp_buf; in intel_pt_get_split_packet()
581 old_len = decoder->len; in intel_pt_get_split_packet()
582 len = decoder->len; in intel_pt_get_split_packet()
583 memcpy(buf, decoder->buf, len); in intel_pt_get_split_packet()
585 ret = intel_pt_get_data(decoder, false); in intel_pt_get_split_packet()
587 decoder->pos += old_len; in intel_pt_get_split_packet()
592 if (n > decoder->len) in intel_pt_get_split_packet()
593 n = decoder->len; in intel_pt_get_split_packet()
594 memcpy(buf + len, decoder->buf, n); in intel_pt_get_split_packet()
597 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_split_packet()
598 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_split_packet()
600 decoder->next_buf = decoder->buf; in intel_pt_get_split_packet()
601 decoder->next_len = decoder->len; in intel_pt_get_split_packet()
602 decoder->buf = buf; in intel_pt_get_split_packet()
603 decoder->len = old_len; in intel_pt_get_split_packet()
604 return intel_pt_bad_packet(decoder); in intel_pt_get_split_packet()
607 decoder->next_buf = decoder->buf + (ret - old_len); in intel_pt_get_split_packet()
608 decoder->next_len = decoder->len - (ret - old_len); in intel_pt_get_split_packet()
610 decoder->buf = buf; in intel_pt_get_split_packet()
611 decoder->len = ret; in intel_pt_get_split_packet()
617 struct intel_pt_decoder *decoder; member
628 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder, in intel_pt_pkt_lookahead() argument
632 const unsigned char *buf = decoder->buf; in intel_pt_pkt_lookahead()
633 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx; in intel_pt_pkt_lookahead()
634 size_t len = decoder->len; in intel_pt_pkt_lookahead()
637 pkt_info.decoder = decoder; in intel_pt_pkt_lookahead()
638 pkt_info.pos = decoder->pos; in intel_pt_pkt_lookahead()
639 pkt_info.pkt_len = decoder->pkt_step; in intel_pt_pkt_lookahead()
640 pkt_info.last_packet_type = decoder->last_packet_type; in intel_pt_pkt_lookahead()
706 struct intel_pt_decoder *decoder = pkt_info->decoder; in intel_pt_calc_cyc_cb() local
739 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) { in intel_pt_calc_cyc_cb()
741 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_cyc_cb()
748 data->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_cyc_cb()
751 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
753 data->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
757 decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
758 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
781 data->timestamp - timestamp < decoder->tsc_slip) in intel_pt_calc_cyc_cb()
798 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_cyc_cb()
803 ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_cyc_cb()
805 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_cyc_cb()
808 if (decoder->tsc_ctc_mult) { in intel_pt_calc_cyc_cb()
809 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_cyc_cb()
812 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n, in intel_pt_calc_cyc_cb()
813 decoder->tsc_ctc_ratio_d); in intel_pt_calc_cyc_cb()
831 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cyc_cb()
847 if (!data->cbr && decoder->cbr) { in intel_pt_calc_cyc_cb()
848 data->cbr = decoder->cbr; in intel_pt_calc_cyc_cb()
849 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_cb()
855 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt; in intel_pt_calc_cyc_cb()
864 decoder->calc_cyc_to_tsc = cyc_to_tsc; in intel_pt_calc_cyc_cb()
865 decoder->have_calc_cyc_to_tsc = true; in intel_pt_calc_cyc_cb()
878 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder, in intel_pt_calc_cyc_to_tsc() argument
884 .last_mtc = decoder->last_mtc, in intel_pt_calc_cyc_to_tsc()
885 .ctc_timestamp = decoder->ctc_timestamp, in intel_pt_calc_cyc_to_tsc()
886 .ctc_delta = decoder->ctc_delta, in intel_pt_calc_cyc_to_tsc()
887 .tsc_timestamp = decoder->tsc_timestamp, in intel_pt_calc_cyc_to_tsc()
888 .timestamp = decoder->timestamp, in intel_pt_calc_cyc_to_tsc()
889 .have_tma = decoder->have_tma, in intel_pt_calc_cyc_to_tsc()
890 .fixup_last_mtc = decoder->fixup_last_mtc, in intel_pt_calc_cyc_to_tsc()
903 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data); in intel_pt_calc_cyc_to_tsc()
906 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder) in intel_pt_get_next_packet() argument
910 decoder->last_packet_type = decoder->packet.type; in intel_pt_get_next_packet()
913 decoder->pos += decoder->pkt_step; in intel_pt_get_next_packet()
914 decoder->buf += decoder->pkt_step; in intel_pt_get_next_packet()
915 decoder->len -= decoder->pkt_step; in intel_pt_get_next_packet()
917 if (!decoder->len) { in intel_pt_get_next_packet()
918 ret = intel_pt_get_next_data(decoder, false); in intel_pt_get_next_packet()
923 decoder->prev_pkt_ctx = decoder->pkt_ctx; in intel_pt_get_next_packet()
924 ret = intel_pt_get_packet(decoder->buf, decoder->len, in intel_pt_get_next_packet()
925 &decoder->packet, &decoder->pkt_ctx); in intel_pt_get_next_packet()
927 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) { in intel_pt_get_next_packet()
928 ret = intel_pt_get_split_packet(decoder); in intel_pt_get_next_packet()
933 return intel_pt_bad_packet(decoder); in intel_pt_get_next_packet()
935 decoder->pkt_len = ret; in intel_pt_get_next_packet()
936 decoder->pkt_step = ret; in intel_pt_get_next_packet()
937 intel_pt_decoder_log_packet(decoder); in intel_pt_get_next_packet()
938 } while (decoder->packet.type == INTEL_PT_PAD); in intel_pt_get_next_packet()
943 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder) in intel_pt_next_period() argument
947 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_next_period()
948 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
949 if (decoder->continuous_period) { in intel_pt_next_period()
950 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_next_period()
954 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_next_period()
955 if (masked_timestamp > decoder->last_masked_timestamp) { in intel_pt_next_period()
956 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_next_period()
957 decoder->continuous_period = true; in intel_pt_next_period()
961 if (masked_timestamp < decoder->last_masked_timestamp) in intel_pt_next_period()
962 return decoder->period_ticks; in intel_pt_next_period()
964 return decoder->period_ticks - (timestamp - masked_timestamp); in intel_pt_next_period()
967 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder) in intel_pt_next_sample() argument
969 switch (decoder->period_type) { in intel_pt_next_sample()
971 return decoder->period - decoder->period_insn_cnt; in intel_pt_next_sample()
973 return intel_pt_next_period(decoder); in intel_pt_next_sample()
981 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder) in intel_pt_sample_insn() argument
985 switch (decoder->period_type) { in intel_pt_sample_insn()
987 decoder->period_insn_cnt = 0; in intel_pt_sample_insn()
990 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt; in intel_pt_sample_insn()
991 masked_timestamp = timestamp & decoder->period_mask; in intel_pt_sample_insn()
992 if (masked_timestamp > decoder->last_masked_timestamp) in intel_pt_sample_insn()
993 decoder->last_masked_timestamp = masked_timestamp; in intel_pt_sample_insn()
995 decoder->last_masked_timestamp += decoder->period_ticks; in intel_pt_sample_insn()
1003 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_sample_insn()
1006 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, in intel_pt_walk_insn() argument
1012 if (!decoder->mtc_insn) in intel_pt_walk_insn()
1013 decoder->mtc_insn = true; in intel_pt_walk_insn()
1015 max_insn_cnt = intel_pt_next_sample(decoder); in intel_pt_walk_insn()
1017 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip, in intel_pt_walk_insn()
1018 max_insn_cnt, decoder->data); in intel_pt_walk_insn()
1020 decoder->tot_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1021 decoder->timestamp_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1022 decoder->sample_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1023 decoder->period_insn_cnt += insn_cnt; in intel_pt_walk_insn()
1026 decoder->no_progress = 0; in intel_pt_walk_insn()
1027 decoder->pkt_state = INTEL_PT_STATE_ERR2; in intel_pt_walk_insn()
1029 decoder->ip); in intel_pt_walk_insn()
1035 if (ip && decoder->ip == ip) { in intel_pt_walk_insn()
1041 intel_pt_sample_insn(decoder); in intel_pt_walk_insn()
1044 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_insn()
1045 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1046 decoder->state.to_ip = 0; in intel_pt_walk_insn()
1047 decoder->ip += intel_pt_insn->length; in intel_pt_walk_insn()
1056 err = intel_pt_push(&decoder->stack, decoder->ip + in intel_pt_walk_insn()
1062 decoder->ret_addr = intel_pt_pop(&decoder->stack); in intel_pt_walk_insn()
1066 int cnt = decoder->no_progress++; in intel_pt_walk_insn()
1068 decoder->state.from_ip = decoder->ip; in intel_pt_walk_insn()
1069 decoder->ip += intel_pt_insn->length + in intel_pt_walk_insn()
1071 decoder->state.to_ip = decoder->ip; in intel_pt_walk_insn()
1083 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1084 decoder->stuck_ip_prd = 1; in intel_pt_walk_insn()
1085 decoder->stuck_ip_cnt = 1; in intel_pt_walk_insn()
1087 decoder->state.to_ip == decoder->stuck_ip) { in intel_pt_walk_insn()
1089 decoder->state.to_ip); in intel_pt_walk_insn()
1090 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_insn()
1093 } else if (!--decoder->stuck_ip_cnt) { in intel_pt_walk_insn()
1094 decoder->stuck_ip_prd += 1; in intel_pt_walk_insn()
1095 decoder->stuck_ip_cnt = decoder->stuck_ip_prd; in intel_pt_walk_insn()
1096 decoder->stuck_ip = decoder->state.to_ip; in intel_pt_walk_insn()
1102 decoder->no_progress = 0; in intel_pt_walk_insn()
1104 decoder->state.insn_op = intel_pt_insn->op; in intel_pt_walk_insn()
1105 decoder->state.insn_len = intel_pt_insn->length; in intel_pt_walk_insn()
1106 memcpy(decoder->state.insn, intel_pt_insn->buf, in intel_pt_walk_insn()
1109 if (decoder->tx_flags & INTEL_PT_IN_TX) in intel_pt_walk_insn()
1110 decoder->state.flags |= INTEL_PT_IN_TX; in intel_pt_walk_insn()
1115 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) in intel_pt_fup_event() argument
1117 enum intel_pt_sample_type type = decoder->state.type; in intel_pt_fup_event()
1120 decoder->state.type &= ~INTEL_PT_BRANCH; in intel_pt_fup_event()
1122 if (decoder->set_fup_tx_flags) { in intel_pt_fup_event()
1123 decoder->set_fup_tx_flags = false; in intel_pt_fup_event()
1124 decoder->tx_flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1125 decoder->state.type |= INTEL_PT_TRANSACTION; in intel_pt_fup_event()
1126 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) in intel_pt_fup_event()
1127 decoder->state.type |= INTEL_PT_BRANCH; in intel_pt_fup_event()
1128 decoder->state.flags = decoder->fup_tx_flags; in intel_pt_fup_event()
1131 if (decoder->set_fup_ptw) { in intel_pt_fup_event()
1132 decoder->set_fup_ptw = false; in intel_pt_fup_event()
1133 decoder->state.type |= INTEL_PT_PTW; in intel_pt_fup_event()
1134 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1135 decoder->state.ptw_payload = decoder->fup_ptw_payload; in intel_pt_fup_event()
1138 if (decoder->set_fup_mwait) { in intel_pt_fup_event()
1139 decoder->set_fup_mwait = false; in intel_pt_fup_event()
1140 decoder->state.type |= INTEL_PT_MWAIT_OP; in intel_pt_fup_event()
1141 decoder->state.mwait_payload = decoder->fup_mwait_payload; in intel_pt_fup_event()
1144 if (decoder->set_fup_pwre) { in intel_pt_fup_event()
1145 decoder->set_fup_pwre = false; in intel_pt_fup_event()
1146 decoder->state.type |= INTEL_PT_PWR_ENTRY; in intel_pt_fup_event()
1147 decoder->state.pwre_payload = decoder->fup_pwre_payload; in intel_pt_fup_event()
1150 if (decoder->set_fup_exstop) { in intel_pt_fup_event()
1151 decoder->set_fup_exstop = false; in intel_pt_fup_event()
1152 decoder->state.type |= INTEL_PT_EX_STOP; in intel_pt_fup_event()
1153 decoder->state.flags |= INTEL_PT_FUP_IP; in intel_pt_fup_event()
1156 if (decoder->set_fup_bep) { in intel_pt_fup_event()
1157 decoder->set_fup_bep = false; in intel_pt_fup_event()
1158 decoder->state.type |= INTEL_PT_BLK_ITEMS; in intel_pt_fup_event()
1161 if (decoder->overflow) { in intel_pt_fup_event()
1162 decoder->overflow = false; in intel_pt_fup_event()
1163 if (!ret && !decoder->pge) { in intel_pt_fup_event()
1164 if (decoder->hop) { in intel_pt_fup_event()
1165 decoder->state.type = 0; in intel_pt_fup_event()
1166 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_fup_event()
1168 decoder->pge = true; in intel_pt_fup_event()
1169 decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN; in intel_pt_fup_event()
1170 decoder->state.from_ip = 0; in intel_pt_fup_event()
1171 decoder->state.to_ip = decoder->ip; in intel_pt_fup_event()
1176 decoder->state.from_ip = decoder->ip; in intel_pt_fup_event()
1177 decoder->state.to_ip = 0; in intel_pt_fup_event()
1179 decoder->state.type = type; in intel_pt_fup_event()
1184 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder, in intel_pt_fup_with_nlip() argument
1188 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err && in intel_pt_fup_with_nlip()
1190 ip == decoder->ip + intel_pt_insn->length; in intel_pt_fup_with_nlip()
1193 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) in intel_pt_walk_fup() argument
1199 ip = decoder->last_ip; in intel_pt_walk_fup()
1202 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip); in intel_pt_walk_fup()
1206 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { in intel_pt_walk_fup()
1207 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; in intel_pt_walk_fup()
1209 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_fup()
1210 if (intel_pt_fup_event(decoder) && no_tip) in intel_pt_walk_fup()
1214 decoder->set_fup_tx_flags = false; in intel_pt_walk_fup()
1220 decoder->ip); in intel_pt_walk_fup()
1221 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1227 decoder->ip); in intel_pt_walk_fup()
1228 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_fup()
1232 intel_pt_bug(decoder); in intel_pt_walk_fup()
1236 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_tip() argument
1241 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tip()
1243 decoder->pgd_ip && in intel_pt_walk_tip()
1244 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1245 (decoder->state.type & INTEL_PT_BRANCH) && in intel_pt_walk_tip()
1246 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) { in intel_pt_walk_tip()
1248 decoder->no_progress = 0; in intel_pt_walk_tip()
1249 decoder->pge = false; in intel_pt_walk_tip()
1250 decoder->continuous_period = false; in intel_pt_walk_tip()
1251 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1252 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1261 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) { in intel_pt_walk_tip()
1262 decoder->pge = false; in intel_pt_walk_tip()
1263 decoder->continuous_period = false; in intel_pt_walk_tip()
1264 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1265 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1266 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1267 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1269 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1270 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1272 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1274 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1275 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1276 if (decoder->packet.count == 0) { in intel_pt_walk_tip()
1277 decoder->state.to_ip = 0; in intel_pt_walk_tip()
1279 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tip()
1280 decoder->ip = decoder->last_ip; in intel_pt_walk_tip()
1287 uint64_t to_ip = decoder->ip + intel_pt_insn.length + in intel_pt_walk_tip()
1290 if (decoder->pgd_ip && in intel_pt_walk_tip()
1291 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD && in intel_pt_walk_tip()
1292 decoder->pgd_ip(to_ip, decoder->data)) { in intel_pt_walk_tip()
1294 decoder->pge = false; in intel_pt_walk_tip()
1295 decoder->continuous_period = false; in intel_pt_walk_tip()
1296 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tip()
1297 decoder->ip = to_ip; in intel_pt_walk_tip()
1298 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tip()
1299 decoder->state.to_ip = to_ip; in intel_pt_walk_tip()
1300 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_tip()
1304 decoder->ip); in intel_pt_walk_tip()
1305 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC; in intel_pt_walk_tip()
1309 return intel_pt_bug(decoder); in intel_pt_walk_tip()
1312 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder) in intel_pt_walk_tnt() argument
1318 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0); in intel_pt_walk_tnt()
1325 if (!decoder->return_compression) { in intel_pt_walk_tnt()
1327 decoder->ip); in intel_pt_walk_tnt()
1328 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1331 if (!decoder->ret_addr) { in intel_pt_walk_tnt()
1333 decoder->ip); in intel_pt_walk_tnt()
1334 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1337 if (!(decoder->tnt.payload & BIT63)) { in intel_pt_walk_tnt()
1339 decoder->ip); in intel_pt_walk_tnt()
1340 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1343 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1344 if (decoder->tnt.count) in intel_pt_walk_tnt()
1345 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1347 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1348 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1349 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1350 decoder->ip = decoder->ret_addr; in intel_pt_walk_tnt()
1351 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1357 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_tnt()
1360 if (decoder->packet.type != INTEL_PT_TIP || in intel_pt_walk_tnt()
1361 decoder->packet.count == 0) { in intel_pt_walk_tnt()
1363 decoder->ip); in intel_pt_walk_tnt()
1364 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_tnt()
1365 decoder->pkt_step = 0; in intel_pt_walk_tnt()
1368 intel_pt_set_last_ip(decoder); in intel_pt_walk_tnt()
1369 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1370 decoder->state.to_ip = decoder->last_ip; in intel_pt_walk_tnt()
1371 decoder->ip = decoder->last_ip; in intel_pt_walk_tnt()
1376 decoder->tnt.count -= 1; in intel_pt_walk_tnt()
1377 if (decoder->tnt.count) in intel_pt_walk_tnt()
1378 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT; in intel_pt_walk_tnt()
1380 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_walk_tnt()
1381 if (decoder->tnt.payload & BIT63) { in intel_pt_walk_tnt()
1382 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1383 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1384 decoder->ip += intel_pt_insn.length + in intel_pt_walk_tnt()
1386 decoder->state.to_ip = decoder->ip; in intel_pt_walk_tnt()
1390 if (decoder->state.type & INTEL_PT_INSTRUCTION) { in intel_pt_walk_tnt()
1391 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1392 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_tnt()
1393 decoder->state.from_ip = decoder->ip; in intel_pt_walk_tnt()
1394 decoder->state.to_ip = 0; in intel_pt_walk_tnt()
1395 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1398 decoder->sample_cyc = false; in intel_pt_walk_tnt()
1399 decoder->ip += intel_pt_insn.length; in intel_pt_walk_tnt()
1400 if (!decoder->tnt.count) { in intel_pt_walk_tnt()
1401 intel_pt_update_sample_time(decoder); in intel_pt_walk_tnt()
1404 decoder->tnt.payload <<= 1; in intel_pt_walk_tnt()
1408 return intel_pt_bug(decoder); in intel_pt_walk_tnt()
1412 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip) in intel_pt_mode_tsx() argument
1417 fup_tx_flags = decoder->packet.payload & in intel_pt_mode_tsx()
1419 err = intel_pt_get_next_packet(decoder); in intel_pt_mode_tsx()
1422 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_mode_tsx()
1423 decoder->fup_tx_flags = fup_tx_flags; in intel_pt_mode_tsx()
1424 decoder->set_fup_tx_flags = true; in intel_pt_mode_tsx()
1425 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX)) in intel_pt_mode_tsx()
1429 decoder->pos); in intel_pt_mode_tsx()
1430 intel_pt_update_in_tx(decoder); in intel_pt_mode_tsx()
1450 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_tsc_timestamp() argument
1454 decoder->have_tma = false; in intel_pt_calc_tsc_timestamp()
1456 if (decoder->ref_timestamp) { in intel_pt_calc_tsc_timestamp()
1457 timestamp = intel_pt_8b_tsc(decoder->packet.payload, in intel_pt_calc_tsc_timestamp()
1458 decoder->ref_timestamp); in intel_pt_calc_tsc_timestamp()
1459 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1460 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1461 decoder->ref_timestamp = 0; in intel_pt_calc_tsc_timestamp()
1462 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1463 } else if (decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1464 timestamp = decoder->packet.payload | in intel_pt_calc_tsc_timestamp()
1465 (decoder->timestamp & (0xffULL << 56)); in intel_pt_calc_tsc_timestamp()
1466 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1467 if (timestamp < decoder->timestamp && in intel_pt_calc_tsc_timestamp()
1468 decoder->timestamp - timestamp < decoder->tsc_slip) { in intel_pt_calc_tsc_timestamp()
1471 timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1473 if (timestamp < decoder->timestamp) { in intel_pt_calc_tsc_timestamp()
1476 decoder->tsc_timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1478 decoder->timestamp = timestamp; in intel_pt_calc_tsc_timestamp()
1479 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_tsc_timestamp()
1482 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_tsc_timestamp()
1483 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_tsc_timestamp()
1484 decoder->cycle_cnt = 0; in intel_pt_calc_tsc_timestamp()
1485 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_tsc_timestamp()
1486 intel_pt_calc_cyc_to_tsc(decoder, false); in intel_pt_calc_tsc_timestamp()
1489 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_tsc_timestamp()
1492 static int intel_pt_overflow(struct intel_pt_decoder *decoder) in intel_pt_overflow() argument
1495 intel_pt_clear_tx_flags(decoder); in intel_pt_overflow()
1496 decoder->timestamp_insn_cnt = 0; in intel_pt_overflow()
1497 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_overflow()
1498 decoder->state.from_ip = decoder->ip; in intel_pt_overflow()
1499 decoder->ip = 0; in intel_pt_overflow()
1500 decoder->pge = false; in intel_pt_overflow()
1501 decoder->set_fup_tx_flags = false; in intel_pt_overflow()
1502 decoder->set_fup_ptw = false; in intel_pt_overflow()
1503 decoder->set_fup_mwait = false; in intel_pt_overflow()
1504 decoder->set_fup_pwre = false; in intel_pt_overflow()
1505 decoder->set_fup_exstop = false; in intel_pt_overflow()
1506 decoder->set_fup_bep = false; in intel_pt_overflow()
1507 decoder->overflow = true; in intel_pt_overflow()
1511 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_pge() argument
1513 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_pge()
1516 decoder->cyc_cnt_timestamp = decoder->timestamp; in intel_pt_mtc_cyc_cnt_pge()
1517 decoder->base_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_pge()
1520 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_cbr() argument
1522 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp; in intel_pt_mtc_cyc_cnt_cbr()
1524 if (decoder->pge) in intel_pt_mtc_cyc_cnt_cbr()
1525 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_mtc_cyc_cnt_cbr()
1528 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder) in intel_pt_mtc_cyc_cnt_upd() argument
1532 if (decoder->have_cyc) in intel_pt_mtc_cyc_cnt_upd()
1535 decoder->sample_cyc = true; in intel_pt_mtc_cyc_cnt_upd()
1537 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp) in intel_pt_mtc_cyc_cnt_upd()
1540 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp; in intel_pt_mtc_cyc_cnt_upd()
1541 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1543 if (tot_cyc_cnt > decoder->tot_cyc_cnt) in intel_pt_mtc_cyc_cnt_upd()
1544 decoder->tot_cyc_cnt = tot_cyc_cnt; in intel_pt_mtc_cyc_cnt_upd()
1547 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) in intel_pt_calc_tma() argument
1549 uint32_t ctc = decoder->packet.payload; in intel_pt_calc_tma()
1550 uint32_t fc = decoder->packet.count; in intel_pt_calc_tma()
1551 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask; in intel_pt_calc_tma()
1553 if (!decoder->tsc_ctc_ratio_d) in intel_pt_calc_tma()
1556 if (decoder->pge && !decoder->in_psb) in intel_pt_calc_tma()
1557 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_calc_tma()
1559 intel_pt_mtc_cyc_cnt_upd(decoder); in intel_pt_calc_tma()
1561 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; in intel_pt_calc_tma()
1562 decoder->ctc_timestamp = decoder->tsc_timestamp - fc; in intel_pt_calc_tma()
1563 if (decoder->tsc_ctc_mult) { in intel_pt_calc_tma()
1564 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; in intel_pt_calc_tma()
1566 decoder->ctc_timestamp -= multdiv(ctc_rem, in intel_pt_calc_tma()
1567 decoder->tsc_ctc_ratio_n, in intel_pt_calc_tma()
1568 decoder->tsc_ctc_ratio_d); in intel_pt_calc_tma()
1570 decoder->ctc_delta = 0; in intel_pt_calc_tma()
1571 decoder->have_tma = true; in intel_pt_calc_tma()
1572 decoder->fixup_last_mtc = true; in intel_pt_calc_tma()
1574 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem); in intel_pt_calc_tma()
1577 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_mtc_timestamp() argument
1582 if (!decoder->have_tma) in intel_pt_calc_mtc_timestamp()
1585 mtc = decoder->packet.payload; in intel_pt_calc_mtc_timestamp()
1587 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) { in intel_pt_calc_mtc_timestamp()
1588 decoder->fixup_last_mtc = false; in intel_pt_calc_mtc_timestamp()
1589 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift, in intel_pt_calc_mtc_timestamp()
1590 &decoder->last_mtc); in intel_pt_calc_mtc_timestamp()
1593 if (mtc > decoder->last_mtc) in intel_pt_calc_mtc_timestamp()
1594 mtc_delta = mtc - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1596 mtc_delta = mtc + 256 - decoder->last_mtc; in intel_pt_calc_mtc_timestamp()
1598 decoder->ctc_delta += mtc_delta << decoder->mtc_shift; in intel_pt_calc_mtc_timestamp()
1600 if (decoder->tsc_ctc_mult) { in intel_pt_calc_mtc_timestamp()
1601 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1602 decoder->ctc_delta * decoder->tsc_ctc_mult; in intel_pt_calc_mtc_timestamp()
1604 timestamp = decoder->ctc_timestamp + in intel_pt_calc_mtc_timestamp()
1605 multdiv(decoder->ctc_delta, in intel_pt_calc_mtc_timestamp()
1606 decoder->tsc_ctc_ratio_n, in intel_pt_calc_mtc_timestamp()
1607 decoder->tsc_ctc_ratio_d); in intel_pt_calc_mtc_timestamp()
1610 if (timestamp < decoder->timestamp) in intel_pt_calc_mtc_timestamp()
1612 timestamp, decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1614 decoder->timestamp = timestamp; in intel_pt_calc_mtc_timestamp()
1616 intel_pt_mtc_cyc_cnt_upd(decoder); in intel_pt_calc_mtc_timestamp()
1618 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_mtc_timestamp()
1619 decoder->last_mtc = mtc; in intel_pt_calc_mtc_timestamp()
1621 if (decoder->last_packet_type == INTEL_PT_CYC) { in intel_pt_calc_mtc_timestamp()
1622 decoder->cyc_ref_timestamp = decoder->timestamp; in intel_pt_calc_mtc_timestamp()
1623 decoder->cycle_cnt = 0; in intel_pt_calc_mtc_timestamp()
1624 decoder->have_calc_cyc_to_tsc = false; in intel_pt_calc_mtc_timestamp()
1625 intel_pt_calc_cyc_to_tsc(decoder, true); in intel_pt_calc_mtc_timestamp()
1628 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_mtc_timestamp()
1631 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder) in intel_pt_calc_cbr() argument
1633 unsigned int cbr = decoder->packet.payload & 0xff; in intel_pt_calc_cbr()
1635 decoder->cbr_payload = decoder->packet.payload; in intel_pt_calc_cbr()
1637 if (decoder->cbr == cbr) in intel_pt_calc_cbr()
1640 decoder->cbr = cbr; in intel_pt_calc_cbr()
1641 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr; in intel_pt_calc_cbr()
1643 intel_pt_mtc_cyc_cnt_cbr(decoder); in intel_pt_calc_cbr()
1646 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder) in intel_pt_calc_cyc_timestamp() argument
1648 uint64_t timestamp = decoder->cyc_ref_timestamp; in intel_pt_calc_cyc_timestamp()
1650 decoder->have_cyc = true; in intel_pt_calc_cyc_timestamp()
1652 decoder->cycle_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
1653 if (decoder->pge) in intel_pt_calc_cyc_timestamp()
1654 decoder->tot_cyc_cnt += decoder->packet.payload; in intel_pt_calc_cyc_timestamp()
1655 decoder->sample_cyc = true; in intel_pt_calc_cyc_timestamp()
1657 if (!decoder->cyc_ref_timestamp) in intel_pt_calc_cyc_timestamp()
1660 if (decoder->have_calc_cyc_to_tsc) in intel_pt_calc_cyc_timestamp()
1661 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1662 else if (decoder->cbr) in intel_pt_calc_cyc_timestamp()
1663 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc; in intel_pt_calc_cyc_timestamp()
1667 if (timestamp < decoder->timestamp) in intel_pt_calc_cyc_timestamp()
1669 timestamp, decoder->timestamp); in intel_pt_calc_cyc_timestamp()
1671 decoder->timestamp = timestamp; in intel_pt_calc_cyc_timestamp()
1673 decoder->timestamp_insn_cnt = 0; in intel_pt_calc_cyc_timestamp()
1675 intel_pt_log_to("Setting timestamp", decoder->timestamp); in intel_pt_calc_cyc_timestamp()
1678 static void intel_pt_bbp(struct intel_pt_decoder *decoder) in intel_pt_bbp() argument
1680 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) { in intel_pt_bbp()
1681 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask)); in intel_pt_bbp()
1682 decoder->state.items.is_32_bit = false; in intel_pt_bbp()
1684 decoder->blk_type = decoder->packet.payload; in intel_pt_bbp()
1685 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type); in intel_pt_bbp()
1686 if (decoder->blk_type == INTEL_PT_GP_REGS) in intel_pt_bbp()
1687 decoder->state.items.is_32_bit = decoder->packet.count; in intel_pt_bbp()
1688 if (decoder->blk_type_pos < 0) { in intel_pt_bbp()
1690 decoder->blk_type); in intel_pt_bbp()
1691 } else if (decoder->state.items.mask[decoder->blk_type_pos]) { in intel_pt_bbp()
1693 decoder->blk_type); in intel_pt_bbp()
1697 static void intel_pt_bip(struct intel_pt_decoder *decoder) in intel_pt_bip() argument
1699 uint32_t id = decoder->packet.count; in intel_pt_bip()
1701 int pos = decoder->blk_type_pos; in intel_pt_bip()
1705 id, decoder->blk_type); in intel_pt_bip()
1709 if (decoder->state.items.mask[pos] & bit) { in intel_pt_bip()
1711 id, decoder->blk_type); in intel_pt_bip()
1714 decoder->state.items.mask[pos] |= bit; in intel_pt_bip()
1715 decoder->state.items.val[pos][id] = decoder->packet.payload; in intel_pt_bip()
1719 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder) in intel_pt_walk_psbend() argument
1723 decoder->in_psb = true; in intel_pt_walk_psbend()
1726 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psbend()
1730 switch (decoder->packet.type) { in intel_pt_walk_psbend()
1753 decoder->have_tma = false; in intel_pt_walk_psbend()
1759 err = intel_pt_overflow(decoder); in intel_pt_walk_psbend()
1763 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psbend()
1767 intel_pt_calc_tma(decoder); in intel_pt_walk_psbend()
1771 intel_pt_calc_cbr(decoder); in intel_pt_walk_psbend()
1775 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psbend()
1779 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psbend()
1783 decoder->pge = true; in intel_pt_walk_psbend()
1784 if (decoder->packet.count) { in intel_pt_walk_psbend()
1785 intel_pt_set_last_ip(decoder); in intel_pt_walk_psbend()
1786 if (decoder->hop) { in intel_pt_walk_psbend()
1788 decoder->ip = decoder->last_ip; in intel_pt_walk_psbend()
1789 decoder->hop_psb_fup = true; in intel_pt_walk_psbend()
1795 intel_pt_update_in_tx(decoder); in intel_pt_walk_psbend()
1799 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psbend()
1800 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_psbend()
1801 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_psbend()
1805 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_psbend()
1816 decoder->in_psb = false; in intel_pt_walk_psbend()
1821 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder) in intel_pt_walk_fup_tip() argument
1825 if (decoder->tx_flags & INTEL_PT_ABORT_TX) { in intel_pt_walk_fup_tip()
1826 decoder->tx_flags = 0; in intel_pt_walk_fup_tip()
1827 decoder->state.flags &= ~INTEL_PT_IN_TX; in intel_pt_walk_fup_tip()
1828 decoder->state.flags |= INTEL_PT_ABORT_TX; in intel_pt_walk_fup_tip()
1830 decoder->state.flags |= INTEL_PT_ASYNC; in intel_pt_walk_fup_tip()
1834 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_fup_tip()
1838 switch (decoder->packet.type) { in intel_pt_walk_fup_tip()
1860 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_fup_tip()
1861 decoder->pkt_step = 0; in intel_pt_walk_fup_tip()
1865 intel_pt_calc_cbr(decoder); in intel_pt_walk_fup_tip()
1869 return intel_pt_overflow(decoder); in intel_pt_walk_fup_tip()
1872 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1873 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1874 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1876 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1877 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1879 decoder->pge = false; in intel_pt_walk_fup_tip()
1880 decoder->continuous_period = false; in intel_pt_walk_fup_tip()
1881 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_fup_tip()
1885 decoder->pge = true; in intel_pt_walk_fup_tip()
1887 decoder->ip); in intel_pt_walk_fup_tip()
1888 decoder->state.from_ip = 0; in intel_pt_walk_fup_tip()
1889 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1890 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1892 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1893 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1895 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_fup_tip()
1896 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_fup_tip()
1900 decoder->state.from_ip = decoder->ip; in intel_pt_walk_fup_tip()
1901 if (decoder->packet.count == 0) { in intel_pt_walk_fup_tip()
1902 decoder->state.to_ip = 0; in intel_pt_walk_fup_tip()
1904 intel_pt_set_ip(decoder); in intel_pt_walk_fup_tip()
1905 decoder->state.to_ip = decoder->ip; in intel_pt_walk_fup_tip()
1910 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_fup_tip()
1914 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_fup_tip()
1915 if (decoder->period_type == INTEL_PT_PERIOD_MTC) in intel_pt_walk_fup_tip()
1916 decoder->state.type |= INTEL_PT_INSTRUCTION; in intel_pt_walk_fup_tip()
1920 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_fup_tip()
1924 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_fup_tip()
1933 return intel_pt_bug(decoder); in intel_pt_walk_fup_tip()
1938 static int intel_pt_resample(struct intel_pt_decoder *decoder) in intel_pt_resample() argument
1940 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_resample()
1941 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_resample()
1942 decoder->state.from_ip = decoder->ip; in intel_pt_resample()
1943 decoder->state.to_ip = 0; in intel_pt_resample()
1952 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
1955 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err) in intel_pt_hop_trace() argument
1960 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) { in intel_pt_hop_trace()
1961 *err = intel_pt_scan_for_psb(decoder); in intel_pt_hop_trace()
1966 switch (decoder->packet.type) { in intel_pt_hop_trace()
1971 decoder->pge = false; in intel_pt_hop_trace()
1972 if (!decoder->packet.count) in intel_pt_hop_trace()
1974 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1975 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_hop_trace()
1976 decoder->state.from_ip = 0; in intel_pt_hop_trace()
1977 decoder->state.to_ip = decoder->ip; in intel_pt_hop_trace()
1981 if (!decoder->packet.count) in intel_pt_hop_trace()
1983 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1984 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
1985 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
1986 decoder->state.to_ip = 0; in intel_pt_hop_trace()
1990 if (!decoder->packet.count) in intel_pt_hop_trace()
1992 intel_pt_set_ip(decoder); in intel_pt_hop_trace()
1993 if (decoder->set_fup_mwait || decoder->set_fup_pwre) in intel_pt_hop_trace()
1995 if (!decoder->branch_enable || !decoder->pge) in intel_pt_hop_trace()
1998 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
1999 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2000 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2001 intel_pt_fup_event(decoder); in intel_pt_hop_trace()
2004 intel_pt_fup_event(decoder); in intel_pt_hop_trace()
2005 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH; in intel_pt_hop_trace()
2006 *err = intel_pt_walk_fup_tip(decoder); in intel_pt_hop_trace()
2007 if (!*err && decoder->state.to_ip) in intel_pt_hop_trace()
2008 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_hop_trace()
2012 decoder->last_ip = 0; in intel_pt_hop_trace()
2013 decoder->have_last_ip = true; in intel_pt_hop_trace()
2014 decoder->hop_psb_fup = false; in intel_pt_hop_trace()
2015 *err = intel_pt_walk_psbend(decoder); in intel_pt_hop_trace()
2020 if (decoder->hop_psb_fup) { in intel_pt_hop_trace()
2021 decoder->hop_psb_fup = false; in intel_pt_hop_trace()
2022 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_hop_trace()
2023 decoder->state.from_ip = decoder->ip; in intel_pt_hop_trace()
2024 decoder->state.to_ip = 0; in intel_pt_hop_trace()
2027 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_hop_trace()
2028 decoder->state.type = 0; in intel_pt_hop_trace()
2065 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder) in intel_pt_walk_trace() argument
2072 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2077 if (decoder->cyc_threshold) { in intel_pt_walk_trace()
2078 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC) in intel_pt_walk_trace()
2079 decoder->sample_cyc = false; in intel_pt_walk_trace()
2080 last_packet_type = decoder->packet.type; in intel_pt_walk_trace()
2083 if (decoder->hop) { in intel_pt_walk_trace()
2084 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) { in intel_pt_walk_trace()
2096 switch (decoder->packet.type) { in intel_pt_walk_trace()
2098 if (!decoder->packet.count) in intel_pt_walk_trace()
2100 decoder->tnt = decoder->packet; in intel_pt_walk_trace()
2101 decoder->pkt_state = INTEL_PT_STATE_TNT; in intel_pt_walk_trace()
2102 err = intel_pt_walk_tnt(decoder); in intel_pt_walk_trace()
2108 if (decoder->packet.count != 0) in intel_pt_walk_trace()
2109 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2110 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD; in intel_pt_walk_trace()
2111 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
2114 decoder->pge = true; in intel_pt_walk_trace()
2115 decoder->overflow = false; in intel_pt_walk_trace()
2116 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_trace()
2117 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
2119 decoder->pos); in intel_pt_walk_trace()
2122 intel_pt_set_ip(decoder); in intel_pt_walk_trace()
2123 decoder->state.from_ip = 0; in intel_pt_walk_trace()
2124 decoder->state.to_ip = decoder->ip; in intel_pt_walk_trace()
2125 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_trace()
2130 if (decoder->hop) in intel_pt_walk_trace()
2131 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_walk_trace()
2136 return intel_pt_overflow(decoder); in intel_pt_walk_trace()
2139 if (decoder->packet.count != 0) in intel_pt_walk_trace()
2140 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2141 decoder->pkt_state = INTEL_PT_STATE_TIP; in intel_pt_walk_trace()
2142 return intel_pt_walk_tip(decoder); in intel_pt_walk_trace()
2145 if (decoder->packet.count == 0) { in intel_pt_walk_trace()
2147 decoder->pos); in intel_pt_walk_trace()
2151 intel_pt_set_last_ip(decoder); in intel_pt_walk_trace()
2152 if (!decoder->branch_enable || !decoder->pge) { in intel_pt_walk_trace()
2153 decoder->ip = decoder->last_ip; in intel_pt_walk_trace()
2154 if (intel_pt_fup_event(decoder)) in intel_pt_walk_trace()
2159 if (decoder->set_fup_mwait) in intel_pt_walk_trace()
2162 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP; in intel_pt_walk_trace()
2164 decoder->pkt_state = INTEL_PT_STATE_FUP; in intel_pt_walk_trace()
2165 err = intel_pt_walk_fup(decoder); in intel_pt_walk_trace()
2172 return intel_pt_walk_fup_tip(decoder); in intel_pt_walk_trace()
2175 decoder->pge = false; in intel_pt_walk_trace()
2176 decoder->continuous_period = false; in intel_pt_walk_trace()
2177 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_trace()
2178 decoder->have_tma = false; in intel_pt_walk_trace()
2182 decoder->last_ip = 0; in intel_pt_walk_trace()
2183 decoder->have_last_ip = true; in intel_pt_walk_trace()
2184 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_trace()
2185 err = intel_pt_walk_psbend(decoder); in intel_pt_walk_trace()
2195 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
2196 decoder->state.type = 0; in intel_pt_walk_trace()
2202 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_trace()
2206 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_trace()
2207 if (decoder->period_type != INTEL_PT_PERIOD_MTC) in intel_pt_walk_trace()
2213 if (!decoder->mtc_insn) in intel_pt_walk_trace()
2215 decoder->mtc_insn = false; in intel_pt_walk_trace()
2217 if (!decoder->timestamp) in intel_pt_walk_trace()
2219 decoder->state.type = INTEL_PT_INSTRUCTION; in intel_pt_walk_trace()
2220 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2221 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2222 decoder->mtc_insn = false; in intel_pt_walk_trace()
2226 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_trace()
2230 intel_pt_calc_tma(decoder); in intel_pt_walk_trace()
2234 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_trace()
2238 intel_pt_calc_cbr(decoder); in intel_pt_walk_trace()
2239 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_walk_trace()
2240 decoder->state.type = 0; in intel_pt_walk_trace()
2246 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_trace()
2251 if (!decoder->pge || decoder->in_psb) { in intel_pt_walk_trace()
2252 intel_pt_update_in_tx(decoder); in intel_pt_walk_trace()
2255 err = intel_pt_mode_tsx(decoder, &no_tip); in intel_pt_walk_trace()
2261 return intel_pt_bug(decoder); in intel_pt_walk_trace()
2270 decoder->fup_ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
2271 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2274 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2275 decoder->set_fup_ptw = true; in intel_pt_walk_trace()
2279 decoder->pos); in intel_pt_walk_trace()
2284 decoder->state.type = INTEL_PT_PTW; in intel_pt_walk_trace()
2285 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2286 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2287 decoder->state.ptw_payload = decoder->packet.payload; in intel_pt_walk_trace()
2291 decoder->fup_mwait_payload = decoder->packet.payload; in intel_pt_walk_trace()
2292 decoder->set_fup_mwait = true; in intel_pt_walk_trace()
2296 if (decoder->set_fup_mwait) { in intel_pt_walk_trace()
2297 decoder->fup_pwre_payload = in intel_pt_walk_trace()
2298 decoder->packet.payload; in intel_pt_walk_trace()
2299 decoder->set_fup_pwre = true; in intel_pt_walk_trace()
2302 decoder->state.type = INTEL_PT_PWR_ENTRY; in intel_pt_walk_trace()
2303 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2304 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2305 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
2309 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2312 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2313 decoder->set_fup_exstop = true; in intel_pt_walk_trace()
2317 decoder->pos); in intel_pt_walk_trace()
2322 decoder->state.type = INTEL_PT_EX_STOP; in intel_pt_walk_trace()
2323 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2324 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2328 decoder->state.type = INTEL_PT_PWR_EXIT; in intel_pt_walk_trace()
2329 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2330 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2331 decoder->state.pwrx_payload = decoder->packet.payload; in intel_pt_walk_trace()
2335 intel_pt_bbp(decoder); in intel_pt_walk_trace()
2339 intel_pt_bip(decoder); in intel_pt_walk_trace()
2343 decoder->state.type = INTEL_PT_BLK_ITEMS; in intel_pt_walk_trace()
2344 decoder->state.from_ip = decoder->ip; in intel_pt_walk_trace()
2345 decoder->state.to_ip = 0; in intel_pt_walk_trace()
2349 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_trace()
2352 if (decoder->packet.type == INTEL_PT_FUP) { in intel_pt_walk_trace()
2353 decoder->set_fup_bep = true; in intel_pt_walk_trace()
2357 decoder->pos); in intel_pt_walk_trace()
2362 return intel_pt_bug(decoder); in intel_pt_walk_trace()
2367 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder) in intel_pt_have_ip() argument
2369 return decoder->packet.count && in intel_pt_have_ip()
2370 (decoder->have_last_ip || decoder->packet.count == 3 || in intel_pt_have_ip()
2371 decoder->packet.count == 6); in intel_pt_have_ip()
2375 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder) in intel_pt_walk_psb() argument
2379 decoder->in_psb = true; in intel_pt_walk_psb()
2382 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_psb()
2386 switch (decoder->packet.type) { in intel_pt_walk_psb()
2388 decoder->continuous_period = false; in intel_pt_walk_psb()
2408 decoder->pge = true; in intel_pt_walk_psb()
2409 if (intel_pt_have_ip(decoder)) { in intel_pt_walk_psb()
2410 uint64_t current_ip = decoder->ip; in intel_pt_walk_psb()
2412 intel_pt_set_ip(decoder); in intel_pt_walk_psb()
2415 decoder->ip); in intel_pt_walk_psb()
2420 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_psb()
2424 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_psb()
2428 intel_pt_calc_tma(decoder); in intel_pt_walk_psb()
2432 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_psb()
2436 intel_pt_calc_cbr(decoder); in intel_pt_walk_psb()
2440 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_psb()
2444 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_psb()
2448 intel_pt_update_in_tx(decoder); in intel_pt_walk_psb()
2452 decoder->pge = false; in intel_pt_walk_psb()
2453 decoder->continuous_period = false; in intel_pt_walk_psb()
2454 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_psb()
2458 decoder->have_tma = false; in intel_pt_walk_psb()
2460 if (decoder->ip) in intel_pt_walk_psb()
2461 decoder->pkt_state = INTEL_PT_STATE_ERR4; in intel_pt_walk_psb()
2463 decoder->pkt_state = INTEL_PT_STATE_ERR3; in intel_pt_walk_psb()
2468 err = intel_pt_bug(decoder); in intel_pt_walk_psb()
2472 err = intel_pt_overflow(decoder); in intel_pt_walk_psb()
2488 decoder->in_psb = false; in intel_pt_walk_psb()
2493 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder) in intel_pt_walk_to_ip() argument
2498 err = intel_pt_get_next_packet(decoder); in intel_pt_walk_to_ip()
2502 switch (decoder->packet.type) { in intel_pt_walk_to_ip()
2504 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2505 decoder->pge = false; in intel_pt_walk_to_ip()
2506 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2507 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2508 if (!decoder->ip) in intel_pt_walk_to_ip()
2510 decoder->state.type |= INTEL_PT_TRACE_END; in intel_pt_walk_to_ip()
2514 decoder->pge = true; in intel_pt_walk_to_ip()
2515 intel_pt_mtc_cyc_cnt_pge(decoder); in intel_pt_walk_to_ip()
2516 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2517 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2518 if (!decoder->ip) in intel_pt_walk_to_ip()
2520 decoder->state.type |= INTEL_PT_TRACE_BEGIN; in intel_pt_walk_to_ip()
2524 decoder->pge = true; in intel_pt_walk_to_ip()
2525 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2526 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2527 if (!decoder->ip) in intel_pt_walk_to_ip()
2532 if (intel_pt_have_ip(decoder)) in intel_pt_walk_to_ip()
2533 intel_pt_set_ip(decoder); in intel_pt_walk_to_ip()
2534 if (decoder->ip) in intel_pt_walk_to_ip()
2539 intel_pt_calc_mtc_timestamp(decoder); in intel_pt_walk_to_ip()
2543 intel_pt_calc_tsc_timestamp(decoder); in intel_pt_walk_to_ip()
2547 intel_pt_calc_tma(decoder); in intel_pt_walk_to_ip()
2551 intel_pt_calc_cyc_timestamp(decoder); in intel_pt_walk_to_ip()
2555 intel_pt_calc_cbr(decoder); in intel_pt_walk_to_ip()
2559 decoder->cr3 = decoder->packet.payload & (BIT63 - 1); in intel_pt_walk_to_ip()
2563 decoder->exec_mode = decoder->packet.payload; in intel_pt_walk_to_ip()
2567 intel_pt_update_in_tx(decoder); in intel_pt_walk_to_ip()
2571 return intel_pt_overflow(decoder); in intel_pt_walk_to_ip()
2574 return intel_pt_bug(decoder); in intel_pt_walk_to_ip()
2577 decoder->pge = false; in intel_pt_walk_to_ip()
2578 decoder->continuous_period = false; in intel_pt_walk_to_ip()
2579 intel_pt_clear_tx_flags(decoder); in intel_pt_walk_to_ip()
2580 decoder->have_tma = false; in intel_pt_walk_to_ip()
2584 decoder->last_ip = 0; in intel_pt_walk_to_ip()
2585 decoder->have_last_ip = true; in intel_pt_walk_to_ip()
2586 intel_pt_clear_stack(&decoder->stack); in intel_pt_walk_to_ip()
2587 err = intel_pt_walk_psb(decoder); in intel_pt_walk_to_ip()
2590 if (decoder->ip) { in intel_pt_walk_to_ip()
2592 decoder->state.type = 0; in intel_pt_walk_to_ip()
2619 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder) in intel_pt_sync_ip() argument
2623 decoder->set_fup_tx_flags = false; in intel_pt_sync_ip()
2624 decoder->set_fup_ptw = false; in intel_pt_sync_ip()
2625 decoder->set_fup_mwait = false; in intel_pt_sync_ip()
2626 decoder->set_fup_pwre = false; in intel_pt_sync_ip()
2627 decoder->set_fup_exstop = false; in intel_pt_sync_ip()
2628 decoder->set_fup_bep = false; in intel_pt_sync_ip()
2629 decoder->overflow = false; in intel_pt_sync_ip()
2631 if (!decoder->branch_enable) { in intel_pt_sync_ip()
2632 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2633 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync_ip()
2638 err = intel_pt_walk_to_ip(decoder); in intel_pt_sync_ip()
2643 if (decoder->hop) in intel_pt_sync_ip()
2644 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync_ip()
2646 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync_ip()
2648 decoder->state.from_ip = 0; in intel_pt_sync_ip()
2649 decoder->state.to_ip = decoder->ip; in intel_pt_sync_ip()
2650 intel_pt_log_to("Setting IP", decoder->ip); in intel_pt_sync_ip()
2655 static int intel_pt_part_psb(struct intel_pt_decoder *decoder) in intel_pt_part_psb() argument
2657 const unsigned char *end = decoder->buf + decoder->len; in intel_pt_part_psb()
2661 if (i > decoder->len) in intel_pt_part_psb()
2669 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb) in intel_pt_rest_psb() argument
2674 if (rest_psb > decoder->len || in intel_pt_rest_psb()
2675 memcmp(decoder->buf, psb + part_psb, rest_psb)) in intel_pt_rest_psb()
2681 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder, in intel_pt_get_split_psb() argument
2686 decoder->pos += decoder->len; in intel_pt_get_split_psb()
2687 decoder->len = 0; in intel_pt_get_split_psb()
2689 ret = intel_pt_get_next_data(decoder, false); in intel_pt_get_split_psb()
2693 rest_psb = intel_pt_rest_psb(decoder, part_psb); in intel_pt_get_split_psb()
2697 decoder->pos -= part_psb; in intel_pt_get_split_psb()
2698 decoder->next_buf = decoder->buf + rest_psb; in intel_pt_get_split_psb()
2699 decoder->next_len = decoder->len - rest_psb; in intel_pt_get_split_psb()
2700 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN); in intel_pt_get_split_psb()
2701 decoder->buf = decoder->temp_buf; in intel_pt_get_split_psb()
2702 decoder->len = INTEL_PT_PSB_LEN; in intel_pt_get_split_psb()
2707 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder) in intel_pt_scan_for_psb() argument
2714 if (!decoder->len) { in intel_pt_scan_for_psb()
2715 ret = intel_pt_get_next_data(decoder, false); in intel_pt_scan_for_psb()
2720 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR, in intel_pt_scan_for_psb()
2725 part_psb = intel_pt_part_psb(decoder); in intel_pt_scan_for_psb()
2727 ret = intel_pt_get_split_psb(decoder, part_psb); in intel_pt_scan_for_psb()
2731 decoder->pos += decoder->len; in intel_pt_scan_for_psb()
2732 decoder->len = 0; in intel_pt_scan_for_psb()
2737 decoder->pkt_step = next - decoder->buf; in intel_pt_scan_for_psb()
2738 return intel_pt_get_next_packet(decoder); in intel_pt_scan_for_psb()
2742 static int intel_pt_sync(struct intel_pt_decoder *decoder) in intel_pt_sync() argument
2746 decoder->pge = false; in intel_pt_sync()
2747 decoder->continuous_period = false; in intel_pt_sync()
2748 decoder->have_last_ip = false; in intel_pt_sync()
2749 decoder->last_ip = 0; in intel_pt_sync()
2750 decoder->ip = 0; in intel_pt_sync()
2751 intel_pt_clear_stack(&decoder->stack); in intel_pt_sync()
2754 err = intel_pt_scan_for_psb(decoder); in intel_pt_sync()
2758 decoder->have_last_ip = true; in intel_pt_sync()
2759 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
2761 err = intel_pt_walk_psb(decoder); in intel_pt_sync()
2765 if (decoder->ip) { in intel_pt_sync()
2766 decoder->state.type = 0; /* Do not have a sample */ in intel_pt_sync()
2771 if (decoder->hop) in intel_pt_sync()
2772 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE; in intel_pt_sync()
2774 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; in intel_pt_sync()
2775 } else if (decoder->leap) { in intel_pt_sync()
2782 return intel_pt_sync_ip(decoder); in intel_pt_sync()
2788 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder) in intel_pt_est_timestamp() argument
2790 uint64_t est = decoder->sample_insn_cnt << 1; in intel_pt_est_timestamp()
2792 if (!decoder->cbr || !decoder->max_non_turbo_ratio) in intel_pt_est_timestamp()
2795 est *= decoder->max_non_turbo_ratio; in intel_pt_est_timestamp()
2796 est /= decoder->cbr; in intel_pt_est_timestamp()
2798 return decoder->sample_timestamp + est; in intel_pt_est_timestamp()
2801 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) in intel_pt_decode() argument
2806 decoder->state.type = INTEL_PT_BRANCH; in intel_pt_decode()
2807 decoder->state.flags = 0; in intel_pt_decode()
2809 switch (decoder->pkt_state) { in intel_pt_decode()
2811 err = intel_pt_sync(decoder); in intel_pt_decode()
2814 decoder->have_last_ip = false; in intel_pt_decode()
2815 decoder->last_ip = 0; in intel_pt_decode()
2816 decoder->ip = 0; in intel_pt_decode()
2819 err = intel_pt_sync_ip(decoder); in intel_pt_decode()
2822 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2826 err = intel_pt_walk_tnt(decoder); in intel_pt_decode()
2828 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2832 err = intel_pt_walk_tip(decoder); in intel_pt_decode()
2835 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2837 err = intel_pt_walk_fup_tip(decoder); in intel_pt_decode()
2840 err = intel_pt_walk_fup(decoder); in intel_pt_decode()
2842 err = intel_pt_walk_trace(decoder); in intel_pt_decode()
2845 err = intel_pt_resample(decoder); in intel_pt_decode()
2848 err = intel_pt_bug(decoder); in intel_pt_decode()
2854 decoder->state.err = intel_pt_ext_err(err); in intel_pt_decode()
2856 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
2857 intel_pt_update_sample_time(decoder); in intel_pt_decode()
2858 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
2860 decoder->state.err = 0; in intel_pt_decode()
2861 if (decoder->cbr != decoder->cbr_seen) { in intel_pt_decode()
2862 decoder->cbr_seen = decoder->cbr; in intel_pt_decode()
2863 if (!decoder->state.type) { in intel_pt_decode()
2864 decoder->state.from_ip = decoder->ip; in intel_pt_decode()
2865 decoder->state.to_ip = 0; in intel_pt_decode()
2867 decoder->state.type |= INTEL_PT_CBR_CHG; in intel_pt_decode()
2868 decoder->state.cbr_payload = decoder->cbr_payload; in intel_pt_decode()
2869 decoder->state.cbr = decoder->cbr; in intel_pt_decode()
2871 if (intel_pt_sample_time(decoder->pkt_state)) { in intel_pt_decode()
2872 intel_pt_update_sample_time(decoder); in intel_pt_decode()
2873 if (decoder->sample_cyc) { in intel_pt_decode()
2874 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt; in intel_pt_decode()
2875 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
2876 decoder->sample_cyc = false; in intel_pt_decode()
2883 if (!decoder->have_cyc) in intel_pt_decode()
2884 decoder->state.flags |= INTEL_PT_SAMPLE_IPC; in intel_pt_decode()
2887 decoder->state.timestamp = decoder->sample_timestamp; in intel_pt_decode()
2888 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder); in intel_pt_decode()
2889 decoder->state.cr3 = decoder->cr3; in intel_pt_decode()
2890 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt; in intel_pt_decode()
2891 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt; in intel_pt_decode()
2893 return &decoder->state; in intel_pt_decode()
3265 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp) in intel_pt_fast_forward() argument
3275 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d); in intel_pt_fast_forward()
3282 decoder->pos += decoder->len; in intel_pt_fast_forward()
3283 decoder->len = 0; in intel_pt_fast_forward()
3284 err = intel_pt_get_next_data(decoder, true); in intel_pt_fast_forward()
3288 } while (decoder->buf_timestamp != d.buf_timestamp); in intel_pt_fast_forward()
3291 if (!decoder->buf) in intel_pt_fast_forward()
3294 buf = (unsigned char *)decoder->buf; in intel_pt_fast_forward()
3295 len = decoder->len; in intel_pt_fast_forward()
3310 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp); in intel_pt_fast_forward()
3319 decoder->pos += decoder->len - len; in intel_pt_fast_forward()
3320 decoder->buf = buf; in intel_pt_fast_forward()
3321 decoder->len = len; in intel_pt_fast_forward()
3322 intel_pt_reposition(decoder); in intel_pt_fast_forward()