tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /*
  45  * Default maximum number of loops with no packets consumed i.e. stuck in a
  46  * loop.
  47  */
  48 #define INTEL_PT_MAX_LOOPS 100000
  49
  50 struct intel_pt_blk {
  51         struct intel_pt_blk *prev;
  52         uint64_t ip[INTEL_PT_BLK_SIZE];
  53 };
  54
  55 struct intel_pt_stack {
  56         struct intel_pt_blk *blk;
  57         struct intel_pt_blk *spare;
  58         int pos;
  59 };
  60
  61 enum intel_pt_p_once {
  62         INTEL_PT_PRT_ONCE_UNK_VMCS,
  63         INTEL_PT_PRT_ONCE_ERANGE,
  64 };
  65
  66 enum intel_pt_pkt_state {
  67         INTEL_PT_STATE_NO_PSB,
  68         INTEL_PT_STATE_NO_IP,
  69         INTEL_PT_STATE_ERR_RESYNC,
  70         INTEL_PT_STATE_IN_SYNC,
  71         INTEL_PT_STATE_TNT_CONT,
  72         INTEL_PT_STATE_TNT,
  73         INTEL_PT_STATE_TIP,
  74         INTEL_PT_STATE_TIP_PGD,
  75         INTEL_PT_STATE_FUP,
  76         INTEL_PT_STATE_FUP_NO_TIP,
  77         INTEL_PT_STATE_FUP_IN_PSB,
  78         INTEL_PT_STATE_RESAMPLE,
  79         INTEL_PT_STATE_VM_TIME_CORRELATION,
  80 };
  81
  82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  83 {
  84         switch (pkt_state) {
  85         case INTEL_PT_STATE_NO_PSB:
  86         case INTEL_PT_STATE_NO_IP:
  87         case INTEL_PT_STATE_ERR_RESYNC:
  88         case INTEL_PT_STATE_IN_SYNC:
  89         case INTEL_PT_STATE_TNT_CONT:
  90         case INTEL_PT_STATE_RESAMPLE:
  91         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  92                 return true;
  93         case INTEL_PT_STATE_TNT:
  94         case INTEL_PT_STATE_TIP:
  95         case INTEL_PT_STATE_TIP_PGD:
  96         case INTEL_PT_STATE_FUP:
  97         case INTEL_PT_STATE_FUP_NO_TIP:
  98         case INTEL_PT_STATE_FUP_IN_PSB:
  99                 return false;
 100         default:
 101                 return true;
 102         };
 103 }
 104
 105 #ifdef INTEL_PT_STRICT
 106 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 107 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 108 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 109 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 110 #else
 111 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 112 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 113 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 114 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 115 #endif
 116
 117 struct intel_pt_decoder {
 118         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 119         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 120                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 121                          uint64_t max_insn_cnt, void *data);
 122         bool (*pgd_ip)(uint64_t ip, void *data);
 123         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 124         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 125         void *data;
 126         struct intel_pt_state state;
 127         const unsigned char *buf;
 128         size_t len;
 129         bool return_compression;
 130         bool branch_enable;
 131         bool mtc_insn;
 132         bool pge;
 133         bool have_tma;
 134         bool have_cyc;
 135         bool fixup_last_mtc;
 136         bool have_last_ip;
 137         bool in_psb;
 138         bool hop;
 139         bool leap;
 140         bool emulated_ptwrite;
 141         bool vm_time_correlation;
 142         bool vm_tm_corr_dry_run;
 143         bool vm_tm_corr_reliable;
 144         bool vm_tm_corr_same_buf;
 145         bool vm_tm_corr_continuous;
 146         bool nr;
 147         bool next_nr;
 148         bool iflag;
 149         bool next_iflag;
 150         enum intel_pt_param_flags flags;
 151         uint64_t pos;
 152         uint64_t last_ip;
 153         uint64_t ip;
 154         uint64_t pip_payload;
 155         uint64_t timestamp;
 156         uint64_t tsc_timestamp;
 157         uint64_t ref_timestamp;
 158         uint64_t buf_timestamp;
 159         uint64_t sample_timestamp;
 160         uint64_t ret_addr;
 161         uint64_t ctc_timestamp;
 162         uint64_t ctc_delta;
 163         uint64_t cycle_cnt;
 164         uint64_t cyc_ref_timestamp;
 165         uint64_t first_timestamp;
 166         uint64_t last_reliable_timestamp;
 167         uint64_t vmcs;
 168         uint64_t print_once;
 169         uint64_t last_ctc;
 170         uint32_t last_mtc;
 171         uint32_t tsc_ctc_ratio_n;
 172         uint32_t tsc_ctc_ratio_d;
 173         uint32_t tsc_ctc_mult;
 174         uint32_t tsc_slip;
 175         uint32_t ctc_rem_mask;
 176         int mtc_shift;
 177         struct intel_pt_stack stack;
 178         enum intel_pt_pkt_state pkt_state;
 179         enum intel_pt_pkt_ctx pkt_ctx;
 180         enum intel_pt_pkt_ctx prev_pkt_ctx;
 181         enum intel_pt_blk_type blk_type;
 182         int blk_type_pos;
 183         struct intel_pt_pkt packet;
 184         struct intel_pt_pkt tnt;
 185         int pkt_step;
 186         int pkt_len;
 187         int last_packet_type;
 188         unsigned int cbr;
 189         unsigned int cbr_seen;
 190         unsigned int max_non_turbo_ratio;
 191         double max_non_turbo_ratio_fp;
 192         double cbr_cyc_to_tsc;
 193         double calc_cyc_to_tsc;
 194         bool have_calc_cyc_to_tsc;
 195         int exec_mode;
 196         unsigned int insn_bytes;
 197         uint64_t period;
 198         enum intel_pt_period_type period_type;
 199         uint64_t tot_insn_cnt;
 200         uint64_t period_insn_cnt;
 201         uint64_t period_mask;
 202         uint64_t period_ticks;
 203         uint64_t last_masked_timestamp;
 204         uint64_t tot_cyc_cnt;
 205         uint64_t sample_tot_cyc_cnt;
 206         uint64_t base_cyc_cnt;
 207         uint64_t cyc_cnt_timestamp;
 208         uint64_t ctl;
 209         uint64_t cyc_threshold;
 210         double tsc_to_cyc;
 211         bool continuous_period;
 212         bool overflow;
 213         bool set_fup_tx_flags;
 214         bool set_fup_ptw;
 215         bool set_fup_mwait;
 216         bool set_fup_pwre;
 217         bool set_fup_exstop;
 218         bool set_fup_bep;
 219         bool set_fup_cfe_ip;
 220         bool set_fup_cfe;
 221         bool set_fup_mode_exec;
 222         bool sample_cyc;
 223         unsigned int fup_tx_flags;
 224         unsigned int tx_flags;
 225         uint64_t fup_ptw_payload;
 226         uint64_t fup_mwait_payload;
 227         uint64_t fup_pwre_payload;
 228         uint64_t cbr_payload;
 229         uint64_t timestamp_insn_cnt;
 230         uint64_t sample_insn_cnt;
 231         uint64_t stuck_ip;
 232         struct intel_pt_pkt fup_cfe_pkt;
 233         int max_loops;
 234         int no_progress;
 235         int stuck_ip_prd;
 236         int stuck_ip_cnt;
 237         uint64_t psb_ip;
 238         const unsigned char *next_buf;
 239         size_t next_len;
 240         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 241         int evd_cnt;
 242         struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];
 243 };
 244
 245 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 246 {
 247         int i;
 248
 249         for (i = 0; x != 1; i++)
 250                 x >>= 1;
 251
 252         return x << i;
 253 }
 254
 255 __printf(1, 2)
 256 static void p_log(const char *fmt, ...)
 257 {
 258         char buf[512];
 259         va_list args;
 260
 261         va_start(args, fmt);
 262         vsnprintf(buf, sizeof(buf), fmt, args);
 263         va_end(args);
 264
 265         fprintf(stderr, "%s\n", buf);
 266         intel_pt_log("%s\n", buf);
 267 }
 268
 269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 270                                 enum intel_pt_p_once id)
 271 {
 272         uint64_t bit = 1ULL << id;
 273
 274         if (decoder->print_once & bit)
 275                 return false;
 276         decoder->print_once |= bit;
 277         return true;
 278 }
 279
 280 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 281 {
 282         if (!(ctl & INTEL_PT_CYC_ENABLE))
 283                 return 0;
 284
 285         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 286 }
 287
 288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 289 {
 290         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 291                 uint64_t period;
 292
 293                 period = intel_pt_lower_power_of_2(decoder->period);
 294                 decoder->period_mask  = ~(period - 1);
 295                 decoder->period_ticks = period;
 296         }
 297 }
 298
 299 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 300 {
 301         if (!d)
 302                 return 0;
 303         return (t / d) * n + ((t % d) * n) / d;
 304 }
 305
 306 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 307 {
 308         struct intel_pt_decoder *decoder;
 309
 310         if (!params->get_trace || !params->walk_insn)
 311                 return NULL;
 312
 313         decoder = zalloc(sizeof(struct intel_pt_decoder));
 314         if (!decoder)
 315                 return NULL;
 316
 317         decoder->get_trace          = params->get_trace;
 318         decoder->walk_insn          = params->walk_insn;
 319         decoder->pgd_ip             = params->pgd_ip;
 320         decoder->lookahead          = params->lookahead;
 321         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 322         decoder->data               = params->data;
 323         decoder->return_compression = params->return_compression;
 324         decoder->branch_enable      = params->branch_enable;
 325         decoder->hop                = params->quick >= 1;
 326         decoder->leap               = params->quick >= 2;
 327         decoder->vm_time_correlation = params->vm_time_correlation;
 328         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 329         decoder->first_timestamp    = params->first_timestamp;
 330         decoder->last_reliable_timestamp = params->first_timestamp;
 331         decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
 332
 333         decoder->flags              = params->flags;
 334
 335         decoder->ctl                = params->ctl;
 336         decoder->period             = params->period;
 337         decoder->period_type        = params->period_type;
 338
 339         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 340         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 341
 342         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 343
 344         intel_pt_setup_period(decoder);
 345
 346         decoder->mtc_shift = params->mtc_period;
 347         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 348
 349         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 350         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 351
 352         if (!decoder->tsc_ctc_ratio_n)
 353                 decoder->tsc_ctc_ratio_d = 0;
 354
 355         if (decoder->tsc_ctc_ratio_d) {
 356                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 357                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 358                                                 decoder->tsc_ctc_ratio_d;
 359         }
 360
 361         /*
 362          * A TSC packet can slip past MTC packets so that the timestamp appears
 363          * to go backwards. One estimate is that can be up to about 40 CPU
 364          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 365          * slippage an order of magnitude more to be on the safe side.
 366          */
 367         decoder->tsc_slip = 0x10000;
 368
 369         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 370         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 371         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 372         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 373         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 374
 375         if (decoder->hop)
 376                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 377
 378         return decoder;
 379 }
 380
 381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 382                                   uint64_t first_timestamp)
 383 {
 384         decoder->first_timestamp = first_timestamp;
 385 }
 386
 387 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 388 {
 389         struct intel_pt_blk *blk = stack->blk;
 390
 391         stack->blk = blk->prev;
 392         if (!stack->spare)
 393                 stack->spare = blk;
 394         else
 395                 free(blk);
 396 }
 397
 398 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 399 {
 400         if (!stack->pos) {
 401                 if (!stack->blk)
 402                         return 0;
 403                 intel_pt_pop_blk(stack);
 404                 if (!stack->blk)
 405                         return 0;
 406                 stack->pos = INTEL_PT_BLK_SIZE;
 407         }
 408         return stack->blk->ip[--stack->pos];
 409 }
 410
 411 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 412 {
 413         struct intel_pt_blk *blk;
 414
 415         if (stack->spare) {
 416                 blk = stack->spare;
 417                 stack->spare = NULL;
 418         } else {
 419                 blk = malloc(sizeof(struct intel_pt_blk));
 420                 if (!blk)
 421                         return -ENOMEM;
 422         }
 423
 424         blk->prev = stack->blk;
 425         stack->blk = blk;
 426         stack->pos = 0;
 427         return 0;
 428 }
 429
 430 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 431 {
 432         int err;
 433
 434         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 435                 err = intel_pt_alloc_blk(stack);
 436                 if (err)
 437                         return err;
 438         }
 439
 440         stack->blk->ip[stack->pos++] = ip;
 441         return 0;
 442 }
 443
 444 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 445 {
 446         while (stack->blk)
 447                 intel_pt_pop_blk(stack);
 448         stack->pos = 0;
 449 }
 450
 451 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 452 {
 453         intel_pt_clear_stack(stack);
 454         zfree(&stack->blk);
 455         zfree(&stack->spare);
 456 }
 457
 458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 459 {
 460         intel_pt_free_stack(&decoder->stack);
 461         free(decoder);
 462 }
 463
 464 static int intel_pt_ext_err(int code)
 465 {
 466         switch (code) {
 467         case -ENOMEM:
 468                 return INTEL_PT_ERR_NOMEM;
 469         case -ENOSYS:
 470                 return INTEL_PT_ERR_INTERN;
 471         case -EBADMSG:
 472                 return INTEL_PT_ERR_BADPKT;
 473         case -ENODATA:
 474                 return INTEL_PT_ERR_NODATA;
 475         case -EILSEQ:
 476                 return INTEL_PT_ERR_NOINSN;
 477         case -ENOENT:
 478                 return INTEL_PT_ERR_MISMAT;
 479         case -EOVERFLOW:
 480                 return INTEL_PT_ERR_OVR;
 481         case -ENOSPC:
 482                 return INTEL_PT_ERR_LOST;
 483         case -ELOOP:
 484                 return INTEL_PT_ERR_NELOOP;
 485         case -ECONNRESET:
 486                 return INTEL_PT_ERR_EPTW;
 487         default:
 488                 return INTEL_PT_ERR_UNK;
 489         }
 490 }
 491
 492 static const char *intel_pt_err_msgs[] = {
 493         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 494         [INTEL_PT_ERR_INTERN] = "Internal error",
 495         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 496         [INTEL_PT_ERR_NODATA] = "No more data",
 497         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 498         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 499         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 500         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 501         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 502         [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
 503         [INTEL_PT_ERR_EPTW]   = "Broken emulated ptwrite",
 504 };
 505
 506 int intel_pt__strerror(int code, char *buf, size_t buflen)
 507 {
 508         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 509                 code = INTEL_PT_ERR_UNK;
 510         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 511         return 0;
 512 }
 513
 514 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 515                                  uint64_t last_ip)
 516 {
 517         uint64_t ip;
 518
 519         switch (packet->count) {
 520         case 1:
 521                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 522                      packet->payload;
 523                 break;
 524         case 2:
 525                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 526                      packet->payload;
 527                 break;
 528         case 3:
 529                 ip = packet->payload;
 530                 /* Sign-extend 6-byte ip */
 531                 if (ip & (uint64_t)0x800000000000ULL)
 532                         ip |= (uint64_t)0xffff000000000000ULL;
 533                 break;
 534         case 4:
 535                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 536                      packet->payload;
 537                 break;
 538         case 6:
 539                 ip = packet->payload;
 540                 break;
 541         default:
 542                 return 0;
 543         }
 544
 545         return ip;
 546 }
 547
 548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 549 {
 550         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 551         decoder->have_last_ip = true;
 552 }
 553
 554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 555 {
 556         intel_pt_set_last_ip(decoder);
 557         decoder->ip = decoder->last_ip;
 558 }
 559
 560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 561 {
 562         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 563                             decoder->buf);
 564 }
 565
 566 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 567 {
 568         intel_pt_log("ERROR: Internal error\n");
 569         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 570         return -ENOSYS;
 571 }
 572
 573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 574 {
 575         decoder->tx_flags = 0;
 576 }
 577
 578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 579 {
 580         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 581 }
 582
 583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 584 {
 585         decoder->pip_payload = decoder->packet.payload;
 586 }
 587
 588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 589 {
 590         decoder->next_nr = decoder->pip_payload & 1;
 591 }
 592
 593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 594 {
 595         decoder->nr = decoder->pip_payload & 1;
 596         decoder->next_nr = decoder->nr;
 597 }
 598
 599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 600 {
 601         intel_pt_update_pip(decoder);
 602         intel_pt_set_nr(decoder);
 603 }
 604
 605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 606 {
 607         intel_pt_clear_tx_flags(decoder);
 608         decoder->have_tma = false;
 609         decoder->pkt_len = 1;
 610         decoder->pkt_step = 1;
 611         intel_pt_decoder_log_packet(decoder);
 612         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 613                 intel_pt_log("ERROR: Bad packet\n");
 614                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 615         }
 616         return -EBADMSG;
 617 }
 618
 619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 620 {
 621         decoder->sample_timestamp = decoder->timestamp;
 622         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 623         decoder->state.cycles = decoder->tot_cyc_cnt;
 624 }
 625
 626 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 627 {
 628         decoder->ip = 0;
 629         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 630         decoder->timestamp = 0;
 631         decoder->have_tma = false;
 632 }
 633
 634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 635 {
 636         struct intel_pt_buffer buffer = { .buf = 0, };
 637         int ret;
 638
 639         decoder->pkt_step = 0;
 640
 641         intel_pt_log("Getting more data\n");
 642         ret = decoder->get_trace(&buffer, decoder->data);
 643         if (ret)
 644                 return ret;
 645         decoder->buf = buffer.buf;
 646         decoder->len = buffer.len;
 647         if (!decoder->len) {
 648                 intel_pt_log("No more data\n");
 649                 return -ENODATA;
 650         }
 651         decoder->buf_timestamp = buffer.ref_timestamp;
 652         if (!buffer.consecutive || reposition) {
 653                 intel_pt_reposition(decoder);
 654                 decoder->ref_timestamp = buffer.ref_timestamp;
 655                 decoder->state.trace_nr = buffer.trace_nr;
 656                 decoder->vm_tm_corr_same_buf = false;
 657                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 658                              decoder->ref_timestamp);
 659                 return -ENOLINK;
 660         }
 661
 662         return 0;
 663 }
 664
 665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 666                                   bool reposition)
 667 {
 668         if (!decoder->next_buf)
 669                 return intel_pt_get_data(decoder, reposition);
 670
 671         decoder->buf = decoder->next_buf;
 672         decoder->len = decoder->next_len;
 673         decoder->next_buf = 0;
 674         decoder->next_len = 0;
 675         return 0;
 676 }
 677
 678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 679 {
 680         unsigned char *buf = decoder->temp_buf;
 681         size_t old_len, len, n;
 682         int ret;
 683
 684         old_len = decoder->len;
 685         len = decoder->len;
 686         memcpy(buf, decoder->buf, len);
 687
 688         ret = intel_pt_get_data(decoder, false);
 689         if (ret) {
 690                 decoder->pos += old_len;
 691                 return ret < 0 ? ret : -EINVAL;
 692         }
 693
 694         n = INTEL_PT_PKT_MAX_SZ - len;
 695         if (n > decoder->len)
 696                 n = decoder->len;
 697         memcpy(buf + len, decoder->buf, n);
 698         len += n;
 699
 700         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 701         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 702         if (ret < (int)old_len) {
 703                 decoder->next_buf = decoder->buf;
 704                 decoder->next_len = decoder->len;
 705                 decoder->buf = buf;
 706                 decoder->len = old_len;
 707                 return intel_pt_bad_packet(decoder);
 708         }
 709
 710         decoder->next_buf = decoder->buf + (ret - old_len);
 711         decoder->next_len = decoder->len - (ret - old_len);
 712
 713         decoder->buf = buf;
 714         decoder->len = ret;
 715
 716         return ret;
 717 }
 718
 719 struct intel_pt_pkt_info {
 720         struct intel_pt_decoder   *decoder;
 721         struct intel_pt_pkt       packet;
 722         uint64_t                  pos;
 723         int                       pkt_len;
 724         int                       last_packet_type;
 725         void                      *data;
 726 };
 727
 728 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 729
 730 /* Lookahead packets in current buffer */
 731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 732                                   intel_pt_pkt_cb_t cb, void *data)
 733 {
 734         struct intel_pt_pkt_info pkt_info;
 735         const unsigned char *buf = decoder->buf;
 736         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 737         size_t len = decoder->len;
 738         int ret;
 739
 740         pkt_info.decoder          = decoder;
 741         pkt_info.pos              = decoder->pos;
 742         pkt_info.pkt_len          = decoder->pkt_step;
 743         pkt_info.last_packet_type = decoder->last_packet_type;
 744         pkt_info.data             = data;
 745
 746         while (1) {
 747                 do {
 748                         pkt_info.pos += pkt_info.pkt_len;
 749                         buf          += pkt_info.pkt_len;
 750                         len          -= pkt_info.pkt_len;
 751
 752                         if (!len)
 753                                 return INTEL_PT_NEED_MORE_BYTES;
 754
 755                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 756                                                   &pkt_ctx);
 757                         if (!ret)
 758                                 return INTEL_PT_NEED_MORE_BYTES;
 759                         if (ret < 0)
 760                                 return ret;
 761
 762                         pkt_info.pkt_len = ret;
 763                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 764
 765                 ret = cb(&pkt_info);
 766                 if (ret)
 767                         return 0;
 768
 769                 pkt_info.last_packet_type = pkt_info.packet.type;
 770         }
 771 }
 772
 773 struct intel_pt_calc_cyc_to_tsc_info {
 774         uint64_t        cycle_cnt;
 775         unsigned int    cbr;
 776         uint32_t        last_mtc;
 777         uint64_t        ctc_timestamp;
 778         uint64_t        ctc_delta;
 779         uint64_t        tsc_timestamp;
 780         uint64_t        timestamp;
 781         bool            have_tma;
 782         bool            fixup_last_mtc;
 783         bool            from_mtc;
 784         double          cbr_cyc_to_tsc;
 785 };
 786
 787 /*
 788  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 789  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 790  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 791  * packet by copying the missing bits from the current MTC assuming the least
 792  * difference between the two, and that the current MTC comes after last_mtc.
 793  */
 794 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 795                                     uint32_t *last_mtc)
 796 {
 797         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 798         uint32_t mask = ~(first_missing_bit - 1);
 799
 800         *last_mtc |= mtc & mask;
 801         if (*last_mtc >= mtc) {
 802                 *last_mtc -= first_missing_bit;
 803                 *last_mtc &= 0xff;
 804         }
 805 }
 806
 807 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 808 {
 809         struct intel_pt_decoder *decoder = pkt_info->decoder;
 810         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 811         uint64_t timestamp;
 812         double cyc_to_tsc;
 813         unsigned int cbr;
 814         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 815
 816         switch (pkt_info->packet.type) {
 817         case INTEL_PT_TNT:
 818         case INTEL_PT_TIP_PGE:
 819         case INTEL_PT_TIP:
 820         case INTEL_PT_FUP:
 821         case INTEL_PT_PSB:
 822         case INTEL_PT_PIP:
 823         case INTEL_PT_MODE_EXEC:
 824         case INTEL_PT_MODE_TSX:
 825         case INTEL_PT_PSBEND:
 826         case INTEL_PT_PAD:
 827         case INTEL_PT_VMCS:
 828         case INTEL_PT_MNT:
 829         case INTEL_PT_PTWRITE:
 830         case INTEL_PT_PTWRITE_IP:
 831         case INTEL_PT_BBP:
 832         case INTEL_PT_BIP:
 833         case INTEL_PT_BEP:
 834         case INTEL_PT_BEP_IP:
 835         case INTEL_PT_CFE:
 836         case INTEL_PT_CFE_IP:
 837         case INTEL_PT_EVD:
 838                 return 0;
 839
 840         case INTEL_PT_MTC:
 841                 if (!data->have_tma)
 842                         return 0;
 843
 844                 mtc = pkt_info->packet.payload;
 845                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 846                         data->fixup_last_mtc = false;
 847                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 848                                                 &data->last_mtc);
 849                 }
 850                 if (mtc > data->last_mtc)
 851                         mtc_delta = mtc - data->last_mtc;
 852                 else
 853                         mtc_delta = mtc + 256 - data->last_mtc;
 854                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 855                 data->last_mtc = mtc;
 856
 857                 if (decoder->tsc_ctc_mult) {
 858                         timestamp = data->ctc_timestamp +
 859                                 data->ctc_delta * decoder->tsc_ctc_mult;
 860                 } else {
 861                         timestamp = data->ctc_timestamp +
 862                                 multdiv(data->ctc_delta,
 863                                         decoder->tsc_ctc_ratio_n,
 864                                         decoder->tsc_ctc_ratio_d);
 865                 }
 866
 867                 if (timestamp < data->timestamp)
 868                         return 1;
 869
 870                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 871                         data->timestamp = timestamp;
 872                         return 0;
 873                 }
 874
 875                 break;
 876
 877         case INTEL_PT_TSC:
 878                 /*
 879                  * For now, do not support using TSC packets - refer
 880                  * intel_pt_calc_cyc_to_tsc().
 881                  */
 882                 if (data->from_mtc)
 883                         return 1;
 884                 timestamp = pkt_info->packet.payload |
 885                             (data->timestamp & (0xffULL << 56));
 886                 if (data->from_mtc && timestamp < data->timestamp &&
 887                     data->timestamp - timestamp < decoder->tsc_slip)
 888                         return 1;
 889                 if (timestamp < data->timestamp)
 890                         timestamp += (1ULL << 56);
 891                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 892                         if (data->from_mtc)
 893                                 return 1;
 894                         data->tsc_timestamp = timestamp;
 895                         data->timestamp = timestamp;
 896                         return 0;
 897                 }
 898                 break;
 899
 900         case INTEL_PT_TMA:
 901                 if (data->from_mtc)
 902                         return 1;
 903
 904                 if (!decoder->tsc_ctc_ratio_d)
 905                         return 0;
 906
 907                 ctc = pkt_info->packet.payload;
 908                 fc = pkt_info->packet.count;
 909                 ctc_rem = ctc & decoder->ctc_rem_mask;
 910
 911                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 912
 913                 data->ctc_timestamp = data->tsc_timestamp - fc;
 914                 if (decoder->tsc_ctc_mult) {
 915                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 916                 } else {
 917                         data->ctc_timestamp -=
 918                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 919                                         decoder->tsc_ctc_ratio_d);
 920                 }
 921
 922                 data->ctc_delta = 0;
 923                 data->have_tma = true;
 924                 data->fixup_last_mtc = true;
 925
 926                 return 0;
 927
 928         case INTEL_PT_CYC:
 929                 data->cycle_cnt += pkt_info->packet.payload;
 930                 return 0;
 931
 932         case INTEL_PT_CBR:
 933                 cbr = pkt_info->packet.payload;
 934                 if (data->cbr && data->cbr != cbr)
 935                         return 1;
 936                 data->cbr = cbr;
 937                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 938                 return 0;
 939
 940         case INTEL_PT_TIP_PGD:
 941         case INTEL_PT_TRACESTOP:
 942         case INTEL_PT_EXSTOP:
 943         case INTEL_PT_EXSTOP_IP:
 944         case INTEL_PT_MWAIT:
 945         case INTEL_PT_PWRE:
 946         case INTEL_PT_PWRX:
 947         case INTEL_PT_OVF:
 948         case INTEL_PT_BAD: /* Does not happen */
 949         default:
 950                 return 1;
 951         }
 952
 953         if (!data->cbr && decoder->cbr) {
 954                 data->cbr = decoder->cbr;
 955                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 956         }
 957
 958         if (!data->cycle_cnt)
 959                 return 1;
 960
 961         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 962
 963         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 964             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 965                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 966                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 967                 return 1;
 968         }
 969
 970         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 971         decoder->have_calc_cyc_to_tsc = true;
 972
 973         if (data->cbr) {
 974                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 975                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 976         } else {
 977                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 978                              cyc_to_tsc, pkt_info->pos);
 979         }
 980
 981         return 1;
 982 }
 983
 984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 985                                      bool from_mtc)
 986 {
 987         struct intel_pt_calc_cyc_to_tsc_info data = {
 988                 .cycle_cnt      = 0,
 989                 .cbr            = 0,
 990                 .last_mtc       = decoder->last_mtc,
 991                 .ctc_timestamp  = decoder->ctc_timestamp,
 992                 .ctc_delta      = decoder->ctc_delta,
 993                 .tsc_timestamp  = decoder->tsc_timestamp,
 994                 .timestamp      = decoder->timestamp,
 995                 .have_tma       = decoder->have_tma,
 996                 .fixup_last_mtc = decoder->fixup_last_mtc,
 997                 .from_mtc       = from_mtc,
 998                 .cbr_cyc_to_tsc = 0,
 999         };
1000
1001         /*
1002          * For now, do not support using TSC packets for at least the reasons:
1003          * 1) timing might have stopped
1004          * 2) TSC packets within PSB+ can slip against CYC packets
1005          */
1006         if (!from_mtc)
1007                 return;
1008
1009         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
1010 }
1011
1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
1013 {
1014         int ret;
1015
1016         decoder->last_packet_type = decoder->packet.type;
1017
1018         do {
1019                 decoder->pos += decoder->pkt_step;
1020                 decoder->buf += decoder->pkt_step;
1021                 decoder->len -= decoder->pkt_step;
1022
1023                 if (!decoder->len) {
1024                         ret = intel_pt_get_next_data(decoder, false);
1025                         if (ret)
1026                                 return ret;
1027                 }
1028
1029                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1030                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1031                                           &decoder->packet, &decoder->pkt_ctx);
1032                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1033                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1034                         ret = intel_pt_get_split_packet(decoder);
1035                         if (ret < 0)
1036                                 return ret;
1037                 }
1038                 if (ret <= 0)
1039                         return intel_pt_bad_packet(decoder);
1040
1041                 decoder->pkt_len = ret;
1042                 decoder->pkt_step = ret;
1043                 intel_pt_decoder_log_packet(decoder);
1044         } while (decoder->packet.type == INTEL_PT_PAD);
1045
1046         return 0;
1047 }
1048
1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1050 {
1051         uint64_t timestamp, masked_timestamp;
1052
1053         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1054         masked_timestamp = timestamp & decoder->period_mask;
1055         if (decoder->continuous_period) {
1056                 if (masked_timestamp > decoder->last_masked_timestamp)
1057                         return 1;
1058         } else {
1059                 timestamp += 1;
1060                 masked_timestamp = timestamp & decoder->period_mask;
1061                 if (masked_timestamp > decoder->last_masked_timestamp) {
1062                         decoder->last_masked_timestamp = masked_timestamp;
1063                         decoder->continuous_period = true;
1064                 }
1065         }
1066
1067         if (masked_timestamp < decoder->last_masked_timestamp)
1068                 return decoder->period_ticks;
1069
1070         return decoder->period_ticks - (timestamp - masked_timestamp);
1071 }
1072
1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1074 {
1075         switch (decoder->period_type) {
1076         case INTEL_PT_PERIOD_INSTRUCTIONS:
1077                 return decoder->period - decoder->period_insn_cnt;
1078         case INTEL_PT_PERIOD_TICKS:
1079                 return intel_pt_next_period(decoder);
1080         case INTEL_PT_PERIOD_NONE:
1081         case INTEL_PT_PERIOD_MTC:
1082         default:
1083                 return 0;
1084         }
1085 }
1086
1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1088 {
1089         uint64_t timestamp, masked_timestamp;
1090
1091         switch (decoder->period_type) {
1092         case INTEL_PT_PERIOD_INSTRUCTIONS:
1093                 decoder->period_insn_cnt = 0;
1094                 break;
1095         case INTEL_PT_PERIOD_TICKS:
1096                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1097                 masked_timestamp = timestamp & decoder->period_mask;
1098                 if (masked_timestamp > decoder->last_masked_timestamp)
1099                         decoder->last_masked_timestamp = masked_timestamp;
1100                 else
1101                         decoder->last_masked_timestamp += decoder->period_ticks;
1102                 break;
1103         case INTEL_PT_PERIOD_NONE:
1104         case INTEL_PT_PERIOD_MTC:
1105         default:
1106                 break;
1107         }
1108
1109         decoder->state.type |= INTEL_PT_INSTRUCTION;
1110 }
1111
1112 /*
1113  * Sample FUP instruction at the same time as reporting the FUP event, so the
1114  * instruction sample gets the same flags as the FUP event.
1115  */
1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)
1117 {
1118         struct intel_pt_insn intel_pt_insn;
1119         uint64_t max_insn_cnt, insn_cnt = 0;
1120         int err;
1121
1122         decoder->state.insn_op = INTEL_PT_OP_OTHER;
1123         decoder->state.insn_len = 0;
1124
1125         if (!decoder->branch_enable || !decoder->pge || decoder->hop ||
1126             decoder->ip != decoder->last_ip)
1127                 return;
1128
1129         if (!decoder->mtc_insn)
1130                 decoder->mtc_insn = true;
1131
1132         max_insn_cnt = intel_pt_next_sample(decoder);
1133         if (max_insn_cnt != 1)
1134                 return;
1135
1136         err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,
1137                                  0, max_insn_cnt, decoder->data);
1138         /* Ignore error, it will be reported next walk anyway */
1139         if (err)
1140                 return;
1141
1142         if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) {
1143                 intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);
1144                 return;
1145         }
1146
1147         decoder->tot_insn_cnt += insn_cnt;
1148         decoder->timestamp_insn_cnt += insn_cnt;
1149         decoder->sample_insn_cnt += insn_cnt;
1150         decoder->period_insn_cnt += insn_cnt;
1151
1152         intel_pt_sample_insn(decoder);
1153
1154         decoder->state.type |= INTEL_PT_INSTRUCTION;
1155         decoder->ip += intel_pt_insn.length;
1156 }
1157
1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1159                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1160 {
1161         uint64_t max_insn_cnt, insn_cnt = 0;
1162         int err;
1163
1164         if (!decoder->mtc_insn)
1165                 decoder->mtc_insn = true;
1166
1167         max_insn_cnt = intel_pt_next_sample(decoder);
1168
1169         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1170                                  max_insn_cnt, decoder->data);
1171
1172         decoder->tot_insn_cnt += insn_cnt;
1173         decoder->timestamp_insn_cnt += insn_cnt;
1174         decoder->sample_insn_cnt += insn_cnt;
1175         decoder->period_insn_cnt += insn_cnt;
1176
1177         if (err) {
1178                 decoder->no_progress = 0;
1179                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1180                 intel_pt_log_at("ERROR: Failed to get instruction",
1181                                 decoder->ip);
1182                 if (err == -ENOENT)
1183                         return -ENOLINK;
1184                 return -EILSEQ;
1185         }
1186
1187         if (ip && decoder->ip == ip) {
1188                 err = -EAGAIN;
1189                 goto out;
1190         }
1191
1192         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1193                 intel_pt_sample_insn(decoder);
1194
1195         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1196                 decoder->state.type = INTEL_PT_INSTRUCTION;
1197                 decoder->state.from_ip = decoder->ip;
1198                 decoder->state.to_ip = 0;
1199                 decoder->ip += intel_pt_insn->length;
1200                 err = INTEL_PT_RETURN;
1201                 goto out;
1202         }
1203
1204         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1205                 /* Zero-length calls are excluded */
1206                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1207                     intel_pt_insn->rel) {
1208                         err = intel_pt_push(&decoder->stack, decoder->ip +
1209                                             intel_pt_insn->length);
1210                         if (err)
1211                                 goto out;
1212                 }
1213         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1214                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1215         }
1216
1217         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1218                 int cnt = decoder->no_progress++;
1219
1220                 decoder->state.from_ip = decoder->ip;
1221                 decoder->ip += intel_pt_insn->length +
1222                                 intel_pt_insn->rel;
1223                 decoder->state.to_ip = decoder->ip;
1224                 err = INTEL_PT_RETURN;
1225
1226                 /*
1227                  * Check for being stuck in a loop.  This can happen if a
1228                  * decoder error results in the decoder erroneously setting the
1229                  * ip to an address that is itself in an infinite loop that
1230                  * consumes no packets.  When that happens, there must be an
1231                  * unconditional branch.
1232                  */
1233                 if (cnt) {
1234                         if (cnt == 1) {
1235                                 decoder->stuck_ip = decoder->state.to_ip;
1236                                 decoder->stuck_ip_prd = 1;
1237                                 decoder->stuck_ip_cnt = 1;
1238                         } else if (cnt > decoder->max_loops ||
1239                                    decoder->state.to_ip == decoder->stuck_ip) {
1240                                 intel_pt_log_at("ERROR: Never-ending loop",
1241                                                 decoder->state.to_ip);
1242                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1243                                 err = -ELOOP;
1244                                 goto out;
1245                         } else if (!--decoder->stuck_ip_cnt) {
1246                                 decoder->stuck_ip_prd += 1;
1247                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1248                                 decoder->stuck_ip = decoder->state.to_ip;
1249                         }
1250                 }
1251                 goto out_no_progress;
1252         }
1253 out:
1254         decoder->no_progress = 0;
1255 out_no_progress:
1256         decoder->state.insn_op = intel_pt_insn->op;
1257         decoder->state.insn_len = intel_pt_insn->length;
1258         memcpy(decoder->state.insn, intel_pt_insn->buf,
1259                INTEL_PT_INSN_BUF_SZ);
1260
1261         if (decoder->tx_flags & INTEL_PT_IN_TX)
1262                 decoder->state.flags |= INTEL_PT_IN_TX;
1263
1264         return err;
1265 }
1266
1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)
1268 {
1269         bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1270
1271         decoder->exec_mode = decoder->packet.payload;
1272         decoder->iflag = iflag;
1273         decoder->next_iflag = iflag;
1274         decoder->state.from_iflag = iflag;
1275         decoder->state.to_iflag = iflag;
1276 }
1277
1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)
1279 {
1280         bool iflag = decoder->packet.count & INTEL_PT_IFLAG;
1281
1282         decoder->exec_mode = decoder->packet.payload;
1283         decoder->next_iflag = iflag;
1284 }
1285
1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)
1287 {
1288         decoder->state.type |= INTEL_PT_IFLAG_CHG;
1289         decoder->state.from_iflag = decoder->iflag;
1290         decoder->state.to_iflag = decoder->next_iflag;
1291         decoder->iflag = decoder->next_iflag;
1292 }
1293
1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)
1295 {
1296         if (decoder->iflag != decoder->next_iflag)
1297                 intel_pt_sample_iflag(decoder);
1298 }
1299
1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)
1301 {
1302         decoder->set_fup_tx_flags = false;
1303         decoder->set_fup_ptw = false;
1304         decoder->set_fup_mwait = false;
1305         decoder->set_fup_pwre = false;
1306         decoder->set_fup_exstop = false;
1307         decoder->set_fup_bep = false;
1308         decoder->set_fup_cfe_ip = false;
1309         decoder->set_fup_cfe = false;
1310         decoder->evd_cnt = 0;
1311         decoder->set_fup_mode_exec = false;
1312         decoder->iflag = decoder->next_iflag;
1313 }
1314
1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)
1316 {
1317         enum intel_pt_sample_type type = decoder->state.type;
1318         bool sample_fup_insn = false;
1319         bool ret = false;
1320
1321         decoder->state.type &= ~INTEL_PT_BRANCH;
1322
1323         if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {
1324                 bool ip = decoder->set_fup_cfe_ip;
1325
1326                 decoder->set_fup_cfe_ip = false;
1327                 decoder->set_fup_cfe = false;
1328                 decoder->state.type |= INTEL_PT_EVT;
1329                 if (!ip && decoder->pge)
1330                         decoder->state.type |= INTEL_PT_BRANCH;
1331                 decoder->state.cfe_type = decoder->fup_cfe_pkt.count;
1332                 decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;
1333                 decoder->state.evd_cnt = decoder->evd_cnt;
1334                 decoder->state.evd = decoder->evd;
1335                 decoder->evd_cnt = 0;
1336                 if (ip || decoder->pge)
1337                         decoder->state.flags |= INTEL_PT_FUP_IP;
1338                 ret = true;
1339         }
1340         if (decoder->set_fup_mode_exec) {
1341                 decoder->set_fup_mode_exec = false;
1342                 intel_pt_sample_iflag(decoder);
1343                 sample_fup_insn = no_tip;
1344                 ret = true;
1345         }
1346         if (decoder->set_fup_tx_flags) {
1347                 decoder->set_fup_tx_flags = false;
1348                 decoder->tx_flags = decoder->fup_tx_flags;
1349                 decoder->state.type |= INTEL_PT_TRANSACTION;
1350                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1351                         decoder->state.type |= INTEL_PT_BRANCH;
1352                 decoder->state.flags = decoder->fup_tx_flags;
1353                 ret = true;
1354         }
1355         if (decoder->set_fup_ptw) {
1356                 decoder->set_fup_ptw = false;
1357                 decoder->state.type |= INTEL_PT_PTW;
1358                 decoder->state.flags |= INTEL_PT_FUP_IP;
1359                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1360                 ret = true;
1361         }
1362         if (decoder->set_fup_mwait) {
1363                 decoder->set_fup_mwait = false;
1364                 decoder->state.type |= INTEL_PT_MWAIT_OP;
1365                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1366                 ret = true;
1367         }
1368         if (decoder->set_fup_pwre) {
1369                 decoder->set_fup_pwre = false;
1370                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1371                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1372                 ret = true;
1373         }
1374         if (decoder->set_fup_exstop) {
1375                 decoder->set_fup_exstop = false;
1376                 decoder->state.type |= INTEL_PT_EX_STOP;
1377                 decoder->state.flags |= INTEL_PT_FUP_IP;
1378                 ret = true;
1379         }
1380         if (decoder->set_fup_bep) {
1381                 decoder->set_fup_bep = false;
1382                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1383                 ret = true;
1384         }
1385         if (decoder->overflow) {
1386                 decoder->overflow = false;
1387                 if (!ret && !decoder->pge) {
1388                         if (decoder->hop) {
1389                                 decoder->state.type = 0;
1390                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1391                         }
1392                         decoder->pge = true;
1393                         decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
1394                         decoder->state.from_ip = 0;
1395                         decoder->state.to_ip = decoder->ip;
1396                         return true;
1397                 }
1398         }
1399         if (ret) {
1400                 decoder->state.from_ip = decoder->ip;
1401                 decoder->state.to_ip = 0;
1402                 if (sample_fup_insn)
1403                         intel_pt_sample_fup_insn(decoder);
1404         } else {
1405                 decoder->state.type = type;
1406         }
1407         return ret;
1408 }
1409
1410 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1411                                           struct intel_pt_insn *intel_pt_insn,
1412                                           uint64_t ip, int err)
1413 {
1414         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1415                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1416                ip == decoder->ip + intel_pt_insn->length;
1417 }
1418
1419 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1420 {
1421         struct intel_pt_insn intel_pt_insn;
1422         uint64_t ip;
1423         int err;
1424
1425         ip = decoder->last_ip;
1426
1427         while (1) {
1428                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1429                 if (err == INTEL_PT_RETURN)
1430                         return 0;
1431                 if (err == -EAGAIN ||
1432                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1433                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1434
1435                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1436                         if (intel_pt_fup_event(decoder, no_tip) && no_tip)
1437                                 return 0;
1438                         return -EAGAIN;
1439                 }
1440                 decoder->set_fup_tx_flags = false;
1441                 if (err)
1442                         return err;
1443
1444                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1445                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1446                                         decoder->ip);
1447                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1448                         return -ENOENT;
1449                 }
1450
1451                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1452                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1453                                         decoder->ip);
1454                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1455                         return -ENOENT;
1456                 }
1457
1458                 intel_pt_bug(decoder);
1459         }
1460 }
1461
1462 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1463 {
1464         struct intel_pt_insn intel_pt_insn;
1465         int err;
1466
1467         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1468         if (err == INTEL_PT_RETURN &&
1469             decoder->pgd_ip &&
1470             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1471             (decoder->state.type & INTEL_PT_BRANCH) &&
1472             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1473                 /* Unconditional branch leaving filter region */
1474                 decoder->no_progress = 0;
1475                 decoder->pge = false;
1476                 decoder->continuous_period = false;
1477                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1478                 decoder->state.type |= INTEL_PT_TRACE_END;
1479                 intel_pt_update_nr(decoder);
1480                 return 0;
1481         }
1482         if (err == INTEL_PT_RETURN)
1483                 return 0;
1484         if (err)
1485                 return err;
1486
1487         intel_pt_update_nr(decoder);
1488         intel_pt_sample_iflag_chg(decoder);
1489
1490         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1491                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1492                         decoder->pge = false;
1493                         decoder->continuous_period = false;
1494                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1495                         decoder->state.from_ip = decoder->ip;
1496                         if (decoder->packet.count == 0) {
1497                                 decoder->state.to_ip = 0;
1498                         } else {
1499                                 decoder->state.to_ip = decoder->last_ip;
1500                                 decoder->ip = decoder->last_ip;
1501                         }
1502                         decoder->state.type |= INTEL_PT_TRACE_END;
1503                 } else {
1504                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1505                         decoder->state.from_ip = decoder->ip;
1506                         if (decoder->packet.count == 0) {
1507                                 decoder->state.to_ip = 0;
1508                         } else {
1509                                 decoder->state.to_ip = decoder->last_ip;
1510                                 decoder->ip = decoder->last_ip;
1511                         }
1512                 }
1513                 return 0;
1514         }
1515
1516         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1517                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1518                                  intel_pt_insn.rel;
1519
1520                 if (decoder->pgd_ip &&
1521                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1522                     decoder->pgd_ip(to_ip, decoder->data)) {
1523                         /* Conditional branch leaving filter region */
1524                         decoder->pge = false;
1525                         decoder->continuous_period = false;
1526                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1527                         decoder->ip = to_ip;
1528                         decoder->state.from_ip = decoder->ip;
1529                         decoder->state.to_ip = to_ip;
1530                         decoder->state.type |= INTEL_PT_TRACE_END;
1531                         return 0;
1532                 }
1533                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1534                                 decoder->ip);
1535                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1536                 return -ENOENT;
1537         }
1538
1539         return intel_pt_bug(decoder);
1540 }
1541
1542 struct eptw_data {
1543         int bit_countdown;
1544         uint64_t payload;
1545 };
1546
1547 static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
1548 {
1549         struct eptw_data *data = pkt_info->data;
1550         int nr_bits;
1551
1552         switch (pkt_info->packet.type) {
1553         case INTEL_PT_PAD:
1554         case INTEL_PT_MNT:
1555         case INTEL_PT_MODE_EXEC:
1556         case INTEL_PT_MODE_TSX:
1557         case INTEL_PT_MTC:
1558         case INTEL_PT_FUP:
1559         case INTEL_PT_CYC:
1560         case INTEL_PT_CBR:
1561         case INTEL_PT_TSC:
1562         case INTEL_PT_TMA:
1563         case INTEL_PT_PIP:
1564         case INTEL_PT_VMCS:
1565         case INTEL_PT_PSB:
1566         case INTEL_PT_PSBEND:
1567         case INTEL_PT_PTWRITE:
1568         case INTEL_PT_PTWRITE_IP:
1569         case INTEL_PT_EXSTOP:
1570         case INTEL_PT_EXSTOP_IP:
1571         case INTEL_PT_MWAIT:
1572         case INTEL_PT_PWRE:
1573         case INTEL_PT_PWRX:
1574         case INTEL_PT_BBP:
1575         case INTEL_PT_BIP:
1576         case INTEL_PT_BEP:
1577         case INTEL_PT_BEP_IP:
1578         case INTEL_PT_CFE:
1579         case INTEL_PT_CFE_IP:
1580         case INTEL_PT_EVD:
1581                 break;
1582
1583         case INTEL_PT_TNT:
1584                 nr_bits = data->bit_countdown;
1585                 if (nr_bits > pkt_info->packet.count)
1586                         nr_bits = pkt_info->packet.count;
1587                 data->payload <<= nr_bits;
1588                 data->payload |= pkt_info->packet.payload >> (64 - nr_bits);
1589                 data->bit_countdown -= nr_bits;
1590                 return !data->bit_countdown;
1591
1592         case INTEL_PT_TIP_PGE:
1593         case INTEL_PT_TIP_PGD:
1594         case INTEL_PT_TIP:
1595         case INTEL_PT_BAD:
1596         case INTEL_PT_OVF:
1597         case INTEL_PT_TRACESTOP:
1598         default:
1599                 return 1;
1600         }
1601
1602         return 0;
1603 }
1604
1605 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)
1606 {
1607         int n = 64 - decoder->tnt.count;
1608         struct eptw_data data = {
1609                 .bit_countdown = n,
1610                 .payload = decoder->tnt.payload >> n,
1611         };
1612
1613         decoder->emulated_ptwrite = false;
1614         intel_pt_log("Emulated ptwrite detected\n");
1615
1616         intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);
1617         if (data.bit_countdown)
1618                 return -ECONNRESET;
1619
1620         decoder->state.type = INTEL_PT_PTW;
1621         decoder->state.from_ip = decoder->ip;
1622         decoder->state.to_ip = 0;
1623         decoder->state.ptw_payload = data.payload;
1624         return 0;
1625 }
1626
1627 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1628 {
1629         struct intel_pt_insn intel_pt_insn;
1630         int err;
1631
1632         while (1) {
1633                 if (decoder->emulated_ptwrite)
1634                         return intel_pt_emulated_ptwrite(decoder);
1635                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1636                 if (err == INTEL_PT_RETURN) {
1637                         decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;
1638                         return 0;
1639                 }
1640                 if (err) {
1641                         decoder->emulated_ptwrite = false;
1642                         return err;
1643                 }
1644
1645                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1646                         if (!decoder->return_compression) {
1647                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1648                                                 decoder->ip);
1649                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1650                                 return -ENOENT;
1651                         }
1652                         if (!decoder->ret_addr) {
1653                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1654                                                 decoder->ip);
1655                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1656                                 return -ENOENT;
1657                         }
1658                         if (!(decoder->tnt.payload & BIT63)) {
1659                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1660                                                 decoder->ip);
1661                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1662                                 return -ENOENT;
1663                         }
1664                         decoder->tnt.count -= 1;
1665                         if (decoder->tnt.count)
1666                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1667                         else
1668                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1669                         decoder->tnt.payload <<= 1;
1670                         decoder->state.from_ip = decoder->ip;
1671                         decoder->ip = decoder->ret_addr;
1672                         decoder->state.to_ip = decoder->ip;
1673                         return 0;
1674                 }
1675
1676                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1677                         /* Handle deferred TIPs */
1678                         err = intel_pt_get_next_packet(decoder);
1679                         if (err)
1680                                 return err;
1681                         if (decoder->packet.type != INTEL_PT_TIP ||
1682                             decoder->packet.count == 0) {
1683                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1684                                                 decoder->ip);
1685                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1686                                 decoder->pkt_step = 0;
1687                                 return -ENOENT;
1688                         }
1689                         intel_pt_set_last_ip(decoder);
1690                         decoder->state.from_ip = decoder->ip;
1691                         decoder->state.to_ip = decoder->last_ip;
1692                         decoder->ip = decoder->last_ip;
1693                         intel_pt_update_nr(decoder);
1694                         intel_pt_sample_iflag_chg(decoder);
1695                         return 0;
1696                 }
1697
1698                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1699                         decoder->tnt.count -= 1;
1700                         if (decoder->tnt.count)
1701                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1702                         else
1703                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1704                         if (decoder->tnt.payload & BIT63) {
1705                                 decoder->tnt.payload <<= 1;
1706                                 decoder->state.from_ip = decoder->ip;
1707                                 decoder->ip += intel_pt_insn.length +
1708                                                intel_pt_insn.rel;
1709                                 decoder->state.to_ip = decoder->ip;
1710                                 return 0;
1711                         }
1712                         /* Instruction sample for a non-taken branch */
1713                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1714                                 decoder->tnt.payload <<= 1;
1715                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1716                                 decoder->state.from_ip = decoder->ip;
1717                                 decoder->state.to_ip = 0;
1718                                 decoder->ip += intel_pt_insn.length;
1719                                 return 0;
1720                         }
1721                         decoder->sample_cyc = false;
1722                         decoder->ip += intel_pt_insn.length;
1723                         if (!decoder->tnt.count) {
1724                                 intel_pt_update_sample_time(decoder);
1725                                 return -EAGAIN;
1726                         }
1727                         decoder->tnt.payload <<= 1;
1728                         continue;
1729                 }
1730
1731                 return intel_pt_bug(decoder);
1732         }
1733 }
1734
1735 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1736 {
1737         unsigned int fup_tx_flags;
1738         int err;
1739
1740         fup_tx_flags = decoder->packet.payload &
1741                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1742         err = intel_pt_get_next_packet(decoder);
1743         if (err)
1744                 return err;
1745         if (decoder->packet.type == INTEL_PT_FUP) {
1746                 decoder->fup_tx_flags = fup_tx_flags;
1747                 decoder->set_fup_tx_flags = true;
1748                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1749                         *no_tip = true;
1750         } else {
1751                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1752                                 decoder->pos);
1753                 intel_pt_update_in_tx(decoder);
1754         }
1755         return 0;
1756 }
1757
1758 static int intel_pt_evd(struct intel_pt_decoder *decoder)
1759 {
1760         if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {
1761                 intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);
1762                 return -ENOSYS;
1763         }
1764         decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){
1765                 .type = decoder->packet.count,
1766                 .payload = decoder->packet.payload,
1767         };
1768         return 0;
1769 }
1770
1771 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1772 {
1773         timestamp |= (ref_timestamp & (0xffULL << 56));
1774
1775         if (timestamp < ref_timestamp) {
1776                 if (ref_timestamp - timestamp > (1ULL << 55))
1777                         timestamp += (1ULL << 56);
1778         } else {
1779                 if (timestamp - ref_timestamp > (1ULL << 55))
1780                         timestamp -= (1ULL << 56);
1781         }
1782
1783         return timestamp;
1784 }
1785
1786 /* For use only when decoder->vm_time_correlation is true */
1787 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1788                                    uint64_t timestamp)
1789 {
1790         uint64_t max_timestamp = decoder->buf_timestamp;
1791
1792         if (!max_timestamp) {
1793                 max_timestamp = decoder->last_reliable_timestamp +
1794                                 0x400000000ULL;
1795         }
1796         return timestamp >= decoder->last_reliable_timestamp &&
1797                timestamp < decoder->buf_timestamp;
1798 }
1799
1800 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1801 {
1802         uint64_t timestamp;
1803         bool bad = false;
1804
1805         decoder->have_tma = false;
1806
1807         if (decoder->ref_timestamp) {
1808                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1809                                             decoder->ref_timestamp);
1810                 decoder->tsc_timestamp = timestamp;
1811                 decoder->timestamp = timestamp;
1812                 decoder->ref_timestamp = 0;
1813                 decoder->timestamp_insn_cnt = 0;
1814         } else if (decoder->timestamp) {
1815                 timestamp = decoder->packet.payload |
1816                             (decoder->timestamp & (0xffULL << 56));
1817                 decoder->tsc_timestamp = timestamp;
1818                 if (timestamp < decoder->timestamp &&
1819                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1820                         intel_pt_log_to("Suppressing backwards timestamp",
1821                                         timestamp);
1822                         timestamp = decoder->timestamp;
1823                 }
1824                 if (timestamp < decoder->timestamp) {
1825                         if (!decoder->buf_timestamp ||
1826                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1827                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1828                                 timestamp += (1ULL << 56);
1829                                 decoder->tsc_timestamp = timestamp;
1830                         } else {
1831                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1832                                 timestamp = decoder->timestamp;
1833                                 bad = true;
1834                         }
1835                 }
1836                 if (decoder->vm_time_correlation &&
1837                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1838                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1839                         p_log("Timestamp out of range");
1840                 decoder->timestamp = timestamp;
1841                 decoder->timestamp_insn_cnt = 0;
1842         }
1843
1844         if (decoder->last_packet_type == INTEL_PT_CYC) {
1845                 decoder->cyc_ref_timestamp = decoder->timestamp;
1846                 decoder->cycle_cnt = 0;
1847                 decoder->have_calc_cyc_to_tsc = false;
1848                 intel_pt_calc_cyc_to_tsc(decoder, false);
1849         }
1850
1851         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1852 }
1853
1854 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1855 {
1856         intel_pt_log("ERROR: Buffer overflow\n");
1857         intel_pt_clear_tx_flags(decoder);
1858         intel_pt_set_nr(decoder);
1859         decoder->timestamp_insn_cnt = 0;
1860         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1861         decoder->state.from_ip = decoder->ip;
1862         decoder->ip = 0;
1863         decoder->pge = false;
1864         intel_pt_clear_fup_event(decoder);
1865         decoder->overflow = true;
1866         return -EOVERFLOW;
1867 }
1868
1869 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1870 {
1871         if (decoder->have_cyc)
1872                 return;
1873
1874         decoder->cyc_cnt_timestamp = decoder->timestamp;
1875         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1876 }
1877
1878 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1879 {
1880         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1881
1882         if (decoder->pge)
1883                 intel_pt_mtc_cyc_cnt_pge(decoder);
1884 }
1885
1886 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1887 {
1888         uint64_t tot_cyc_cnt, tsc_delta;
1889
1890         if (decoder->have_cyc)
1891                 return;
1892
1893         decoder->sample_cyc = true;
1894
1895         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1896                 return;
1897
1898         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1899         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1900
1901         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1902                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1903 }
1904
1905 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1906 {
1907         uint32_t ctc = decoder->packet.payload;
1908         uint32_t fc = decoder->packet.count;
1909         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1910
1911         if (!decoder->tsc_ctc_ratio_d)
1912                 return;
1913
1914         if (decoder->pge && !decoder->in_psb)
1915                 intel_pt_mtc_cyc_cnt_pge(decoder);
1916         else
1917                 intel_pt_mtc_cyc_cnt_upd(decoder);
1918
1919         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1920         decoder->last_ctc = ctc - ctc_rem;
1921         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1922         if (decoder->tsc_ctc_mult) {
1923                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1924         } else {
1925                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1926                                                   decoder->tsc_ctc_ratio_n,
1927                                                   decoder->tsc_ctc_ratio_d);
1928         }
1929         decoder->ctc_delta = 0;
1930         decoder->have_tma = true;
1931         decoder->fixup_last_mtc = true;
1932         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1933                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1934 }
1935
1936 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1937 {
1938         uint64_t timestamp;
1939         uint32_t mtc, mtc_delta;
1940
1941         if (!decoder->have_tma)
1942                 return;
1943
1944         mtc = decoder->packet.payload;
1945
1946         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1947                 decoder->fixup_last_mtc = false;
1948                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1949                                         &decoder->last_mtc);
1950         }
1951
1952         if (mtc > decoder->last_mtc)
1953                 mtc_delta = mtc - decoder->last_mtc;
1954         else
1955                 mtc_delta = mtc + 256 - decoder->last_mtc;
1956
1957         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1958
1959         if (decoder->tsc_ctc_mult) {
1960                 timestamp = decoder->ctc_timestamp +
1961                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1962         } else {
1963                 timestamp = decoder->ctc_timestamp +
1964                             multdiv(decoder->ctc_delta,
1965                                     decoder->tsc_ctc_ratio_n,
1966                                     decoder->tsc_ctc_ratio_d);
1967         }
1968
1969         if (timestamp < decoder->timestamp)
1970                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1971                              timestamp, decoder->timestamp);
1972         else
1973                 decoder->timestamp = timestamp;
1974
1975         intel_pt_mtc_cyc_cnt_upd(decoder);
1976
1977         decoder->timestamp_insn_cnt = 0;
1978         decoder->last_mtc = mtc;
1979
1980         if (decoder->last_packet_type == INTEL_PT_CYC) {
1981                 decoder->cyc_ref_timestamp = decoder->timestamp;
1982                 decoder->cycle_cnt = 0;
1983                 decoder->have_calc_cyc_to_tsc = false;
1984                 intel_pt_calc_cyc_to_tsc(decoder, true);
1985         }
1986
1987         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1988 }
1989
1990 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1991 {
1992         unsigned int cbr = decoder->packet.payload & 0xff;
1993
1994         decoder->cbr_payload = decoder->packet.payload;
1995
1996         if (decoder->cbr == cbr)
1997                 return;
1998
1999         decoder->cbr = cbr;
2000         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
2001         decoder->cyc_ref_timestamp = decoder->timestamp;
2002         decoder->cycle_cnt = 0;
2003
2004         intel_pt_mtc_cyc_cnt_cbr(decoder);
2005 }
2006
2007 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
2008 {
2009         uint64_t timestamp = decoder->cyc_ref_timestamp;
2010
2011         decoder->have_cyc = true;
2012
2013         decoder->cycle_cnt += decoder->packet.payload;
2014         if (decoder->pge)
2015                 decoder->tot_cyc_cnt += decoder->packet.payload;
2016         decoder->sample_cyc = true;
2017
2018         if (!decoder->cyc_ref_timestamp)
2019                 return;
2020
2021         if (decoder->have_calc_cyc_to_tsc)
2022                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
2023         else if (decoder->cbr)
2024                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
2025         else
2026                 return;
2027
2028         if (timestamp < decoder->timestamp)
2029                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
2030                              timestamp, decoder->timestamp);
2031         else
2032                 decoder->timestamp = timestamp;
2033
2034         decoder->timestamp_insn_cnt = 0;
2035
2036         intel_pt_log_to("Setting timestamp", decoder->timestamp);
2037 }
2038
2039 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
2040 {
2041         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
2042                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
2043                 decoder->state.items.is_32_bit = false;
2044         }
2045         decoder->blk_type = decoder->packet.payload;
2046         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
2047         if (decoder->blk_type == INTEL_PT_GP_REGS)
2048                 decoder->state.items.is_32_bit = decoder->packet.count;
2049         if (decoder->blk_type_pos < 0) {
2050                 intel_pt_log("WARNING: Unknown block type %u\n",
2051                              decoder->blk_type);
2052         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
2053                 intel_pt_log("WARNING: Duplicate block type %u\n",
2054                              decoder->blk_type);
2055         }
2056 }
2057
2058 static void intel_pt_bip(struct intel_pt_decoder *decoder)
2059 {
2060         uint32_t id = decoder->packet.count;
2061         uint32_t bit = 1 << id;
2062         int pos = decoder->blk_type_pos;
2063
2064         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
2065                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
2066                              id, decoder->blk_type);
2067                 return;
2068         }
2069
2070         if (decoder->state.items.mask[pos] & bit) {
2071                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
2072                              id, decoder->blk_type);
2073         }
2074
2075         decoder->state.items.mask[pos] |= bit;
2076         decoder->state.items.val[pos][id] = decoder->packet.payload;
2077 }
2078
2079 /* Walk PSB+ packets when already in sync. */
2080 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
2081 {
2082         int err;
2083
2084         decoder->in_psb = true;
2085
2086         while (1) {
2087                 err = intel_pt_get_next_packet(decoder);
2088                 if (err)
2089                         goto out;
2090
2091                 switch (decoder->packet.type) {
2092                 case INTEL_PT_PSBEND:
2093                         err = 0;
2094                         goto out;
2095
2096                 case INTEL_PT_TIP_PGD:
2097                 case INTEL_PT_TIP_PGE:
2098                 case INTEL_PT_TIP:
2099                 case INTEL_PT_TNT:
2100                 case INTEL_PT_TRACESTOP:
2101                 case INTEL_PT_BAD:
2102                 case INTEL_PT_PSB:
2103                 case INTEL_PT_PTWRITE:
2104                 case INTEL_PT_PTWRITE_IP:
2105                 case INTEL_PT_EXSTOP:
2106                 case INTEL_PT_EXSTOP_IP:
2107                 case INTEL_PT_MWAIT:
2108                 case INTEL_PT_PWRE:
2109                 case INTEL_PT_PWRX:
2110                 case INTEL_PT_BBP:
2111                 case INTEL_PT_BIP:
2112                 case INTEL_PT_BEP:
2113                 case INTEL_PT_BEP_IP:
2114                 case INTEL_PT_CFE:
2115                 case INTEL_PT_CFE_IP:
2116                 case INTEL_PT_EVD:
2117                         decoder->have_tma = false;
2118                         intel_pt_log("ERROR: Unexpected packet\n");
2119                         err = -EAGAIN;
2120                         goto out;
2121
2122                 case INTEL_PT_OVF:
2123                         err = intel_pt_overflow(decoder);
2124                         goto out;
2125
2126                 case INTEL_PT_TSC:
2127                         intel_pt_calc_tsc_timestamp(decoder);
2128                         break;
2129
2130                 case INTEL_PT_TMA:
2131                         intel_pt_calc_tma(decoder);
2132                         break;
2133
2134                 case INTEL_PT_CBR:
2135                         intel_pt_calc_cbr(decoder);
2136                         break;
2137
2138                 case INTEL_PT_MODE_EXEC:
2139                         intel_pt_mode_exec_status(decoder);
2140                         break;
2141
2142                 case INTEL_PT_PIP:
2143                         intel_pt_set_pip(decoder);
2144                         break;
2145
2146                 case INTEL_PT_FUP:
2147                         decoder->pge = true;
2148                         if (decoder->packet.count) {
2149                                 intel_pt_set_last_ip(decoder);
2150                                 decoder->psb_ip = decoder->last_ip;
2151                         }
2152                         break;
2153
2154                 case INTEL_PT_MODE_TSX:
2155                         intel_pt_update_in_tx(decoder);
2156                         break;
2157
2158                 case INTEL_PT_MTC:
2159                         intel_pt_calc_mtc_timestamp(decoder);
2160                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2161                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2162                         break;
2163
2164                 case INTEL_PT_CYC:
2165                         intel_pt_calc_cyc_timestamp(decoder);
2166                         break;
2167
2168                 case INTEL_PT_VMCS:
2169                 case INTEL_PT_MNT:
2170                 case INTEL_PT_PAD:
2171                 default:
2172                         break;
2173                 }
2174         }
2175 out:
2176         decoder->in_psb = false;
2177
2178         return err;
2179 }
2180
2181 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
2182 {
2183         int err;
2184
2185         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
2186                 decoder->tx_flags = 0;
2187                 decoder->state.flags &= ~INTEL_PT_IN_TX;
2188                 decoder->state.flags |= INTEL_PT_ABORT_TX;
2189         } else {
2190                 decoder->state.flags |= INTEL_PT_ASYNC;
2191         }
2192
2193         while (1) {
2194                 err = intel_pt_get_next_packet(decoder);
2195                 if (err)
2196                         return err;
2197
2198                 switch (decoder->packet.type) {
2199                 case INTEL_PT_TNT:
2200                 case INTEL_PT_FUP:
2201                 case INTEL_PT_TRACESTOP:
2202                 case INTEL_PT_PSB:
2203                 case INTEL_PT_TSC:
2204                 case INTEL_PT_TMA:
2205                 case INTEL_PT_MODE_TSX:
2206                 case INTEL_PT_BAD:
2207                 case INTEL_PT_PSBEND:
2208                 case INTEL_PT_PTWRITE:
2209                 case INTEL_PT_PTWRITE_IP:
2210                 case INTEL_PT_EXSTOP:
2211                 case INTEL_PT_EXSTOP_IP:
2212                 case INTEL_PT_MWAIT:
2213                 case INTEL_PT_PWRE:
2214                 case INTEL_PT_PWRX:
2215                 case INTEL_PT_BBP:
2216                 case INTEL_PT_BIP:
2217                 case INTEL_PT_BEP:
2218                 case INTEL_PT_BEP_IP:
2219                 case INTEL_PT_CFE:
2220                 case INTEL_PT_CFE_IP:
2221                 case INTEL_PT_EVD:
2222                         intel_pt_log("ERROR: Missing TIP after FUP\n");
2223                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
2224                         decoder->pkt_step = 0;
2225                         return -ENOENT;
2226
2227                 case INTEL_PT_CBR:
2228                         intel_pt_calc_cbr(decoder);
2229                         break;
2230
2231                 case INTEL_PT_OVF:
2232                         return intel_pt_overflow(decoder);
2233
2234                 case INTEL_PT_TIP_PGD:
2235                         decoder->state.from_ip = decoder->ip;
2236                         if (decoder->packet.count == 0) {
2237                                 decoder->state.to_ip = 0;
2238                         } else {
2239                                 intel_pt_set_ip(decoder);
2240                                 decoder->state.to_ip = decoder->ip;
2241                         }
2242                         decoder->pge = false;
2243                         decoder->continuous_period = false;
2244                         decoder->state.type |= INTEL_PT_TRACE_END;
2245                         intel_pt_update_nr(decoder);
2246                         return 0;
2247
2248                 case INTEL_PT_TIP_PGE:
2249                         decoder->pge = true;
2250                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2251                                      decoder->ip);
2252                         decoder->state.from_ip = 0;
2253                         if (decoder->packet.count == 0) {
2254                                 decoder->state.to_ip = 0;
2255                         } else {
2256                                 intel_pt_set_ip(decoder);
2257                                 decoder->state.to_ip = decoder->ip;
2258                         }
2259                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2260                         intel_pt_mtc_cyc_cnt_pge(decoder);
2261                         intel_pt_set_nr(decoder);
2262                         return 0;
2263
2264                 case INTEL_PT_TIP:
2265                         decoder->state.from_ip = decoder->ip;
2266                         if (decoder->packet.count == 0) {
2267                                 decoder->state.to_ip = 0;
2268                         } else {
2269                                 intel_pt_set_ip(decoder);
2270                                 decoder->state.to_ip = decoder->ip;
2271                         }
2272                         intel_pt_update_nr(decoder);
2273                         intel_pt_sample_iflag_chg(decoder);
2274                         return 0;
2275
2276                 case INTEL_PT_PIP:
2277                         intel_pt_update_pip(decoder);
2278                         break;
2279
2280                 case INTEL_PT_MTC:
2281                         intel_pt_calc_mtc_timestamp(decoder);
2282                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2283                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2284                         break;
2285
2286                 case INTEL_PT_CYC:
2287                         intel_pt_calc_cyc_timestamp(decoder);
2288                         break;
2289
2290                 case INTEL_PT_MODE_EXEC:
2291                         intel_pt_mode_exec(decoder);
2292                         break;
2293
2294                 case INTEL_PT_VMCS:
2295                 case INTEL_PT_MNT:
2296                 case INTEL_PT_PAD:
2297                         break;
2298
2299                 default:
2300                         return intel_pt_bug(decoder);
2301                 }
2302         }
2303 }
2304
2305 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2306 {
2307         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2308         decoder->state.type = INTEL_PT_INSTRUCTION;
2309         decoder->state.from_ip = decoder->ip;
2310         decoder->state.to_ip = 0;
2311         return 0;
2312 }
2313
2314 struct intel_pt_vm_tsc_info {
2315         struct intel_pt_pkt pip_packet;
2316         struct intel_pt_pkt vmcs_packet;
2317         struct intel_pt_pkt tma_packet;
2318         bool tsc, pip, vmcs, tma, psbend;
2319         uint64_t ctc_delta;
2320         uint64_t last_ctc;
2321         int max_lookahead;
2322 };
2323
2324 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2325 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2326 {
2327         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2328
2329         switch (pkt_info->packet.type) {
2330         case INTEL_PT_PAD:
2331         case INTEL_PT_MNT:
2332         case INTEL_PT_MODE_EXEC:
2333         case INTEL_PT_MODE_TSX:
2334         case INTEL_PT_MTC:
2335         case INTEL_PT_FUP:
2336         case INTEL_PT_CYC:
2337         case INTEL_PT_CBR:
2338                 break;
2339
2340         case INTEL_PT_TSC:
2341                 data->tsc = true;
2342                 break;
2343
2344         case INTEL_PT_TMA:
2345                 data->tma_packet = pkt_info->packet;
2346                 data->tma = true;
2347                 break;
2348
2349         case INTEL_PT_PIP:
2350                 data->pip_packet = pkt_info->packet;
2351                 data->pip = true;
2352                 break;
2353
2354         case INTEL_PT_VMCS:
2355                 data->vmcs_packet = pkt_info->packet;
2356                 data->vmcs = true;
2357                 break;
2358
2359         case INTEL_PT_PSBEND:
2360                 data->psbend = true;
2361                 return 1;
2362
2363         case INTEL_PT_TIP_PGE:
2364         case INTEL_PT_PTWRITE:
2365         case INTEL_PT_PTWRITE_IP:
2366         case INTEL_PT_EXSTOP:
2367         case INTEL_PT_EXSTOP_IP:
2368         case INTEL_PT_MWAIT:
2369         case INTEL_PT_PWRE:
2370         case INTEL_PT_PWRX:
2371         case INTEL_PT_BBP:
2372         case INTEL_PT_BIP:
2373         case INTEL_PT_BEP:
2374         case INTEL_PT_BEP_IP:
2375         case INTEL_PT_OVF:
2376         case INTEL_PT_BAD:
2377         case INTEL_PT_TNT:
2378         case INTEL_PT_TIP_PGD:
2379         case INTEL_PT_TIP:
2380         case INTEL_PT_PSB:
2381         case INTEL_PT_TRACESTOP:
2382         case INTEL_PT_CFE:
2383         case INTEL_PT_CFE_IP:
2384         case INTEL_PT_EVD:
2385         default:
2386                 return 1;
2387         }
2388
2389         return 0;
2390 }
2391
2392 struct intel_pt_ovf_fup_info {
2393         int max_lookahead;
2394         bool found;
2395 };
2396
2397 /* Lookahead to detect a FUP packet after OVF */
2398 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2399 {
2400         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2401
2402         if (pkt_info->packet.type == INTEL_PT_CYC ||
2403             pkt_info->packet.type == INTEL_PT_MTC ||
2404             pkt_info->packet.type == INTEL_PT_TSC)
2405                 return !--(data->max_lookahead);
2406         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2407         return 1;
2408 }
2409
2410 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2411 {
2412         struct intel_pt_ovf_fup_info data = {
2413                 .max_lookahead = 16,
2414                 .found = false,
2415         };
2416
2417         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2418         return data.found;
2419 }
2420
2421 /* Lookahead and get the TMA packet after TSC */
2422 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2423 {
2424         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2425
2426         if (pkt_info->packet.type == INTEL_PT_CYC ||
2427             pkt_info->packet.type == INTEL_PT_MTC)
2428                 return !--(data->max_lookahead);
2429
2430         if (pkt_info->packet.type == INTEL_PT_TMA) {
2431                 data->tma_packet = pkt_info->packet;
2432                 data->tma = true;
2433         }
2434         return 1;
2435 }
2436
2437 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2438 {
2439         if (decoder->tsc_ctc_mult)
2440                 return ctc * decoder->tsc_ctc_mult;
2441         else
2442                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2443 }
2444
2445 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2446                                            uint32_t ctc,
2447                                            uint32_t fc,
2448                                            uint64_t last_ctc_timestamp,
2449                                            uint64_t ctc_delta,
2450                                            uint32_t last_ctc)
2451 {
2452         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2453         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2454         /*
2455          * Number of CTC ticks from there until current TMA packet. We would
2456          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2457          * past an MTC, so a sign-extended value is used.
2458          */
2459         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2460         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2461         uint64_t new_ctc_delta = ctc_delta + delta;
2462         uint64_t expected_tsc;
2463
2464         /*
2465          * Convert CTC ticks to TSC ticks, add the starting point
2466          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2467          */
2468         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2469
2470         if (intel_pt_enable_logging) {
2471                 intel_pt_log_x64(last_mtc_ctc);
2472                 intel_pt_log_x32(last_ctc);
2473                 intel_pt_log_x64(ctc_delta);
2474                 intel_pt_log_x64(delta);
2475                 intel_pt_log_x32(ctc);
2476                 intel_pt_log_x64(new_ctc_delta);
2477                 intel_pt_log_x64(last_ctc_timestamp);
2478                 intel_pt_log_x32(fc);
2479                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2480                 intel_pt_log_x64(expected_tsc);
2481         }
2482
2483         return expected_tsc;
2484 }
2485
2486 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2487                                       struct intel_pt_vm_tsc_info *data)
2488 {
2489         uint32_t ctc = data->tma_packet.payload;
2490         uint32_t fc = data->tma_packet.count;
2491
2492         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2493                                           decoder->ctc_timestamp,
2494                                           data->ctc_delta, data->last_ctc);
2495 }
2496
2497 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2498                                       struct intel_pt_vmcs_info *vmcs_info)
2499 {
2500         uint64_t payload = decoder->packet.payload;
2501
2502         /* VMX adds the TSC Offset, so subtract to get host TSC */
2503         decoder->packet.payload -= vmcs_info->tsc_offset;
2504         /* TSC packet has only 7 bytes */
2505         decoder->packet.payload &= SEVEN_BYTES;
2506
2507         /*
2508          * The buffer is mmapped from the data file, so this also updates the
2509          * data file.
2510          */
2511         if (!decoder->vm_tm_corr_dry_run)
2512                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2513
2514         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2515                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2516                      payload, decoder->packet.payload, vmcs_info->vmcs,
2517                      vmcs_info->tsc_offset);
2518 }
2519
2520 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2521                                              uint64_t tsc_offset)
2522 {
2523         struct intel_pt_vmcs_info vmcs_info = {
2524                 .vmcs = NO_VMCS,
2525                 .tsc_offset = tsc_offset
2526         };
2527
2528         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2529 }
2530
2531 static inline bool in_vm(uint64_t pip_payload)
2532 {
2533         return pip_payload & 1;
2534 }
2535
2536 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2537 {
2538         return pip_packet->payload & 1;
2539 }
2540
2541 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2542 {
2543         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2544               vmcs_info->vmcs, vmcs_info->tsc_offset);
2545 }
2546
2547 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2548                                     struct intel_pt_vm_tsc_info *data)
2549 {
2550         memset(data, 0, sizeof(*data));
2551         data->ctc_delta = decoder->ctc_delta;
2552         data->last_ctc = decoder->last_ctc;
2553         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2554         if (data->tsc && !data->psbend)
2555                 p_log("ERROR: PSB without PSBEND");
2556         decoder->in_psb = data->psbend;
2557 }
2558
2559 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2560                                           struct intel_pt_vm_tsc_info *data,
2561                                           struct intel_pt_vmcs_info *vmcs_info,
2562                                           uint64_t host_tsc)
2563 {
2564         if (!decoder->in_psb) {
2565                 /* Can't happen */
2566                 p_log("ERROR: First TSC is not in PSB+");
2567         }
2568
2569         if (data->pip) {
2570                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2571                         if (vmcs_info && vmcs_info->tsc_offset) {
2572                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2573                                 decoder->vm_tm_corr_reliable = true;
2574                         } else {
2575                                 p_log("ERROR: First TSC, unknown TSC Offset");
2576                         }
2577                 } else { /* Host */
2578                         decoder->vm_tm_corr_reliable = true;
2579                 }
2580         } else { /* Host or Guest */
2581                 decoder->vm_tm_corr_reliable = false;
2582                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2583                         /* Assume Host */
2584                 } else {
2585                         /* Assume Guest */
2586                         if (vmcs_info && vmcs_info->tsc_offset)
2587                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2588                         else
2589                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2590                 }
2591         }
2592 }
2593
2594 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2595                                     struct intel_pt_vm_tsc_info *data)
2596 {
2597         struct intel_pt_vmcs_info *vmcs_info;
2598         uint64_t tsc_offset = 0;
2599         uint64_t vmcs;
2600         bool reliable = true;
2601         uint64_t expected_tsc;
2602         uint64_t host_tsc;
2603         uint64_t ref_timestamp;
2604
2605         bool assign = false;
2606         bool assign_reliable = false;
2607
2608         /* Already have 'data' for the in_psb case */
2609         if (!decoder->in_psb) {
2610                 memset(data, 0, sizeof(*data));
2611                 data->ctc_delta = decoder->ctc_delta;
2612                 data->last_ctc = decoder->last_ctc;
2613                 data->max_lookahead = 16;
2614                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2615                 if (decoder->pge) {
2616                         data->pip = true;
2617                         data->pip_packet.payload = decoder->pip_payload;
2618                 }
2619         }
2620
2621         /* Calculations depend on having TMA packets */
2622         if (!data->tma) {
2623                 p_log("ERROR: TSC without TMA");
2624                 return;
2625         }
2626
2627         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2628         if (vmcs == NO_VMCS)
2629                 vmcs = 0;
2630
2631         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2632
2633         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2634         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2635
2636         if (!decoder->ctc_timestamp) {
2637                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2638                 return;
2639         }
2640
2641         expected_tsc = intel_pt_expected_tsc(decoder, data);
2642
2643         tsc_offset = host_tsc - expected_tsc;
2644
2645         /* Determine if TSC is from Host or Guest */
2646         if (data->pip) {
2647                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2648                         if (!vmcs_info) {
2649                                 /* PIP NR=1 without VMCS cannot happen */
2650                                 p_log("ERROR: Missing VMCS");
2651                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2652                                 decoder->vm_tm_corr_reliable = false;
2653                                 return;
2654                         }
2655                 } else { /* Host */
2656                         decoder->last_reliable_timestamp = host_tsc;
2657                         decoder->vm_tm_corr_reliable = true;
2658                         return;
2659                 }
2660         } else { /* Host or Guest */
2661                 reliable = false; /* Host/Guest is a guess, so not reliable */
2662                 if (decoder->in_psb) {
2663                         if (!tsc_offset)
2664                                 return; /* Zero TSC Offset, assume Host */
2665                         /*
2666                          * TSC packet has only 7 bytes of TSC. We have no
2667                          * information about the Guest's 8th byte, but it
2668                          * doesn't matter because we only need 7 bytes.
2669                          * Here, since the 8th byte is unreliable and
2670                          * irrelevant, compare only 7 byes.
2671                          */
2672                         if (vmcs_info &&
2673                             (tsc_offset & SEVEN_BYTES) ==
2674                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2675                                 /* Same TSC Offset as last VMCS, assume Guest */
2676                                 goto guest;
2677                         }
2678                 }
2679                 /*
2680                  * Check if the host_tsc is within the expected range.
2681                  * Note, we could narrow the range more by looking ahead for
2682                  * the next host TSC in the same buffer, but we don't bother to
2683                  * do that because this is probably good enough.
2684                  */
2685                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2686                         /* Within expected range for Host TSC, assume Host */
2687                         decoder->vm_tm_corr_reliable = false;
2688                         return;
2689                 }
2690         }
2691
2692 guest: /* Assuming Guest */
2693
2694         /* Determine whether to assign TSC Offset */
2695         if (vmcs_info && vmcs_info->vmcs) {
2696                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2697                         assign = false;
2698                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2699                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2700                         /* Continuous tracing, TSC in a PSB is not a time loss */
2701                         assign = true;
2702                         assign_reliable = true;
2703                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2704                         /*
2705                          * Unlikely to be a time loss TSC in a PSB which is not
2706                          * at the start of a buffer.
2707                          */
2708                         assign = true;
2709                         assign_reliable = false;
2710                 }
2711         }
2712
2713         /* Record VMCS TSC Offset */
2714         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2715                        vmcs_info->reliable != assign_reliable)) {
2716                 bool print = vmcs_info->tsc_offset != tsc_offset;
2717
2718                 vmcs_info->tsc_offset = tsc_offset;
2719                 vmcs_info->reliable = assign_reliable;
2720                 if (print)
2721                         intel_pt_print_vmcs_info(vmcs_info);
2722         }
2723
2724         /* Determine what TSC Offset to use */
2725         if (vmcs_info && vmcs_info->tsc_offset) {
2726                 if (!vmcs_info->reliable)
2727                         reliable = false;
2728                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2729         } else {
2730                 reliable = false;
2731                 if (vmcs_info) {
2732                         if (!vmcs_info->error_printed) {
2733                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2734                                       vmcs_info->vmcs);
2735                                 vmcs_info->error_printed = true;
2736                         }
2737                 } else {
2738                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2739                                 p_log("ERROR: Unknown VMCS");
2740                 }
2741                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2742         }
2743
2744         decoder->vm_tm_corr_reliable = reliable;
2745 }
2746
2747 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2748 {
2749         uint64_t host_tsc = decoder->packet.payload;
2750         uint64_t guest_tsc = decoder->packet.payload;
2751         struct intel_pt_vmcs_info *vmcs_info;
2752         uint64_t vmcs;
2753
2754         vmcs = decoder->vmcs;
2755         if (vmcs == NO_VMCS)
2756                 vmcs = 0;
2757
2758         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2759
2760         if (decoder->pge) {
2761                 if (in_vm(decoder->pip_payload)) { /* Guest */
2762                         if (!vmcs_info) {
2763                                 /* PIP NR=1 without VMCS cannot happen */
2764                                 p_log("ERROR: Missing VMCS");
2765                         }
2766                 } else { /* Host */
2767                         return;
2768                 }
2769         } else { /* Host or Guest */
2770                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2771                         /* Within expected range for Host TSC, assume Host */
2772                         return;
2773                 }
2774         }
2775
2776         if (vmcs_info) {
2777                 /* Translate Guest TSC to Host TSC */
2778                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2779                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2780                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2781                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2782                              guest_tsc, host_tsc, vmcs_info->vmcs,
2783                              vmcs_info->tsc_offset);
2784                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2785                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2786                         p_log("Timestamp out of range");
2787         } else {
2788                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2789                         p_log("ERROR: Unknown VMCS");
2790                 host_tsc = decoder->timestamp;
2791         }
2792
2793         decoder->packet.payload = host_tsc;
2794
2795         if (!decoder->vm_tm_corr_dry_run)
2796                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2797 }
2798
2799 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2800 {
2801         struct intel_pt_vm_tsc_info data = { .psbend = false };
2802         bool pge;
2803         int err;
2804
2805         if (decoder->in_psb)
2806                 intel_pt_vm_tm_corr_psb(decoder, &data);
2807
2808         while (1) {
2809                 err = intel_pt_get_next_packet(decoder);
2810                 if (err == -ENOLINK)
2811                         continue;
2812                 if (err)
2813                         break;
2814
2815                 switch (decoder->packet.type) {
2816                 case INTEL_PT_TIP_PGD:
2817                         decoder->pge = false;
2818                         decoder->vm_tm_corr_continuous = false;
2819                         break;
2820
2821                 case INTEL_PT_TNT:
2822                 case INTEL_PT_TIP:
2823                 case INTEL_PT_TIP_PGE:
2824                         decoder->pge = true;
2825                         break;
2826
2827                 case INTEL_PT_OVF:
2828                         decoder->in_psb = false;
2829                         pge = decoder->pge;
2830                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2831                         if (pge != decoder->pge)
2832                                 intel_pt_log("Surprising PGE change in OVF!");
2833                         if (!decoder->pge)
2834                                 decoder->vm_tm_corr_continuous = false;
2835                         break;
2836
2837                 case INTEL_PT_FUP:
2838                         if (decoder->in_psb)
2839                                 decoder->pge = true;
2840                         break;
2841
2842                 case INTEL_PT_TRACESTOP:
2843                         decoder->pge = false;
2844                         decoder->vm_tm_corr_continuous = false;
2845                         decoder->have_tma = false;
2846                         break;
2847
2848                 case INTEL_PT_PSB:
2849                         intel_pt_vm_tm_corr_psb(decoder, &data);
2850                         break;
2851
2852                 case INTEL_PT_PIP:
2853                         decoder->pip_payload = decoder->packet.payload;
2854                         break;
2855
2856                 case INTEL_PT_MTC:
2857                         intel_pt_calc_mtc_timestamp(decoder);
2858                         break;
2859
2860                 case INTEL_PT_TSC:
2861                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2862                         intel_pt_calc_tsc_timestamp(decoder);
2863                         decoder->vm_tm_corr_same_buf = true;
2864                         decoder->vm_tm_corr_continuous = decoder->pge;
2865                         break;
2866
2867                 case INTEL_PT_TMA:
2868                         intel_pt_calc_tma(decoder);
2869                         break;
2870
2871                 case INTEL_PT_CYC:
2872                         intel_pt_calc_cyc_timestamp(decoder);
2873                         break;
2874
2875                 case INTEL_PT_CBR:
2876                         intel_pt_calc_cbr(decoder);
2877                         break;
2878
2879                 case INTEL_PT_PSBEND:
2880                         decoder->in_psb = false;
2881                         data.psbend = false;
2882                         break;
2883
2884                 case INTEL_PT_VMCS:
2885                         if (decoder->packet.payload != NO_VMCS)
2886                                 decoder->vmcs = decoder->packet.payload;
2887                         break;
2888
2889                 case INTEL_PT_BBP:
2890                         decoder->blk_type = decoder->packet.payload;
2891                         break;
2892
2893                 case INTEL_PT_BIP:
2894                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2895                             decoder->packet.count == 2)
2896                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2897                         break;
2898
2899                 case INTEL_PT_BEP:
2900                 case INTEL_PT_BEP_IP:
2901                         decoder->blk_type = 0;
2902                         break;
2903
2904                 case INTEL_PT_CFE:
2905                 case INTEL_PT_CFE_IP:
2906                 case INTEL_PT_EVD:
2907                 case INTEL_PT_MODE_EXEC:
2908                 case INTEL_PT_MODE_TSX:
2909                 case INTEL_PT_MNT:
2910                 case INTEL_PT_PAD:
2911                 case INTEL_PT_PTWRITE_IP:
2912                 case INTEL_PT_PTWRITE:
2913                 case INTEL_PT_MWAIT:
2914                 case INTEL_PT_PWRE:
2915                 case INTEL_PT_EXSTOP_IP:
2916                 case INTEL_PT_EXSTOP:
2917                 case INTEL_PT_PWRX:
2918                 case INTEL_PT_BAD: /* Does not happen */
2919                 default:
2920                         break;
2921                 }
2922         }
2923
2924         return err;
2925 }
2926
2927 #define HOP_PROCESS     0
2928 #define HOP_IGNORE      1
2929 #define HOP_RETURN      2
2930 #define HOP_AGAIN       3
2931
2932 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2933
2934 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2935 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2936 {
2937         *err = 0;
2938
2939         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2940         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2941                 *err = intel_pt_scan_for_psb(decoder);
2942                 if (*err)
2943                         return HOP_RETURN;
2944         }
2945
2946         switch (decoder->packet.type) {
2947         case INTEL_PT_TNT:
2948                 return HOP_IGNORE;
2949
2950         case INTEL_PT_TIP_PGD:
2951                 decoder->pge = false;
2952                 if (!decoder->packet.count) {
2953                         intel_pt_set_nr(decoder);
2954                         return HOP_IGNORE;
2955                 }
2956                 intel_pt_set_ip(decoder);
2957                 decoder->state.type |= INTEL_PT_TRACE_END;
2958                 decoder->state.from_ip = 0;
2959                 decoder->state.to_ip = decoder->ip;
2960                 intel_pt_update_nr(decoder);
2961                 return HOP_RETURN;
2962
2963         case INTEL_PT_TIP:
2964                 if (!decoder->packet.count) {
2965                         intel_pt_set_nr(decoder);
2966                         return HOP_IGNORE;
2967                 }
2968                 intel_pt_set_ip(decoder);
2969                 decoder->state.type = INTEL_PT_INSTRUCTION;
2970                 decoder->state.from_ip = decoder->ip;
2971                 decoder->state.to_ip = 0;
2972                 intel_pt_update_nr(decoder);
2973                 intel_pt_sample_iflag_chg(decoder);
2974                 return HOP_RETURN;
2975
2976         case INTEL_PT_FUP:
2977                 if (!decoder->packet.count)
2978                         return HOP_IGNORE;
2979                 intel_pt_set_ip(decoder);
2980                 if (decoder->set_fup_mwait || decoder->set_fup_pwre)
2981                         *no_tip = true;
2982                 if (!decoder->branch_enable || !decoder->pge)
2983                         *no_tip = true;
2984                 if (*no_tip) {
2985                         decoder->state.type = INTEL_PT_INSTRUCTION;
2986                         decoder->state.from_ip = decoder->ip;
2987                         decoder->state.to_ip = 0;
2988                         intel_pt_fup_event(decoder, *no_tip);
2989                         return HOP_RETURN;
2990                 }
2991                 intel_pt_fup_event(decoder, *no_tip);
2992                 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;
2993                 *err = intel_pt_walk_fup_tip(decoder);
2994                 if (!*err && decoder->state.to_ip)
2995                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2996                 return HOP_RETURN;
2997
2998         case INTEL_PT_PSB:
2999                 decoder->state.psb_offset = decoder->pos;
3000                 decoder->psb_ip = 0;
3001                 decoder->last_ip = 0;
3002                 decoder->have_last_ip = true;
3003                 *err = intel_pt_walk_psbend(decoder);
3004                 if (*err == -EAGAIN)
3005                         return HOP_AGAIN;
3006                 if (*err)
3007                         return HOP_RETURN;
3008                 decoder->state.type = INTEL_PT_PSB_EVT;
3009                 if (decoder->psb_ip) {
3010                         decoder->state.type |= INTEL_PT_INSTRUCTION;
3011                         decoder->ip = decoder->psb_ip;
3012                 }
3013                 decoder->state.from_ip = decoder->psb_ip;
3014                 decoder->state.to_ip = 0;
3015                 return HOP_RETURN;
3016
3017         case INTEL_PT_BAD:
3018         case INTEL_PT_PAD:
3019         case INTEL_PT_TIP_PGE:
3020         case INTEL_PT_TSC:
3021         case INTEL_PT_TMA:
3022         case INTEL_PT_MODE_EXEC:
3023         case INTEL_PT_MODE_TSX:
3024         case INTEL_PT_MTC:
3025         case INTEL_PT_CYC:
3026         case INTEL_PT_VMCS:
3027         case INTEL_PT_PSBEND:
3028         case INTEL_PT_CBR:
3029         case INTEL_PT_TRACESTOP:
3030         case INTEL_PT_PIP:
3031         case INTEL_PT_OVF:
3032         case INTEL_PT_MNT:
3033         case INTEL_PT_PTWRITE:
3034         case INTEL_PT_PTWRITE_IP:
3035         case INTEL_PT_EXSTOP:
3036         case INTEL_PT_EXSTOP_IP:
3037         case INTEL_PT_MWAIT:
3038         case INTEL_PT_PWRE:
3039         case INTEL_PT_PWRX:
3040         case INTEL_PT_BBP:
3041         case INTEL_PT_BIP:
3042         case INTEL_PT_BEP:
3043         case INTEL_PT_BEP_IP:
3044         case INTEL_PT_CFE:
3045         case INTEL_PT_CFE_IP:
3046         case INTEL_PT_EVD:
3047         default:
3048                 return HOP_PROCESS;
3049         }
3050 }
3051
3052 struct intel_pt_psb_info {
3053         struct intel_pt_pkt fup_packet;
3054         bool fup;
3055         int after_psbend;
3056 };
3057
3058 /* Lookahead and get the FUP packet from PSB+ */
3059 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
3060 {
3061         struct intel_pt_psb_info *data = pkt_info->data;
3062
3063         switch (pkt_info->packet.type) {
3064         case INTEL_PT_PAD:
3065         case INTEL_PT_MNT:
3066         case INTEL_PT_TSC:
3067         case INTEL_PT_TMA:
3068         case INTEL_PT_MODE_EXEC:
3069         case INTEL_PT_MODE_TSX:
3070         case INTEL_PT_MTC:
3071         case INTEL_PT_CYC:
3072         case INTEL_PT_VMCS:
3073         case INTEL_PT_CBR:
3074         case INTEL_PT_PIP:
3075                 if (data->after_psbend) {
3076                         data->after_psbend -= 1;
3077                         if (!data->after_psbend)
3078                                 return 1;
3079                 }
3080                 break;
3081
3082         case INTEL_PT_FUP:
3083                 if (data->after_psbend)
3084                         return 1;
3085                 if (data->fup || pkt_info->packet.count == 0)
3086                         return 1;
3087                 data->fup_packet = pkt_info->packet;
3088                 data->fup = true;
3089                 break;
3090
3091         case INTEL_PT_PSBEND:
3092                 if (!data->fup)
3093                         return 1;
3094                 /* Keep going to check for a TIP.PGE */
3095                 data->after_psbend = 6;
3096                 break;
3097
3098         case INTEL_PT_TIP_PGE:
3099                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
3100                 if (data->after_psbend)
3101                         data->fup = false;
3102                 return 1;
3103
3104         case INTEL_PT_PTWRITE:
3105         case INTEL_PT_PTWRITE_IP:
3106         case INTEL_PT_EXSTOP:
3107         case INTEL_PT_EXSTOP_IP:
3108         case INTEL_PT_MWAIT:
3109         case INTEL_PT_PWRE:
3110         case INTEL_PT_PWRX:
3111         case INTEL_PT_BBP:
3112         case INTEL_PT_BIP:
3113         case INTEL_PT_BEP:
3114         case INTEL_PT_BEP_IP:
3115         case INTEL_PT_CFE:
3116         case INTEL_PT_CFE_IP:
3117         case INTEL_PT_EVD:
3118                 if (data->after_psbend) {
3119                         data->after_psbend -= 1;
3120                         if (!data->after_psbend)
3121                                 return 1;
3122                         break;
3123                 }
3124                 return 1;
3125
3126         case INTEL_PT_OVF:
3127         case INTEL_PT_BAD:
3128         case INTEL_PT_TNT:
3129         case INTEL_PT_TIP_PGD:
3130         case INTEL_PT_TIP:
3131         case INTEL_PT_PSB:
3132         case INTEL_PT_TRACESTOP:
3133         default:
3134                 return 1;
3135         }
3136
3137         return 0;
3138 }
3139
3140 static int intel_pt_psb(struct intel_pt_decoder *decoder)
3141 {
3142         int err;
3143
3144         decoder->last_ip = 0;
3145         decoder->psb_ip = 0;
3146         decoder->have_last_ip = true;
3147         intel_pt_clear_stack(&decoder->stack);
3148         err = intel_pt_walk_psbend(decoder);
3149         if (err)
3150                 return err;
3151         decoder->state.type = INTEL_PT_PSB_EVT;
3152         decoder->state.from_ip = decoder->psb_ip;
3153         decoder->state.to_ip = 0;
3154         return 0;
3155 }
3156
3157 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
3158 {
3159         int err;
3160
3161         if (decoder->ip != decoder->last_ip) {
3162                 err = intel_pt_walk_fup(decoder);
3163                 if (!err || err != -EAGAIN)
3164                         return err;
3165         }
3166
3167         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3168         err = intel_pt_psb(decoder);
3169         if (err) {
3170                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3171                 return -ENOENT;
3172         }
3173
3174         return 0;
3175 }
3176
3177 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
3178 {
3179         struct intel_pt_psb_info data = { .fup = false };
3180
3181         if (!decoder->branch_enable)
3182                 return false;
3183
3184         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
3185         if (!data.fup)
3186                 return false;
3187
3188         decoder->packet = data.fup_packet;
3189         intel_pt_set_last_ip(decoder);
3190         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
3191
3192         *err = intel_pt_fup_in_psb(decoder);
3193
3194         return true;
3195 }
3196
3197 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
3198 {
3199         int last_packet_type = INTEL_PT_PAD;
3200         bool no_tip = false;
3201         int err;
3202
3203         while (1) {
3204                 err = intel_pt_get_next_packet(decoder);
3205                 if (err)
3206                         return err;
3207 next:
3208                 err = 0;
3209                 if (decoder->cyc_threshold) {
3210                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
3211                                 decoder->sample_cyc = false;
3212                         last_packet_type = decoder->packet.type;
3213                 }
3214
3215                 if (decoder->hop) {
3216                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
3217                         case HOP_IGNORE:
3218                                 continue;
3219                         case HOP_RETURN:
3220                                 return err;
3221                         case HOP_AGAIN:
3222                                 goto next;
3223                         default:
3224                                 break;
3225                         }
3226                 }
3227
3228                 switch (decoder->packet.type) {
3229                 case INTEL_PT_TNT:
3230                         if (!decoder->packet.count)
3231                                 break;
3232                         decoder->tnt = decoder->packet;
3233                         decoder->pkt_state = INTEL_PT_STATE_TNT;
3234                         err = intel_pt_walk_tnt(decoder);
3235                         if (err == -EAGAIN)
3236                                 break;
3237                         return err;
3238
3239                 case INTEL_PT_TIP_PGD:
3240                         if (decoder->packet.count != 0)
3241                                 intel_pt_set_last_ip(decoder);
3242                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
3243                         return intel_pt_walk_tip(decoder);
3244
3245                 case INTEL_PT_TIP_PGE: {
3246                         decoder->pge = true;
3247                         decoder->overflow = false;
3248                         intel_pt_mtc_cyc_cnt_pge(decoder);
3249                         intel_pt_set_nr(decoder);
3250                         if (decoder->packet.count == 0) {
3251                                 intel_pt_log_at("Skipping zero TIP.PGE",
3252                                                 decoder->pos);
3253                                 break;
3254                         }
3255                         intel_pt_sample_iflag_chg(decoder);
3256                         intel_pt_set_ip(decoder);
3257                         decoder->state.from_ip = 0;
3258                         decoder->state.to_ip = decoder->ip;
3259                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3260                         /*
3261                          * In hop mode, resample to get the to_ip as an
3262                          * "instruction" sample.
3263                          */
3264                         if (decoder->hop)
3265                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3266                         return 0;
3267                 }
3268
3269                 case INTEL_PT_OVF:
3270                         return intel_pt_overflow(decoder);
3271
3272                 case INTEL_PT_TIP:
3273                         if (decoder->packet.count != 0)
3274                                 intel_pt_set_last_ip(decoder);
3275                         decoder->pkt_state = INTEL_PT_STATE_TIP;
3276                         return intel_pt_walk_tip(decoder);
3277
3278                 case INTEL_PT_FUP:
3279                         if (decoder->packet.count == 0) {
3280                                 intel_pt_log_at("Skipping zero FUP",
3281                                                 decoder->pos);
3282                                 no_tip = false;
3283                                 break;
3284                         }
3285                         intel_pt_set_last_ip(decoder);
3286                         if (!decoder->branch_enable || !decoder->pge) {
3287                                 decoder->ip = decoder->last_ip;
3288                                 if (intel_pt_fup_event(decoder, no_tip))
3289                                         return 0;
3290                                 no_tip = false;
3291                                 break;
3292                         }
3293                         if (decoder->set_fup_mwait)
3294                                 no_tip = true;
3295                         if (no_tip)
3296                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3297                         else
3298                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3299                         err = intel_pt_walk_fup(decoder);
3300                         if (err != -EAGAIN)
3301                                 return err;
3302                         if (no_tip) {
3303                                 no_tip = false;
3304                                 break;
3305                         }
3306                         return intel_pt_walk_fup_tip(decoder);
3307
3308                 case INTEL_PT_TRACESTOP:
3309                         decoder->pge = false;
3310                         decoder->continuous_period = false;
3311                         intel_pt_clear_tx_flags(decoder);
3312                         decoder->have_tma = false;
3313                         break;
3314
3315                 case INTEL_PT_PSB:
3316                         decoder->state.psb_offset = decoder->pos;
3317                         decoder->psb_ip = 0;
3318                         if (intel_pt_psb_with_fup(decoder, &err))
3319                                 return err;
3320                         err = intel_pt_psb(decoder);
3321                         if (err == -EAGAIN)
3322                                 goto next;
3323                         return err;
3324
3325                 case INTEL_PT_PIP:
3326                         intel_pt_update_pip(decoder);
3327                         break;
3328
3329                 case INTEL_PT_MTC:
3330                         intel_pt_calc_mtc_timestamp(decoder);
3331                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3332                                 break;
3333                         /*
3334                          * Ensure that there has been an instruction since the
3335                          * last MTC.
3336                          */
3337                         if (!decoder->mtc_insn)
3338                                 break;
3339                         decoder->mtc_insn = false;
3340                         /* Ensure that there is a timestamp */
3341                         if (!decoder->timestamp)
3342                                 break;
3343                         decoder->state.type = INTEL_PT_INSTRUCTION;
3344                         decoder->state.from_ip = decoder->ip;
3345                         decoder->state.to_ip = 0;
3346                         decoder->mtc_insn = false;
3347                         return 0;
3348
3349                 case INTEL_PT_TSC:
3350                         intel_pt_calc_tsc_timestamp(decoder);
3351                         break;
3352
3353                 case INTEL_PT_TMA:
3354                         intel_pt_calc_tma(decoder);
3355                         break;
3356
3357                 case INTEL_PT_CYC:
3358                         intel_pt_calc_cyc_timestamp(decoder);
3359                         break;
3360
3361                 case INTEL_PT_CBR:
3362                         intel_pt_calc_cbr(decoder);
3363                         if (decoder->cbr != decoder->cbr_seen) {
3364                                 decoder->state.type = 0;
3365                                 return 0;
3366                         }
3367                         break;
3368
3369                 case INTEL_PT_MODE_EXEC:
3370                         intel_pt_mode_exec(decoder);
3371                         err = intel_pt_get_next_packet(decoder);
3372                         if (err)
3373                                 return err;
3374                         if (decoder->packet.type == INTEL_PT_FUP) {
3375                                 decoder->set_fup_mode_exec = true;
3376                                 no_tip = true;
3377                         }
3378                         goto next;
3379
3380                 case INTEL_PT_MODE_TSX:
3381                         /* MODE_TSX need not be followed by FUP */
3382                         if (!decoder->pge || decoder->in_psb) {
3383                                 intel_pt_update_in_tx(decoder);
3384                                 break;
3385                         }
3386                         err = intel_pt_mode_tsx(decoder, &no_tip);
3387                         if (err)
3388                                 return err;
3389                         goto next;
3390
3391                 case INTEL_PT_BAD: /* Does not happen */
3392                         return intel_pt_bug(decoder);
3393
3394                 case INTEL_PT_PSBEND:
3395                 case INTEL_PT_VMCS:
3396                 case INTEL_PT_MNT:
3397                 case INTEL_PT_PAD:
3398                         break;
3399
3400                 case INTEL_PT_PTWRITE_IP:
3401                         decoder->fup_ptw_payload = decoder->packet.payload;
3402                         err = intel_pt_get_next_packet(decoder);
3403                         if (err)
3404                                 return err;
3405                         if (decoder->packet.type == INTEL_PT_FUP) {
3406                                 decoder->set_fup_ptw = true;
3407                                 no_tip = true;
3408                         } else {
3409                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3410                                                 decoder->pos);
3411                         }
3412                         goto next;
3413
3414                 case INTEL_PT_PTWRITE:
3415                         decoder->state.type = INTEL_PT_PTW;
3416                         decoder->state.from_ip = decoder->ip;
3417                         decoder->state.to_ip = 0;
3418                         decoder->state.ptw_payload = decoder->packet.payload;
3419                         return 0;
3420
3421                 case INTEL_PT_MWAIT:
3422                         decoder->fup_mwait_payload = decoder->packet.payload;
3423                         decoder->set_fup_mwait = true;
3424                         break;
3425
3426                 case INTEL_PT_PWRE:
3427                         if (decoder->set_fup_mwait) {
3428                                 decoder->fup_pwre_payload =
3429                                                         decoder->packet.payload;
3430                                 decoder->set_fup_pwre = true;
3431                                 break;
3432                         }
3433                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3434                         decoder->state.from_ip = decoder->ip;
3435                         decoder->state.to_ip = 0;
3436                         decoder->state.pwrx_payload = decoder->packet.payload;
3437                         return 0;
3438
3439                 case INTEL_PT_EXSTOP_IP:
3440                         err = intel_pt_get_next_packet(decoder);
3441                         if (err)
3442                                 return err;
3443                         if (decoder->packet.type == INTEL_PT_FUP) {
3444                                 decoder->set_fup_exstop = true;
3445                                 no_tip = true;
3446                         } else {
3447                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3448                                                 decoder->pos);
3449                         }
3450                         goto next;
3451
3452                 case INTEL_PT_EXSTOP:
3453                         decoder->state.type = INTEL_PT_EX_STOP;
3454                         decoder->state.from_ip = decoder->ip;
3455                         decoder->state.to_ip = 0;
3456                         return 0;
3457
3458                 case INTEL_PT_PWRX:
3459                         decoder->state.type = INTEL_PT_PWR_EXIT;
3460                         decoder->state.from_ip = decoder->ip;
3461                         decoder->state.to_ip = 0;
3462                         decoder->state.pwrx_payload = decoder->packet.payload;
3463                         return 0;
3464
3465                 case INTEL_PT_BBP:
3466                         intel_pt_bbp(decoder);
3467                         break;
3468
3469                 case INTEL_PT_BIP:
3470                         intel_pt_bip(decoder);
3471                         break;
3472
3473                 case INTEL_PT_BEP:
3474                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3475                         decoder->state.from_ip = decoder->ip;
3476                         decoder->state.to_ip = 0;
3477                         return 0;
3478
3479                 case INTEL_PT_BEP_IP:
3480                         err = intel_pt_get_next_packet(decoder);
3481                         if (err)
3482                                 return err;
3483                         if (decoder->packet.type == INTEL_PT_FUP) {
3484                                 decoder->set_fup_bep = true;
3485                                 no_tip = true;
3486                         } else {
3487                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3488                                                 decoder->pos);
3489                         }
3490                         goto next;
3491
3492                 case INTEL_PT_CFE:
3493                         decoder->fup_cfe_pkt = decoder->packet;
3494                         decoder->set_fup_cfe = true;
3495                         if (!decoder->pge) {
3496                                 intel_pt_fup_event(decoder, true);
3497                                 return 0;
3498                         }
3499                         break;
3500
3501                 case INTEL_PT_CFE_IP:
3502                         decoder->fup_cfe_pkt = decoder->packet;
3503                         err = intel_pt_get_next_packet(decoder);
3504                         if (err)
3505                                 return err;
3506                         if (decoder->packet.type == INTEL_PT_FUP) {
3507                                 decoder->set_fup_cfe_ip = true;
3508                                 no_tip = true;
3509                         } else {
3510                                 intel_pt_log_at("ERROR: Missing FUP after CFE",
3511                                                 decoder->pos);
3512                         }
3513                         goto next;
3514
3515                 case INTEL_PT_EVD:
3516                         err = intel_pt_evd(decoder);
3517                         if (err)
3518                                 return err;
3519                         break;
3520
3521                 default:
3522                         return intel_pt_bug(decoder);
3523                 }
3524         }
3525 }
3526
3527 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3528 {
3529         return decoder->packet.count &&
3530                (decoder->have_last_ip || decoder->packet.count == 3 ||
3531                 decoder->packet.count == 6);
3532 }
3533
3534 /* Walk PSB+ packets to get in sync. */
3535 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3536 {
3537         int err;
3538
3539         decoder->in_psb = true;
3540
3541         while (1) {
3542                 err = intel_pt_get_next_packet(decoder);
3543                 if (err)
3544                         goto out;
3545
3546                 switch (decoder->packet.type) {
3547                 case INTEL_PT_TIP_PGD:
3548                         decoder->continuous_period = false;
3549                         fallthrough;
3550                 case INTEL_PT_TIP_PGE:
3551                 case INTEL_PT_TIP:
3552                 case INTEL_PT_PTWRITE:
3553                 case INTEL_PT_PTWRITE_IP:
3554                 case INTEL_PT_EXSTOP:
3555                 case INTEL_PT_EXSTOP_IP:
3556                 case INTEL_PT_MWAIT:
3557                 case INTEL_PT_PWRE:
3558                 case INTEL_PT_PWRX:
3559                 case INTEL_PT_BBP:
3560                 case INTEL_PT_BIP:
3561                 case INTEL_PT_BEP:
3562                 case INTEL_PT_BEP_IP:
3563                 case INTEL_PT_CFE:
3564                 case INTEL_PT_CFE_IP:
3565                 case INTEL_PT_EVD:
3566                         intel_pt_log("ERROR: Unexpected packet\n");
3567                         err = -ENOENT;
3568                         goto out;
3569
3570                 case INTEL_PT_FUP:
3571                         decoder->pge = true;
3572                         if (intel_pt_have_ip(decoder)) {
3573                                 uint64_t current_ip = decoder->ip;
3574
3575                                 intel_pt_set_ip(decoder);
3576                                 decoder->psb_ip = decoder->ip;
3577                                 if (current_ip)
3578                                         intel_pt_log_to("Setting IP",
3579                                                         decoder->ip);
3580                         }
3581                         break;
3582
3583                 case INTEL_PT_MTC:
3584                         intel_pt_calc_mtc_timestamp(decoder);
3585                         break;
3586
3587                 case INTEL_PT_TSC:
3588                         intel_pt_calc_tsc_timestamp(decoder);
3589                         break;
3590
3591                 case INTEL_PT_TMA:
3592                         intel_pt_calc_tma(decoder);
3593                         break;
3594
3595                 case INTEL_PT_CYC:
3596                         intel_pt_calc_cyc_timestamp(decoder);
3597                         break;
3598
3599                 case INTEL_PT_CBR:
3600                         intel_pt_calc_cbr(decoder);
3601                         break;
3602
3603                 case INTEL_PT_PIP:
3604                         intel_pt_set_pip(decoder);
3605                         break;
3606
3607                 case INTEL_PT_MODE_EXEC:
3608                         intel_pt_mode_exec_status(decoder);
3609                         break;
3610
3611                 case INTEL_PT_MODE_TSX:
3612                         intel_pt_update_in_tx(decoder);
3613                         break;
3614
3615                 case INTEL_PT_TRACESTOP:
3616                         decoder->pge = false;
3617                         decoder->continuous_period = false;
3618                         intel_pt_clear_tx_flags(decoder);
3619                         fallthrough;
3620
3621                 case INTEL_PT_TNT:
3622                         decoder->have_tma = false;
3623                         intel_pt_log("ERROR: Unexpected packet\n");
3624                         if (decoder->ip)
3625                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3626                         else
3627                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3628                         err = -ENOENT;
3629                         goto out;
3630
3631                 case INTEL_PT_BAD: /* Does not happen */
3632                         err = intel_pt_bug(decoder);
3633                         goto out;
3634
3635                 case INTEL_PT_OVF:
3636                         err = intel_pt_overflow(decoder);
3637                         goto out;
3638
3639                 case INTEL_PT_PSBEND:
3640                         err = 0;
3641                         goto out;
3642
3643                 case INTEL_PT_PSB:
3644                 case INTEL_PT_VMCS:
3645                 case INTEL_PT_MNT:
3646                 case INTEL_PT_PAD:
3647                 default:
3648                         break;
3649                 }
3650         }
3651 out:
3652         decoder->in_psb = false;
3653
3654         return err;
3655 }
3656
3657 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3658 {
3659         int err;
3660
3661         while (1) {
3662                 err = intel_pt_get_next_packet(decoder);
3663                 if (err)
3664                         return err;
3665
3666                 switch (decoder->packet.type) {
3667                 case INTEL_PT_TIP_PGD:
3668                         decoder->continuous_period = false;
3669                         decoder->pge = false;
3670                         if (intel_pt_have_ip(decoder))
3671                                 intel_pt_set_ip(decoder);
3672                         if (!decoder->ip)
3673                                 break;
3674                         decoder->state.type |= INTEL_PT_TRACE_END;
3675                         return 0;
3676
3677                 case INTEL_PT_TIP_PGE:
3678                         decoder->pge = true;
3679                         intel_pt_mtc_cyc_cnt_pge(decoder);
3680                         if (intel_pt_have_ip(decoder))
3681                                 intel_pt_set_ip(decoder);
3682                         if (!decoder->ip)
3683                                 break;
3684                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3685                         return 0;
3686
3687                 case INTEL_PT_TIP:
3688                         decoder->pge = true;
3689                         if (intel_pt_have_ip(decoder))
3690                                 intel_pt_set_ip(decoder);
3691                         if (!decoder->ip)
3692                                 break;
3693                         return 0;
3694
3695                 case INTEL_PT_FUP:
3696                         if (intel_pt_have_ip(decoder))
3697                                 intel_pt_set_ip(decoder);
3698                         if (decoder->ip)
3699                                 return 0;
3700                         break;
3701
3702                 case INTEL_PT_MTC:
3703                         intel_pt_calc_mtc_timestamp(decoder);
3704                         break;
3705
3706                 case INTEL_PT_TSC:
3707                         intel_pt_calc_tsc_timestamp(decoder);
3708                         break;
3709
3710                 case INTEL_PT_TMA:
3711                         intel_pt_calc_tma(decoder);
3712                         break;
3713
3714                 case INTEL_PT_CYC:
3715                         intel_pt_calc_cyc_timestamp(decoder);
3716                         break;
3717
3718                 case INTEL_PT_CBR:
3719                         intel_pt_calc_cbr(decoder);
3720                         break;
3721
3722                 case INTEL_PT_PIP:
3723                         intel_pt_set_pip(decoder);
3724                         break;
3725
3726                 case INTEL_PT_MODE_EXEC:
3727                         intel_pt_mode_exec_status(decoder);
3728                         break;
3729
3730                 case INTEL_PT_MODE_TSX:
3731                         intel_pt_update_in_tx(decoder);
3732                         break;
3733
3734                 case INTEL_PT_OVF:
3735                         return intel_pt_overflow(decoder);
3736
3737                 case INTEL_PT_BAD: /* Does not happen */
3738                         return intel_pt_bug(decoder);
3739
3740                 case INTEL_PT_TRACESTOP:
3741                         decoder->pge = false;
3742                         decoder->continuous_period = false;
3743                         intel_pt_clear_tx_flags(decoder);
3744                         decoder->have_tma = false;
3745                         break;
3746
3747                 case INTEL_PT_PSB:
3748                         decoder->state.psb_offset = decoder->pos;
3749                         decoder->psb_ip = 0;
3750                         decoder->last_ip = 0;
3751                         decoder->have_last_ip = true;
3752                         intel_pt_clear_stack(&decoder->stack);
3753                         err = intel_pt_walk_psb(decoder);
3754                         if (err)
3755                                 return err;
3756                         decoder->state.type = INTEL_PT_PSB_EVT;
3757                         decoder->state.from_ip = decoder->psb_ip;
3758                         decoder->state.to_ip = 0;
3759                         return 0;
3760
3761                 case INTEL_PT_TNT:
3762                 case INTEL_PT_PSBEND:
3763                 case INTEL_PT_VMCS:
3764                 case INTEL_PT_MNT:
3765                 case INTEL_PT_PAD:
3766                 case INTEL_PT_PTWRITE:
3767                 case INTEL_PT_PTWRITE_IP:
3768                 case INTEL_PT_EXSTOP:
3769                 case INTEL_PT_EXSTOP_IP:
3770                 case INTEL_PT_MWAIT:
3771                 case INTEL_PT_PWRE:
3772                 case INTEL_PT_PWRX:
3773                 case INTEL_PT_BBP:
3774                 case INTEL_PT_BIP:
3775                 case INTEL_PT_BEP:
3776                 case INTEL_PT_BEP_IP:
3777                 case INTEL_PT_CFE:
3778                 case INTEL_PT_CFE_IP:
3779                 case INTEL_PT_EVD:
3780                 default:
3781                         break;
3782                 }
3783         }
3784 }
3785
3786 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3787 {
3788         int err;
3789
3790         intel_pt_clear_fup_event(decoder);
3791         decoder->overflow = false;
3792
3793         if (!decoder->branch_enable) {
3794                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3795                 decoder->state.type = 0; /* Do not have a sample */
3796                 return 0;
3797         }
3798
3799         intel_pt_log("Scanning for full IP\n");
3800         err = intel_pt_walk_to_ip(decoder);
3801         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3802                 return err;
3803
3804         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3805         if (decoder->hop)
3806                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3807         else
3808                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3809
3810         decoder->state.from_ip = 0;
3811         decoder->state.to_ip = decoder->ip;
3812         intel_pt_log_to("Setting IP", decoder->ip);
3813
3814         return 0;
3815 }
3816
3817 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3818 {
3819         const unsigned char *end = decoder->buf + decoder->len;
3820         size_t i;
3821
3822         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3823                 if (i > decoder->len)
3824                         continue;
3825                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3826                         return i;
3827         }
3828         return 0;
3829 }
3830
3831 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3832 {
3833         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3834         const char *psb = INTEL_PT_PSB_STR;
3835
3836         if (rest_psb > decoder->len ||
3837             memcmp(decoder->buf, psb + part_psb, rest_psb))
3838                 return 0;
3839
3840         return rest_psb;
3841 }
3842
3843 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3844                                   int part_psb)
3845 {
3846         int rest_psb, ret;
3847
3848         decoder->pos += decoder->len;
3849         decoder->len = 0;
3850
3851         ret = intel_pt_get_next_data(decoder, false);
3852         if (ret)
3853                 return ret;
3854
3855         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3856         if (!rest_psb)
3857                 return 0;
3858
3859         decoder->pos -= part_psb;
3860         decoder->next_buf = decoder->buf + rest_psb;
3861         decoder->next_len = decoder->len - rest_psb;
3862         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3863         decoder->buf = decoder->temp_buf;
3864         decoder->len = INTEL_PT_PSB_LEN;
3865
3866         return 0;
3867 }
3868
3869 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3870 {
3871         unsigned char *next;
3872         int ret;
3873
3874         intel_pt_log("Scanning for PSB\n");
3875         while (1) {
3876                 if (!decoder->len) {
3877                         ret = intel_pt_get_next_data(decoder, false);
3878                         if (ret)
3879                                 return ret;
3880                 }
3881
3882                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3883                               INTEL_PT_PSB_LEN);
3884                 if (!next) {
3885                         int part_psb;
3886
3887                         part_psb = intel_pt_part_psb(decoder);
3888                         if (part_psb) {
3889                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3890                                 if (ret)
3891                                         return ret;
3892                         } else {
3893                                 decoder->pos += decoder->len;
3894                                 decoder->len = 0;
3895                         }
3896                         continue;
3897                 }
3898
3899                 decoder->pkt_step = next - decoder->buf;
3900                 return intel_pt_get_next_packet(decoder);
3901         }
3902 }
3903
3904 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3905 {
3906         int err;
3907
3908         decoder->pge = false;
3909         decoder->continuous_period = false;
3910         decoder->have_last_ip = false;
3911         decoder->last_ip = 0;
3912         decoder->psb_ip = 0;
3913         decoder->ip = 0;
3914         intel_pt_clear_stack(&decoder->stack);
3915
3916         err = intel_pt_scan_for_psb(decoder);
3917         if (err)
3918                 return err;
3919
3920         if (decoder->vm_time_correlation) {
3921                 decoder->in_psb = true;
3922                 if (!decoder->timestamp)
3923                         decoder->timestamp = 1;
3924                 decoder->state.type = 0;
3925                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3926                 return 0;
3927         }
3928
3929         decoder->have_last_ip = true;
3930         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3931
3932         err = intel_pt_walk_psb(decoder);
3933         if (err)
3934                 return err;
3935
3936         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3937         decoder->state.from_ip = decoder->psb_ip;
3938         decoder->state.to_ip = 0;
3939
3940         if (decoder->ip) {
3941                 /*
3942                  * In hop mode, resample to get the PSB FUP ip as an
3943                  * "instruction" sample.
3944                  */
3945                 if (decoder->hop)
3946                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3947                 else
3948                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3949         }
3950
3951         return 0;
3952 }
3953
3954 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3955 {
3956         uint64_t est = decoder->sample_insn_cnt << 1;
3957
3958         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3959                 goto out;
3960
3961         est *= decoder->max_non_turbo_ratio;
3962         est /= decoder->cbr;
3963 out:
3964         return decoder->sample_timestamp + est;
3965 }
3966
3967 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3968 {
3969         int err;
3970
3971         do {
3972                 decoder->state.type = INTEL_PT_BRANCH;
3973                 decoder->state.flags = 0;
3974
3975                 switch (decoder->pkt_state) {
3976                 case INTEL_PT_STATE_NO_PSB:
3977                         err = intel_pt_sync(decoder);
3978                         break;
3979                 case INTEL_PT_STATE_NO_IP:
3980                         decoder->have_last_ip = false;
3981                         decoder->last_ip = 0;
3982                         decoder->ip = 0;
3983                         fallthrough;
3984                 case INTEL_PT_STATE_ERR_RESYNC:
3985                         err = intel_pt_sync_ip(decoder);
3986                         break;
3987                 case INTEL_PT_STATE_IN_SYNC:
3988                         err = intel_pt_walk_trace(decoder);
3989                         break;
3990                 case INTEL_PT_STATE_TNT:
3991                 case INTEL_PT_STATE_TNT_CONT:
3992                         err = intel_pt_walk_tnt(decoder);
3993                         if (err == -EAGAIN)
3994                                 err = intel_pt_walk_trace(decoder);
3995                         break;
3996                 case INTEL_PT_STATE_TIP:
3997                 case INTEL_PT_STATE_TIP_PGD:
3998                         err = intel_pt_walk_tip(decoder);
3999                         break;
4000                 case INTEL_PT_STATE_FUP:
4001                         err = intel_pt_walk_fup(decoder);
4002                         if (err == -EAGAIN)
4003                                 err = intel_pt_walk_fup_tip(decoder);
4004                         break;
4005                 case INTEL_PT_STATE_FUP_NO_TIP:
4006                         err = intel_pt_walk_fup(decoder);
4007                         if (err == -EAGAIN)
4008                                 err = intel_pt_walk_trace(decoder);
4009                         break;
4010                 case INTEL_PT_STATE_FUP_IN_PSB:
4011                         err = intel_pt_fup_in_psb(decoder);
4012                         break;
4013                 case INTEL_PT_STATE_RESAMPLE:
4014                         err = intel_pt_resample(decoder);
4015                         break;
4016                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
4017                         err = intel_pt_vm_time_correlation(decoder);
4018                         break;
4019                 default:
4020                         err = intel_pt_bug(decoder);
4021                         break;
4022                 }
4023         } while (err == -ENOLINK);
4024
4025         if (err) {
4026                 decoder->state.err = intel_pt_ext_err(err);
4027                 if (err != -EOVERFLOW)
4028                         decoder->state.from_ip = decoder->ip;
4029                 intel_pt_update_sample_time(decoder);
4030                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4031                 intel_pt_set_nr(decoder);
4032         } else {
4033                 decoder->state.err = 0;
4034                 if (decoder->cbr != decoder->cbr_seen) {
4035                         decoder->cbr_seen = decoder->cbr;
4036                         if (!decoder->state.type) {
4037                                 decoder->state.from_ip = decoder->ip;
4038                                 decoder->state.to_ip = 0;
4039                         }
4040                         decoder->state.type |= INTEL_PT_CBR_CHG;
4041                         decoder->state.cbr_payload = decoder->cbr_payload;
4042                         decoder->state.cbr = decoder->cbr;
4043                 }
4044                 if (intel_pt_sample_time(decoder->pkt_state)) {
4045                         intel_pt_update_sample_time(decoder);
4046                         if (decoder->sample_cyc) {
4047                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
4048                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4049                                 decoder->sample_cyc = false;
4050                         }
4051                 }
4052                 /*
4053                  * When using only TSC/MTC to compute cycles, IPC can be
4054                  * sampled as soon as the cycle count changes.
4055                  */
4056                 if (!decoder->have_cyc)
4057                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
4058         }
4059
4060          /* Let PSB event always have TSC timestamp */
4061         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
4062                 decoder->sample_timestamp = decoder->tsc_timestamp;
4063
4064         decoder->state.from_nr = decoder->nr;
4065         decoder->state.to_nr = decoder->next_nr;
4066         decoder->nr = decoder->next_nr;
4067
4068         decoder->state.timestamp = decoder->sample_timestamp;
4069         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
4070         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
4071         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
4072
4073         return &decoder->state;
4074 }
4075
4076 /**
4077  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
4078  * @buf: pointer to buffer pointer
4079  * @len: size of buffer
4080  *
4081  * Updates the buffer pointer to point to the start of the next PSB packet if
4082  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
4083  * @len is adjusted accordingly.
4084  *
4085  * Return: %true if a PSB packet is found, %false otherwise.
4086  */
4087 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
4088 {
4089         unsigned char *next;
4090
4091         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4092         if (next) {
4093                 *len -= next - *buf;
4094                 *buf = next;
4095                 return true;
4096         }
4097         return false;
4098 }
4099
4100 /**
4101  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
4102  *                     packet.
4103  * @buf: pointer to buffer pointer
4104  * @len: size of buffer
4105  *
4106  * Updates the buffer pointer to point to the start of the following PSB packet
4107  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
4108  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
4109  *
4110  * Return: %true if a PSB packet is found, %false otherwise.
4111  */
4112 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
4113 {
4114         unsigned char *next;
4115
4116         if (!*len)
4117                 return false;
4118
4119         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
4120         if (next) {
4121                 *len -= next - *buf;
4122                 *buf = next;
4123                 return true;
4124         }
4125         return false;
4126 }
4127
4128 /**
4129  * intel_pt_last_psb - find the last PSB packet in a buffer.
4130  * @buf: buffer
4131  * @len: size of buffer
4132  *
4133  * This function finds the last PSB in a buffer.
4134  *
4135  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
4136  */
4137 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
4138 {
4139         const char *n = INTEL_PT_PSB_STR;
4140         unsigned char *p;
4141         size_t k;
4142
4143         if (len < INTEL_PT_PSB_LEN)
4144                 return NULL;
4145
4146         k = len - INTEL_PT_PSB_LEN + 1;
4147         while (1) {
4148                 p = memrchr(buf, n[0], k);
4149                 if (!p)
4150                         return NULL;
4151                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
4152                         return p;
4153                 k = p - buf;
4154                 if (!k)
4155                         return NULL;
4156         }
4157 }
4158
4159 /**
4160  * intel_pt_next_tsc - find and return next TSC.
4161  * @buf: buffer
4162  * @len: size of buffer
4163  * @tsc: TSC value returned
4164  * @rem: returns remaining size when TSC is found
4165  *
4166  * Find a TSC packet in @buf and return the TSC value.  This function assumes
4167  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
4168  * PSBEND packet is found.
4169  *
4170  * Return: %true if TSC is found, false otherwise.
4171  */
4172 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
4173                               size_t *rem)
4174 {
4175         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
4176         struct intel_pt_pkt packet;
4177         int ret;
4178
4179         while (len) {
4180                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
4181                 if (ret <= 0)
4182                         return false;
4183                 if (packet.type == INTEL_PT_TSC) {
4184                         *tsc = packet.payload;
4185                         *rem = len;
4186                         return true;
4187                 }
4188                 if (packet.type == INTEL_PT_PSBEND)
4189                         return false;
4190                 buf += ret;
4191                 len -= ret;
4192         }
4193         return false;
4194 }
4195
4196 /**
4197  * intel_pt_tsc_cmp - compare 7-byte TSCs.
4198  * @tsc1: first TSC to compare
4199  * @tsc2: second TSC to compare
4200  *
4201  * This function compares 7-byte TSC values allowing for the possibility that
4202  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
4203  * around so for that purpose this function assumes the absolute difference is
4204  * less than half the maximum difference.
4205  *
4206  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
4207  * after @tsc2.
4208  */
4209 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
4210 {
4211         const uint64_t halfway = (1ULL << 55);
4212
4213         if (tsc1 == tsc2)
4214                 return 0;
4215
4216         if (tsc1 < tsc2) {
4217                 if (tsc2 - tsc1 < halfway)
4218                         return -1;
4219                 else
4220                         return 1;
4221         } else {
4222                 if (tsc1 - tsc2 < halfway)
4223                         return 1;
4224                 else
4225                         return -1;
4226         }
4227 }
4228
4229 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
4230
4231 /**
4232  * adj_for_padding - adjust overlap to account for padding.
4233  * @buf_b: second buffer
4234  * @buf_a: first buffer
4235  * @len_a: size of first buffer
4236  *
4237  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
4238  * accordingly.
4239  *
4240  * Return: A pointer into @buf_b from where non-overlapped data starts
4241  */
4242 static unsigned char *adj_for_padding(unsigned char *buf_b,
4243                                       unsigned char *buf_a, size_t len_a)
4244 {
4245         unsigned char *p = buf_b - MAX_PADDING;
4246         unsigned char *q = buf_a + len_a - MAX_PADDING;
4247         int i;
4248
4249         for (i = MAX_PADDING; i; i--, p++, q++) {
4250                 if (*p != *q)
4251                         break;
4252         }
4253
4254         return p;
4255 }
4256
4257 /**
4258  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
4259  *                             using TSC.
4260  * @buf_a: first buffer
4261  * @len_a: size of first buffer
4262  * @buf_b: second buffer
4263  * @len_b: size of second buffer
4264  * @consecutive: returns true if there is data in buf_b that is consecutive
4265  *               to buf_a
4266  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4267  *
4268  * If the trace contains TSC we can look at the last TSC of @buf_a and the
4269  * first TSC of @buf_b in order to determine if the buffers overlap, and then
4270  * walk forward in @buf_b until a later TSC is found.  A precondition is that
4271  * @buf_a and @buf_b are positioned at a PSB.
4272  *
4273  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4274  * @buf_b + @len_b if there is no non-overlapped data.
4275  */
4276 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
4277                                                 size_t len_a,
4278                                                 unsigned char *buf_b,
4279                                                 size_t len_b, bool *consecutive,
4280                                                 bool ooo_tsc)
4281 {
4282         uint64_t tsc_a, tsc_b;
4283         unsigned char *p;
4284         size_t len, rem_a, rem_b;
4285
4286         p = intel_pt_last_psb(buf_a, len_a);
4287         if (!p)
4288                 return buf_b; /* No PSB in buf_a => no overlap */
4289
4290         len = len_a - (p - buf_a);
4291         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
4292                 /* The last PSB+ in buf_a is incomplete, so go back one more */
4293                 len_a -= len;
4294                 p = intel_pt_last_psb(buf_a, len_a);
4295                 if (!p)
4296                         return buf_b; /* No full PSB+ => assume no overlap */
4297                 len = len_a - (p - buf_a);
4298                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
4299                         return buf_b; /* No TSC in buf_a => assume no overlap */
4300         }
4301
4302         while (1) {
4303                 /* Ignore PSB+ with no TSC */
4304                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4305                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4306
4307                         /* Same TSC, so buffers are consecutive */
4308                         if (!cmp && rem_b >= rem_a) {
4309                                 unsigned char *start;
4310
4311                                 *consecutive = true;
4312                                 start = buf_b + len_b - (rem_b - rem_a);
4313                                 return adj_for_padding(start, buf_a, len_a);
4314                         }
4315                         if (cmp < 0 && !ooo_tsc)
4316                                 return buf_b; /* tsc_a < tsc_b => no overlap */
4317                 }
4318
4319                 if (!intel_pt_step_psb(&buf_b, &len_b))
4320                         return buf_b + len_b; /* No PSB in buf_b => no data */
4321         }
4322 }
4323
4324 /**
4325  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4326  * @buf_a: first buffer
4327  * @len_a: size of first buffer
4328  * @buf_b: second buffer
4329  * @len_b: size of second buffer
4330  * @have_tsc: can use TSC packets to detect overlap
4331  * @consecutive: returns true if there is data in buf_b that is consecutive
4332  *               to buf_a
4333  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4334  *
4335  * When trace samples or snapshots are recorded there is the possibility that
4336  * the data overlaps.  Note that, for the purposes of decoding, data is only
4337  * useful if it begins with a PSB packet.
4338  *
4339  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4340  * @buf_b + @len_b if there is no non-overlapped data.
4341  */
4342 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4343                                      unsigned char *buf_b, size_t len_b,
4344                                      bool have_tsc, bool *consecutive,
4345                                      bool ooo_tsc)
4346 {
4347         unsigned char *found;
4348
4349         /* Buffer 'b' must start at PSB so throw away everything before that */
4350         if (!intel_pt_next_psb(&buf_b, &len_b))
4351                 return buf_b + len_b; /* No PSB */
4352
4353         if (!intel_pt_next_psb(&buf_a, &len_a))
4354                 return buf_b; /* No overlap */
4355
4356         if (have_tsc) {
4357                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4358                                                   consecutive, ooo_tsc);
4359                 if (found)
4360                         return found;
4361         }
4362
4363         /*
4364          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4365          * we can ignore the first part of buffer 'a'.
4366          */
4367         while (len_b < len_a) {
4368                 if (!intel_pt_step_psb(&buf_a, &len_a))
4369                         return buf_b; /* No overlap */
4370         }
4371
4372         /* Now len_b >= len_a */
4373         while (1) {
4374                 /* Potential overlap so check the bytes */
4375                 found = memmem(buf_a, len_a, buf_b, len_a);
4376                 if (found) {
4377                         *consecutive = true;
4378                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4379                 }
4380
4381                 /* Try again at next PSB in buffer 'a' */
4382                 if (!intel_pt_step_psb(&buf_a, &len_a))
4383                         return buf_b; /* No overlap */
4384         }
4385 }
4386
4387 /**
4388  * struct fast_forward_data - data used by intel_pt_ff_cb().
4389  * @timestamp: timestamp to fast forward towards
4390  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4391  *                 the fast forward timestamp.
4392  */
4393 struct fast_forward_data {
4394         uint64_t timestamp;
4395         uint64_t buf_timestamp;
4396 };
4397
4398 /**
4399  * intel_pt_ff_cb - fast forward lookahead callback.
4400  * @buffer: Intel PT trace buffer
4401  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4402  *
4403  * Determine if @buffer trace is past the fast forward timestamp.
4404  *
4405  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4406  *         timestamp, and 0 otherwise.
4407  */
4408 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4409 {
4410         struct fast_forward_data *d = data;
4411         unsigned char *buf;
4412         uint64_t tsc;
4413         size_t rem;
4414         size_t len;
4415
4416         buf = (unsigned char *)buffer->buf;
4417         len = buffer->len;
4418
4419         if (!intel_pt_next_psb(&buf, &len) ||
4420             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4421                 return 0;
4422
4423         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4424
4425         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4426                      tsc, buffer->ref_timestamp);
4427
4428         /*
4429          * If the buffer contains a timestamp earlier that the fast forward
4430          * timestamp, then record it, else stop.
4431          */
4432         if (tsc < d->timestamp)
4433                 d->buf_timestamp = buffer->ref_timestamp;
4434         else
4435                 return 1;
4436
4437         return 0;
4438 }
4439
4440 /**
4441  * intel_pt_fast_forward - reposition decoder forwards.
4442  * @decoder: Intel PT decoder
4443  * @timestamp: timestamp to fast forward towards
4444  *
4445  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4446  *
4447  * Return: 0 on success or negative error code on failure.
4448  */
4449 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4450 {
4451         struct fast_forward_data d = { .timestamp = timestamp };
4452         unsigned char *buf;
4453         size_t len;
4454         int err;
4455
4456         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4457
4458         /* Find buffer timestamp of buffer to fast forward to */
4459         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4460         if (err < 0)
4461                 return err;
4462
4463         /* Walk to buffer with same buffer timestamp */
4464         if (d.buf_timestamp) {
4465                 do {
4466                         decoder->pos += decoder->len;
4467                         decoder->len = 0;
4468                         err = intel_pt_get_next_data(decoder, true);
4469                         /* -ENOLINK means non-consecutive trace */
4470                         if (err && err != -ENOLINK)
4471                                 return err;
4472                 } while (decoder->buf_timestamp != d.buf_timestamp);
4473         }
4474
4475         if (!decoder->buf)
4476                 return 0;
4477
4478         buf = (unsigned char *)decoder->buf;
4479         len = decoder->len;
4480
4481         if (!intel_pt_next_psb(&buf, &len))
4482                 return 0;
4483
4484         /*
4485          * Walk PSBs while the PSB timestamp is less than the fast forward
4486          * timestamp.
4487          */
4488         do {
4489                 uint64_t tsc;
4490                 size_t rem;
4491
4492                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4493                         break;
4494                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4495                 /*
4496                  * A TSC packet can slip past MTC packets but, after fast
4497                  * forward, decoding starts at the TSC timestamp. That means
4498                  * the timestamps may not be exactly the same as the timestamps
4499                  * that would have been decoded without fast forward.
4500                  */
4501                 if (tsc < timestamp) {
4502                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4503                         decoder->pos += decoder->len - len;
4504                         decoder->buf = buf;
4505                         decoder->len = len;
4506                         intel_pt_reposition(decoder);
4507                 } else {
4508                         break;
4509                 }
4510         } while (intel_pt_step_psb(&buf, &len));
4511
4512         return 0;
4513 }