tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /*
  45  * Default maximum number of loops with no packets consumed i.e. stuck in a
  46  * loop.
  47  */
  48 #define INTEL_PT_MAX_LOOPS 100000
  49
  50 struct intel_pt_blk {
  51         struct intel_pt_blk *prev;
  52         uint64_t ip[INTEL_PT_BLK_SIZE];
  53 };
  54
  55 struct intel_pt_stack {
  56         struct intel_pt_blk *blk;
  57         struct intel_pt_blk *spare;
  58         int pos;
  59 };
  60
  61 enum intel_pt_p_once {
  62         INTEL_PT_PRT_ONCE_UNK_VMCS,
  63         INTEL_PT_PRT_ONCE_ERANGE,
  64 };
  65
  66 enum intel_pt_pkt_state {
  67         INTEL_PT_STATE_NO_PSB,
  68         INTEL_PT_STATE_NO_IP,
  69         INTEL_PT_STATE_ERR_RESYNC,
  70         INTEL_PT_STATE_IN_SYNC,
  71         INTEL_PT_STATE_TNT_CONT,
  72         INTEL_PT_STATE_TNT,
  73         INTEL_PT_STATE_TIP,
  74         INTEL_PT_STATE_TIP_PGD,
  75         INTEL_PT_STATE_FUP,
  76         INTEL_PT_STATE_FUP_NO_TIP,
  77         INTEL_PT_STATE_FUP_IN_PSB,
  78         INTEL_PT_STATE_RESAMPLE,
  79         INTEL_PT_STATE_VM_TIME_CORRELATION,
  80 };
  81
  82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  83 {
  84         switch (pkt_state) {
  85         case INTEL_PT_STATE_NO_PSB:
  86         case INTEL_PT_STATE_NO_IP:
  87         case INTEL_PT_STATE_ERR_RESYNC:
  88         case INTEL_PT_STATE_IN_SYNC:
  89         case INTEL_PT_STATE_TNT_CONT:
  90         case INTEL_PT_STATE_RESAMPLE:
  91         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  92                 return true;
  93         case INTEL_PT_STATE_TNT:
  94         case INTEL_PT_STATE_TIP:
  95         case INTEL_PT_STATE_TIP_PGD:
  96         case INTEL_PT_STATE_FUP:
  97         case INTEL_PT_STATE_FUP_NO_TIP:
  98         case INTEL_PT_STATE_FUP_IN_PSB:
  99                 return false;
 100         default:
 101                 return true;
 102         };
 103 }
 104
 105 #ifdef INTEL_PT_STRICT
 106 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 107 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 108 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 109 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 110 #else
 111 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 112 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 113 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 114 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 115 #endif
 116
 117 struct intel_pt_decoder {
 118         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 119         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 120                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 121                          uint64_t max_insn_cnt, void *data);
 122         bool (*pgd_ip)(uint64_t ip, void *data);
 123         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 124         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 125         void *data;
 126         struct intel_pt_state state;
 127         const unsigned char *buf;
 128         size_t len;
 129         bool return_compression;
 130         bool branch_enable;
 131         bool mtc_insn;
 132         bool pge;
 133         bool have_tma;
 134         bool have_cyc;
 135         bool fixup_last_mtc;
 136         bool have_last_ip;
 137         bool in_psb;
 138         bool hop;
 139         bool leap;
 140         bool vm_time_correlation;
 141         bool vm_tm_corr_dry_run;
 142         bool vm_tm_corr_reliable;
 143         bool vm_tm_corr_same_buf;
 144         bool vm_tm_corr_continuous;
 145         bool nr;
 146         bool next_nr;
 147         enum intel_pt_param_flags flags;
 148         uint64_t pos;
 149         uint64_t last_ip;
 150         uint64_t ip;
 151         uint64_t pip_payload;
 152         uint64_t timestamp;
 153         uint64_t tsc_timestamp;
 154         uint64_t ref_timestamp;
 155         uint64_t buf_timestamp;
 156         uint64_t sample_timestamp;
 157         uint64_t ret_addr;
 158         uint64_t ctc_timestamp;
 159         uint64_t ctc_delta;
 160         uint64_t cycle_cnt;
 161         uint64_t cyc_ref_timestamp;
 162         uint64_t first_timestamp;
 163         uint64_t last_reliable_timestamp;
 164         uint64_t vmcs;
 165         uint64_t print_once;
 166         uint64_t last_ctc;
 167         uint32_t last_mtc;
 168         uint32_t tsc_ctc_ratio_n;
 169         uint32_t tsc_ctc_ratio_d;
 170         uint32_t tsc_ctc_mult;
 171         uint32_t tsc_slip;
 172         uint32_t ctc_rem_mask;
 173         int mtc_shift;
 174         struct intel_pt_stack stack;
 175         enum intel_pt_pkt_state pkt_state;
 176         enum intel_pt_pkt_ctx pkt_ctx;
 177         enum intel_pt_pkt_ctx prev_pkt_ctx;
 178         enum intel_pt_blk_type blk_type;
 179         int blk_type_pos;
 180         struct intel_pt_pkt packet;
 181         struct intel_pt_pkt tnt;
 182         int pkt_step;
 183         int pkt_len;
 184         int last_packet_type;
 185         unsigned int cbr;
 186         unsigned int cbr_seen;
 187         unsigned int max_non_turbo_ratio;
 188         double max_non_turbo_ratio_fp;
 189         double cbr_cyc_to_tsc;
 190         double calc_cyc_to_tsc;
 191         bool have_calc_cyc_to_tsc;
 192         int exec_mode;
 193         unsigned int insn_bytes;
 194         uint64_t period;
 195         enum intel_pt_period_type period_type;
 196         uint64_t tot_insn_cnt;
 197         uint64_t period_insn_cnt;
 198         uint64_t period_mask;
 199         uint64_t period_ticks;
 200         uint64_t last_masked_timestamp;
 201         uint64_t tot_cyc_cnt;
 202         uint64_t sample_tot_cyc_cnt;
 203         uint64_t base_cyc_cnt;
 204         uint64_t cyc_cnt_timestamp;
 205         uint64_t ctl;
 206         uint64_t cyc_threshold;
 207         double tsc_to_cyc;
 208         bool continuous_period;
 209         bool overflow;
 210         bool set_fup_tx_flags;
 211         bool set_fup_ptw;
 212         bool set_fup_mwait;
 213         bool set_fup_pwre;
 214         bool set_fup_exstop;
 215         bool set_fup_bep;
 216         bool sample_cyc;
 217         unsigned int fup_tx_flags;
 218         unsigned int tx_flags;
 219         uint64_t fup_ptw_payload;
 220         uint64_t fup_mwait_payload;
 221         uint64_t fup_pwre_payload;
 222         uint64_t cbr_payload;
 223         uint64_t timestamp_insn_cnt;
 224         uint64_t sample_insn_cnt;
 225         uint64_t stuck_ip;
 226         int max_loops;
 227         int no_progress;
 228         int stuck_ip_prd;
 229         int stuck_ip_cnt;
 230         uint64_t psb_ip;
 231         const unsigned char *next_buf;
 232         size_t next_len;
 233         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 234 };
 235
 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 237 {
 238         int i;
 239
 240         for (i = 0; x != 1; i++)
 241                 x >>= 1;
 242
 243         return x << i;
 244 }
 245
 246 __printf(1, 2)
 247 static void p_log(const char *fmt, ...)
 248 {
 249         char buf[512];
 250         va_list args;
 251
 252         va_start(args, fmt);
 253         vsnprintf(buf, sizeof(buf), fmt, args);
 254         va_end(args);
 255
 256         fprintf(stderr, "%s\n", buf);
 257         intel_pt_log("%s\n", buf);
 258 }
 259
 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 261                                 enum intel_pt_p_once id)
 262 {
 263         uint64_t bit = 1ULL << id;
 264
 265         if (decoder->print_once & bit)
 266                 return false;
 267         decoder->print_once |= bit;
 268         return true;
 269 }
 270
 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 272 {
 273         if (!(ctl & INTEL_PT_CYC_ENABLE))
 274                 return 0;
 275
 276         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 277 }
 278
 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 280 {
 281         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 282                 uint64_t period;
 283
 284                 period = intel_pt_lower_power_of_2(decoder->period);
 285                 decoder->period_mask  = ~(period - 1);
 286                 decoder->period_ticks = period;
 287         }
 288 }
 289
 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 291 {
 292         if (!d)
 293                 return 0;
 294         return (t / d) * n + ((t % d) * n) / d;
 295 }
 296
 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 298 {
 299         struct intel_pt_decoder *decoder;
 300
 301         if (!params->get_trace || !params->walk_insn)
 302                 return NULL;
 303
 304         decoder = zalloc(sizeof(struct intel_pt_decoder));
 305         if (!decoder)
 306                 return NULL;
 307
 308         decoder->get_trace          = params->get_trace;
 309         decoder->walk_insn          = params->walk_insn;
 310         decoder->pgd_ip             = params->pgd_ip;
 311         decoder->lookahead          = params->lookahead;
 312         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 313         decoder->data               = params->data;
 314         decoder->return_compression = params->return_compression;
 315         decoder->branch_enable      = params->branch_enable;
 316         decoder->hop                = params->quick >= 1;
 317         decoder->leap               = params->quick >= 2;
 318         decoder->vm_time_correlation = params->vm_time_correlation;
 319         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 320         decoder->first_timestamp    = params->first_timestamp;
 321         decoder->last_reliable_timestamp = params->first_timestamp;
 322         decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
 323
 324         decoder->flags              = params->flags;
 325
 326         decoder->ctl                = params->ctl;
 327         decoder->period             = params->period;
 328         decoder->period_type        = params->period_type;
 329
 330         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 331         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 332
 333         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 334
 335         intel_pt_setup_period(decoder);
 336
 337         decoder->mtc_shift = params->mtc_period;
 338         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 339
 340         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 341         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 342
 343         if (!decoder->tsc_ctc_ratio_n)
 344                 decoder->tsc_ctc_ratio_d = 0;
 345
 346         if (decoder->tsc_ctc_ratio_d) {
 347                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 348                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 349                                                 decoder->tsc_ctc_ratio_d;
 350         }
 351
 352         /*
 353          * A TSC packet can slip past MTC packets so that the timestamp appears
 354          * to go backwards. One estimate is that can be up to about 40 CPU
 355          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 356          * slippage an order of magnitude more to be on the safe side.
 357          */
 358         decoder->tsc_slip = 0x10000;
 359
 360         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 361         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 362         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 363         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 364         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 365
 366         if (decoder->hop)
 367                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 368
 369         return decoder;
 370 }
 371
 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 373                                   uint64_t first_timestamp)
 374 {
 375         decoder->first_timestamp = first_timestamp;
 376 }
 377
 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 379 {
 380         struct intel_pt_blk *blk = stack->blk;
 381
 382         stack->blk = blk->prev;
 383         if (!stack->spare)
 384                 stack->spare = blk;
 385         else
 386                 free(blk);
 387 }
 388
 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 390 {
 391         if (!stack->pos) {
 392                 if (!stack->blk)
 393                         return 0;
 394                 intel_pt_pop_blk(stack);
 395                 if (!stack->blk)
 396                         return 0;
 397                 stack->pos = INTEL_PT_BLK_SIZE;
 398         }
 399         return stack->blk->ip[--stack->pos];
 400 }
 401
 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 403 {
 404         struct intel_pt_blk *blk;
 405
 406         if (stack->spare) {
 407                 blk = stack->spare;
 408                 stack->spare = NULL;
 409         } else {
 410                 blk = malloc(sizeof(struct intel_pt_blk));
 411                 if (!blk)
 412                         return -ENOMEM;
 413         }
 414
 415         blk->prev = stack->blk;
 416         stack->blk = blk;
 417         stack->pos = 0;
 418         return 0;
 419 }
 420
 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 422 {
 423         int err;
 424
 425         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 426                 err = intel_pt_alloc_blk(stack);
 427                 if (err)
 428                         return err;
 429         }
 430
 431         stack->blk->ip[stack->pos++] = ip;
 432         return 0;
 433 }
 434
 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 436 {
 437         while (stack->blk)
 438                 intel_pt_pop_blk(stack);
 439         stack->pos = 0;
 440 }
 441
 442 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 443 {
 444         intel_pt_clear_stack(stack);
 445         zfree(&stack->blk);
 446         zfree(&stack->spare);
 447 }
 448
 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 450 {
 451         intel_pt_free_stack(&decoder->stack);
 452         free(decoder);
 453 }
 454
 455 static int intel_pt_ext_err(int code)
 456 {
 457         switch (code) {
 458         case -ENOMEM:
 459                 return INTEL_PT_ERR_NOMEM;
 460         case -ENOSYS:
 461                 return INTEL_PT_ERR_INTERN;
 462         case -EBADMSG:
 463                 return INTEL_PT_ERR_BADPKT;
 464         case -ENODATA:
 465                 return INTEL_PT_ERR_NODATA;
 466         case -EILSEQ:
 467                 return INTEL_PT_ERR_NOINSN;
 468         case -ENOENT:
 469                 return INTEL_PT_ERR_MISMAT;
 470         case -EOVERFLOW:
 471                 return INTEL_PT_ERR_OVR;
 472         case -ENOSPC:
 473                 return INTEL_PT_ERR_LOST;
 474         case -ELOOP:
 475                 return INTEL_PT_ERR_NELOOP;
 476         default:
 477                 return INTEL_PT_ERR_UNK;
 478         }
 479 }
 480
 481 static const char *intel_pt_err_msgs[] = {
 482         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 483         [INTEL_PT_ERR_INTERN] = "Internal error",
 484         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 485         [INTEL_PT_ERR_NODATA] = "No more data",
 486         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 487         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 488         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 489         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 490         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 491         [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
 492 };
 493
 494 int intel_pt__strerror(int code, char *buf, size_t buflen)
 495 {
 496         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 497                 code = INTEL_PT_ERR_UNK;
 498         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 499         return 0;
 500 }
 501
 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 503                                  uint64_t last_ip)
 504 {
 505         uint64_t ip;
 506
 507         switch (packet->count) {
 508         case 1:
 509                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 510                      packet->payload;
 511                 break;
 512         case 2:
 513                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 514                      packet->payload;
 515                 break;
 516         case 3:
 517                 ip = packet->payload;
 518                 /* Sign-extend 6-byte ip */
 519                 if (ip & (uint64_t)0x800000000000ULL)
 520                         ip |= (uint64_t)0xffff000000000000ULL;
 521                 break;
 522         case 4:
 523                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 524                      packet->payload;
 525                 break;
 526         case 6:
 527                 ip = packet->payload;
 528                 break;
 529         default:
 530                 return 0;
 531         }
 532
 533         return ip;
 534 }
 535
 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 537 {
 538         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 539         decoder->have_last_ip = true;
 540 }
 541
 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 543 {
 544         intel_pt_set_last_ip(decoder);
 545         decoder->ip = decoder->last_ip;
 546 }
 547
 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 549 {
 550         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 551                             decoder->buf);
 552 }
 553
 554 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 555 {
 556         intel_pt_log("ERROR: Internal error\n");
 557         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 558         return -ENOSYS;
 559 }
 560
 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 562 {
 563         decoder->tx_flags = 0;
 564 }
 565
 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 567 {
 568         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 569 }
 570
 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 572 {
 573         decoder->pip_payload = decoder->packet.payload;
 574 }
 575
 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 577 {
 578         decoder->next_nr = decoder->pip_payload & 1;
 579 }
 580
 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 582 {
 583         decoder->nr = decoder->pip_payload & 1;
 584         decoder->next_nr = decoder->nr;
 585 }
 586
 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 588 {
 589         intel_pt_update_pip(decoder);
 590         intel_pt_set_nr(decoder);
 591 }
 592
 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 594 {
 595         intel_pt_clear_tx_flags(decoder);
 596         decoder->have_tma = false;
 597         decoder->pkt_len = 1;
 598         decoder->pkt_step = 1;
 599         intel_pt_decoder_log_packet(decoder);
 600         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 601                 intel_pt_log("ERROR: Bad packet\n");
 602                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 603         }
 604         return -EBADMSG;
 605 }
 606
 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 608 {
 609         decoder->sample_timestamp = decoder->timestamp;
 610         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 611 }
 612
 613 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 614 {
 615         decoder->ip = 0;
 616         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 617         decoder->timestamp = 0;
 618         decoder->have_tma = false;
 619 }
 620
 621 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 622 {
 623         struct intel_pt_buffer buffer = { .buf = 0, };
 624         int ret;
 625
 626         decoder->pkt_step = 0;
 627
 628         intel_pt_log("Getting more data\n");
 629         ret = decoder->get_trace(&buffer, decoder->data);
 630         if (ret)
 631                 return ret;
 632         decoder->buf = buffer.buf;
 633         decoder->len = buffer.len;
 634         if (!decoder->len) {
 635                 intel_pt_log("No more data\n");
 636                 return -ENODATA;
 637         }
 638         decoder->buf_timestamp = buffer.ref_timestamp;
 639         if (!buffer.consecutive || reposition) {
 640                 intel_pt_reposition(decoder);
 641                 decoder->ref_timestamp = buffer.ref_timestamp;
 642                 decoder->state.trace_nr = buffer.trace_nr;
 643                 decoder->vm_tm_corr_same_buf = false;
 644                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 645                              decoder->ref_timestamp);
 646                 return -ENOLINK;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 653                                   bool reposition)
 654 {
 655         if (!decoder->next_buf)
 656                 return intel_pt_get_data(decoder, reposition);
 657
 658         decoder->buf = decoder->next_buf;
 659         decoder->len = decoder->next_len;
 660         decoder->next_buf = 0;
 661         decoder->next_len = 0;
 662         return 0;
 663 }
 664
 665 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 666 {
 667         unsigned char *buf = decoder->temp_buf;
 668         size_t old_len, len, n;
 669         int ret;
 670
 671         old_len = decoder->len;
 672         len = decoder->len;
 673         memcpy(buf, decoder->buf, len);
 674
 675         ret = intel_pt_get_data(decoder, false);
 676         if (ret) {
 677                 decoder->pos += old_len;
 678                 return ret < 0 ? ret : -EINVAL;
 679         }
 680
 681         n = INTEL_PT_PKT_MAX_SZ - len;
 682         if (n > decoder->len)
 683                 n = decoder->len;
 684         memcpy(buf + len, decoder->buf, n);
 685         len += n;
 686
 687         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 688         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 689         if (ret < (int)old_len) {
 690                 decoder->next_buf = decoder->buf;
 691                 decoder->next_len = decoder->len;
 692                 decoder->buf = buf;
 693                 decoder->len = old_len;
 694                 return intel_pt_bad_packet(decoder);
 695         }
 696
 697         decoder->next_buf = decoder->buf + (ret - old_len);
 698         decoder->next_len = decoder->len - (ret - old_len);
 699
 700         decoder->buf = buf;
 701         decoder->len = ret;
 702
 703         return ret;
 704 }
 705
 706 struct intel_pt_pkt_info {
 707         struct intel_pt_decoder   *decoder;
 708         struct intel_pt_pkt       packet;
 709         uint64_t                  pos;
 710         int                       pkt_len;
 711         int                       last_packet_type;
 712         void                      *data;
 713 };
 714
 715 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 716
 717 /* Lookahead packets in current buffer */
 718 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 719                                   intel_pt_pkt_cb_t cb, void *data)
 720 {
 721         struct intel_pt_pkt_info pkt_info;
 722         const unsigned char *buf = decoder->buf;
 723         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 724         size_t len = decoder->len;
 725         int ret;
 726
 727         pkt_info.decoder          = decoder;
 728         pkt_info.pos              = decoder->pos;
 729         pkt_info.pkt_len          = decoder->pkt_step;
 730         pkt_info.last_packet_type = decoder->last_packet_type;
 731         pkt_info.data             = data;
 732
 733         while (1) {
 734                 do {
 735                         pkt_info.pos += pkt_info.pkt_len;
 736                         buf          += pkt_info.pkt_len;
 737                         len          -= pkt_info.pkt_len;
 738
 739                         if (!len)
 740                                 return INTEL_PT_NEED_MORE_BYTES;
 741
 742                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 743                                                   &pkt_ctx);
 744                         if (!ret)
 745                                 return INTEL_PT_NEED_MORE_BYTES;
 746                         if (ret < 0)
 747                                 return ret;
 748
 749                         pkt_info.pkt_len = ret;
 750                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 751
 752                 ret = cb(&pkt_info);
 753                 if (ret)
 754                         return 0;
 755
 756                 pkt_info.last_packet_type = pkt_info.packet.type;
 757         }
 758 }
 759
 760 struct intel_pt_calc_cyc_to_tsc_info {
 761         uint64_t        cycle_cnt;
 762         unsigned int    cbr;
 763         uint32_t        last_mtc;
 764         uint64_t        ctc_timestamp;
 765         uint64_t        ctc_delta;
 766         uint64_t        tsc_timestamp;
 767         uint64_t        timestamp;
 768         bool            have_tma;
 769         bool            fixup_last_mtc;
 770         bool            from_mtc;
 771         double          cbr_cyc_to_tsc;
 772 };
 773
 774 /*
 775  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 776  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 777  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 778  * packet by copying the missing bits from the current MTC assuming the least
 779  * difference between the two, and that the current MTC comes after last_mtc.
 780  */
 781 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 782                                     uint32_t *last_mtc)
 783 {
 784         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 785         uint32_t mask = ~(first_missing_bit - 1);
 786
 787         *last_mtc |= mtc & mask;
 788         if (*last_mtc >= mtc) {
 789                 *last_mtc -= first_missing_bit;
 790                 *last_mtc &= 0xff;
 791         }
 792 }
 793
 794 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 795 {
 796         struct intel_pt_decoder *decoder = pkt_info->decoder;
 797         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 798         uint64_t timestamp;
 799         double cyc_to_tsc;
 800         unsigned int cbr;
 801         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 802
 803         switch (pkt_info->packet.type) {
 804         case INTEL_PT_TNT:
 805         case INTEL_PT_TIP_PGE:
 806         case INTEL_PT_TIP:
 807         case INTEL_PT_FUP:
 808         case INTEL_PT_PSB:
 809         case INTEL_PT_PIP:
 810         case INTEL_PT_MODE_EXEC:
 811         case INTEL_PT_MODE_TSX:
 812         case INTEL_PT_PSBEND:
 813         case INTEL_PT_PAD:
 814         case INTEL_PT_VMCS:
 815         case INTEL_PT_MNT:
 816         case INTEL_PT_PTWRITE:
 817         case INTEL_PT_PTWRITE_IP:
 818         case INTEL_PT_BBP:
 819         case INTEL_PT_BIP:
 820         case INTEL_PT_BEP:
 821         case INTEL_PT_BEP_IP:
 822                 return 0;
 823
 824         case INTEL_PT_MTC:
 825                 if (!data->have_tma)
 826                         return 0;
 827
 828                 mtc = pkt_info->packet.payload;
 829                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 830                         data->fixup_last_mtc = false;
 831                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 832                                                 &data->last_mtc);
 833                 }
 834                 if (mtc > data->last_mtc)
 835                         mtc_delta = mtc - data->last_mtc;
 836                 else
 837                         mtc_delta = mtc + 256 - data->last_mtc;
 838                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 839                 data->last_mtc = mtc;
 840
 841                 if (decoder->tsc_ctc_mult) {
 842                         timestamp = data->ctc_timestamp +
 843                                 data->ctc_delta * decoder->tsc_ctc_mult;
 844                 } else {
 845                         timestamp = data->ctc_timestamp +
 846                                 multdiv(data->ctc_delta,
 847                                         decoder->tsc_ctc_ratio_n,
 848                                         decoder->tsc_ctc_ratio_d);
 849                 }
 850
 851                 if (timestamp < data->timestamp)
 852                         return 1;
 853
 854                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 855                         data->timestamp = timestamp;
 856                         return 0;
 857                 }
 858
 859                 break;
 860
 861         case INTEL_PT_TSC:
 862                 /*
 863                  * For now, do not support using TSC packets - refer
 864                  * intel_pt_calc_cyc_to_tsc().
 865                  */
 866                 if (data->from_mtc)
 867                         return 1;
 868                 timestamp = pkt_info->packet.payload |
 869                             (data->timestamp & (0xffULL << 56));
 870                 if (data->from_mtc && timestamp < data->timestamp &&
 871                     data->timestamp - timestamp < decoder->tsc_slip)
 872                         return 1;
 873                 if (timestamp < data->timestamp)
 874                         timestamp += (1ULL << 56);
 875                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 876                         if (data->from_mtc)
 877                                 return 1;
 878                         data->tsc_timestamp = timestamp;
 879                         data->timestamp = timestamp;
 880                         return 0;
 881                 }
 882                 break;
 883
 884         case INTEL_PT_TMA:
 885                 if (data->from_mtc)
 886                         return 1;
 887
 888                 if (!decoder->tsc_ctc_ratio_d)
 889                         return 0;
 890
 891                 ctc = pkt_info->packet.payload;
 892                 fc = pkt_info->packet.count;
 893                 ctc_rem = ctc & decoder->ctc_rem_mask;
 894
 895                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 896
 897                 data->ctc_timestamp = data->tsc_timestamp - fc;
 898                 if (decoder->tsc_ctc_mult) {
 899                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 900                 } else {
 901                         data->ctc_timestamp -=
 902                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 903                                         decoder->tsc_ctc_ratio_d);
 904                 }
 905
 906                 data->ctc_delta = 0;
 907                 data->have_tma = true;
 908                 data->fixup_last_mtc = true;
 909
 910                 return 0;
 911
 912         case INTEL_PT_CYC:
 913                 data->cycle_cnt += pkt_info->packet.payload;
 914                 return 0;
 915
 916         case INTEL_PT_CBR:
 917                 cbr = pkt_info->packet.payload;
 918                 if (data->cbr && data->cbr != cbr)
 919                         return 1;
 920                 data->cbr = cbr;
 921                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 922                 return 0;
 923
 924         case INTEL_PT_TIP_PGD:
 925         case INTEL_PT_TRACESTOP:
 926         case INTEL_PT_EXSTOP:
 927         case INTEL_PT_EXSTOP_IP:
 928         case INTEL_PT_MWAIT:
 929         case INTEL_PT_PWRE:
 930         case INTEL_PT_PWRX:
 931         case INTEL_PT_OVF:
 932         case INTEL_PT_BAD: /* Does not happen */
 933         default:
 934                 return 1;
 935         }
 936
 937         if (!data->cbr && decoder->cbr) {
 938                 data->cbr = decoder->cbr;
 939                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 940         }
 941
 942         if (!data->cycle_cnt)
 943                 return 1;
 944
 945         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 946
 947         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 948             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 949                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 950                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 951                 return 1;
 952         }
 953
 954         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 955         decoder->have_calc_cyc_to_tsc = true;
 956
 957         if (data->cbr) {
 958                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 959                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 960         } else {
 961                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 962                              cyc_to_tsc, pkt_info->pos);
 963         }
 964
 965         return 1;
 966 }
 967
 968 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 969                                      bool from_mtc)
 970 {
 971         struct intel_pt_calc_cyc_to_tsc_info data = {
 972                 .cycle_cnt      = 0,
 973                 .cbr            = 0,
 974                 .last_mtc       = decoder->last_mtc,
 975                 .ctc_timestamp  = decoder->ctc_timestamp,
 976                 .ctc_delta      = decoder->ctc_delta,
 977                 .tsc_timestamp  = decoder->tsc_timestamp,
 978                 .timestamp      = decoder->timestamp,
 979                 .have_tma       = decoder->have_tma,
 980                 .fixup_last_mtc = decoder->fixup_last_mtc,
 981                 .from_mtc       = from_mtc,
 982                 .cbr_cyc_to_tsc = 0,
 983         };
 984
 985         /*
 986          * For now, do not support using TSC packets for at least the reasons:
 987          * 1) timing might have stopped
 988          * 2) TSC packets within PSB+ can slip against CYC packets
 989          */
 990         if (!from_mtc)
 991                 return;
 992
 993         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 994 }
 995
 996 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 997 {
 998         int ret;
 999
1000         decoder->last_packet_type = decoder->packet.type;
1001
1002         do {
1003                 decoder->pos += decoder->pkt_step;
1004                 decoder->buf += decoder->pkt_step;
1005                 decoder->len -= decoder->pkt_step;
1006
1007                 if (!decoder->len) {
1008                         ret = intel_pt_get_next_data(decoder, false);
1009                         if (ret)
1010                                 return ret;
1011                 }
1012
1013                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1014                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1015                                           &decoder->packet, &decoder->pkt_ctx);
1016                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1017                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1018                         ret = intel_pt_get_split_packet(decoder);
1019                         if (ret < 0)
1020                                 return ret;
1021                 }
1022                 if (ret <= 0)
1023                         return intel_pt_bad_packet(decoder);
1024
1025                 decoder->pkt_len = ret;
1026                 decoder->pkt_step = ret;
1027                 intel_pt_decoder_log_packet(decoder);
1028         } while (decoder->packet.type == INTEL_PT_PAD);
1029
1030         return 0;
1031 }
1032
1033 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1034 {
1035         uint64_t timestamp, masked_timestamp;
1036
1037         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1038         masked_timestamp = timestamp & decoder->period_mask;
1039         if (decoder->continuous_period) {
1040                 if (masked_timestamp > decoder->last_masked_timestamp)
1041                         return 1;
1042         } else {
1043                 timestamp += 1;
1044                 masked_timestamp = timestamp & decoder->period_mask;
1045                 if (masked_timestamp > decoder->last_masked_timestamp) {
1046                         decoder->last_masked_timestamp = masked_timestamp;
1047                         decoder->continuous_period = true;
1048                 }
1049         }
1050
1051         if (masked_timestamp < decoder->last_masked_timestamp)
1052                 return decoder->period_ticks;
1053
1054         return decoder->period_ticks - (timestamp - masked_timestamp);
1055 }
1056
1057 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1058 {
1059         switch (decoder->period_type) {
1060         case INTEL_PT_PERIOD_INSTRUCTIONS:
1061                 return decoder->period - decoder->period_insn_cnt;
1062         case INTEL_PT_PERIOD_TICKS:
1063                 return intel_pt_next_period(decoder);
1064         case INTEL_PT_PERIOD_NONE:
1065         case INTEL_PT_PERIOD_MTC:
1066         default:
1067                 return 0;
1068         }
1069 }
1070
1071 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1072 {
1073         uint64_t timestamp, masked_timestamp;
1074
1075         switch (decoder->period_type) {
1076         case INTEL_PT_PERIOD_INSTRUCTIONS:
1077                 decoder->period_insn_cnt = 0;
1078                 break;
1079         case INTEL_PT_PERIOD_TICKS:
1080                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1081                 masked_timestamp = timestamp & decoder->period_mask;
1082                 if (masked_timestamp > decoder->last_masked_timestamp)
1083                         decoder->last_masked_timestamp = masked_timestamp;
1084                 else
1085                         decoder->last_masked_timestamp += decoder->period_ticks;
1086                 break;
1087         case INTEL_PT_PERIOD_NONE:
1088         case INTEL_PT_PERIOD_MTC:
1089         default:
1090                 break;
1091         }
1092
1093         decoder->state.type |= INTEL_PT_INSTRUCTION;
1094 }
1095
1096 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1097                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1098 {
1099         uint64_t max_insn_cnt, insn_cnt = 0;
1100         int err;
1101
1102         if (!decoder->mtc_insn)
1103                 decoder->mtc_insn = true;
1104
1105         max_insn_cnt = intel_pt_next_sample(decoder);
1106
1107         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1108                                  max_insn_cnt, decoder->data);
1109
1110         decoder->tot_insn_cnt += insn_cnt;
1111         decoder->timestamp_insn_cnt += insn_cnt;
1112         decoder->sample_insn_cnt += insn_cnt;
1113         decoder->period_insn_cnt += insn_cnt;
1114
1115         if (err) {
1116                 decoder->no_progress = 0;
1117                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1118                 intel_pt_log_at("ERROR: Failed to get instruction",
1119                                 decoder->ip);
1120                 if (err == -ENOENT)
1121                         return -ENOLINK;
1122                 return -EILSEQ;
1123         }
1124
1125         if (ip && decoder->ip == ip) {
1126                 err = -EAGAIN;
1127                 goto out;
1128         }
1129
1130         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1131                 intel_pt_sample_insn(decoder);
1132
1133         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1134                 decoder->state.type = INTEL_PT_INSTRUCTION;
1135                 decoder->state.from_ip = decoder->ip;
1136                 decoder->state.to_ip = 0;
1137                 decoder->ip += intel_pt_insn->length;
1138                 err = INTEL_PT_RETURN;
1139                 goto out;
1140         }
1141
1142         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1143                 /* Zero-length calls are excluded */
1144                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1145                     intel_pt_insn->rel) {
1146                         err = intel_pt_push(&decoder->stack, decoder->ip +
1147                                             intel_pt_insn->length);
1148                         if (err)
1149                                 goto out;
1150                 }
1151         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1152                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1153         }
1154
1155         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1156                 int cnt = decoder->no_progress++;
1157
1158                 decoder->state.from_ip = decoder->ip;
1159                 decoder->ip += intel_pt_insn->length +
1160                                 intel_pt_insn->rel;
1161                 decoder->state.to_ip = decoder->ip;
1162                 err = INTEL_PT_RETURN;
1163
1164                 /*
1165                  * Check for being stuck in a loop.  This can happen if a
1166                  * decoder error results in the decoder erroneously setting the
1167                  * ip to an address that is itself in an infinite loop that
1168                  * consumes no packets.  When that happens, there must be an
1169                  * unconditional branch.
1170                  */
1171                 if (cnt) {
1172                         if (cnt == 1) {
1173                                 decoder->stuck_ip = decoder->state.to_ip;
1174                                 decoder->stuck_ip_prd = 1;
1175                                 decoder->stuck_ip_cnt = 1;
1176                         } else if (cnt > decoder->max_loops ||
1177                                    decoder->state.to_ip == decoder->stuck_ip) {
1178                                 intel_pt_log_at("ERROR: Never-ending loop",
1179                                                 decoder->state.to_ip);
1180                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1181                                 err = -ELOOP;
1182                                 goto out;
1183                         } else if (!--decoder->stuck_ip_cnt) {
1184                                 decoder->stuck_ip_prd += 1;
1185                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1186                                 decoder->stuck_ip = decoder->state.to_ip;
1187                         }
1188                 }
1189                 goto out_no_progress;
1190         }
1191 out:
1192         decoder->no_progress = 0;
1193 out_no_progress:
1194         decoder->state.insn_op = intel_pt_insn->op;
1195         decoder->state.insn_len = intel_pt_insn->length;
1196         memcpy(decoder->state.insn, intel_pt_insn->buf,
1197                INTEL_PT_INSN_BUF_SZ);
1198
1199         if (decoder->tx_flags & INTEL_PT_IN_TX)
1200                 decoder->state.flags |= INTEL_PT_IN_TX;
1201
1202         return err;
1203 }
1204
1205 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1206 {
1207         enum intel_pt_sample_type type = decoder->state.type;
1208         bool ret = false;
1209
1210         decoder->state.type &= ~INTEL_PT_BRANCH;
1211
1212         if (decoder->set_fup_tx_flags) {
1213                 decoder->set_fup_tx_flags = false;
1214                 decoder->tx_flags = decoder->fup_tx_flags;
1215                 decoder->state.type |= INTEL_PT_TRANSACTION;
1216                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1217                         decoder->state.type |= INTEL_PT_BRANCH;
1218                 decoder->state.flags = decoder->fup_tx_flags;
1219                 ret = true;
1220         }
1221         if (decoder->set_fup_ptw) {
1222                 decoder->set_fup_ptw = false;
1223                 decoder->state.type |= INTEL_PT_PTW;
1224                 decoder->state.flags |= INTEL_PT_FUP_IP;
1225                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1226                 ret = true;
1227         }
1228         if (decoder->set_fup_mwait) {
1229                 decoder->set_fup_mwait = false;
1230                 decoder->state.type |= INTEL_PT_MWAIT_OP;
1231                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1232                 ret = true;
1233         }
1234         if (decoder->set_fup_pwre) {
1235                 decoder->set_fup_pwre = false;
1236                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1237                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1238                 ret = true;
1239         }
1240         if (decoder->set_fup_exstop) {
1241                 decoder->set_fup_exstop = false;
1242                 decoder->state.type |= INTEL_PT_EX_STOP;
1243                 decoder->state.flags |= INTEL_PT_FUP_IP;
1244                 ret = true;
1245         }
1246         if (decoder->set_fup_bep) {
1247                 decoder->set_fup_bep = false;
1248                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1249                 ret = true;
1250         }
1251         if (decoder->overflow) {
1252                 decoder->overflow = false;
1253                 if (!ret && !decoder->pge) {
1254                         if (decoder->hop) {
1255                                 decoder->state.type = 0;
1256                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1257                         }
1258                         decoder->pge = true;
1259                         decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
1260                         decoder->state.from_ip = 0;
1261                         decoder->state.to_ip = decoder->ip;
1262                         return true;
1263                 }
1264         }
1265         if (ret) {
1266                 decoder->state.from_ip = decoder->ip;
1267                 decoder->state.to_ip = 0;
1268         } else {
1269                 decoder->state.type = type;
1270         }
1271         return ret;
1272 }
1273
1274 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1275                                           struct intel_pt_insn *intel_pt_insn,
1276                                           uint64_t ip, int err)
1277 {
1278         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1279                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1280                ip == decoder->ip + intel_pt_insn->length;
1281 }
1282
1283 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1284 {
1285         struct intel_pt_insn intel_pt_insn;
1286         uint64_t ip;
1287         int err;
1288
1289         ip = decoder->last_ip;
1290
1291         while (1) {
1292                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1293                 if (err == INTEL_PT_RETURN)
1294                         return 0;
1295                 if (err == -EAGAIN ||
1296                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1297                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1298
1299                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1300                         if (intel_pt_fup_event(decoder) && no_tip)
1301                                 return 0;
1302                         return -EAGAIN;
1303                 }
1304                 decoder->set_fup_tx_flags = false;
1305                 if (err)
1306                         return err;
1307
1308                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1309                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1310                                         decoder->ip);
1311                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1312                         return -ENOENT;
1313                 }
1314
1315                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1316                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1317                                         decoder->ip);
1318                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1319                         return -ENOENT;
1320                 }
1321
1322                 intel_pt_bug(decoder);
1323         }
1324 }
1325
1326 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1327 {
1328         struct intel_pt_insn intel_pt_insn;
1329         int err;
1330
1331         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1332         if (err == INTEL_PT_RETURN &&
1333             decoder->pgd_ip &&
1334             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1335             (decoder->state.type & INTEL_PT_BRANCH) &&
1336             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1337                 /* Unconditional branch leaving filter region */
1338                 decoder->no_progress = 0;
1339                 decoder->pge = false;
1340                 decoder->continuous_period = false;
1341                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1342                 decoder->state.type |= INTEL_PT_TRACE_END;
1343                 intel_pt_update_nr(decoder);
1344                 return 0;
1345         }
1346         if (err == INTEL_PT_RETURN)
1347                 return 0;
1348         if (err)
1349                 return err;
1350
1351         intel_pt_update_nr(decoder);
1352
1353         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1354                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1355                         decoder->pge = false;
1356                         decoder->continuous_period = false;
1357                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1358                         decoder->state.from_ip = decoder->ip;
1359                         if (decoder->packet.count == 0) {
1360                                 decoder->state.to_ip = 0;
1361                         } else {
1362                                 decoder->state.to_ip = decoder->last_ip;
1363                                 decoder->ip = decoder->last_ip;
1364                         }
1365                         decoder->state.type |= INTEL_PT_TRACE_END;
1366                 } else {
1367                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1368                         decoder->state.from_ip = decoder->ip;
1369                         if (decoder->packet.count == 0) {
1370                                 decoder->state.to_ip = 0;
1371                         } else {
1372                                 decoder->state.to_ip = decoder->last_ip;
1373                                 decoder->ip = decoder->last_ip;
1374                         }
1375                 }
1376                 return 0;
1377         }
1378
1379         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1380                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1381                                  intel_pt_insn.rel;
1382
1383                 if (decoder->pgd_ip &&
1384                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1385                     decoder->pgd_ip(to_ip, decoder->data)) {
1386                         /* Conditional branch leaving filter region */
1387                         decoder->pge = false;
1388                         decoder->continuous_period = false;
1389                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1390                         decoder->ip = to_ip;
1391                         decoder->state.from_ip = decoder->ip;
1392                         decoder->state.to_ip = to_ip;
1393                         decoder->state.type |= INTEL_PT_TRACE_END;
1394                         return 0;
1395                 }
1396                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1397                                 decoder->ip);
1398                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1399                 return -ENOENT;
1400         }
1401
1402         return intel_pt_bug(decoder);
1403 }
1404
1405 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1406 {
1407         struct intel_pt_insn intel_pt_insn;
1408         int err;
1409
1410         while (1) {
1411                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1412                 if (err == INTEL_PT_RETURN)
1413                         return 0;
1414                 if (err)
1415                         return err;
1416
1417                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1418                         if (!decoder->return_compression) {
1419                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1420                                                 decoder->ip);
1421                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1422                                 return -ENOENT;
1423                         }
1424                         if (!decoder->ret_addr) {
1425                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1426                                                 decoder->ip);
1427                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1428                                 return -ENOENT;
1429                         }
1430                         if (!(decoder->tnt.payload & BIT63)) {
1431                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1432                                                 decoder->ip);
1433                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1434                                 return -ENOENT;
1435                         }
1436                         decoder->tnt.count -= 1;
1437                         if (decoder->tnt.count)
1438                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1439                         else
1440                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1441                         decoder->tnt.payload <<= 1;
1442                         decoder->state.from_ip = decoder->ip;
1443                         decoder->ip = decoder->ret_addr;
1444                         decoder->state.to_ip = decoder->ip;
1445                         return 0;
1446                 }
1447
1448                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1449                         /* Handle deferred TIPs */
1450                         err = intel_pt_get_next_packet(decoder);
1451                         if (err)
1452                                 return err;
1453                         if (decoder->packet.type != INTEL_PT_TIP ||
1454                             decoder->packet.count == 0) {
1455                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1456                                                 decoder->ip);
1457                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1458                                 decoder->pkt_step = 0;
1459                                 return -ENOENT;
1460                         }
1461                         intel_pt_set_last_ip(decoder);
1462                         decoder->state.from_ip = decoder->ip;
1463                         decoder->state.to_ip = decoder->last_ip;
1464                         decoder->ip = decoder->last_ip;
1465                         intel_pt_update_nr(decoder);
1466                         return 0;
1467                 }
1468
1469                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1470                         decoder->tnt.count -= 1;
1471                         if (decoder->tnt.count)
1472                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1473                         else
1474                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1475                         if (decoder->tnt.payload & BIT63) {
1476                                 decoder->tnt.payload <<= 1;
1477                                 decoder->state.from_ip = decoder->ip;
1478                                 decoder->ip += intel_pt_insn.length +
1479                                                intel_pt_insn.rel;
1480                                 decoder->state.to_ip = decoder->ip;
1481                                 return 0;
1482                         }
1483                         /* Instruction sample for a non-taken branch */
1484                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1485                                 decoder->tnt.payload <<= 1;
1486                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1487                                 decoder->state.from_ip = decoder->ip;
1488                                 decoder->state.to_ip = 0;
1489                                 decoder->ip += intel_pt_insn.length;
1490                                 return 0;
1491                         }
1492                         decoder->sample_cyc = false;
1493                         decoder->ip += intel_pt_insn.length;
1494                         if (!decoder->tnt.count) {
1495                                 intel_pt_update_sample_time(decoder);
1496                                 return -EAGAIN;
1497                         }
1498                         decoder->tnt.payload <<= 1;
1499                         continue;
1500                 }
1501
1502                 return intel_pt_bug(decoder);
1503         }
1504 }
1505
1506 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1507 {
1508         unsigned int fup_tx_flags;
1509         int err;
1510
1511         fup_tx_flags = decoder->packet.payload &
1512                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1513         err = intel_pt_get_next_packet(decoder);
1514         if (err)
1515                 return err;
1516         if (decoder->packet.type == INTEL_PT_FUP) {
1517                 decoder->fup_tx_flags = fup_tx_flags;
1518                 decoder->set_fup_tx_flags = true;
1519                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1520                         *no_tip = true;
1521         } else {
1522                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1523                                 decoder->pos);
1524                 intel_pt_update_in_tx(decoder);
1525         }
1526         return 0;
1527 }
1528
1529 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1530 {
1531         timestamp |= (ref_timestamp & (0xffULL << 56));
1532
1533         if (timestamp < ref_timestamp) {
1534                 if (ref_timestamp - timestamp > (1ULL << 55))
1535                         timestamp += (1ULL << 56);
1536         } else {
1537                 if (timestamp - ref_timestamp > (1ULL << 55))
1538                         timestamp -= (1ULL << 56);
1539         }
1540
1541         return timestamp;
1542 }
1543
1544 /* For use only when decoder->vm_time_correlation is true */
1545 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1546                                    uint64_t timestamp)
1547 {
1548         uint64_t max_timestamp = decoder->buf_timestamp;
1549
1550         if (!max_timestamp) {
1551                 max_timestamp = decoder->last_reliable_timestamp +
1552                                 0x400000000ULL;
1553         }
1554         return timestamp >= decoder->last_reliable_timestamp &&
1555                timestamp < decoder->buf_timestamp;
1556 }
1557
1558 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1559 {
1560         uint64_t timestamp;
1561         bool bad = false;
1562
1563         decoder->have_tma = false;
1564
1565         if (decoder->ref_timestamp) {
1566                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1567                                             decoder->ref_timestamp);
1568                 decoder->tsc_timestamp = timestamp;
1569                 decoder->timestamp = timestamp;
1570                 decoder->ref_timestamp = 0;
1571                 decoder->timestamp_insn_cnt = 0;
1572         } else if (decoder->timestamp) {
1573                 timestamp = decoder->packet.payload |
1574                             (decoder->timestamp & (0xffULL << 56));
1575                 decoder->tsc_timestamp = timestamp;
1576                 if (timestamp < decoder->timestamp &&
1577                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1578                         intel_pt_log_to("Suppressing backwards timestamp",
1579                                         timestamp);
1580                         timestamp = decoder->timestamp;
1581                 }
1582                 if (timestamp < decoder->timestamp) {
1583                         if (!decoder->buf_timestamp ||
1584                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1585                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1586                                 timestamp += (1ULL << 56);
1587                                 decoder->tsc_timestamp = timestamp;
1588                         } else {
1589                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1590                                 timestamp = decoder->timestamp;
1591                                 bad = true;
1592                         }
1593                 }
1594                 if (decoder->vm_time_correlation &&
1595                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1596                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1597                         p_log("Timestamp out of range");
1598                 decoder->timestamp = timestamp;
1599                 decoder->timestamp_insn_cnt = 0;
1600         }
1601
1602         if (decoder->last_packet_type == INTEL_PT_CYC) {
1603                 decoder->cyc_ref_timestamp = decoder->timestamp;
1604                 decoder->cycle_cnt = 0;
1605                 decoder->have_calc_cyc_to_tsc = false;
1606                 intel_pt_calc_cyc_to_tsc(decoder, false);
1607         }
1608
1609         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1610 }
1611
1612 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1613 {
1614         intel_pt_log("ERROR: Buffer overflow\n");
1615         intel_pt_clear_tx_flags(decoder);
1616         intel_pt_set_nr(decoder);
1617         decoder->timestamp_insn_cnt = 0;
1618         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1619         decoder->state.from_ip = decoder->ip;
1620         decoder->ip = 0;
1621         decoder->pge = false;
1622         decoder->set_fup_tx_flags = false;
1623         decoder->set_fup_ptw = false;
1624         decoder->set_fup_mwait = false;
1625         decoder->set_fup_pwre = false;
1626         decoder->set_fup_exstop = false;
1627         decoder->set_fup_bep = false;
1628         decoder->overflow = true;
1629         return -EOVERFLOW;
1630 }
1631
1632 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1633 {
1634         if (decoder->have_cyc)
1635                 return;
1636
1637         decoder->cyc_cnt_timestamp = decoder->timestamp;
1638         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1639 }
1640
1641 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1642 {
1643         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1644
1645         if (decoder->pge)
1646                 intel_pt_mtc_cyc_cnt_pge(decoder);
1647 }
1648
1649 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1650 {
1651         uint64_t tot_cyc_cnt, tsc_delta;
1652
1653         if (decoder->have_cyc)
1654                 return;
1655
1656         decoder->sample_cyc = true;
1657
1658         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1659                 return;
1660
1661         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1662         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1663
1664         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1665                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1666 }
1667
1668 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1669 {
1670         uint32_t ctc = decoder->packet.payload;
1671         uint32_t fc = decoder->packet.count;
1672         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1673
1674         if (!decoder->tsc_ctc_ratio_d)
1675                 return;
1676
1677         if (decoder->pge && !decoder->in_psb)
1678                 intel_pt_mtc_cyc_cnt_pge(decoder);
1679         else
1680                 intel_pt_mtc_cyc_cnt_upd(decoder);
1681
1682         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1683         decoder->last_ctc = ctc - ctc_rem;
1684         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1685         if (decoder->tsc_ctc_mult) {
1686                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1687         } else {
1688                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1689                                                   decoder->tsc_ctc_ratio_n,
1690                                                   decoder->tsc_ctc_ratio_d);
1691         }
1692         decoder->ctc_delta = 0;
1693         decoder->have_tma = true;
1694         decoder->fixup_last_mtc = true;
1695         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1696                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1697 }
1698
1699 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1700 {
1701         uint64_t timestamp;
1702         uint32_t mtc, mtc_delta;
1703
1704         if (!decoder->have_tma)
1705                 return;
1706
1707         mtc = decoder->packet.payload;
1708
1709         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1710                 decoder->fixup_last_mtc = false;
1711                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1712                                         &decoder->last_mtc);
1713         }
1714
1715         if (mtc > decoder->last_mtc)
1716                 mtc_delta = mtc - decoder->last_mtc;
1717         else
1718                 mtc_delta = mtc + 256 - decoder->last_mtc;
1719
1720         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1721
1722         if (decoder->tsc_ctc_mult) {
1723                 timestamp = decoder->ctc_timestamp +
1724                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1725         } else {
1726                 timestamp = decoder->ctc_timestamp +
1727                             multdiv(decoder->ctc_delta,
1728                                     decoder->tsc_ctc_ratio_n,
1729                                     decoder->tsc_ctc_ratio_d);
1730         }
1731
1732         if (timestamp < decoder->timestamp)
1733                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1734                              timestamp, decoder->timestamp);
1735         else
1736                 decoder->timestamp = timestamp;
1737
1738         intel_pt_mtc_cyc_cnt_upd(decoder);
1739
1740         decoder->timestamp_insn_cnt = 0;
1741         decoder->last_mtc = mtc;
1742
1743         if (decoder->last_packet_type == INTEL_PT_CYC) {
1744                 decoder->cyc_ref_timestamp = decoder->timestamp;
1745                 decoder->cycle_cnt = 0;
1746                 decoder->have_calc_cyc_to_tsc = false;
1747                 intel_pt_calc_cyc_to_tsc(decoder, true);
1748         }
1749
1750         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1751 }
1752
1753 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1754 {
1755         unsigned int cbr = decoder->packet.payload & 0xff;
1756
1757         decoder->cbr_payload = decoder->packet.payload;
1758
1759         if (decoder->cbr == cbr)
1760                 return;
1761
1762         decoder->cbr = cbr;
1763         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1764
1765         intel_pt_mtc_cyc_cnt_cbr(decoder);
1766 }
1767
1768 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1769 {
1770         uint64_t timestamp = decoder->cyc_ref_timestamp;
1771
1772         decoder->have_cyc = true;
1773
1774         decoder->cycle_cnt += decoder->packet.payload;
1775         if (decoder->pge)
1776                 decoder->tot_cyc_cnt += decoder->packet.payload;
1777         decoder->sample_cyc = true;
1778
1779         if (!decoder->cyc_ref_timestamp)
1780                 return;
1781
1782         if (decoder->have_calc_cyc_to_tsc)
1783                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1784         else if (decoder->cbr)
1785                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1786         else
1787                 return;
1788
1789         if (timestamp < decoder->timestamp)
1790                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1791                              timestamp, decoder->timestamp);
1792         else
1793                 decoder->timestamp = timestamp;
1794
1795         decoder->timestamp_insn_cnt = 0;
1796
1797         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1798 }
1799
1800 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1801 {
1802         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1803                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1804                 decoder->state.items.is_32_bit = false;
1805         }
1806         decoder->blk_type = decoder->packet.payload;
1807         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1808         if (decoder->blk_type == INTEL_PT_GP_REGS)
1809                 decoder->state.items.is_32_bit = decoder->packet.count;
1810         if (decoder->blk_type_pos < 0) {
1811                 intel_pt_log("WARNING: Unknown block type %u\n",
1812                              decoder->blk_type);
1813         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1814                 intel_pt_log("WARNING: Duplicate block type %u\n",
1815                              decoder->blk_type);
1816         }
1817 }
1818
1819 static void intel_pt_bip(struct intel_pt_decoder *decoder)
1820 {
1821         uint32_t id = decoder->packet.count;
1822         uint32_t bit = 1 << id;
1823         int pos = decoder->blk_type_pos;
1824
1825         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1826                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1827                              id, decoder->blk_type);
1828                 return;
1829         }
1830
1831         if (decoder->state.items.mask[pos] & bit) {
1832                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1833                              id, decoder->blk_type);
1834         }
1835
1836         decoder->state.items.mask[pos] |= bit;
1837         decoder->state.items.val[pos][id] = decoder->packet.payload;
1838 }
1839
1840 /* Walk PSB+ packets when already in sync. */
1841 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1842 {
1843         int err;
1844
1845         decoder->in_psb = true;
1846
1847         while (1) {
1848                 err = intel_pt_get_next_packet(decoder);
1849                 if (err)
1850                         goto out;
1851
1852                 switch (decoder->packet.type) {
1853                 case INTEL_PT_PSBEND:
1854                         err = 0;
1855                         goto out;
1856
1857                 case INTEL_PT_TIP_PGD:
1858                 case INTEL_PT_TIP_PGE:
1859                 case INTEL_PT_TIP:
1860                 case INTEL_PT_TNT:
1861                 case INTEL_PT_TRACESTOP:
1862                 case INTEL_PT_BAD:
1863                 case INTEL_PT_PSB:
1864                 case INTEL_PT_PTWRITE:
1865                 case INTEL_PT_PTWRITE_IP:
1866                 case INTEL_PT_EXSTOP:
1867                 case INTEL_PT_EXSTOP_IP:
1868                 case INTEL_PT_MWAIT:
1869                 case INTEL_PT_PWRE:
1870                 case INTEL_PT_PWRX:
1871                 case INTEL_PT_BBP:
1872                 case INTEL_PT_BIP:
1873                 case INTEL_PT_BEP:
1874                 case INTEL_PT_BEP_IP:
1875                         decoder->have_tma = false;
1876                         intel_pt_log("ERROR: Unexpected packet\n");
1877                         err = -EAGAIN;
1878                         goto out;
1879
1880                 case INTEL_PT_OVF:
1881                         err = intel_pt_overflow(decoder);
1882                         goto out;
1883
1884                 case INTEL_PT_TSC:
1885                         intel_pt_calc_tsc_timestamp(decoder);
1886                         break;
1887
1888                 case INTEL_PT_TMA:
1889                         intel_pt_calc_tma(decoder);
1890                         break;
1891
1892                 case INTEL_PT_CBR:
1893                         intel_pt_calc_cbr(decoder);
1894                         break;
1895
1896                 case INTEL_PT_MODE_EXEC:
1897                         decoder->exec_mode = decoder->packet.payload;
1898                         break;
1899
1900                 case INTEL_PT_PIP:
1901                         intel_pt_set_pip(decoder);
1902                         break;
1903
1904                 case INTEL_PT_FUP:
1905                         decoder->pge = true;
1906                         if (decoder->packet.count) {
1907                                 intel_pt_set_last_ip(decoder);
1908                                 decoder->psb_ip = decoder->last_ip;
1909                         }
1910                         break;
1911
1912                 case INTEL_PT_MODE_TSX:
1913                         intel_pt_update_in_tx(decoder);
1914                         break;
1915
1916                 case INTEL_PT_MTC:
1917                         intel_pt_calc_mtc_timestamp(decoder);
1918                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1919                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1920                         break;
1921
1922                 case INTEL_PT_CYC:
1923                         intel_pt_calc_cyc_timestamp(decoder);
1924                         break;
1925
1926                 case INTEL_PT_VMCS:
1927                 case INTEL_PT_MNT:
1928                 case INTEL_PT_PAD:
1929                 default:
1930                         break;
1931                 }
1932         }
1933 out:
1934         decoder->in_psb = false;
1935
1936         return err;
1937 }
1938
1939 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1940 {
1941         int err;
1942
1943         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1944                 decoder->tx_flags = 0;
1945                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1946                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1947         } else {
1948                 decoder->state.flags |= INTEL_PT_ASYNC;
1949         }
1950
1951         while (1) {
1952                 err = intel_pt_get_next_packet(decoder);
1953                 if (err)
1954                         return err;
1955
1956                 switch (decoder->packet.type) {
1957                 case INTEL_PT_TNT:
1958                 case INTEL_PT_FUP:
1959                 case INTEL_PT_TRACESTOP:
1960                 case INTEL_PT_PSB:
1961                 case INTEL_PT_TSC:
1962                 case INTEL_PT_TMA:
1963                 case INTEL_PT_MODE_TSX:
1964                 case INTEL_PT_BAD:
1965                 case INTEL_PT_PSBEND:
1966                 case INTEL_PT_PTWRITE:
1967                 case INTEL_PT_PTWRITE_IP:
1968                 case INTEL_PT_EXSTOP:
1969                 case INTEL_PT_EXSTOP_IP:
1970                 case INTEL_PT_MWAIT:
1971                 case INTEL_PT_PWRE:
1972                 case INTEL_PT_PWRX:
1973                 case INTEL_PT_BBP:
1974                 case INTEL_PT_BIP:
1975                 case INTEL_PT_BEP:
1976                 case INTEL_PT_BEP_IP:
1977                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1978                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1979                         decoder->pkt_step = 0;
1980                         return -ENOENT;
1981
1982                 case INTEL_PT_CBR:
1983                         intel_pt_calc_cbr(decoder);
1984                         break;
1985
1986                 case INTEL_PT_OVF:
1987                         return intel_pt_overflow(decoder);
1988
1989                 case INTEL_PT_TIP_PGD:
1990                         decoder->state.from_ip = decoder->ip;
1991                         if (decoder->packet.count == 0) {
1992                                 decoder->state.to_ip = 0;
1993                         } else {
1994                                 intel_pt_set_ip(decoder);
1995                                 decoder->state.to_ip = decoder->ip;
1996                         }
1997                         decoder->pge = false;
1998                         decoder->continuous_period = false;
1999                         decoder->state.type |= INTEL_PT_TRACE_END;
2000                         intel_pt_update_nr(decoder);
2001                         return 0;
2002
2003                 case INTEL_PT_TIP_PGE:
2004                         decoder->pge = true;
2005                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2006                                      decoder->ip);
2007                         decoder->state.from_ip = 0;
2008                         if (decoder->packet.count == 0) {
2009                                 decoder->state.to_ip = 0;
2010                         } else {
2011                                 intel_pt_set_ip(decoder);
2012                                 decoder->state.to_ip = decoder->ip;
2013                         }
2014                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2015                         intel_pt_mtc_cyc_cnt_pge(decoder);
2016                         intel_pt_set_nr(decoder);
2017                         return 0;
2018
2019                 case INTEL_PT_TIP:
2020                         decoder->state.from_ip = decoder->ip;
2021                         if (decoder->packet.count == 0) {
2022                                 decoder->state.to_ip = 0;
2023                         } else {
2024                                 intel_pt_set_ip(decoder);
2025                                 decoder->state.to_ip = decoder->ip;
2026                         }
2027                         intel_pt_update_nr(decoder);
2028                         return 0;
2029
2030                 case INTEL_PT_PIP:
2031                         intel_pt_update_pip(decoder);
2032                         break;
2033
2034                 case INTEL_PT_MTC:
2035                         intel_pt_calc_mtc_timestamp(decoder);
2036                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2037                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2038                         break;
2039
2040                 case INTEL_PT_CYC:
2041                         intel_pt_calc_cyc_timestamp(decoder);
2042                         break;
2043
2044                 case INTEL_PT_MODE_EXEC:
2045                         decoder->exec_mode = decoder->packet.payload;
2046                         break;
2047
2048                 case INTEL_PT_VMCS:
2049                 case INTEL_PT_MNT:
2050                 case INTEL_PT_PAD:
2051                         break;
2052
2053                 default:
2054                         return intel_pt_bug(decoder);
2055                 }
2056         }
2057 }
2058
2059 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2060 {
2061         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2062         decoder->state.type = INTEL_PT_INSTRUCTION;
2063         decoder->state.from_ip = decoder->ip;
2064         decoder->state.to_ip = 0;
2065         return 0;
2066 }
2067
2068 struct intel_pt_vm_tsc_info {
2069         struct intel_pt_pkt pip_packet;
2070         struct intel_pt_pkt vmcs_packet;
2071         struct intel_pt_pkt tma_packet;
2072         bool tsc, pip, vmcs, tma, psbend;
2073         uint64_t ctc_delta;
2074         uint64_t last_ctc;
2075         int max_lookahead;
2076 };
2077
2078 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2079 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2080 {
2081         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2082
2083         switch (pkt_info->packet.type) {
2084         case INTEL_PT_PAD:
2085         case INTEL_PT_MNT:
2086         case INTEL_PT_MODE_EXEC:
2087         case INTEL_PT_MODE_TSX:
2088         case INTEL_PT_MTC:
2089         case INTEL_PT_FUP:
2090         case INTEL_PT_CYC:
2091         case INTEL_PT_CBR:
2092                 break;
2093
2094         case INTEL_PT_TSC:
2095                 data->tsc = true;
2096                 break;
2097
2098         case INTEL_PT_TMA:
2099                 data->tma_packet = pkt_info->packet;
2100                 data->tma = true;
2101                 break;
2102
2103         case INTEL_PT_PIP:
2104                 data->pip_packet = pkt_info->packet;
2105                 data->pip = true;
2106                 break;
2107
2108         case INTEL_PT_VMCS:
2109                 data->vmcs_packet = pkt_info->packet;
2110                 data->vmcs = true;
2111                 break;
2112
2113         case INTEL_PT_PSBEND:
2114                 data->psbend = true;
2115                 return 1;
2116
2117         case INTEL_PT_TIP_PGE:
2118         case INTEL_PT_PTWRITE:
2119         case INTEL_PT_PTWRITE_IP:
2120         case INTEL_PT_EXSTOP:
2121         case INTEL_PT_EXSTOP_IP:
2122         case INTEL_PT_MWAIT:
2123         case INTEL_PT_PWRE:
2124         case INTEL_PT_PWRX:
2125         case INTEL_PT_BBP:
2126         case INTEL_PT_BIP:
2127         case INTEL_PT_BEP:
2128         case INTEL_PT_BEP_IP:
2129         case INTEL_PT_OVF:
2130         case INTEL_PT_BAD:
2131         case INTEL_PT_TNT:
2132         case INTEL_PT_TIP_PGD:
2133         case INTEL_PT_TIP:
2134         case INTEL_PT_PSB:
2135         case INTEL_PT_TRACESTOP:
2136         default:
2137                 return 1;
2138         }
2139
2140         return 0;
2141 }
2142
2143 struct intel_pt_ovf_fup_info {
2144         int max_lookahead;
2145         bool found;
2146 };
2147
2148 /* Lookahead to detect a FUP packet after OVF */
2149 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2150 {
2151         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2152
2153         if (pkt_info->packet.type == INTEL_PT_CYC ||
2154             pkt_info->packet.type == INTEL_PT_MTC ||
2155             pkt_info->packet.type == INTEL_PT_TSC)
2156                 return !--(data->max_lookahead);
2157         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2158         return 1;
2159 }
2160
2161 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2162 {
2163         struct intel_pt_ovf_fup_info data = {
2164                 .max_lookahead = 16,
2165                 .found = false,
2166         };
2167
2168         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2169         return data.found;
2170 }
2171
2172 /* Lookahead and get the TMA packet after TSC */
2173 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2174 {
2175         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2176
2177         if (pkt_info->packet.type == INTEL_PT_CYC ||
2178             pkt_info->packet.type == INTEL_PT_MTC)
2179                 return !--(data->max_lookahead);
2180
2181         if (pkt_info->packet.type == INTEL_PT_TMA) {
2182                 data->tma_packet = pkt_info->packet;
2183                 data->tma = true;
2184         }
2185         return 1;
2186 }
2187
2188 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2189 {
2190         if (decoder->tsc_ctc_mult)
2191                 return ctc * decoder->tsc_ctc_mult;
2192         else
2193                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2194 }
2195
2196 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2197                                            uint32_t ctc,
2198                                            uint32_t fc,
2199                                            uint64_t last_ctc_timestamp,
2200                                            uint64_t ctc_delta,
2201                                            uint32_t last_ctc)
2202 {
2203         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2204         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2205         /*
2206          * Number of CTC ticks from there until current TMA packet. We would
2207          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2208          * past an MTC, so a sign-extended value is used.
2209          */
2210         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2211         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2212         uint64_t new_ctc_delta = ctc_delta + delta;
2213         uint64_t expected_tsc;
2214
2215         /*
2216          * Convert CTC ticks to TSC ticks, add the starting point
2217          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2218          */
2219         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2220
2221         if (intel_pt_enable_logging) {
2222                 intel_pt_log_x64(last_mtc_ctc);
2223                 intel_pt_log_x32(last_ctc);
2224                 intel_pt_log_x64(ctc_delta);
2225                 intel_pt_log_x64(delta);
2226                 intel_pt_log_x32(ctc);
2227                 intel_pt_log_x64(new_ctc_delta);
2228                 intel_pt_log_x64(last_ctc_timestamp);
2229                 intel_pt_log_x32(fc);
2230                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2231                 intel_pt_log_x64(expected_tsc);
2232         }
2233
2234         return expected_tsc;
2235 }
2236
2237 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2238                                       struct intel_pt_vm_tsc_info *data)
2239 {
2240         uint32_t ctc = data->tma_packet.payload;
2241         uint32_t fc = data->tma_packet.count;
2242
2243         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2244                                           decoder->ctc_timestamp,
2245                                           data->ctc_delta, data->last_ctc);
2246 }
2247
2248 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2249                                       struct intel_pt_vmcs_info *vmcs_info)
2250 {
2251         uint64_t payload = decoder->packet.payload;
2252
2253         /* VMX adds the TSC Offset, so subtract to get host TSC */
2254         decoder->packet.payload -= vmcs_info->tsc_offset;
2255         /* TSC packet has only 7 bytes */
2256         decoder->packet.payload &= SEVEN_BYTES;
2257
2258         /*
2259          * The buffer is mmapped from the data file, so this also updates the
2260          * data file.
2261          */
2262         if (!decoder->vm_tm_corr_dry_run)
2263                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2264
2265         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2266                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2267                      payload, decoder->packet.payload, vmcs_info->vmcs,
2268                      vmcs_info->tsc_offset);
2269 }
2270
2271 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2272                                              uint64_t tsc_offset)
2273 {
2274         struct intel_pt_vmcs_info vmcs_info = {
2275                 .vmcs = NO_VMCS,
2276                 .tsc_offset = tsc_offset
2277         };
2278
2279         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2280 }
2281
2282 static inline bool in_vm(uint64_t pip_payload)
2283 {
2284         return pip_payload & 1;
2285 }
2286
2287 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2288 {
2289         return pip_packet->payload & 1;
2290 }
2291
2292 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2293 {
2294         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2295               vmcs_info->vmcs, vmcs_info->tsc_offset);
2296 }
2297
2298 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2299                                     struct intel_pt_vm_tsc_info *data)
2300 {
2301         memset(data, 0, sizeof(*data));
2302         data->ctc_delta = decoder->ctc_delta;
2303         data->last_ctc = decoder->last_ctc;
2304         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2305         if (data->tsc && !data->psbend)
2306                 p_log("ERROR: PSB without PSBEND");
2307         decoder->in_psb = data->psbend;
2308 }
2309
2310 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2311                                           struct intel_pt_vm_tsc_info *data,
2312                                           struct intel_pt_vmcs_info *vmcs_info,
2313                                           uint64_t host_tsc)
2314 {
2315         if (!decoder->in_psb) {
2316                 /* Can't happen */
2317                 p_log("ERROR: First TSC is not in PSB+");
2318         }
2319
2320         if (data->pip) {
2321                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2322                         if (vmcs_info && vmcs_info->tsc_offset) {
2323                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2324                                 decoder->vm_tm_corr_reliable = true;
2325                         } else {
2326                                 p_log("ERROR: First TSC, unknown TSC Offset");
2327                         }
2328                 } else { /* Host */
2329                         decoder->vm_tm_corr_reliable = true;
2330                 }
2331         } else { /* Host or Guest */
2332                 decoder->vm_tm_corr_reliable = false;
2333                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2334                         /* Assume Host */
2335                 } else {
2336                         /* Assume Guest */
2337                         if (vmcs_info && vmcs_info->tsc_offset)
2338                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2339                         else
2340                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2341                 }
2342         }
2343 }
2344
2345 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2346                                     struct intel_pt_vm_tsc_info *data)
2347 {
2348         struct intel_pt_vmcs_info *vmcs_info;
2349         uint64_t tsc_offset = 0;
2350         uint64_t vmcs;
2351         bool reliable = true;
2352         uint64_t expected_tsc;
2353         uint64_t host_tsc;
2354         uint64_t ref_timestamp;
2355
2356         bool assign = false;
2357         bool assign_reliable = false;
2358
2359         /* Already have 'data' for the in_psb case */
2360         if (!decoder->in_psb) {
2361                 memset(data, 0, sizeof(*data));
2362                 data->ctc_delta = decoder->ctc_delta;
2363                 data->last_ctc = decoder->last_ctc;
2364                 data->max_lookahead = 16;
2365                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2366                 if (decoder->pge) {
2367                         data->pip = true;
2368                         data->pip_packet.payload = decoder->pip_payload;
2369                 }
2370         }
2371
2372         /* Calculations depend on having TMA packets */
2373         if (!data->tma) {
2374                 p_log("ERROR: TSC without TMA");
2375                 return;
2376         }
2377
2378         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2379         if (vmcs == NO_VMCS)
2380                 vmcs = 0;
2381
2382         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2383
2384         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2385         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2386
2387         if (!decoder->ctc_timestamp) {
2388                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2389                 return;
2390         }
2391
2392         expected_tsc = intel_pt_expected_tsc(decoder, data);
2393
2394         tsc_offset = host_tsc - expected_tsc;
2395
2396         /* Determine if TSC is from Host or Guest */
2397         if (data->pip) {
2398                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2399                         if (!vmcs_info) {
2400                                 /* PIP NR=1 without VMCS cannot happen */
2401                                 p_log("ERROR: Missing VMCS");
2402                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2403                                 decoder->vm_tm_corr_reliable = false;
2404                                 return;
2405                         }
2406                 } else { /* Host */
2407                         decoder->last_reliable_timestamp = host_tsc;
2408                         decoder->vm_tm_corr_reliable = true;
2409                         return;
2410                 }
2411         } else { /* Host or Guest */
2412                 reliable = false; /* Host/Guest is a guess, so not reliable */
2413                 if (decoder->in_psb) {
2414                         if (!tsc_offset)
2415                                 return; /* Zero TSC Offset, assume Host */
2416                         /*
2417                          * TSC packet has only 7 bytes of TSC. We have no
2418                          * information about the Guest's 8th byte, but it
2419                          * doesn't matter because we only need 7 bytes.
2420                          * Here, since the 8th byte is unreliable and
2421                          * irrelevant, compare only 7 byes.
2422                          */
2423                         if (vmcs_info &&
2424                             (tsc_offset & SEVEN_BYTES) ==
2425                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2426                                 /* Same TSC Offset as last VMCS, assume Guest */
2427                                 goto guest;
2428                         }
2429                 }
2430                 /*
2431                  * Check if the host_tsc is within the expected range.
2432                  * Note, we could narrow the range more by looking ahead for
2433                  * the next host TSC in the same buffer, but we don't bother to
2434                  * do that because this is probably good enough.
2435                  */
2436                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2437                         /* Within expected range for Host TSC, assume Host */
2438                         decoder->vm_tm_corr_reliable = false;
2439                         return;
2440                 }
2441         }
2442
2443 guest: /* Assuming Guest */
2444
2445         /* Determine whether to assign TSC Offset */
2446         if (vmcs_info && vmcs_info->vmcs) {
2447                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2448                         assign = false;
2449                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2450                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2451                         /* Continuous tracing, TSC in a PSB is not a time loss */
2452                         assign = true;
2453                         assign_reliable = true;
2454                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2455                         /*
2456                          * Unlikely to be a time loss TSC in a PSB which is not
2457                          * at the start of a buffer.
2458                          */
2459                         assign = true;
2460                         assign_reliable = false;
2461                 }
2462         }
2463
2464         /* Record VMCS TSC Offset */
2465         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2466                        vmcs_info->reliable != assign_reliable)) {
2467                 bool print = vmcs_info->tsc_offset != tsc_offset;
2468
2469                 vmcs_info->tsc_offset = tsc_offset;
2470                 vmcs_info->reliable = assign_reliable;
2471                 if (print)
2472                         intel_pt_print_vmcs_info(vmcs_info);
2473         }
2474
2475         /* Determine what TSC Offset to use */
2476         if (vmcs_info && vmcs_info->tsc_offset) {
2477                 if (!vmcs_info->reliable)
2478                         reliable = false;
2479                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2480         } else {
2481                 reliable = false;
2482                 if (vmcs_info) {
2483                         if (!vmcs_info->error_printed) {
2484                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2485                                       vmcs_info->vmcs);
2486                                 vmcs_info->error_printed = true;
2487                         }
2488                 } else {
2489                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2490                                 p_log("ERROR: Unknown VMCS");
2491                 }
2492                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2493         }
2494
2495         decoder->vm_tm_corr_reliable = reliable;
2496 }
2497
2498 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2499 {
2500         uint64_t host_tsc = decoder->packet.payload;
2501         uint64_t guest_tsc = decoder->packet.payload;
2502         struct intel_pt_vmcs_info *vmcs_info;
2503         uint64_t vmcs;
2504
2505         vmcs = decoder->vmcs;
2506         if (vmcs == NO_VMCS)
2507                 vmcs = 0;
2508
2509         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2510
2511         if (decoder->pge) {
2512                 if (in_vm(decoder->pip_payload)) { /* Guest */
2513                         if (!vmcs_info) {
2514                                 /* PIP NR=1 without VMCS cannot happen */
2515                                 p_log("ERROR: Missing VMCS");
2516                         }
2517                 } else { /* Host */
2518                         return;
2519                 }
2520         } else { /* Host or Guest */
2521                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2522                         /* Within expected range for Host TSC, assume Host */
2523                         return;
2524                 }
2525         }
2526
2527         if (vmcs_info) {
2528                 /* Translate Guest TSC to Host TSC */
2529                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2530                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2531                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2532                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2533                              guest_tsc, host_tsc, vmcs_info->vmcs,
2534                              vmcs_info->tsc_offset);
2535                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2536                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2537                         p_log("Timestamp out of range");
2538         } else {
2539                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2540                         p_log("ERROR: Unknown VMCS");
2541                 host_tsc = decoder->timestamp;
2542         }
2543
2544         decoder->packet.payload = host_tsc;
2545
2546         if (!decoder->vm_tm_corr_dry_run)
2547                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2548 }
2549
2550 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2551 {
2552         struct intel_pt_vm_tsc_info data = { .psbend = false };
2553         bool pge;
2554         int err;
2555
2556         if (decoder->in_psb)
2557                 intel_pt_vm_tm_corr_psb(decoder, &data);
2558
2559         while (1) {
2560                 err = intel_pt_get_next_packet(decoder);
2561                 if (err == -ENOLINK)
2562                         continue;
2563                 if (err)
2564                         break;
2565
2566                 switch (decoder->packet.type) {
2567                 case INTEL_PT_TIP_PGD:
2568                         decoder->pge = false;
2569                         decoder->vm_tm_corr_continuous = false;
2570                         break;
2571
2572                 case INTEL_PT_TNT:
2573                 case INTEL_PT_TIP:
2574                 case INTEL_PT_TIP_PGE:
2575                         decoder->pge = true;
2576                         break;
2577
2578                 case INTEL_PT_OVF:
2579                         decoder->in_psb = false;
2580                         pge = decoder->pge;
2581                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2582                         if (pge != decoder->pge)
2583                                 intel_pt_log("Surprising PGE change in OVF!");
2584                         if (!decoder->pge)
2585                                 decoder->vm_tm_corr_continuous = false;
2586                         break;
2587
2588                 case INTEL_PT_FUP:
2589                         if (decoder->in_psb)
2590                                 decoder->pge = true;
2591                         break;
2592
2593                 case INTEL_PT_TRACESTOP:
2594                         decoder->pge = false;
2595                         decoder->vm_tm_corr_continuous = false;
2596                         decoder->have_tma = false;
2597                         break;
2598
2599                 case INTEL_PT_PSB:
2600                         intel_pt_vm_tm_corr_psb(decoder, &data);
2601                         break;
2602
2603                 case INTEL_PT_PIP:
2604                         decoder->pip_payload = decoder->packet.payload;
2605                         break;
2606
2607                 case INTEL_PT_MTC:
2608                         intel_pt_calc_mtc_timestamp(decoder);
2609                         break;
2610
2611                 case INTEL_PT_TSC:
2612                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2613                         intel_pt_calc_tsc_timestamp(decoder);
2614                         decoder->vm_tm_corr_same_buf = true;
2615                         decoder->vm_tm_corr_continuous = decoder->pge;
2616                         break;
2617
2618                 case INTEL_PT_TMA:
2619                         intel_pt_calc_tma(decoder);
2620                         break;
2621
2622                 case INTEL_PT_CYC:
2623                         intel_pt_calc_cyc_timestamp(decoder);
2624                         break;
2625
2626                 case INTEL_PT_CBR:
2627                         intel_pt_calc_cbr(decoder);
2628                         break;
2629
2630                 case INTEL_PT_PSBEND:
2631                         decoder->in_psb = false;
2632                         data.psbend = false;
2633                         break;
2634
2635                 case INTEL_PT_VMCS:
2636                         if (decoder->packet.payload != NO_VMCS)
2637                                 decoder->vmcs = decoder->packet.payload;
2638                         break;
2639
2640                 case INTEL_PT_BBP:
2641                         decoder->blk_type = decoder->packet.payload;
2642                         break;
2643
2644                 case INTEL_PT_BIP:
2645                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2646                             decoder->packet.count == 2)
2647                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2648                         break;
2649
2650                 case INTEL_PT_BEP:
2651                 case INTEL_PT_BEP_IP:
2652                         decoder->blk_type = 0;
2653                         break;
2654
2655                 case INTEL_PT_MODE_EXEC:
2656                 case INTEL_PT_MODE_TSX:
2657                 case INTEL_PT_MNT:
2658                 case INTEL_PT_PAD:
2659                 case INTEL_PT_PTWRITE_IP:
2660                 case INTEL_PT_PTWRITE:
2661                 case INTEL_PT_MWAIT:
2662                 case INTEL_PT_PWRE:
2663                 case INTEL_PT_EXSTOP_IP:
2664                 case INTEL_PT_EXSTOP:
2665                 case INTEL_PT_PWRX:
2666                 case INTEL_PT_BAD: /* Does not happen */
2667                 default:
2668                         break;
2669                 }
2670         }
2671
2672         return err;
2673 }
2674
2675 #define HOP_PROCESS     0
2676 #define HOP_IGNORE      1
2677 #define HOP_RETURN      2
2678 #define HOP_AGAIN       3
2679
2680 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2681
2682 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2683 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2684 {
2685         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2686         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2687                 *err = intel_pt_scan_for_psb(decoder);
2688                 if (*err)
2689                         return HOP_RETURN;
2690         }
2691
2692         switch (decoder->packet.type) {
2693         case INTEL_PT_TNT:
2694                 return HOP_IGNORE;
2695
2696         case INTEL_PT_TIP_PGD:
2697                 decoder->pge = false;
2698                 if (!decoder->packet.count) {
2699                         intel_pt_set_nr(decoder);
2700                         return HOP_IGNORE;
2701                 }
2702                 intel_pt_set_ip(decoder);
2703                 decoder->state.type |= INTEL_PT_TRACE_END;
2704                 decoder->state.from_ip = 0;
2705                 decoder->state.to_ip = decoder->ip;
2706                 intel_pt_update_nr(decoder);
2707                 return HOP_RETURN;
2708
2709         case INTEL_PT_TIP:
2710                 if (!decoder->packet.count) {
2711                         intel_pt_set_nr(decoder);
2712                         return HOP_IGNORE;
2713                 }
2714                 intel_pt_set_ip(decoder);
2715                 decoder->state.type = INTEL_PT_INSTRUCTION;
2716                 decoder->state.from_ip = decoder->ip;
2717                 decoder->state.to_ip = 0;
2718                 intel_pt_update_nr(decoder);
2719                 return HOP_RETURN;
2720
2721         case INTEL_PT_FUP:
2722                 if (!decoder->packet.count)
2723                         return HOP_IGNORE;
2724                 intel_pt_set_ip(decoder);
2725                 if (intel_pt_fup_event(decoder))
2726                         return HOP_RETURN;
2727                 if (!decoder->branch_enable || !decoder->pge)
2728                         *no_tip = true;
2729                 if (*no_tip) {
2730                         decoder->state.type = INTEL_PT_INSTRUCTION;
2731                         decoder->state.from_ip = decoder->ip;
2732                         decoder->state.to_ip = 0;
2733                         return HOP_RETURN;
2734                 }
2735                 *err = intel_pt_walk_fup_tip(decoder);
2736                 if (!*err)
2737                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2738                 return HOP_RETURN;
2739
2740         case INTEL_PT_PSB:
2741                 decoder->state.psb_offset = decoder->pos;
2742                 decoder->psb_ip = 0;
2743                 decoder->last_ip = 0;
2744                 decoder->have_last_ip = true;
2745                 *err = intel_pt_walk_psbend(decoder);
2746                 if (*err == -EAGAIN)
2747                         return HOP_AGAIN;
2748                 if (*err)
2749                         return HOP_RETURN;
2750                 decoder->state.type = INTEL_PT_PSB_EVT;
2751                 if (decoder->psb_ip) {
2752                         decoder->state.type |= INTEL_PT_INSTRUCTION;
2753                         decoder->ip = decoder->psb_ip;
2754                 }
2755                 decoder->state.from_ip = decoder->psb_ip;
2756                 decoder->state.to_ip = 0;
2757                 return HOP_RETURN;
2758
2759         case INTEL_PT_BAD:
2760         case INTEL_PT_PAD:
2761         case INTEL_PT_TIP_PGE:
2762         case INTEL_PT_TSC:
2763         case INTEL_PT_TMA:
2764         case INTEL_PT_MODE_EXEC:
2765         case INTEL_PT_MODE_TSX:
2766         case INTEL_PT_MTC:
2767         case INTEL_PT_CYC:
2768         case INTEL_PT_VMCS:
2769         case INTEL_PT_PSBEND:
2770         case INTEL_PT_CBR:
2771         case INTEL_PT_TRACESTOP:
2772         case INTEL_PT_PIP:
2773         case INTEL_PT_OVF:
2774         case INTEL_PT_MNT:
2775         case INTEL_PT_PTWRITE:
2776         case INTEL_PT_PTWRITE_IP:
2777         case INTEL_PT_EXSTOP:
2778         case INTEL_PT_EXSTOP_IP:
2779         case INTEL_PT_MWAIT:
2780         case INTEL_PT_PWRE:
2781         case INTEL_PT_PWRX:
2782         case INTEL_PT_BBP:
2783         case INTEL_PT_BIP:
2784         case INTEL_PT_BEP:
2785         case INTEL_PT_BEP_IP:
2786         default:
2787                 return HOP_PROCESS;
2788         }
2789 }
2790
2791 struct intel_pt_psb_info {
2792         struct intel_pt_pkt fup_packet;
2793         bool fup;
2794         int after_psbend;
2795 };
2796
2797 /* Lookahead and get the FUP packet from PSB+ */
2798 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2799 {
2800         struct intel_pt_psb_info *data = pkt_info->data;
2801
2802         switch (pkt_info->packet.type) {
2803         case INTEL_PT_PAD:
2804         case INTEL_PT_MNT:
2805         case INTEL_PT_TSC:
2806         case INTEL_PT_TMA:
2807         case INTEL_PT_MODE_EXEC:
2808         case INTEL_PT_MODE_TSX:
2809         case INTEL_PT_MTC:
2810         case INTEL_PT_CYC:
2811         case INTEL_PT_VMCS:
2812         case INTEL_PT_CBR:
2813         case INTEL_PT_PIP:
2814                 if (data->after_psbend) {
2815                         data->after_psbend -= 1;
2816                         if (!data->after_psbend)
2817                                 return 1;
2818                 }
2819                 break;
2820
2821         case INTEL_PT_FUP:
2822                 if (data->after_psbend)
2823                         return 1;
2824                 if (data->fup || pkt_info->packet.count == 0)
2825                         return 1;
2826                 data->fup_packet = pkt_info->packet;
2827                 data->fup = true;
2828                 break;
2829
2830         case INTEL_PT_PSBEND:
2831                 if (!data->fup)
2832                         return 1;
2833                 /* Keep going to check for a TIP.PGE */
2834                 data->after_psbend = 6;
2835                 break;
2836
2837         case INTEL_PT_TIP_PGE:
2838                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
2839                 if (data->after_psbend)
2840                         data->fup = false;
2841                 return 1;
2842
2843         case INTEL_PT_PTWRITE:
2844         case INTEL_PT_PTWRITE_IP:
2845         case INTEL_PT_EXSTOP:
2846         case INTEL_PT_EXSTOP_IP:
2847         case INTEL_PT_MWAIT:
2848         case INTEL_PT_PWRE:
2849         case INTEL_PT_PWRX:
2850         case INTEL_PT_BBP:
2851         case INTEL_PT_BIP:
2852         case INTEL_PT_BEP:
2853         case INTEL_PT_BEP_IP:
2854                 if (data->after_psbend) {
2855                         data->after_psbend -= 1;
2856                         if (!data->after_psbend)
2857                                 return 1;
2858                         break;
2859                 }
2860                 return 1;
2861
2862         case INTEL_PT_OVF:
2863         case INTEL_PT_BAD:
2864         case INTEL_PT_TNT:
2865         case INTEL_PT_TIP_PGD:
2866         case INTEL_PT_TIP:
2867         case INTEL_PT_PSB:
2868         case INTEL_PT_TRACESTOP:
2869         default:
2870                 return 1;
2871         }
2872
2873         return 0;
2874 }
2875
2876 static int intel_pt_psb(struct intel_pt_decoder *decoder)
2877 {
2878         int err;
2879
2880         decoder->last_ip = 0;
2881         decoder->psb_ip = 0;
2882         decoder->have_last_ip = true;
2883         intel_pt_clear_stack(&decoder->stack);
2884         err = intel_pt_walk_psbend(decoder);
2885         if (err)
2886                 return err;
2887         decoder->state.type = INTEL_PT_PSB_EVT;
2888         decoder->state.from_ip = decoder->psb_ip;
2889         decoder->state.to_ip = 0;
2890         return 0;
2891 }
2892
2893 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2894 {
2895         int err;
2896
2897         if (decoder->ip != decoder->last_ip) {
2898                 err = intel_pt_walk_fup(decoder);
2899                 if (!err || err != -EAGAIN)
2900                         return err;
2901         }
2902
2903         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2904         err = intel_pt_psb(decoder);
2905         if (err) {
2906                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2907                 return -ENOENT;
2908         }
2909
2910         return 0;
2911 }
2912
2913 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2914 {
2915         struct intel_pt_psb_info data = { .fup = false };
2916
2917         if (!decoder->branch_enable)
2918                 return false;
2919
2920         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2921         if (!data.fup)
2922                 return false;
2923
2924         decoder->packet = data.fup_packet;
2925         intel_pt_set_last_ip(decoder);
2926         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2927
2928         *err = intel_pt_fup_in_psb(decoder);
2929
2930         return true;
2931 }
2932
2933 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2934 {
2935         int last_packet_type = INTEL_PT_PAD;
2936         bool no_tip = false;
2937         int err;
2938
2939         while (1) {
2940                 err = intel_pt_get_next_packet(decoder);
2941                 if (err)
2942                         return err;
2943 next:
2944                 err = 0;
2945                 if (decoder->cyc_threshold) {
2946                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2947                                 decoder->sample_cyc = false;
2948                         last_packet_type = decoder->packet.type;
2949                 }
2950
2951                 if (decoder->hop) {
2952                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2953                         case HOP_IGNORE:
2954                                 continue;
2955                         case HOP_RETURN:
2956                                 return err;
2957                         case HOP_AGAIN:
2958                                 goto next;
2959                         default:
2960                                 break;
2961                         }
2962                 }
2963
2964                 switch (decoder->packet.type) {
2965                 case INTEL_PT_TNT:
2966                         if (!decoder->packet.count)
2967                                 break;
2968                         decoder->tnt = decoder->packet;
2969                         decoder->pkt_state = INTEL_PT_STATE_TNT;
2970                         err = intel_pt_walk_tnt(decoder);
2971                         if (err == -EAGAIN)
2972                                 break;
2973                         return err;
2974
2975                 case INTEL_PT_TIP_PGD:
2976                         if (decoder->packet.count != 0)
2977                                 intel_pt_set_last_ip(decoder);
2978                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2979                         return intel_pt_walk_tip(decoder);
2980
2981                 case INTEL_PT_TIP_PGE: {
2982                         decoder->pge = true;
2983                         decoder->overflow = false;
2984                         intel_pt_mtc_cyc_cnt_pge(decoder);
2985                         intel_pt_set_nr(decoder);
2986                         if (decoder->packet.count == 0) {
2987                                 intel_pt_log_at("Skipping zero TIP.PGE",
2988                                                 decoder->pos);
2989                                 break;
2990                         }
2991                         intel_pt_set_ip(decoder);
2992                         decoder->state.from_ip = 0;
2993                         decoder->state.to_ip = decoder->ip;
2994                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2995                         /*
2996                          * In hop mode, resample to get the to_ip as an
2997                          * "instruction" sample.
2998                          */
2999                         if (decoder->hop)
3000                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3001                         return 0;
3002                 }
3003
3004                 case INTEL_PT_OVF:
3005                         return intel_pt_overflow(decoder);
3006
3007                 case INTEL_PT_TIP:
3008                         if (decoder->packet.count != 0)
3009                                 intel_pt_set_last_ip(decoder);
3010                         decoder->pkt_state = INTEL_PT_STATE_TIP;
3011                         return intel_pt_walk_tip(decoder);
3012
3013                 case INTEL_PT_FUP:
3014                         if (decoder->packet.count == 0) {
3015                                 intel_pt_log_at("Skipping zero FUP",
3016                                                 decoder->pos);
3017                                 no_tip = false;
3018                                 break;
3019                         }
3020                         intel_pt_set_last_ip(decoder);
3021                         if (!decoder->branch_enable || !decoder->pge) {
3022                                 decoder->ip = decoder->last_ip;
3023                                 if (intel_pt_fup_event(decoder))
3024                                         return 0;
3025                                 no_tip = false;
3026                                 break;
3027                         }
3028                         if (decoder->set_fup_mwait)
3029                                 no_tip = true;
3030                         if (no_tip)
3031                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3032                         else
3033                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3034                         err = intel_pt_walk_fup(decoder);
3035                         if (err != -EAGAIN)
3036                                 return err;
3037                         if (no_tip) {
3038                                 no_tip = false;
3039                                 break;
3040                         }
3041                         return intel_pt_walk_fup_tip(decoder);
3042
3043                 case INTEL_PT_TRACESTOP:
3044                         decoder->pge = false;
3045                         decoder->continuous_period = false;
3046                         intel_pt_clear_tx_flags(decoder);
3047                         decoder->have_tma = false;
3048                         break;
3049
3050                 case INTEL_PT_PSB:
3051                         decoder->state.psb_offset = decoder->pos;
3052                         decoder->psb_ip = 0;
3053                         if (intel_pt_psb_with_fup(decoder, &err))
3054                                 return err;
3055                         err = intel_pt_psb(decoder);
3056                         if (err == -EAGAIN)
3057                                 goto next;
3058                         return err;
3059
3060                 case INTEL_PT_PIP:
3061                         intel_pt_update_pip(decoder);
3062                         break;
3063
3064                 case INTEL_PT_MTC:
3065                         intel_pt_calc_mtc_timestamp(decoder);
3066                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3067                                 break;
3068                         /*
3069                          * Ensure that there has been an instruction since the
3070                          * last MTC.
3071                          */
3072                         if (!decoder->mtc_insn)
3073                                 break;
3074                         decoder->mtc_insn = false;
3075                         /* Ensure that there is a timestamp */
3076                         if (!decoder->timestamp)
3077                                 break;
3078                         decoder->state.type = INTEL_PT_INSTRUCTION;
3079                         decoder->state.from_ip = decoder->ip;
3080                         decoder->state.to_ip = 0;
3081                         decoder->mtc_insn = false;
3082                         return 0;
3083
3084                 case INTEL_PT_TSC:
3085                         intel_pt_calc_tsc_timestamp(decoder);
3086                         break;
3087
3088                 case INTEL_PT_TMA:
3089                         intel_pt_calc_tma(decoder);
3090                         break;
3091
3092                 case INTEL_PT_CYC:
3093                         intel_pt_calc_cyc_timestamp(decoder);
3094                         break;
3095
3096                 case INTEL_PT_CBR:
3097                         intel_pt_calc_cbr(decoder);
3098                         if (decoder->cbr != decoder->cbr_seen) {
3099                                 decoder->state.type = 0;
3100                                 return 0;
3101                         }
3102                         break;
3103
3104                 case INTEL_PT_MODE_EXEC:
3105                         decoder->exec_mode = decoder->packet.payload;
3106                         break;
3107
3108                 case INTEL_PT_MODE_TSX:
3109                         /* MODE_TSX need not be followed by FUP */
3110                         if (!decoder->pge || decoder->in_psb) {
3111                                 intel_pt_update_in_tx(decoder);
3112                                 break;
3113                         }
3114                         err = intel_pt_mode_tsx(decoder, &no_tip);
3115                         if (err)
3116                                 return err;
3117                         goto next;
3118
3119                 case INTEL_PT_BAD: /* Does not happen */
3120                         return intel_pt_bug(decoder);
3121
3122                 case INTEL_PT_PSBEND:
3123                 case INTEL_PT_VMCS:
3124                 case INTEL_PT_MNT:
3125                 case INTEL_PT_PAD:
3126                         break;
3127
3128                 case INTEL_PT_PTWRITE_IP:
3129                         decoder->fup_ptw_payload = decoder->packet.payload;
3130                         err = intel_pt_get_next_packet(decoder);
3131                         if (err)
3132                                 return err;
3133                         if (decoder->packet.type == INTEL_PT_FUP) {
3134                                 decoder->set_fup_ptw = true;
3135                                 no_tip = true;
3136                         } else {
3137                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3138                                                 decoder->pos);
3139                         }
3140                         goto next;
3141
3142                 case INTEL_PT_PTWRITE:
3143                         decoder->state.type = INTEL_PT_PTW;
3144                         decoder->state.from_ip = decoder->ip;
3145                         decoder->state.to_ip = 0;
3146                         decoder->state.ptw_payload = decoder->packet.payload;
3147                         return 0;
3148
3149                 case INTEL_PT_MWAIT:
3150                         decoder->fup_mwait_payload = decoder->packet.payload;
3151                         decoder->set_fup_mwait = true;
3152                         break;
3153
3154                 case INTEL_PT_PWRE:
3155                         if (decoder->set_fup_mwait) {
3156                                 decoder->fup_pwre_payload =
3157                                                         decoder->packet.payload;
3158                                 decoder->set_fup_pwre = true;
3159                                 break;
3160                         }
3161                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3162                         decoder->state.from_ip = decoder->ip;
3163                         decoder->state.to_ip = 0;
3164                         decoder->state.pwrx_payload = decoder->packet.payload;
3165                         return 0;
3166
3167                 case INTEL_PT_EXSTOP_IP:
3168                         err = intel_pt_get_next_packet(decoder);
3169                         if (err)
3170                                 return err;
3171                         if (decoder->packet.type == INTEL_PT_FUP) {
3172                                 decoder->set_fup_exstop = true;
3173                                 no_tip = true;
3174                         } else {
3175                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3176                                                 decoder->pos);
3177                         }
3178                         goto next;
3179
3180                 case INTEL_PT_EXSTOP:
3181                         decoder->state.type = INTEL_PT_EX_STOP;
3182                         decoder->state.from_ip = decoder->ip;
3183                         decoder->state.to_ip = 0;
3184                         return 0;
3185
3186                 case INTEL_PT_PWRX:
3187                         decoder->state.type = INTEL_PT_PWR_EXIT;
3188                         decoder->state.from_ip = decoder->ip;
3189                         decoder->state.to_ip = 0;
3190                         decoder->state.pwrx_payload = decoder->packet.payload;
3191                         return 0;
3192
3193                 case INTEL_PT_BBP:
3194                         intel_pt_bbp(decoder);
3195                         break;
3196
3197                 case INTEL_PT_BIP:
3198                         intel_pt_bip(decoder);
3199                         break;
3200
3201                 case INTEL_PT_BEP:
3202                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3203                         decoder->state.from_ip = decoder->ip;
3204                         decoder->state.to_ip = 0;
3205                         return 0;
3206
3207                 case INTEL_PT_BEP_IP:
3208                         err = intel_pt_get_next_packet(decoder);
3209                         if (err)
3210                                 return err;
3211                         if (decoder->packet.type == INTEL_PT_FUP) {
3212                                 decoder->set_fup_bep = true;
3213                                 no_tip = true;
3214                         } else {
3215                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3216                                                 decoder->pos);
3217                         }
3218                         goto next;
3219
3220                 default:
3221                         return intel_pt_bug(decoder);
3222                 }
3223         }
3224 }
3225
3226 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3227 {
3228         return decoder->packet.count &&
3229                (decoder->have_last_ip || decoder->packet.count == 3 ||
3230                 decoder->packet.count == 6);
3231 }
3232
3233 /* Walk PSB+ packets to get in sync. */
3234 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3235 {
3236         int err;
3237
3238         decoder->in_psb = true;
3239
3240         while (1) {
3241                 err = intel_pt_get_next_packet(decoder);
3242                 if (err)
3243                         goto out;
3244
3245                 switch (decoder->packet.type) {
3246                 case INTEL_PT_TIP_PGD:
3247                         decoder->continuous_period = false;
3248                         __fallthrough;
3249                 case INTEL_PT_TIP_PGE:
3250                 case INTEL_PT_TIP:
3251                 case INTEL_PT_PTWRITE:
3252                 case INTEL_PT_PTWRITE_IP:
3253                 case INTEL_PT_EXSTOP:
3254                 case INTEL_PT_EXSTOP_IP:
3255                 case INTEL_PT_MWAIT:
3256                 case INTEL_PT_PWRE:
3257                 case INTEL_PT_PWRX:
3258                 case INTEL_PT_BBP:
3259                 case INTEL_PT_BIP:
3260                 case INTEL_PT_BEP:
3261                 case INTEL_PT_BEP_IP:
3262                         intel_pt_log("ERROR: Unexpected packet\n");
3263                         err = -ENOENT;
3264                         goto out;
3265
3266                 case INTEL_PT_FUP:
3267                         decoder->pge = true;
3268                         if (intel_pt_have_ip(decoder)) {
3269                                 uint64_t current_ip = decoder->ip;
3270
3271                                 intel_pt_set_ip(decoder);
3272                                 decoder->psb_ip = decoder->ip;
3273                                 if (current_ip)
3274                                         intel_pt_log_to("Setting IP",
3275                                                         decoder->ip);
3276                         }
3277                         break;
3278
3279                 case INTEL_PT_MTC:
3280                         intel_pt_calc_mtc_timestamp(decoder);
3281                         break;
3282
3283                 case INTEL_PT_TSC:
3284                         intel_pt_calc_tsc_timestamp(decoder);
3285                         break;
3286
3287                 case INTEL_PT_TMA:
3288                         intel_pt_calc_tma(decoder);
3289                         break;
3290
3291                 case INTEL_PT_CYC:
3292                         intel_pt_calc_cyc_timestamp(decoder);
3293                         break;
3294
3295                 case INTEL_PT_CBR:
3296                         intel_pt_calc_cbr(decoder);
3297                         break;
3298
3299                 case INTEL_PT_PIP:
3300                         intel_pt_set_pip(decoder);
3301                         break;
3302
3303                 case INTEL_PT_MODE_EXEC:
3304                         decoder->exec_mode = decoder->packet.payload;
3305                         break;
3306
3307                 case INTEL_PT_MODE_TSX:
3308                         intel_pt_update_in_tx(decoder);
3309                         break;
3310
3311                 case INTEL_PT_TRACESTOP:
3312                         decoder->pge = false;
3313                         decoder->continuous_period = false;
3314                         intel_pt_clear_tx_flags(decoder);
3315                         __fallthrough;
3316
3317                 case INTEL_PT_TNT:
3318                         decoder->have_tma = false;
3319                         intel_pt_log("ERROR: Unexpected packet\n");
3320                         if (decoder->ip)
3321                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3322                         else
3323                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3324                         err = -ENOENT;
3325                         goto out;
3326
3327                 case INTEL_PT_BAD: /* Does not happen */
3328                         err = intel_pt_bug(decoder);
3329                         goto out;
3330
3331                 case INTEL_PT_OVF:
3332                         err = intel_pt_overflow(decoder);
3333                         goto out;
3334
3335                 case INTEL_PT_PSBEND:
3336                         err = 0;
3337                         goto out;
3338
3339                 case INTEL_PT_PSB:
3340                 case INTEL_PT_VMCS:
3341                 case INTEL_PT_MNT:
3342                 case INTEL_PT_PAD:
3343                 default:
3344                         break;
3345                 }
3346         }
3347 out:
3348         decoder->in_psb = false;
3349
3350         return err;
3351 }
3352
3353 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3354 {
3355         int err;
3356
3357         while (1) {
3358                 err = intel_pt_get_next_packet(decoder);
3359                 if (err)
3360                         return err;
3361
3362                 switch (decoder->packet.type) {
3363                 case INTEL_PT_TIP_PGD:
3364                         decoder->continuous_period = false;
3365                         decoder->pge = false;
3366                         if (intel_pt_have_ip(decoder))
3367                                 intel_pt_set_ip(decoder);
3368                         if (!decoder->ip)
3369                                 break;
3370                         decoder->state.type |= INTEL_PT_TRACE_END;
3371                         return 0;
3372
3373                 case INTEL_PT_TIP_PGE:
3374                         decoder->pge = true;
3375                         intel_pt_mtc_cyc_cnt_pge(decoder);
3376                         if (intel_pt_have_ip(decoder))
3377                                 intel_pt_set_ip(decoder);
3378                         if (!decoder->ip)
3379                                 break;
3380                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3381                         return 0;
3382
3383                 case INTEL_PT_TIP:
3384                         decoder->pge = true;
3385                         if (intel_pt_have_ip(decoder))
3386                                 intel_pt_set_ip(decoder);
3387                         if (!decoder->ip)
3388                                 break;
3389                         return 0;
3390
3391                 case INTEL_PT_FUP:
3392                         if (intel_pt_have_ip(decoder))
3393                                 intel_pt_set_ip(decoder);
3394                         if (decoder->ip)
3395                                 return 0;
3396                         break;
3397
3398                 case INTEL_PT_MTC:
3399                         intel_pt_calc_mtc_timestamp(decoder);
3400                         break;
3401
3402                 case INTEL_PT_TSC:
3403                         intel_pt_calc_tsc_timestamp(decoder);
3404                         break;
3405
3406                 case INTEL_PT_TMA:
3407                         intel_pt_calc_tma(decoder);
3408                         break;
3409
3410                 case INTEL_PT_CYC:
3411                         intel_pt_calc_cyc_timestamp(decoder);
3412                         break;
3413
3414                 case INTEL_PT_CBR:
3415                         intel_pt_calc_cbr(decoder);
3416                         break;
3417
3418                 case INTEL_PT_PIP:
3419                         intel_pt_set_pip(decoder);
3420                         break;
3421
3422                 case INTEL_PT_MODE_EXEC:
3423                         decoder->exec_mode = decoder->packet.payload;
3424                         break;
3425
3426                 case INTEL_PT_MODE_TSX:
3427                         intel_pt_update_in_tx(decoder);
3428                         break;
3429
3430                 case INTEL_PT_OVF:
3431                         return intel_pt_overflow(decoder);
3432
3433                 case INTEL_PT_BAD: /* Does not happen */
3434                         return intel_pt_bug(decoder);
3435
3436                 case INTEL_PT_TRACESTOP:
3437                         decoder->pge = false;
3438                         decoder->continuous_period = false;
3439                         intel_pt_clear_tx_flags(decoder);
3440                         decoder->have_tma = false;
3441                         break;
3442
3443                 case INTEL_PT_PSB:
3444                         decoder->state.psb_offset = decoder->pos;
3445                         decoder->psb_ip = 0;
3446                         decoder->last_ip = 0;
3447                         decoder->have_last_ip = true;
3448                         intel_pt_clear_stack(&decoder->stack);
3449                         err = intel_pt_walk_psb(decoder);
3450                         if (err)
3451                                 return err;
3452                         decoder->state.type = INTEL_PT_PSB_EVT;
3453                         decoder->state.from_ip = decoder->psb_ip;
3454                         decoder->state.to_ip = 0;
3455                         return 0;
3456
3457                 case INTEL_PT_TNT:
3458                 case INTEL_PT_PSBEND:
3459                 case INTEL_PT_VMCS:
3460                 case INTEL_PT_MNT:
3461                 case INTEL_PT_PAD:
3462                 case INTEL_PT_PTWRITE:
3463                 case INTEL_PT_PTWRITE_IP:
3464                 case INTEL_PT_EXSTOP:
3465                 case INTEL_PT_EXSTOP_IP:
3466                 case INTEL_PT_MWAIT:
3467                 case INTEL_PT_PWRE:
3468                 case INTEL_PT_PWRX:
3469                 case INTEL_PT_BBP:
3470                 case INTEL_PT_BIP:
3471                 case INTEL_PT_BEP:
3472                 case INTEL_PT_BEP_IP:
3473                 default:
3474                         break;
3475                 }
3476         }
3477 }
3478
3479 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3480 {
3481         int err;
3482
3483         decoder->set_fup_tx_flags = false;
3484         decoder->set_fup_ptw = false;
3485         decoder->set_fup_mwait = false;
3486         decoder->set_fup_pwre = false;
3487         decoder->set_fup_exstop = false;
3488         decoder->set_fup_bep = false;
3489         decoder->overflow = false;
3490
3491         if (!decoder->branch_enable) {
3492                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3493                 decoder->state.type = 0; /* Do not have a sample */
3494                 return 0;
3495         }
3496
3497         intel_pt_log("Scanning for full IP\n");
3498         err = intel_pt_walk_to_ip(decoder);
3499         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3500                 return err;
3501
3502         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3503         if (decoder->hop)
3504                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3505         else
3506                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3507
3508         decoder->state.from_ip = 0;
3509         decoder->state.to_ip = decoder->ip;
3510         intel_pt_log_to("Setting IP", decoder->ip);
3511
3512         return 0;
3513 }
3514
3515 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3516 {
3517         const unsigned char *end = decoder->buf + decoder->len;
3518         size_t i;
3519
3520         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3521                 if (i > decoder->len)
3522                         continue;
3523                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3524                         return i;
3525         }
3526         return 0;
3527 }
3528
3529 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3530 {
3531         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3532         const char *psb = INTEL_PT_PSB_STR;
3533
3534         if (rest_psb > decoder->len ||
3535             memcmp(decoder->buf, psb + part_psb, rest_psb))
3536                 return 0;
3537
3538         return rest_psb;
3539 }
3540
3541 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3542                                   int part_psb)
3543 {
3544         int rest_psb, ret;
3545
3546         decoder->pos += decoder->len;
3547         decoder->len = 0;
3548
3549         ret = intel_pt_get_next_data(decoder, false);
3550         if (ret)
3551                 return ret;
3552
3553         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3554         if (!rest_psb)
3555                 return 0;
3556
3557         decoder->pos -= part_psb;
3558         decoder->next_buf = decoder->buf + rest_psb;
3559         decoder->next_len = decoder->len - rest_psb;
3560         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3561         decoder->buf = decoder->temp_buf;
3562         decoder->len = INTEL_PT_PSB_LEN;
3563
3564         return 0;
3565 }
3566
3567 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3568 {
3569         unsigned char *next;
3570         int ret;
3571
3572         intel_pt_log("Scanning for PSB\n");
3573         while (1) {
3574                 if (!decoder->len) {
3575                         ret = intel_pt_get_next_data(decoder, false);
3576                         if (ret)
3577                                 return ret;
3578                 }
3579
3580                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3581                               INTEL_PT_PSB_LEN);
3582                 if (!next) {
3583                         int part_psb;
3584
3585                         part_psb = intel_pt_part_psb(decoder);
3586                         if (part_psb) {
3587                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3588                                 if (ret)
3589                                         return ret;
3590                         } else {
3591                                 decoder->pos += decoder->len;
3592                                 decoder->len = 0;
3593                         }
3594                         continue;
3595                 }
3596
3597                 decoder->pkt_step = next - decoder->buf;
3598                 return intel_pt_get_next_packet(decoder);
3599         }
3600 }
3601
3602 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3603 {
3604         int err;
3605
3606         decoder->pge = false;
3607         decoder->continuous_period = false;
3608         decoder->have_last_ip = false;
3609         decoder->last_ip = 0;
3610         decoder->psb_ip = 0;
3611         decoder->ip = 0;
3612         intel_pt_clear_stack(&decoder->stack);
3613
3614         err = intel_pt_scan_for_psb(decoder);
3615         if (err)
3616                 return err;
3617
3618         if (decoder->vm_time_correlation) {
3619                 decoder->in_psb = true;
3620                 if (!decoder->timestamp)
3621                         decoder->timestamp = 1;
3622                 decoder->state.type = 0;
3623                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3624                 return 0;
3625         }
3626
3627         decoder->have_last_ip = true;
3628         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3629
3630         err = intel_pt_walk_psb(decoder);
3631         if (err)
3632                 return err;
3633
3634         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3635         decoder->state.from_ip = decoder->psb_ip;
3636         decoder->state.to_ip = 0;
3637
3638         if (decoder->ip) {
3639                 /*
3640                  * In hop mode, resample to get the PSB FUP ip as an
3641                  * "instruction" sample.
3642                  */
3643                 if (decoder->hop)
3644                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3645                 else
3646                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3647         }
3648
3649         return 0;
3650 }
3651
3652 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3653 {
3654         uint64_t est = decoder->sample_insn_cnt << 1;
3655
3656         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3657                 goto out;
3658
3659         est *= decoder->max_non_turbo_ratio;
3660         est /= decoder->cbr;
3661 out:
3662         return decoder->sample_timestamp + est;
3663 }
3664
3665 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3666 {
3667         int err;
3668
3669         do {
3670                 decoder->state.type = INTEL_PT_BRANCH;
3671                 decoder->state.flags = 0;
3672
3673                 switch (decoder->pkt_state) {
3674                 case INTEL_PT_STATE_NO_PSB:
3675                         err = intel_pt_sync(decoder);
3676                         break;
3677                 case INTEL_PT_STATE_NO_IP:
3678                         decoder->have_last_ip = false;
3679                         decoder->last_ip = 0;
3680                         decoder->ip = 0;
3681                         __fallthrough;
3682                 case INTEL_PT_STATE_ERR_RESYNC:
3683                         err = intel_pt_sync_ip(decoder);
3684                         break;
3685                 case INTEL_PT_STATE_IN_SYNC:
3686                         err = intel_pt_walk_trace(decoder);
3687                         break;
3688                 case INTEL_PT_STATE_TNT:
3689                 case INTEL_PT_STATE_TNT_CONT:
3690                         err = intel_pt_walk_tnt(decoder);
3691                         if (err == -EAGAIN)
3692                                 err = intel_pt_walk_trace(decoder);
3693                         break;
3694                 case INTEL_PT_STATE_TIP:
3695                 case INTEL_PT_STATE_TIP_PGD:
3696                         err = intel_pt_walk_tip(decoder);
3697                         break;
3698                 case INTEL_PT_STATE_FUP:
3699                         err = intel_pt_walk_fup(decoder);
3700                         if (err == -EAGAIN)
3701                                 err = intel_pt_walk_fup_tip(decoder);
3702                         break;
3703                 case INTEL_PT_STATE_FUP_NO_TIP:
3704                         err = intel_pt_walk_fup(decoder);
3705                         if (err == -EAGAIN)
3706                                 err = intel_pt_walk_trace(decoder);
3707                         break;
3708                 case INTEL_PT_STATE_FUP_IN_PSB:
3709                         err = intel_pt_fup_in_psb(decoder);
3710                         break;
3711                 case INTEL_PT_STATE_RESAMPLE:
3712                         err = intel_pt_resample(decoder);
3713                         break;
3714                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
3715                         err = intel_pt_vm_time_correlation(decoder);
3716                         break;
3717                 default:
3718                         err = intel_pt_bug(decoder);
3719                         break;
3720                 }
3721         } while (err == -ENOLINK);
3722
3723         if (err) {
3724                 decoder->state.err = intel_pt_ext_err(err);
3725                 if (err != -EOVERFLOW)
3726                         decoder->state.from_ip = decoder->ip;
3727                 intel_pt_update_sample_time(decoder);
3728                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3729                 intel_pt_set_nr(decoder);
3730         } else {
3731                 decoder->state.err = 0;
3732                 if (decoder->cbr != decoder->cbr_seen) {
3733                         decoder->cbr_seen = decoder->cbr;
3734                         if (!decoder->state.type) {
3735                                 decoder->state.from_ip = decoder->ip;
3736                                 decoder->state.to_ip = 0;
3737                         }
3738                         decoder->state.type |= INTEL_PT_CBR_CHG;
3739                         decoder->state.cbr_payload = decoder->cbr_payload;
3740                         decoder->state.cbr = decoder->cbr;
3741                 }
3742                 if (intel_pt_sample_time(decoder->pkt_state)) {
3743                         intel_pt_update_sample_time(decoder);
3744                         if (decoder->sample_cyc) {
3745                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3746                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3747                                 decoder->sample_cyc = false;
3748                         }
3749                 }
3750                 /*
3751                  * When using only TSC/MTC to compute cycles, IPC can be
3752                  * sampled as soon as the cycle count changes.
3753                  */
3754                 if (!decoder->have_cyc)
3755                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3756         }
3757
3758          /* Let PSB event always have TSC timestamp */
3759         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3760                 decoder->sample_timestamp = decoder->tsc_timestamp;
3761
3762         decoder->state.from_nr = decoder->nr;
3763         decoder->state.to_nr = decoder->next_nr;
3764         decoder->nr = decoder->next_nr;
3765
3766         decoder->state.timestamp = decoder->sample_timestamp;
3767         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3768         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3769         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3770
3771         return &decoder->state;
3772 }
3773
3774 /**
3775  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
3776  * @buf: pointer to buffer pointer
3777  * @len: size of buffer
3778  *
3779  * Updates the buffer pointer to point to the start of the next PSB packet if
3780  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
3781  * @len is adjusted accordingly.
3782  *
3783  * Return: %true if a PSB packet is found, %false otherwise.
3784  */
3785 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3786 {
3787         unsigned char *next;
3788
3789         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3790         if (next) {
3791                 *len -= next - *buf;
3792                 *buf = next;
3793                 return true;
3794         }
3795         return false;
3796 }
3797
3798 /**
3799  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
3800  *                     packet.
3801  * @buf: pointer to buffer pointer
3802  * @len: size of buffer
3803  *
3804  * Updates the buffer pointer to point to the start of the following PSB packet
3805  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
3806  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
3807  *
3808  * Return: %true if a PSB packet is found, %false otherwise.
3809  */
3810 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3811 {
3812         unsigned char *next;
3813
3814         if (!*len)
3815                 return false;
3816
3817         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3818         if (next) {
3819                 *len -= next - *buf;
3820                 *buf = next;
3821                 return true;
3822         }
3823         return false;
3824 }
3825
3826 /**
3827  * intel_pt_last_psb - find the last PSB packet in a buffer.
3828  * @buf: buffer
3829  * @len: size of buffer
3830  *
3831  * This function finds the last PSB in a buffer.
3832  *
3833  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
3834  */
3835 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3836 {
3837         const char *n = INTEL_PT_PSB_STR;
3838         unsigned char *p;
3839         size_t k;
3840
3841         if (len < INTEL_PT_PSB_LEN)
3842                 return NULL;
3843
3844         k = len - INTEL_PT_PSB_LEN + 1;
3845         while (1) {
3846                 p = memrchr(buf, n[0], k);
3847                 if (!p)
3848                         return NULL;
3849                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3850                         return p;
3851                 k = p - buf;
3852                 if (!k)
3853                         return NULL;
3854         }
3855 }
3856
3857 /**
3858  * intel_pt_next_tsc - find and return next TSC.
3859  * @buf: buffer
3860  * @len: size of buffer
3861  * @tsc: TSC value returned
3862  * @rem: returns remaining size when TSC is found
3863  *
3864  * Find a TSC packet in @buf and return the TSC value.  This function assumes
3865  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
3866  * PSBEND packet is found.
3867  *
3868  * Return: %true if TSC is found, false otherwise.
3869  */
3870 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3871                               size_t *rem)
3872 {
3873         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3874         struct intel_pt_pkt packet;
3875         int ret;
3876
3877         while (len) {
3878                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3879                 if (ret <= 0)
3880                         return false;
3881                 if (packet.type == INTEL_PT_TSC) {
3882                         *tsc = packet.payload;
3883                         *rem = len;
3884                         return true;
3885                 }
3886                 if (packet.type == INTEL_PT_PSBEND)
3887                         return false;
3888                 buf += ret;
3889                 len -= ret;
3890         }
3891         return false;
3892 }
3893
3894 /**
3895  * intel_pt_tsc_cmp - compare 7-byte TSCs.
3896  * @tsc1: first TSC to compare
3897  * @tsc2: second TSC to compare
3898  *
3899  * This function compares 7-byte TSC values allowing for the possibility that
3900  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
3901  * around so for that purpose this function assumes the absolute difference is
3902  * less than half the maximum difference.
3903  *
3904  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
3905  * after @tsc2.
3906  */
3907 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3908 {
3909         const uint64_t halfway = (1ULL << 55);
3910
3911         if (tsc1 == tsc2)
3912                 return 0;
3913
3914         if (tsc1 < tsc2) {
3915                 if (tsc2 - tsc1 < halfway)
3916                         return -1;
3917                 else
3918                         return 1;
3919         } else {
3920                 if (tsc1 - tsc2 < halfway)
3921                         return 1;
3922                 else
3923                         return -1;
3924         }
3925 }
3926
3927 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3928
3929 /**
3930  * adj_for_padding - adjust overlap to account for padding.
3931  * @buf_b: second buffer
3932  * @buf_a: first buffer
3933  * @len_a: size of first buffer
3934  *
3935  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
3936  * accordingly.
3937  *
3938  * Return: A pointer into @buf_b from where non-overlapped data starts
3939  */
3940 static unsigned char *adj_for_padding(unsigned char *buf_b,
3941                                       unsigned char *buf_a, size_t len_a)
3942 {
3943         unsigned char *p = buf_b - MAX_PADDING;
3944         unsigned char *q = buf_a + len_a - MAX_PADDING;
3945         int i;
3946
3947         for (i = MAX_PADDING; i; i--, p++, q++) {
3948                 if (*p != *q)
3949                         break;
3950         }
3951
3952         return p;
3953 }
3954
3955 /**
3956  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
3957  *                             using TSC.
3958  * @buf_a: first buffer
3959  * @len_a: size of first buffer
3960  * @buf_b: second buffer
3961  * @len_b: size of second buffer
3962  * @consecutive: returns true if there is data in buf_b that is consecutive
3963  *               to buf_a
3964  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
3965  *
3966  * If the trace contains TSC we can look at the last TSC of @buf_a and the
3967  * first TSC of @buf_b in order to determine if the buffers overlap, and then
3968  * walk forward in @buf_b until a later TSC is found.  A precondition is that
3969  * @buf_a and @buf_b are positioned at a PSB.
3970  *
3971  * Return: A pointer into @buf_b from where non-overlapped data starts, or
3972  * @buf_b + @len_b if there is no non-overlapped data.
3973  */
3974 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3975                                                 size_t len_a,
3976                                                 unsigned char *buf_b,
3977                                                 size_t len_b, bool *consecutive,
3978                                                 bool ooo_tsc)
3979 {
3980         uint64_t tsc_a, tsc_b;
3981         unsigned char *p;
3982         size_t len, rem_a, rem_b;
3983
3984         p = intel_pt_last_psb(buf_a, len_a);
3985         if (!p)
3986                 return buf_b; /* No PSB in buf_a => no overlap */
3987
3988         len = len_a - (p - buf_a);
3989         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3990                 /* The last PSB+ in buf_a is incomplete, so go back one more */
3991                 len_a -= len;
3992                 p = intel_pt_last_psb(buf_a, len_a);
3993                 if (!p)
3994                         return buf_b; /* No full PSB+ => assume no overlap */
3995                 len = len_a - (p - buf_a);
3996                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3997                         return buf_b; /* No TSC in buf_a => assume no overlap */
3998         }
3999
4000         while (1) {
4001                 /* Ignore PSB+ with no TSC */
4002                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4003                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4004
4005                         /* Same TSC, so buffers are consecutive */
4006                         if (!cmp && rem_b >= rem_a) {
4007                                 unsigned char *start;
4008
4009                                 *consecutive = true;
4010                                 start = buf_b + len_b - (rem_b - rem_a);
4011                                 return adj_for_padding(start, buf_a, len_a);
4012                         }
4013                         if (cmp < 0 && !ooo_tsc)
4014                                 return buf_b; /* tsc_a < tsc_b => no overlap */
4015                 }
4016
4017                 if (!intel_pt_step_psb(&buf_b, &len_b))
4018                         return buf_b + len_b; /* No PSB in buf_b => no data */
4019         }
4020 }
4021
4022 /**
4023  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4024  * @buf_a: first buffer
4025  * @len_a: size of first buffer
4026  * @buf_b: second buffer
4027  * @len_b: size of second buffer
4028  * @have_tsc: can use TSC packets to detect overlap
4029  * @consecutive: returns true if there is data in buf_b that is consecutive
4030  *               to buf_a
4031  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4032  *
4033  * When trace samples or snapshots are recorded there is the possibility that
4034  * the data overlaps.  Note that, for the purposes of decoding, data is only
4035  * useful if it begins with a PSB packet.
4036  *
4037  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4038  * @buf_b + @len_b if there is no non-overlapped data.
4039  */
4040 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4041                                      unsigned char *buf_b, size_t len_b,
4042                                      bool have_tsc, bool *consecutive,
4043                                      bool ooo_tsc)
4044 {
4045         unsigned char *found;
4046
4047         /* Buffer 'b' must start at PSB so throw away everything before that */
4048         if (!intel_pt_next_psb(&buf_b, &len_b))
4049                 return buf_b + len_b; /* No PSB */
4050
4051         if (!intel_pt_next_psb(&buf_a, &len_a))
4052                 return buf_b; /* No overlap */
4053
4054         if (have_tsc) {
4055                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4056                                                   consecutive, ooo_tsc);
4057                 if (found)
4058                         return found;
4059         }
4060
4061         /*
4062          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4063          * we can ignore the first part of buffer 'a'.
4064          */
4065         while (len_b < len_a) {
4066                 if (!intel_pt_step_psb(&buf_a, &len_a))
4067                         return buf_b; /* No overlap */
4068         }
4069
4070         /* Now len_b >= len_a */
4071         while (1) {
4072                 /* Potential overlap so check the bytes */
4073                 found = memmem(buf_a, len_a, buf_b, len_a);
4074                 if (found) {
4075                         *consecutive = true;
4076                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4077                 }
4078
4079                 /* Try again at next PSB in buffer 'a' */
4080                 if (!intel_pt_step_psb(&buf_a, &len_a))
4081                         return buf_b; /* No overlap */
4082         }
4083 }
4084
4085 /**
4086  * struct fast_forward_data - data used by intel_pt_ff_cb().
4087  * @timestamp: timestamp to fast forward towards
4088  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4089  *                 the fast forward timestamp.
4090  */
4091 struct fast_forward_data {
4092         uint64_t timestamp;
4093         uint64_t buf_timestamp;
4094 };
4095
4096 /**
4097  * intel_pt_ff_cb - fast forward lookahead callback.
4098  * @buffer: Intel PT trace buffer
4099  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4100  *
4101  * Determine if @buffer trace is past the fast forward timestamp.
4102  *
4103  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4104  *         timestamp, and 0 otherwise.
4105  */
4106 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4107 {
4108         struct fast_forward_data *d = data;
4109         unsigned char *buf;
4110         uint64_t tsc;
4111         size_t rem;
4112         size_t len;
4113
4114         buf = (unsigned char *)buffer->buf;
4115         len = buffer->len;
4116
4117         if (!intel_pt_next_psb(&buf, &len) ||
4118             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4119                 return 0;
4120
4121         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4122
4123         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4124                      tsc, buffer->ref_timestamp);
4125
4126         /*
4127          * If the buffer contains a timestamp earlier that the fast forward
4128          * timestamp, then record it, else stop.
4129          */
4130         if (tsc < d->timestamp)
4131                 d->buf_timestamp = buffer->ref_timestamp;
4132         else
4133                 return 1;
4134
4135         return 0;
4136 }
4137
4138 /**
4139  * intel_pt_fast_forward - reposition decoder forwards.
4140  * @decoder: Intel PT decoder
4141  * @timestamp: timestamp to fast forward towards
4142  *
4143  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4144  *
4145  * Return: 0 on success or negative error code on failure.
4146  */
4147 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4148 {
4149         struct fast_forward_data d = { .timestamp = timestamp };
4150         unsigned char *buf;
4151         size_t len;
4152         int err;
4153
4154         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4155
4156         /* Find buffer timestamp of buffer to fast forward to */
4157         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4158         if (err < 0)
4159                 return err;
4160
4161         /* Walk to buffer with same buffer timestamp */
4162         if (d.buf_timestamp) {
4163                 do {
4164                         decoder->pos += decoder->len;
4165                         decoder->len = 0;
4166                         err = intel_pt_get_next_data(decoder, true);
4167                         /* -ENOLINK means non-consecutive trace */
4168                         if (err && err != -ENOLINK)
4169                                 return err;
4170                 } while (decoder->buf_timestamp != d.buf_timestamp);
4171         }
4172
4173         if (!decoder->buf)
4174                 return 0;
4175
4176         buf = (unsigned char *)decoder->buf;
4177         len = decoder->len;
4178
4179         if (!intel_pt_next_psb(&buf, &len))
4180                 return 0;
4181
4182         /*
4183          * Walk PSBs while the PSB timestamp is less than the fast forward
4184          * timestamp.
4185          */
4186         do {
4187                 uint64_t tsc;
4188                 size_t rem;
4189
4190                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4191                         break;
4192                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4193                 /*
4194                  * A TSC packet can slip past MTC packets but, after fast
4195                  * forward, decoding starts at the TSC timestamp. That means
4196                  * the timestamps may not be exactly the same as the timestamps
4197                  * that would have been decoded without fast forward.
4198                  */
4199                 if (tsc < timestamp) {
4200                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4201                         decoder->pos += decoder->len - len;
4202                         decoder->buf = buf;
4203                         decoder->len = len;
4204                         intel_pt_reposition(decoder);
4205                 } else {
4206                         break;
4207                 }
4208         } while (intel_pt_step_psb(&buf, &len));
4209
4210         return 0;
4211 }