tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt_decoder.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #ifndef _GNU_SOURCE
   8 #define _GNU_SOURCE
   9 #endif
  10 #include <stdlib.h>
  11 #include <stdbool.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <linux/compiler.h>
  17 #include <linux/string.h>
  18 #include <linux/zalloc.h>
  19
  20 #include "../auxtrace.h"
  21
  22 #include "intel-pt-insn-decoder.h"
  23 #include "intel-pt-pkt-decoder.h"
  24 #include "intel-pt-decoder.h"
  25 #include "intel-pt-log.h"
  26
  27 #define BITULL(x) (1ULL << (x))
  28
  29 /* IA32_RTIT_CTL MSR bits */
  30 #define INTEL_PT_CYC_ENABLE             BITULL(1)
  31 #define INTEL_PT_CYC_THRESHOLD          (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
  32 #define INTEL_PT_CYC_THRESHOLD_SHIFT    19
  33
  34 #define INTEL_PT_BLK_SIZE 1024
  35
  36 #define BIT63 (((uint64_t)1 << 63))
  37
  38 #define SEVEN_BYTES 0xffffffffffffffULL
  39
  40 #define NO_VMCS 0xffffffffffULL
  41
  42 #define INTEL_PT_RETURN 1
  43
  44 /*
  45  * Default maximum number of loops with no packets consumed i.e. stuck in a
  46  * loop.
  47  */
  48 #define INTEL_PT_MAX_LOOPS 100000
  49
  50 struct intel_pt_blk {
  51         struct intel_pt_blk *prev;
  52         uint64_t ip[INTEL_PT_BLK_SIZE];
  53 };
  54
  55 struct intel_pt_stack {
  56         struct intel_pt_blk *blk;
  57         struct intel_pt_blk *spare;
  58         int pos;
  59 };
  60
  61 enum intel_pt_p_once {
  62         INTEL_PT_PRT_ONCE_UNK_VMCS,
  63         INTEL_PT_PRT_ONCE_ERANGE,
  64 };
  65
  66 enum intel_pt_pkt_state {
  67         INTEL_PT_STATE_NO_PSB,
  68         INTEL_PT_STATE_NO_IP,
  69         INTEL_PT_STATE_ERR_RESYNC,
  70         INTEL_PT_STATE_IN_SYNC,
  71         INTEL_PT_STATE_TNT_CONT,
  72         INTEL_PT_STATE_TNT,
  73         INTEL_PT_STATE_TIP,
  74         INTEL_PT_STATE_TIP_PGD,
  75         INTEL_PT_STATE_FUP,
  76         INTEL_PT_STATE_FUP_NO_TIP,
  77         INTEL_PT_STATE_FUP_IN_PSB,
  78         INTEL_PT_STATE_RESAMPLE,
  79         INTEL_PT_STATE_VM_TIME_CORRELATION,
  80 };
  81
  82 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  83 {
  84         switch (pkt_state) {
  85         case INTEL_PT_STATE_NO_PSB:
  86         case INTEL_PT_STATE_NO_IP:
  87         case INTEL_PT_STATE_ERR_RESYNC:
  88         case INTEL_PT_STATE_IN_SYNC:
  89         case INTEL_PT_STATE_TNT_CONT:
  90         case INTEL_PT_STATE_RESAMPLE:
  91         case INTEL_PT_STATE_VM_TIME_CORRELATION:
  92                 return true;
  93         case INTEL_PT_STATE_TNT:
  94         case INTEL_PT_STATE_TIP:
  95         case INTEL_PT_STATE_TIP_PGD:
  96         case INTEL_PT_STATE_FUP:
  97         case INTEL_PT_STATE_FUP_NO_TIP:
  98         case INTEL_PT_STATE_FUP_IN_PSB:
  99                 return false;
 100         default:
 101                 return true;
 102         };
 103 }
 104
 105 #ifdef INTEL_PT_STRICT
 106 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
 107 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
 108 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
 109 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
 110 #else
 111 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
 112 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
 113 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
 114 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
 115 #endif
 116
 117 struct intel_pt_decoder {
 118         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 119         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 120                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 121                          uint64_t max_insn_cnt, void *data);
 122         bool (*pgd_ip)(uint64_t ip, void *data);
 123         int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
 124         struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
 125         void *data;
 126         struct intel_pt_state state;
 127         const unsigned char *buf;
 128         size_t len;
 129         bool return_compression;
 130         bool branch_enable;
 131         bool mtc_insn;
 132         bool pge;
 133         bool have_tma;
 134         bool have_cyc;
 135         bool fixup_last_mtc;
 136         bool have_last_ip;
 137         bool in_psb;
 138         bool hop;
 139         bool leap;
 140         bool vm_time_correlation;
 141         bool vm_tm_corr_dry_run;
 142         bool vm_tm_corr_reliable;
 143         bool vm_tm_corr_same_buf;
 144         bool vm_tm_corr_continuous;
 145         bool nr;
 146         bool next_nr;
 147         enum intel_pt_param_flags flags;
 148         uint64_t pos;
 149         uint64_t last_ip;
 150         uint64_t ip;
 151         uint64_t pip_payload;
 152         uint64_t timestamp;
 153         uint64_t tsc_timestamp;
 154         uint64_t ref_timestamp;
 155         uint64_t buf_timestamp;
 156         uint64_t sample_timestamp;
 157         uint64_t ret_addr;
 158         uint64_t ctc_timestamp;
 159         uint64_t ctc_delta;
 160         uint64_t cycle_cnt;
 161         uint64_t cyc_ref_timestamp;
 162         uint64_t first_timestamp;
 163         uint64_t last_reliable_timestamp;
 164         uint64_t vmcs;
 165         uint64_t print_once;
 166         uint64_t last_ctc;
 167         uint32_t last_mtc;
 168         uint32_t tsc_ctc_ratio_n;
 169         uint32_t tsc_ctc_ratio_d;
 170         uint32_t tsc_ctc_mult;
 171         uint32_t tsc_slip;
 172         uint32_t ctc_rem_mask;
 173         int mtc_shift;
 174         struct intel_pt_stack stack;
 175         enum intel_pt_pkt_state pkt_state;
 176         enum intel_pt_pkt_ctx pkt_ctx;
 177         enum intel_pt_pkt_ctx prev_pkt_ctx;
 178         enum intel_pt_blk_type blk_type;
 179         int blk_type_pos;
 180         struct intel_pt_pkt packet;
 181         struct intel_pt_pkt tnt;
 182         int pkt_step;
 183         int pkt_len;
 184         int last_packet_type;
 185         unsigned int cbr;
 186         unsigned int cbr_seen;
 187         unsigned int max_non_turbo_ratio;
 188         double max_non_turbo_ratio_fp;
 189         double cbr_cyc_to_tsc;
 190         double calc_cyc_to_tsc;
 191         bool have_calc_cyc_to_tsc;
 192         int exec_mode;
 193         unsigned int insn_bytes;
 194         uint64_t period;
 195         enum intel_pt_period_type period_type;
 196         uint64_t tot_insn_cnt;
 197         uint64_t period_insn_cnt;
 198         uint64_t period_mask;
 199         uint64_t period_ticks;
 200         uint64_t last_masked_timestamp;
 201         uint64_t tot_cyc_cnt;
 202         uint64_t sample_tot_cyc_cnt;
 203         uint64_t base_cyc_cnt;
 204         uint64_t cyc_cnt_timestamp;
 205         uint64_t ctl;
 206         uint64_t cyc_threshold;
 207         double tsc_to_cyc;
 208         bool continuous_period;
 209         bool overflow;
 210         bool set_fup_tx_flags;
 211         bool set_fup_ptw;
 212         bool set_fup_mwait;
 213         bool set_fup_pwre;
 214         bool set_fup_exstop;
 215         bool set_fup_bep;
 216         bool sample_cyc;
 217         unsigned int fup_tx_flags;
 218         unsigned int tx_flags;
 219         uint64_t fup_ptw_payload;
 220         uint64_t fup_mwait_payload;
 221         uint64_t fup_pwre_payload;
 222         uint64_t cbr_payload;
 223         uint64_t timestamp_insn_cnt;
 224         uint64_t sample_insn_cnt;
 225         uint64_t stuck_ip;
 226         int max_loops;
 227         int no_progress;
 228         int stuck_ip_prd;
 229         int stuck_ip_cnt;
 230         uint64_t psb_ip;
 231         const unsigned char *next_buf;
 232         size_t next_len;
 233         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 234 };
 235
 236 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 237 {
 238         int i;
 239
 240         for (i = 0; x != 1; i++)
 241                 x >>= 1;
 242
 243         return x << i;
 244 }
 245
 246 __printf(1, 2)
 247 static void p_log(const char *fmt, ...)
 248 {
 249         char buf[512];
 250         va_list args;
 251
 252         va_start(args, fmt);
 253         vsnprintf(buf, sizeof(buf), fmt, args);
 254         va_end(args);
 255
 256         fprintf(stderr, "%s\n", buf);
 257         intel_pt_log("%s\n", buf);
 258 }
 259
 260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
 261                                 enum intel_pt_p_once id)
 262 {
 263         uint64_t bit = 1ULL << id;
 264
 265         if (decoder->print_once & bit)
 266                 return false;
 267         decoder->print_once |= bit;
 268         return true;
 269 }
 270
 271 static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
 272 {
 273         if (!(ctl & INTEL_PT_CYC_ENABLE))
 274                 return 0;
 275
 276         return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
 277 }
 278
 279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 280 {
 281         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 282                 uint64_t period;
 283
 284                 period = intel_pt_lower_power_of_2(decoder->period);
 285                 decoder->period_mask  = ~(period - 1);
 286                 decoder->period_ticks = period;
 287         }
 288 }
 289
 290 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 291 {
 292         if (!d)
 293                 return 0;
 294         return (t / d) * n + ((t % d) * n) / d;
 295 }
 296
 297 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 298 {
 299         struct intel_pt_decoder *decoder;
 300
 301         if (!params->get_trace || !params->walk_insn)
 302                 return NULL;
 303
 304         decoder = zalloc(sizeof(struct intel_pt_decoder));
 305         if (!decoder)
 306                 return NULL;
 307
 308         decoder->get_trace          = params->get_trace;
 309         decoder->walk_insn          = params->walk_insn;
 310         decoder->pgd_ip             = params->pgd_ip;
 311         decoder->lookahead          = params->lookahead;
 312         decoder->findnew_vmcs_info  = params->findnew_vmcs_info;
 313         decoder->data               = params->data;
 314         decoder->return_compression = params->return_compression;
 315         decoder->branch_enable      = params->branch_enable;
 316         decoder->hop                = params->quick >= 1;
 317         decoder->leap               = params->quick >= 2;
 318         decoder->vm_time_correlation = params->vm_time_correlation;
 319         decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
 320         decoder->first_timestamp    = params->first_timestamp;
 321         decoder->last_reliable_timestamp = params->first_timestamp;
 322         decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
 323
 324         decoder->flags              = params->flags;
 325
 326         decoder->ctl                = params->ctl;
 327         decoder->period             = params->period;
 328         decoder->period_type        = params->period_type;
 329
 330         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 331         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 332
 333         decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
 334
 335         intel_pt_setup_period(decoder);
 336
 337         decoder->mtc_shift = params->mtc_period;
 338         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 339
 340         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 341         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 342
 343         if (!decoder->tsc_ctc_ratio_n)
 344                 decoder->tsc_ctc_ratio_d = 0;
 345
 346         if (decoder->tsc_ctc_ratio_d) {
 347                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 348                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 349                                                 decoder->tsc_ctc_ratio_d;
 350         }
 351
 352         /*
 353          * A TSC packet can slip past MTC packets so that the timestamp appears
 354          * to go backwards. One estimate is that can be up to about 40 CPU
 355          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 356          * slippage an order of magnitude more to be on the safe side.
 357          */
 358         decoder->tsc_slip = 0x10000;
 359
 360         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 361         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 362         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 363         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 364         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 365
 366         if (decoder->hop)
 367                 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
 368
 369         return decoder;
 370 }
 371
 372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
 373                                   uint64_t first_timestamp)
 374 {
 375         decoder->first_timestamp = first_timestamp;
 376 }
 377
 378 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 379 {
 380         struct intel_pt_blk *blk = stack->blk;
 381
 382         stack->blk = blk->prev;
 383         if (!stack->spare)
 384                 stack->spare = blk;
 385         else
 386                 free(blk);
 387 }
 388
 389 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 390 {
 391         if (!stack->pos) {
 392                 if (!stack->blk)
 393                         return 0;
 394                 intel_pt_pop_blk(stack);
 395                 if (!stack->blk)
 396                         return 0;
 397                 stack->pos = INTEL_PT_BLK_SIZE;
 398         }
 399         return stack->blk->ip[--stack->pos];
 400 }
 401
 402 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 403 {
 404         struct intel_pt_blk *blk;
 405
 406         if (stack->spare) {
 407                 blk = stack->spare;
 408                 stack->spare = NULL;
 409         } else {
 410                 blk = malloc(sizeof(struct intel_pt_blk));
 411                 if (!blk)
 412                         return -ENOMEM;
 413         }
 414
 415         blk->prev = stack->blk;
 416         stack->blk = blk;
 417         stack->pos = 0;
 418         return 0;
 419 }
 420
 421 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 422 {
 423         int err;
 424
 425         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 426                 err = intel_pt_alloc_blk(stack);
 427                 if (err)
 428                         return err;
 429         }
 430
 431         stack->blk->ip[stack->pos++] = ip;
 432         return 0;
 433 }
 434
 435 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 436 {
 437         while (stack->blk)
 438                 intel_pt_pop_blk(stack);
 439         stack->pos = 0;
 440 }
 441
 442 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 443 {
 444         intel_pt_clear_stack(stack);
 445         zfree(&stack->blk);
 446         zfree(&stack->spare);
 447 }
 448
 449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 450 {
 451         intel_pt_free_stack(&decoder->stack);
 452         free(decoder);
 453 }
 454
 455 static int intel_pt_ext_err(int code)
 456 {
 457         switch (code) {
 458         case -ENOMEM:
 459                 return INTEL_PT_ERR_NOMEM;
 460         case -ENOSYS:
 461                 return INTEL_PT_ERR_INTERN;
 462         case -EBADMSG:
 463                 return INTEL_PT_ERR_BADPKT;
 464         case -ENODATA:
 465                 return INTEL_PT_ERR_NODATA;
 466         case -EILSEQ:
 467                 return INTEL_PT_ERR_NOINSN;
 468         case -ENOENT:
 469                 return INTEL_PT_ERR_MISMAT;
 470         case -EOVERFLOW:
 471                 return INTEL_PT_ERR_OVR;
 472         case -ENOSPC:
 473                 return INTEL_PT_ERR_LOST;
 474         case -ELOOP:
 475                 return INTEL_PT_ERR_NELOOP;
 476         default:
 477                 return INTEL_PT_ERR_UNK;
 478         }
 479 }
 480
 481 static const char *intel_pt_err_msgs[] = {
 482         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 483         [INTEL_PT_ERR_INTERN] = "Internal error",
 484         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 485         [INTEL_PT_ERR_NODATA] = "No more data",
 486         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 487         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 488         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 489         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 490         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 491         [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
 492 };
 493
 494 int intel_pt__strerror(int code, char *buf, size_t buflen)
 495 {
 496         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 497                 code = INTEL_PT_ERR_UNK;
 498         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 499         return 0;
 500 }
 501
 502 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 503                                  uint64_t last_ip)
 504 {
 505         uint64_t ip;
 506
 507         switch (packet->count) {
 508         case 1:
 509                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 510                      packet->payload;
 511                 break;
 512         case 2:
 513                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 514                      packet->payload;
 515                 break;
 516         case 3:
 517                 ip = packet->payload;
 518                 /* Sign-extend 6-byte ip */
 519                 if (ip & (uint64_t)0x800000000000ULL)
 520                         ip |= (uint64_t)0xffff000000000000ULL;
 521                 break;
 522         case 4:
 523                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 524                      packet->payload;
 525                 break;
 526         case 6:
 527                 ip = packet->payload;
 528                 break;
 529         default:
 530                 return 0;
 531         }
 532
 533         return ip;
 534 }
 535
 536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 537 {
 538         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 539         decoder->have_last_ip = true;
 540 }
 541
 542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 543 {
 544         intel_pt_set_last_ip(decoder);
 545         decoder->ip = decoder->last_ip;
 546 }
 547
 548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 549 {
 550         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 551                             decoder->buf);
 552 }
 553
 554 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 555 {
 556         intel_pt_log("ERROR: Internal error\n");
 557         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 558         return -ENOSYS;
 559 }
 560
 561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 562 {
 563         decoder->tx_flags = 0;
 564 }
 565
 566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 567 {
 568         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 569 }
 570
 571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
 572 {
 573         decoder->pip_payload = decoder->packet.payload;
 574 }
 575
 576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
 577 {
 578         decoder->next_nr = decoder->pip_payload & 1;
 579 }
 580
 581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
 582 {
 583         decoder->nr = decoder->pip_payload & 1;
 584         decoder->next_nr = decoder->nr;
 585 }
 586
 587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
 588 {
 589         intel_pt_update_pip(decoder);
 590         intel_pt_set_nr(decoder);
 591 }
 592
 593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 594 {
 595         intel_pt_clear_tx_flags(decoder);
 596         decoder->have_tma = false;
 597         decoder->pkt_len = 1;
 598         decoder->pkt_step = 1;
 599         intel_pt_decoder_log_packet(decoder);
 600         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 601                 intel_pt_log("ERROR: Bad packet\n");
 602                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 603         }
 604         return -EBADMSG;
 605 }
 606
 607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 608 {
 609         decoder->sample_timestamp = decoder->timestamp;
 610         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 611 }
 612
 613 static void intel_pt_reposition(struct intel_pt_decoder *decoder)
 614 {
 615         decoder->ip = 0;
 616         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 617         decoder->timestamp = 0;
 618         decoder->have_tma = false;
 619 }
 620
 621 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
 622 {
 623         struct intel_pt_buffer buffer = { .buf = 0, };
 624         int ret;
 625
 626         decoder->pkt_step = 0;
 627
 628         intel_pt_log("Getting more data\n");
 629         ret = decoder->get_trace(&buffer, decoder->data);
 630         if (ret)
 631                 return ret;
 632         decoder->buf = buffer.buf;
 633         decoder->len = buffer.len;
 634         if (!decoder->len) {
 635                 intel_pt_log("No more data\n");
 636                 return -ENODATA;
 637         }
 638         decoder->buf_timestamp = buffer.ref_timestamp;
 639         if (!buffer.consecutive || reposition) {
 640                 intel_pt_reposition(decoder);
 641                 decoder->ref_timestamp = buffer.ref_timestamp;
 642                 decoder->state.trace_nr = buffer.trace_nr;
 643                 decoder->vm_tm_corr_same_buf = false;
 644                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 645                              decoder->ref_timestamp);
 646                 return -ENOLINK;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
 653                                   bool reposition)
 654 {
 655         if (!decoder->next_buf)
 656                 return intel_pt_get_data(decoder, reposition);
 657
 658         decoder->buf = decoder->next_buf;
 659         decoder->len = decoder->next_len;
 660         decoder->next_buf = 0;
 661         decoder->next_len = 0;
 662         return 0;
 663 }
 664
 665 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 666 {
 667         unsigned char *buf = decoder->temp_buf;
 668         size_t old_len, len, n;
 669         int ret;
 670
 671         old_len = decoder->len;
 672         len = decoder->len;
 673         memcpy(buf, decoder->buf, len);
 674
 675         ret = intel_pt_get_data(decoder, false);
 676         if (ret) {
 677                 decoder->pos += old_len;
 678                 return ret < 0 ? ret : -EINVAL;
 679         }
 680
 681         n = INTEL_PT_PKT_MAX_SZ - len;
 682         if (n > decoder->len)
 683                 n = decoder->len;
 684         memcpy(buf + len, decoder->buf, n);
 685         len += n;
 686
 687         decoder->prev_pkt_ctx = decoder->pkt_ctx;
 688         ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
 689         if (ret < (int)old_len) {
 690                 decoder->next_buf = decoder->buf;
 691                 decoder->next_len = decoder->len;
 692                 decoder->buf = buf;
 693                 decoder->len = old_len;
 694                 return intel_pt_bad_packet(decoder);
 695         }
 696
 697         decoder->next_buf = decoder->buf + (ret - old_len);
 698         decoder->next_len = decoder->len - (ret - old_len);
 699
 700         decoder->buf = buf;
 701         decoder->len = ret;
 702
 703         return ret;
 704 }
 705
 706 struct intel_pt_pkt_info {
 707         struct intel_pt_decoder   *decoder;
 708         struct intel_pt_pkt       packet;
 709         uint64_t                  pos;
 710         int                       pkt_len;
 711         int                       last_packet_type;
 712         void                      *data;
 713 };
 714
 715 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 716
 717 /* Lookahead packets in current buffer */
 718 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 719                                   intel_pt_pkt_cb_t cb, void *data)
 720 {
 721         struct intel_pt_pkt_info pkt_info;
 722         const unsigned char *buf = decoder->buf;
 723         enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
 724         size_t len = decoder->len;
 725         int ret;
 726
 727         pkt_info.decoder          = decoder;
 728         pkt_info.pos              = decoder->pos;
 729         pkt_info.pkt_len          = decoder->pkt_step;
 730         pkt_info.last_packet_type = decoder->last_packet_type;
 731         pkt_info.data             = data;
 732
 733         while (1) {
 734                 do {
 735                         pkt_info.pos += pkt_info.pkt_len;
 736                         buf          += pkt_info.pkt_len;
 737                         len          -= pkt_info.pkt_len;
 738
 739                         if (!len)
 740                                 return INTEL_PT_NEED_MORE_BYTES;
 741
 742                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
 743                                                   &pkt_ctx);
 744                         if (!ret)
 745                                 return INTEL_PT_NEED_MORE_BYTES;
 746                         if (ret < 0)
 747                                 return ret;
 748
 749                         pkt_info.pkt_len = ret;
 750                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 751
 752                 ret = cb(&pkt_info);
 753                 if (ret)
 754                         return 0;
 755
 756                 pkt_info.last_packet_type = pkt_info.packet.type;
 757         }
 758 }
 759
 760 struct intel_pt_calc_cyc_to_tsc_info {
 761         uint64_t        cycle_cnt;
 762         unsigned int    cbr;
 763         uint32_t        last_mtc;
 764         uint64_t        ctc_timestamp;
 765         uint64_t        ctc_delta;
 766         uint64_t        tsc_timestamp;
 767         uint64_t        timestamp;
 768         bool            have_tma;
 769         bool            fixup_last_mtc;
 770         bool            from_mtc;
 771         double          cbr_cyc_to_tsc;
 772 };
 773
 774 /*
 775  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 776  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 777  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 778  * packet by copying the missing bits from the current MTC assuming the least
 779  * difference between the two, and that the current MTC comes after last_mtc.
 780  */
 781 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 782                                     uint32_t *last_mtc)
 783 {
 784         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 785         uint32_t mask = ~(first_missing_bit - 1);
 786
 787         *last_mtc |= mtc & mask;
 788         if (*last_mtc >= mtc) {
 789                 *last_mtc -= first_missing_bit;
 790                 *last_mtc &= 0xff;
 791         }
 792 }
 793
 794 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 795 {
 796         struct intel_pt_decoder *decoder = pkt_info->decoder;
 797         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 798         uint64_t timestamp;
 799         double cyc_to_tsc;
 800         unsigned int cbr;
 801         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 802
 803         switch (pkt_info->packet.type) {
 804         case INTEL_PT_TNT:
 805         case INTEL_PT_TIP_PGE:
 806         case INTEL_PT_TIP:
 807         case INTEL_PT_FUP:
 808         case INTEL_PT_PSB:
 809         case INTEL_PT_PIP:
 810         case INTEL_PT_MODE_EXEC:
 811         case INTEL_PT_MODE_TSX:
 812         case INTEL_PT_PSBEND:
 813         case INTEL_PT_PAD:
 814         case INTEL_PT_VMCS:
 815         case INTEL_PT_MNT:
 816         case INTEL_PT_PTWRITE:
 817         case INTEL_PT_PTWRITE_IP:
 818         case INTEL_PT_BBP:
 819         case INTEL_PT_BIP:
 820         case INTEL_PT_BEP:
 821         case INTEL_PT_BEP_IP:
 822                 return 0;
 823
 824         case INTEL_PT_MTC:
 825                 if (!data->have_tma)
 826                         return 0;
 827
 828                 mtc = pkt_info->packet.payload;
 829                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 830                         data->fixup_last_mtc = false;
 831                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 832                                                 &data->last_mtc);
 833                 }
 834                 if (mtc > data->last_mtc)
 835                         mtc_delta = mtc - data->last_mtc;
 836                 else
 837                         mtc_delta = mtc + 256 - data->last_mtc;
 838                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 839                 data->last_mtc = mtc;
 840
 841                 if (decoder->tsc_ctc_mult) {
 842                         timestamp = data->ctc_timestamp +
 843                                 data->ctc_delta * decoder->tsc_ctc_mult;
 844                 } else {
 845                         timestamp = data->ctc_timestamp +
 846                                 multdiv(data->ctc_delta,
 847                                         decoder->tsc_ctc_ratio_n,
 848                                         decoder->tsc_ctc_ratio_d);
 849                 }
 850
 851                 if (timestamp < data->timestamp)
 852                         return 1;
 853
 854                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 855                         data->timestamp = timestamp;
 856                         return 0;
 857                 }
 858
 859                 break;
 860
 861         case INTEL_PT_TSC:
 862                 /*
 863                  * For now, do not support using TSC packets - refer
 864                  * intel_pt_calc_cyc_to_tsc().
 865                  */
 866                 if (data->from_mtc)
 867                         return 1;
 868                 timestamp = pkt_info->packet.payload |
 869                             (data->timestamp & (0xffULL << 56));
 870                 if (data->from_mtc && timestamp < data->timestamp &&
 871                     data->timestamp - timestamp < decoder->tsc_slip)
 872                         return 1;
 873                 if (timestamp < data->timestamp)
 874                         timestamp += (1ULL << 56);
 875                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 876                         if (data->from_mtc)
 877                                 return 1;
 878                         data->tsc_timestamp = timestamp;
 879                         data->timestamp = timestamp;
 880                         return 0;
 881                 }
 882                 break;
 883
 884         case INTEL_PT_TMA:
 885                 if (data->from_mtc)
 886                         return 1;
 887
 888                 if (!decoder->tsc_ctc_ratio_d)
 889                         return 0;
 890
 891                 ctc = pkt_info->packet.payload;
 892                 fc = pkt_info->packet.count;
 893                 ctc_rem = ctc & decoder->ctc_rem_mask;
 894
 895                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 896
 897                 data->ctc_timestamp = data->tsc_timestamp - fc;
 898                 if (decoder->tsc_ctc_mult) {
 899                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 900                 } else {
 901                         data->ctc_timestamp -=
 902                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 903                                         decoder->tsc_ctc_ratio_d);
 904                 }
 905
 906                 data->ctc_delta = 0;
 907                 data->have_tma = true;
 908                 data->fixup_last_mtc = true;
 909
 910                 return 0;
 911
 912         case INTEL_PT_CYC:
 913                 data->cycle_cnt += pkt_info->packet.payload;
 914                 return 0;
 915
 916         case INTEL_PT_CBR:
 917                 cbr = pkt_info->packet.payload;
 918                 if (data->cbr && data->cbr != cbr)
 919                         return 1;
 920                 data->cbr = cbr;
 921                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 922                 return 0;
 923
 924         case INTEL_PT_TIP_PGD:
 925         case INTEL_PT_TRACESTOP:
 926         case INTEL_PT_EXSTOP:
 927         case INTEL_PT_EXSTOP_IP:
 928         case INTEL_PT_MWAIT:
 929         case INTEL_PT_PWRE:
 930         case INTEL_PT_PWRX:
 931         case INTEL_PT_OVF:
 932         case INTEL_PT_BAD: /* Does not happen */
 933         default:
 934                 return 1;
 935         }
 936
 937         if (!data->cbr && decoder->cbr) {
 938                 data->cbr = decoder->cbr;
 939                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 940         }
 941
 942         if (!data->cycle_cnt)
 943                 return 1;
 944
 945         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 946
 947         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 948             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 949                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 950                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 951                 return 1;
 952         }
 953
 954         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 955         decoder->have_calc_cyc_to_tsc = true;
 956
 957         if (data->cbr) {
 958                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 959                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 960         } else {
 961                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 962                              cyc_to_tsc, pkt_info->pos);
 963         }
 964
 965         return 1;
 966 }
 967
 968 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 969                                      bool from_mtc)
 970 {
 971         struct intel_pt_calc_cyc_to_tsc_info data = {
 972                 .cycle_cnt      = 0,
 973                 .cbr            = 0,
 974                 .last_mtc       = decoder->last_mtc,
 975                 .ctc_timestamp  = decoder->ctc_timestamp,
 976                 .ctc_delta      = decoder->ctc_delta,
 977                 .tsc_timestamp  = decoder->tsc_timestamp,
 978                 .timestamp      = decoder->timestamp,
 979                 .have_tma       = decoder->have_tma,
 980                 .fixup_last_mtc = decoder->fixup_last_mtc,
 981                 .from_mtc       = from_mtc,
 982                 .cbr_cyc_to_tsc = 0,
 983         };
 984
 985         /*
 986          * For now, do not support using TSC packets for at least the reasons:
 987          * 1) timing might have stopped
 988          * 2) TSC packets within PSB+ can slip against CYC packets
 989          */
 990         if (!from_mtc)
 991                 return;
 992
 993         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 994 }
 995
 996 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 997 {
 998         int ret;
 999
1000         decoder->last_packet_type = decoder->packet.type;
1001
1002         do {
1003                 decoder->pos += decoder->pkt_step;
1004                 decoder->buf += decoder->pkt_step;
1005                 decoder->len -= decoder->pkt_step;
1006
1007                 if (!decoder->len) {
1008                         ret = intel_pt_get_next_data(decoder, false);
1009                         if (ret)
1010                                 return ret;
1011                 }
1012
1013                 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1014                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1015                                           &decoder->packet, &decoder->pkt_ctx);
1016                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1017                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1018                         ret = intel_pt_get_split_packet(decoder);
1019                         if (ret < 0)
1020                                 return ret;
1021                 }
1022                 if (ret <= 0)
1023                         return intel_pt_bad_packet(decoder);
1024
1025                 decoder->pkt_len = ret;
1026                 decoder->pkt_step = ret;
1027                 intel_pt_decoder_log_packet(decoder);
1028         } while (decoder->packet.type == INTEL_PT_PAD);
1029
1030         return 0;
1031 }
1032
1033 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1034 {
1035         uint64_t timestamp, masked_timestamp;
1036
1037         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1038         masked_timestamp = timestamp & decoder->period_mask;
1039         if (decoder->continuous_period) {
1040                 if (masked_timestamp > decoder->last_masked_timestamp)
1041                         return 1;
1042         } else {
1043                 timestamp += 1;
1044                 masked_timestamp = timestamp & decoder->period_mask;
1045                 if (masked_timestamp > decoder->last_masked_timestamp) {
1046                         decoder->last_masked_timestamp = masked_timestamp;
1047                         decoder->continuous_period = true;
1048                 }
1049         }
1050
1051         if (masked_timestamp < decoder->last_masked_timestamp)
1052                 return decoder->period_ticks;
1053
1054         return decoder->period_ticks - (timestamp - masked_timestamp);
1055 }
1056
1057 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1058 {
1059         switch (decoder->period_type) {
1060         case INTEL_PT_PERIOD_INSTRUCTIONS:
1061                 return decoder->period - decoder->period_insn_cnt;
1062         case INTEL_PT_PERIOD_TICKS:
1063                 return intel_pt_next_period(decoder);
1064         case INTEL_PT_PERIOD_NONE:
1065         case INTEL_PT_PERIOD_MTC:
1066         default:
1067                 return 0;
1068         }
1069 }
1070
1071 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1072 {
1073         uint64_t timestamp, masked_timestamp;
1074
1075         switch (decoder->period_type) {
1076         case INTEL_PT_PERIOD_INSTRUCTIONS:
1077                 decoder->period_insn_cnt = 0;
1078                 break;
1079         case INTEL_PT_PERIOD_TICKS:
1080                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1081                 masked_timestamp = timestamp & decoder->period_mask;
1082                 if (masked_timestamp > decoder->last_masked_timestamp)
1083                         decoder->last_masked_timestamp = masked_timestamp;
1084                 else
1085                         decoder->last_masked_timestamp += decoder->period_ticks;
1086                 break;
1087         case INTEL_PT_PERIOD_NONE:
1088         case INTEL_PT_PERIOD_MTC:
1089         default:
1090                 break;
1091         }
1092
1093         decoder->state.type |= INTEL_PT_INSTRUCTION;
1094 }
1095
1096 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1097                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1098 {
1099         uint64_t max_insn_cnt, insn_cnt = 0;
1100         int err;
1101
1102         if (!decoder->mtc_insn)
1103                 decoder->mtc_insn = true;
1104
1105         max_insn_cnt = intel_pt_next_sample(decoder);
1106
1107         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1108                                  max_insn_cnt, decoder->data);
1109
1110         decoder->tot_insn_cnt += insn_cnt;
1111         decoder->timestamp_insn_cnt += insn_cnt;
1112         decoder->sample_insn_cnt += insn_cnt;
1113         decoder->period_insn_cnt += insn_cnt;
1114
1115         if (err) {
1116                 decoder->no_progress = 0;
1117                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1118                 intel_pt_log_at("ERROR: Failed to get instruction",
1119                                 decoder->ip);
1120                 if (err == -ENOENT)
1121                         return -ENOLINK;
1122                 return -EILSEQ;
1123         }
1124
1125         if (ip && decoder->ip == ip) {
1126                 err = -EAGAIN;
1127                 goto out;
1128         }
1129
1130         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1131                 intel_pt_sample_insn(decoder);
1132
1133         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1134                 decoder->state.type = INTEL_PT_INSTRUCTION;
1135                 decoder->state.from_ip = decoder->ip;
1136                 decoder->state.to_ip = 0;
1137                 decoder->ip += intel_pt_insn->length;
1138                 err = INTEL_PT_RETURN;
1139                 goto out;
1140         }
1141
1142         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1143                 /* Zero-length calls are excluded */
1144                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1145                     intel_pt_insn->rel) {
1146                         err = intel_pt_push(&decoder->stack, decoder->ip +
1147                                             intel_pt_insn->length);
1148                         if (err)
1149                                 goto out;
1150                 }
1151         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1152                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1153         }
1154
1155         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1156                 int cnt = decoder->no_progress++;
1157
1158                 decoder->state.from_ip = decoder->ip;
1159                 decoder->ip += intel_pt_insn->length +
1160                                 intel_pt_insn->rel;
1161                 decoder->state.to_ip = decoder->ip;
1162                 err = INTEL_PT_RETURN;
1163
1164                 /*
1165                  * Check for being stuck in a loop.  This can happen if a
1166                  * decoder error results in the decoder erroneously setting the
1167                  * ip to an address that is itself in an infinite loop that
1168                  * consumes no packets.  When that happens, there must be an
1169                  * unconditional branch.
1170                  */
1171                 if (cnt) {
1172                         if (cnt == 1) {
1173                                 decoder->stuck_ip = decoder->state.to_ip;
1174                                 decoder->stuck_ip_prd = 1;
1175                                 decoder->stuck_ip_cnt = 1;
1176                         } else if (cnt > decoder->max_loops ||
1177                                    decoder->state.to_ip == decoder->stuck_ip) {
1178                                 intel_pt_log_at("ERROR: Never-ending loop",
1179                                                 decoder->state.to_ip);
1180                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1181                                 err = -ELOOP;
1182                                 goto out;
1183                         } else if (!--decoder->stuck_ip_cnt) {
1184                                 decoder->stuck_ip_prd += 1;
1185                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1186                                 decoder->stuck_ip = decoder->state.to_ip;
1187                         }
1188                 }
1189                 goto out_no_progress;
1190         }
1191 out:
1192         decoder->no_progress = 0;
1193 out_no_progress:
1194         decoder->state.insn_op = intel_pt_insn->op;
1195         decoder->state.insn_len = intel_pt_insn->length;
1196         memcpy(decoder->state.insn, intel_pt_insn->buf,
1197                INTEL_PT_INSN_BUF_SZ);
1198
1199         if (decoder->tx_flags & INTEL_PT_IN_TX)
1200                 decoder->state.flags |= INTEL_PT_IN_TX;
1201
1202         return err;
1203 }
1204
1205 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1206 {
1207         enum intel_pt_sample_type type = decoder->state.type;
1208         bool ret = false;
1209
1210         decoder->state.type &= ~INTEL_PT_BRANCH;
1211
1212         if (decoder->set_fup_tx_flags) {
1213                 decoder->set_fup_tx_flags = false;
1214                 decoder->tx_flags = decoder->fup_tx_flags;
1215                 decoder->state.type |= INTEL_PT_TRANSACTION;
1216                 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1217                         decoder->state.type |= INTEL_PT_BRANCH;
1218                 decoder->state.flags = decoder->fup_tx_flags;
1219                 ret = true;
1220         }
1221         if (decoder->set_fup_ptw) {
1222                 decoder->set_fup_ptw = false;
1223                 decoder->state.type |= INTEL_PT_PTW;
1224                 decoder->state.flags |= INTEL_PT_FUP_IP;
1225                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1226                 ret = true;
1227         }
1228         if (decoder->set_fup_mwait) {
1229                 decoder->set_fup_mwait = false;
1230                 decoder->state.type |= INTEL_PT_MWAIT_OP;
1231                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1232                 ret = true;
1233         }
1234         if (decoder->set_fup_pwre) {
1235                 decoder->set_fup_pwre = false;
1236                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1237                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1238                 ret = true;
1239         }
1240         if (decoder->set_fup_exstop) {
1241                 decoder->set_fup_exstop = false;
1242                 decoder->state.type |= INTEL_PT_EX_STOP;
1243                 decoder->state.flags |= INTEL_PT_FUP_IP;
1244                 ret = true;
1245         }
1246         if (decoder->set_fup_bep) {
1247                 decoder->set_fup_bep = false;
1248                 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1249                 ret = true;
1250         }
1251         if (decoder->overflow) {
1252                 decoder->overflow = false;
1253                 if (!ret && !decoder->pge) {
1254                         if (decoder->hop) {
1255                                 decoder->state.type = 0;
1256                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
1257                         }
1258                         decoder->pge = true;
1259                         decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;
1260                         decoder->state.from_ip = 0;
1261                         decoder->state.to_ip = decoder->ip;
1262                         return true;
1263                 }
1264         }
1265         if (ret) {
1266                 decoder->state.from_ip = decoder->ip;
1267                 decoder->state.to_ip = 0;
1268         } else {
1269                 decoder->state.type = type;
1270         }
1271         return ret;
1272 }
1273
1274 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1275                                           struct intel_pt_insn *intel_pt_insn,
1276                                           uint64_t ip, int err)
1277 {
1278         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1279                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1280                ip == decoder->ip + intel_pt_insn->length;
1281 }
1282
1283 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1284 {
1285         struct intel_pt_insn intel_pt_insn;
1286         uint64_t ip;
1287         int err;
1288
1289         ip = decoder->last_ip;
1290
1291         while (1) {
1292                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1293                 if (err == INTEL_PT_RETURN)
1294                         return 0;
1295                 if (err == -EAGAIN ||
1296                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1297                         bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1298
1299                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1300                         if (intel_pt_fup_event(decoder) && no_tip)
1301                                 return 0;
1302                         return -EAGAIN;
1303                 }
1304                 decoder->set_fup_tx_flags = false;
1305                 if (err)
1306                         return err;
1307
1308                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1309                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1310                                         decoder->ip);
1311                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1312                         return -ENOENT;
1313                 }
1314
1315                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1316                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1317                                         decoder->ip);
1318                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1319                         return -ENOENT;
1320                 }
1321
1322                 intel_pt_bug(decoder);
1323         }
1324 }
1325
1326 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1327 {
1328         struct intel_pt_insn intel_pt_insn;
1329         int err;
1330
1331         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1332         if (err == INTEL_PT_RETURN &&
1333             decoder->pgd_ip &&
1334             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1335             (decoder->state.type & INTEL_PT_BRANCH) &&
1336             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1337                 /* Unconditional branch leaving filter region */
1338                 decoder->no_progress = 0;
1339                 decoder->pge = false;
1340                 decoder->continuous_period = false;
1341                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1342                 decoder->state.type |= INTEL_PT_TRACE_END;
1343                 intel_pt_update_nr(decoder);
1344                 return 0;
1345         }
1346         if (err == INTEL_PT_RETURN)
1347                 return 0;
1348         if (err)
1349                 return err;
1350
1351         intel_pt_update_nr(decoder);
1352
1353         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1354                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1355                         decoder->pge = false;
1356                         decoder->continuous_period = false;
1357                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1358                         decoder->state.from_ip = decoder->ip;
1359                         if (decoder->packet.count == 0) {
1360                                 decoder->state.to_ip = 0;
1361                         } else {
1362                                 decoder->state.to_ip = decoder->last_ip;
1363                                 decoder->ip = decoder->last_ip;
1364                         }
1365                         decoder->state.type |= INTEL_PT_TRACE_END;
1366                 } else {
1367                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1368                         decoder->state.from_ip = decoder->ip;
1369                         if (decoder->packet.count == 0) {
1370                                 decoder->state.to_ip = 0;
1371                         } else {
1372                                 decoder->state.to_ip = decoder->last_ip;
1373                                 decoder->ip = decoder->last_ip;
1374                         }
1375                 }
1376                 return 0;
1377         }
1378
1379         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1380                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1381                                  intel_pt_insn.rel;
1382
1383                 if (decoder->pgd_ip &&
1384                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1385                     decoder->pgd_ip(to_ip, decoder->data)) {
1386                         /* Conditional branch leaving filter region */
1387                         decoder->pge = false;
1388                         decoder->continuous_period = false;
1389                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1390                         decoder->ip = to_ip;
1391                         decoder->state.from_ip = decoder->ip;
1392                         decoder->state.to_ip = to_ip;
1393                         decoder->state.type |= INTEL_PT_TRACE_END;
1394                         return 0;
1395                 }
1396                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1397                                 decoder->ip);
1398                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1399                 return -ENOENT;
1400         }
1401
1402         return intel_pt_bug(decoder);
1403 }
1404
1405 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1406 {
1407         struct intel_pt_insn intel_pt_insn;
1408         int err;
1409
1410         while (1) {
1411                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1412                 if (err == INTEL_PT_RETURN)
1413                         return 0;
1414                 if (err)
1415                         return err;
1416
1417                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1418                         if (!decoder->return_compression) {
1419                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1420                                                 decoder->ip);
1421                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1422                                 return -ENOENT;
1423                         }
1424                         if (!decoder->ret_addr) {
1425                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1426                                                 decoder->ip);
1427                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1428                                 return -ENOENT;
1429                         }
1430                         if (!(decoder->tnt.payload & BIT63)) {
1431                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1432                                                 decoder->ip);
1433                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1434                                 return -ENOENT;
1435                         }
1436                         decoder->tnt.count -= 1;
1437                         if (decoder->tnt.count)
1438                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1439                         else
1440                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1441                         decoder->tnt.payload <<= 1;
1442                         decoder->state.from_ip = decoder->ip;
1443                         decoder->ip = decoder->ret_addr;
1444                         decoder->state.to_ip = decoder->ip;
1445                         return 0;
1446                 }
1447
1448                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1449                         /* Handle deferred TIPs */
1450                         err = intel_pt_get_next_packet(decoder);
1451                         if (err)
1452                                 return err;
1453                         if (decoder->packet.type != INTEL_PT_TIP ||
1454                             decoder->packet.count == 0) {
1455                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1456                                                 decoder->ip);
1457                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1458                                 decoder->pkt_step = 0;
1459                                 return -ENOENT;
1460                         }
1461                         intel_pt_set_last_ip(decoder);
1462                         decoder->state.from_ip = decoder->ip;
1463                         decoder->state.to_ip = decoder->last_ip;
1464                         decoder->ip = decoder->last_ip;
1465                         intel_pt_update_nr(decoder);
1466                         return 0;
1467                 }
1468
1469                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1470                         decoder->tnt.count -= 1;
1471                         if (decoder->tnt.count)
1472                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1473                         else
1474                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1475                         if (decoder->tnt.payload & BIT63) {
1476                                 decoder->tnt.payload <<= 1;
1477                                 decoder->state.from_ip = decoder->ip;
1478                                 decoder->ip += intel_pt_insn.length +
1479                                                intel_pt_insn.rel;
1480                                 decoder->state.to_ip = decoder->ip;
1481                                 return 0;
1482                         }
1483                         /* Instruction sample for a non-taken branch */
1484                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1485                                 decoder->tnt.payload <<= 1;
1486                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1487                                 decoder->state.from_ip = decoder->ip;
1488                                 decoder->state.to_ip = 0;
1489                                 decoder->ip += intel_pt_insn.length;
1490                                 return 0;
1491                         }
1492                         decoder->sample_cyc = false;
1493                         decoder->ip += intel_pt_insn.length;
1494                         if (!decoder->tnt.count) {
1495                                 intel_pt_update_sample_time(decoder);
1496                                 return -EAGAIN;
1497                         }
1498                         decoder->tnt.payload <<= 1;
1499                         continue;
1500                 }
1501
1502                 return intel_pt_bug(decoder);
1503         }
1504 }
1505
1506 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1507 {
1508         unsigned int fup_tx_flags;
1509         int err;
1510
1511         fup_tx_flags = decoder->packet.payload &
1512                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1513         err = intel_pt_get_next_packet(decoder);
1514         if (err)
1515                 return err;
1516         if (decoder->packet.type == INTEL_PT_FUP) {
1517                 decoder->fup_tx_flags = fup_tx_flags;
1518                 decoder->set_fup_tx_flags = true;
1519                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1520                         *no_tip = true;
1521         } else {
1522                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1523                                 decoder->pos);
1524                 intel_pt_update_in_tx(decoder);
1525         }
1526         return 0;
1527 }
1528
1529 static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1530 {
1531         timestamp |= (ref_timestamp & (0xffULL << 56));
1532
1533         if (timestamp < ref_timestamp) {
1534                 if (ref_timestamp - timestamp > (1ULL << 55))
1535                         timestamp += (1ULL << 56);
1536         } else {
1537                 if (timestamp - ref_timestamp > (1ULL << 55))
1538                         timestamp -= (1ULL << 56);
1539         }
1540
1541         return timestamp;
1542 }
1543
1544 /* For use only when decoder->vm_time_correlation is true */
1545 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1546                                    uint64_t timestamp)
1547 {
1548         uint64_t max_timestamp = decoder->buf_timestamp;
1549
1550         if (!max_timestamp) {
1551                 max_timestamp = decoder->last_reliable_timestamp +
1552                                 0x400000000ULL;
1553         }
1554         return timestamp >= decoder->last_reliable_timestamp &&
1555                timestamp < decoder->buf_timestamp;
1556 }
1557
1558 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1559 {
1560         uint64_t timestamp;
1561         bool bad = false;
1562
1563         decoder->have_tma = false;
1564
1565         if (decoder->ref_timestamp) {
1566                 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1567                                             decoder->ref_timestamp);
1568                 decoder->tsc_timestamp = timestamp;
1569                 decoder->timestamp = timestamp;
1570                 decoder->ref_timestamp = 0;
1571                 decoder->timestamp_insn_cnt = 0;
1572         } else if (decoder->timestamp) {
1573                 timestamp = decoder->packet.payload |
1574                             (decoder->timestamp & (0xffULL << 56));
1575                 decoder->tsc_timestamp = timestamp;
1576                 if (timestamp < decoder->timestamp &&
1577                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1578                         intel_pt_log_to("Suppressing backwards timestamp",
1579                                         timestamp);
1580                         timestamp = decoder->timestamp;
1581                 }
1582                 if (timestamp < decoder->timestamp) {
1583                         if (!decoder->buf_timestamp ||
1584                             (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1585                                 intel_pt_log_to("Wraparound timestamp", timestamp);
1586                                 timestamp += (1ULL << 56);
1587                                 decoder->tsc_timestamp = timestamp;
1588                         } else {
1589                                 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1590                                 timestamp = decoder->timestamp;
1591                                 bad = true;
1592                         }
1593                 }
1594                 if (decoder->vm_time_correlation &&
1595                     (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1596                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1597                         p_log("Timestamp out of range");
1598                 decoder->timestamp = timestamp;
1599                 decoder->timestamp_insn_cnt = 0;
1600         }
1601
1602         if (decoder->last_packet_type == INTEL_PT_CYC) {
1603                 decoder->cyc_ref_timestamp = decoder->timestamp;
1604                 decoder->cycle_cnt = 0;
1605                 decoder->have_calc_cyc_to_tsc = false;
1606                 intel_pt_calc_cyc_to_tsc(decoder, false);
1607         }
1608
1609         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1610 }
1611
1612 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1613 {
1614         intel_pt_log("ERROR: Buffer overflow\n");
1615         intel_pt_clear_tx_flags(decoder);
1616         intel_pt_set_nr(decoder);
1617         decoder->timestamp_insn_cnt = 0;
1618         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1619         decoder->state.from_ip = decoder->ip;
1620         decoder->ip = 0;
1621         decoder->pge = false;
1622         decoder->set_fup_tx_flags = false;
1623         decoder->set_fup_ptw = false;
1624         decoder->set_fup_mwait = false;
1625         decoder->set_fup_pwre = false;
1626         decoder->set_fup_exstop = false;
1627         decoder->set_fup_bep = false;
1628         decoder->overflow = true;
1629         return -EOVERFLOW;
1630 }
1631
1632 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1633 {
1634         if (decoder->have_cyc)
1635                 return;
1636
1637         decoder->cyc_cnt_timestamp = decoder->timestamp;
1638         decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1639 }
1640
1641 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1642 {
1643         decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1644
1645         if (decoder->pge)
1646                 intel_pt_mtc_cyc_cnt_pge(decoder);
1647 }
1648
1649 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1650 {
1651         uint64_t tot_cyc_cnt, tsc_delta;
1652
1653         if (decoder->have_cyc)
1654                 return;
1655
1656         decoder->sample_cyc = true;
1657
1658         if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1659                 return;
1660
1661         tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1662         tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1663
1664         if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1665                 decoder->tot_cyc_cnt = tot_cyc_cnt;
1666 }
1667
1668 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1669 {
1670         uint32_t ctc = decoder->packet.payload;
1671         uint32_t fc = decoder->packet.count;
1672         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1673
1674         if (!decoder->tsc_ctc_ratio_d)
1675                 return;
1676
1677         if (decoder->pge && !decoder->in_psb)
1678                 intel_pt_mtc_cyc_cnt_pge(decoder);
1679         else
1680                 intel_pt_mtc_cyc_cnt_upd(decoder);
1681
1682         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1683         decoder->last_ctc = ctc - ctc_rem;
1684         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1685         if (decoder->tsc_ctc_mult) {
1686                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1687         } else {
1688                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1689                                                   decoder->tsc_ctc_ratio_n,
1690                                                   decoder->tsc_ctc_ratio_d);
1691         }
1692         decoder->ctc_delta = 0;
1693         decoder->have_tma = true;
1694         decoder->fixup_last_mtc = true;
1695         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1696                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1697 }
1698
1699 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1700 {
1701         uint64_t timestamp;
1702         uint32_t mtc, mtc_delta;
1703
1704         if (!decoder->have_tma)
1705                 return;
1706
1707         mtc = decoder->packet.payload;
1708
1709         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1710                 decoder->fixup_last_mtc = false;
1711                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1712                                         &decoder->last_mtc);
1713         }
1714
1715         if (mtc > decoder->last_mtc)
1716                 mtc_delta = mtc - decoder->last_mtc;
1717         else
1718                 mtc_delta = mtc + 256 - decoder->last_mtc;
1719
1720         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1721
1722         if (decoder->tsc_ctc_mult) {
1723                 timestamp = decoder->ctc_timestamp +
1724                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1725         } else {
1726                 timestamp = decoder->ctc_timestamp +
1727                             multdiv(decoder->ctc_delta,
1728                                     decoder->tsc_ctc_ratio_n,
1729                                     decoder->tsc_ctc_ratio_d);
1730         }
1731
1732         if (timestamp < decoder->timestamp)
1733                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1734                              timestamp, decoder->timestamp);
1735         else
1736                 decoder->timestamp = timestamp;
1737
1738         intel_pt_mtc_cyc_cnt_upd(decoder);
1739
1740         decoder->timestamp_insn_cnt = 0;
1741         decoder->last_mtc = mtc;
1742
1743         if (decoder->last_packet_type == INTEL_PT_CYC) {
1744                 decoder->cyc_ref_timestamp = decoder->timestamp;
1745                 decoder->cycle_cnt = 0;
1746                 decoder->have_calc_cyc_to_tsc = false;
1747                 intel_pt_calc_cyc_to_tsc(decoder, true);
1748         }
1749
1750         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1751 }
1752
1753 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1754 {
1755         unsigned int cbr = decoder->packet.payload & 0xff;
1756
1757         decoder->cbr_payload = decoder->packet.payload;
1758
1759         if (decoder->cbr == cbr)
1760                 return;
1761
1762         decoder->cbr = cbr;
1763         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1764
1765         intel_pt_mtc_cyc_cnt_cbr(decoder);
1766 }
1767
1768 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1769 {
1770         uint64_t timestamp = decoder->cyc_ref_timestamp;
1771
1772         decoder->have_cyc = true;
1773
1774         decoder->cycle_cnt += decoder->packet.payload;
1775         if (decoder->pge)
1776                 decoder->tot_cyc_cnt += decoder->packet.payload;
1777         decoder->sample_cyc = true;
1778
1779         if (!decoder->cyc_ref_timestamp)
1780                 return;
1781
1782         if (decoder->have_calc_cyc_to_tsc)
1783                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1784         else if (decoder->cbr)
1785                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1786         else
1787                 return;
1788
1789         if (timestamp < decoder->timestamp)
1790                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1791                              timestamp, decoder->timestamp);
1792         else
1793                 decoder->timestamp = timestamp;
1794
1795         decoder->timestamp_insn_cnt = 0;
1796
1797         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1798 }
1799
1800 static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1801 {
1802         if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1803                 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1804                 decoder->state.items.is_32_bit = false;
1805         }
1806         decoder->blk_type = decoder->packet.payload;
1807         decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1808         if (decoder->blk_type == INTEL_PT_GP_REGS)
1809                 decoder->state.items.is_32_bit = decoder->packet.count;
1810         if (decoder->blk_type_pos < 0) {
1811                 intel_pt_log("WARNING: Unknown block type %u\n",
1812                              decoder->blk_type);
1813         } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1814                 intel_pt_log("WARNING: Duplicate block type %u\n",
1815                              decoder->blk_type);
1816         }
1817 }
1818
1819 static void intel_pt_bip(struct intel_pt_decoder *decoder)
1820 {
1821         uint32_t id = decoder->packet.count;
1822         uint32_t bit = 1 << id;
1823         int pos = decoder->blk_type_pos;
1824
1825         if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1826                 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1827                              id, decoder->blk_type);
1828                 return;
1829         }
1830
1831         if (decoder->state.items.mask[pos] & bit) {
1832                 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1833                              id, decoder->blk_type);
1834         }
1835
1836         decoder->state.items.mask[pos] |= bit;
1837         decoder->state.items.val[pos][id] = decoder->packet.payload;
1838 }
1839
1840 /* Walk PSB+ packets when already in sync. */
1841 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1842 {
1843         int err;
1844
1845         decoder->in_psb = true;
1846
1847         while (1) {
1848                 err = intel_pt_get_next_packet(decoder);
1849                 if (err)
1850                         goto out;
1851
1852                 switch (decoder->packet.type) {
1853                 case INTEL_PT_PSBEND:
1854                         err = 0;
1855                         goto out;
1856
1857                 case INTEL_PT_TIP_PGD:
1858                 case INTEL_PT_TIP_PGE:
1859                 case INTEL_PT_TIP:
1860                 case INTEL_PT_TNT:
1861                 case INTEL_PT_TRACESTOP:
1862                 case INTEL_PT_BAD:
1863                 case INTEL_PT_PSB:
1864                 case INTEL_PT_PTWRITE:
1865                 case INTEL_PT_PTWRITE_IP:
1866                 case INTEL_PT_EXSTOP:
1867                 case INTEL_PT_EXSTOP_IP:
1868                 case INTEL_PT_MWAIT:
1869                 case INTEL_PT_PWRE:
1870                 case INTEL_PT_PWRX:
1871                 case INTEL_PT_BBP:
1872                 case INTEL_PT_BIP:
1873                 case INTEL_PT_BEP:
1874                 case INTEL_PT_BEP_IP:
1875                         decoder->have_tma = false;
1876                         intel_pt_log("ERROR: Unexpected packet\n");
1877                         err = -EAGAIN;
1878                         goto out;
1879
1880                 case INTEL_PT_OVF:
1881                         err = intel_pt_overflow(decoder);
1882                         goto out;
1883
1884                 case INTEL_PT_TSC:
1885                         intel_pt_calc_tsc_timestamp(decoder);
1886                         break;
1887
1888                 case INTEL_PT_TMA:
1889                         intel_pt_calc_tma(decoder);
1890                         break;
1891
1892                 case INTEL_PT_CBR:
1893                         intel_pt_calc_cbr(decoder);
1894                         break;
1895
1896                 case INTEL_PT_MODE_EXEC:
1897                         decoder->exec_mode = decoder->packet.payload;
1898                         break;
1899
1900                 case INTEL_PT_PIP:
1901                         intel_pt_set_pip(decoder);
1902                         break;
1903
1904                 case INTEL_PT_FUP:
1905                         decoder->pge = true;
1906                         if (decoder->packet.count) {
1907                                 intel_pt_set_last_ip(decoder);
1908                                 decoder->psb_ip = decoder->last_ip;
1909                         }
1910                         break;
1911
1912                 case INTEL_PT_MODE_TSX:
1913                         intel_pt_update_in_tx(decoder);
1914                         break;
1915
1916                 case INTEL_PT_MTC:
1917                         intel_pt_calc_mtc_timestamp(decoder);
1918                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1919                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1920                         break;
1921
1922                 case INTEL_PT_CYC:
1923                         intel_pt_calc_cyc_timestamp(decoder);
1924                         break;
1925
1926                 case INTEL_PT_VMCS:
1927                 case INTEL_PT_MNT:
1928                 case INTEL_PT_PAD:
1929                 default:
1930                         break;
1931                 }
1932         }
1933 out:
1934         decoder->in_psb = false;
1935
1936         return err;
1937 }
1938
1939 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1940 {
1941         int err;
1942
1943         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1944                 decoder->tx_flags = 0;
1945                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1946                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1947         } else {
1948                 decoder->state.flags |= INTEL_PT_ASYNC;
1949         }
1950
1951         while (1) {
1952                 err = intel_pt_get_next_packet(decoder);
1953                 if (err)
1954                         return err;
1955
1956                 switch (decoder->packet.type) {
1957                 case INTEL_PT_TNT:
1958                 case INTEL_PT_FUP:
1959                 case INTEL_PT_TRACESTOP:
1960                 case INTEL_PT_PSB:
1961                 case INTEL_PT_TSC:
1962                 case INTEL_PT_TMA:
1963                 case INTEL_PT_MODE_TSX:
1964                 case INTEL_PT_BAD:
1965                 case INTEL_PT_PSBEND:
1966                 case INTEL_PT_PTWRITE:
1967                 case INTEL_PT_PTWRITE_IP:
1968                 case INTEL_PT_EXSTOP:
1969                 case INTEL_PT_EXSTOP_IP:
1970                 case INTEL_PT_MWAIT:
1971                 case INTEL_PT_PWRE:
1972                 case INTEL_PT_PWRX:
1973                 case INTEL_PT_BBP:
1974                 case INTEL_PT_BIP:
1975                 case INTEL_PT_BEP:
1976                 case INTEL_PT_BEP_IP:
1977                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1978                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1979                         decoder->pkt_step = 0;
1980                         return -ENOENT;
1981
1982                 case INTEL_PT_CBR:
1983                         intel_pt_calc_cbr(decoder);
1984                         break;
1985
1986                 case INTEL_PT_OVF:
1987                         return intel_pt_overflow(decoder);
1988
1989                 case INTEL_PT_TIP_PGD:
1990                         decoder->state.from_ip = decoder->ip;
1991                         if (decoder->packet.count == 0) {
1992                                 decoder->state.to_ip = 0;
1993                         } else {
1994                                 intel_pt_set_ip(decoder);
1995                                 decoder->state.to_ip = decoder->ip;
1996                         }
1997                         decoder->pge = false;
1998                         decoder->continuous_period = false;
1999                         decoder->state.type |= INTEL_PT_TRACE_END;
2000                         intel_pt_update_nr(decoder);
2001                         return 0;
2002
2003                 case INTEL_PT_TIP_PGE:
2004                         decoder->pge = true;
2005                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
2006                                      decoder->ip);
2007                         decoder->state.from_ip = 0;
2008                         if (decoder->packet.count == 0) {
2009                                 decoder->state.to_ip = 0;
2010                         } else {
2011                                 intel_pt_set_ip(decoder);
2012                                 decoder->state.to_ip = decoder->ip;
2013                         }
2014                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2015                         intel_pt_mtc_cyc_cnt_pge(decoder);
2016                         intel_pt_set_nr(decoder);
2017                         return 0;
2018
2019                 case INTEL_PT_TIP:
2020                         decoder->state.from_ip = decoder->ip;
2021                         if (decoder->packet.count == 0) {
2022                                 decoder->state.to_ip = 0;
2023                         } else {
2024                                 intel_pt_set_ip(decoder);
2025                                 decoder->state.to_ip = decoder->ip;
2026                         }
2027                         intel_pt_update_nr(decoder);
2028                         return 0;
2029
2030                 case INTEL_PT_PIP:
2031                         intel_pt_update_pip(decoder);
2032                         break;
2033
2034                 case INTEL_PT_MTC:
2035                         intel_pt_calc_mtc_timestamp(decoder);
2036                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2037                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
2038                         break;
2039
2040                 case INTEL_PT_CYC:
2041                         intel_pt_calc_cyc_timestamp(decoder);
2042                         break;
2043
2044                 case INTEL_PT_MODE_EXEC:
2045                         decoder->exec_mode = decoder->packet.payload;
2046                         break;
2047
2048                 case INTEL_PT_VMCS:
2049                 case INTEL_PT_MNT:
2050                 case INTEL_PT_PAD:
2051                         break;
2052
2053                 default:
2054                         return intel_pt_bug(decoder);
2055                 }
2056         }
2057 }
2058
2059 static int intel_pt_resample(struct intel_pt_decoder *decoder)
2060 {
2061         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2062         decoder->state.type = INTEL_PT_INSTRUCTION;
2063         decoder->state.from_ip = decoder->ip;
2064         decoder->state.to_ip = 0;
2065         return 0;
2066 }
2067
2068 struct intel_pt_vm_tsc_info {
2069         struct intel_pt_pkt pip_packet;
2070         struct intel_pt_pkt vmcs_packet;
2071         struct intel_pt_pkt tma_packet;
2072         bool tsc, pip, vmcs, tma, psbend;
2073         uint64_t ctc_delta;
2074         uint64_t last_ctc;
2075         int max_lookahead;
2076 };
2077
2078 /* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */
2079 static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2080 {
2081         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2082
2083         switch (pkt_info->packet.type) {
2084         case INTEL_PT_PAD:
2085         case INTEL_PT_MNT:
2086         case INTEL_PT_MODE_EXEC:
2087         case INTEL_PT_MODE_TSX:
2088         case INTEL_PT_MTC:
2089         case INTEL_PT_FUP:
2090         case INTEL_PT_CYC:
2091         case INTEL_PT_CBR:
2092                 break;
2093
2094         case INTEL_PT_TSC:
2095                 data->tsc = true;
2096                 break;
2097
2098         case INTEL_PT_TMA:
2099                 data->tma_packet = pkt_info->packet;
2100                 data->tma = true;
2101                 break;
2102
2103         case INTEL_PT_PIP:
2104                 data->pip_packet = pkt_info->packet;
2105                 data->pip = true;
2106                 break;
2107
2108         case INTEL_PT_VMCS:
2109                 data->vmcs_packet = pkt_info->packet;
2110                 data->vmcs = true;
2111                 break;
2112
2113         case INTEL_PT_PSBEND:
2114                 data->psbend = true;
2115                 return 1;
2116
2117         case INTEL_PT_TIP_PGE:
2118         case INTEL_PT_PTWRITE:
2119         case INTEL_PT_PTWRITE_IP:
2120         case INTEL_PT_EXSTOP:
2121         case INTEL_PT_EXSTOP_IP:
2122         case INTEL_PT_MWAIT:
2123         case INTEL_PT_PWRE:
2124         case INTEL_PT_PWRX:
2125         case INTEL_PT_BBP:
2126         case INTEL_PT_BIP:
2127         case INTEL_PT_BEP:
2128         case INTEL_PT_BEP_IP:
2129         case INTEL_PT_OVF:
2130         case INTEL_PT_BAD:
2131         case INTEL_PT_TNT:
2132         case INTEL_PT_TIP_PGD:
2133         case INTEL_PT_TIP:
2134         case INTEL_PT_PSB:
2135         case INTEL_PT_TRACESTOP:
2136         default:
2137                 return 1;
2138         }
2139
2140         return 0;
2141 }
2142
2143 struct intel_pt_ovf_fup_info {
2144         int max_lookahead;
2145         bool found;
2146 };
2147
2148 /* Lookahead to detect a FUP packet after OVF */
2149 static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2150 {
2151         struct intel_pt_ovf_fup_info *data = pkt_info->data;
2152
2153         if (pkt_info->packet.type == INTEL_PT_CYC ||
2154             pkt_info->packet.type == INTEL_PT_MTC ||
2155             pkt_info->packet.type == INTEL_PT_TSC)
2156                 return !--(data->max_lookahead);
2157         data->found = pkt_info->packet.type == INTEL_PT_FUP;
2158         return 1;
2159 }
2160
2161 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2162 {
2163         struct intel_pt_ovf_fup_info data = {
2164                 .max_lookahead = 16,
2165                 .found = false,
2166         };
2167
2168         intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2169         return data.found;
2170 }
2171
2172 /* Lookahead and get the TMA packet after TSC */
2173 static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2174 {
2175         struct intel_pt_vm_tsc_info *data = pkt_info->data;
2176
2177         if (pkt_info->packet.type == INTEL_PT_CYC ||
2178             pkt_info->packet.type == INTEL_PT_MTC)
2179                 return !--(data->max_lookahead);
2180
2181         if (pkt_info->packet.type == INTEL_PT_TMA) {
2182                 data->tma_packet = pkt_info->packet;
2183                 data->tma = true;
2184         }
2185         return 1;
2186 }
2187
2188 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2189 {
2190         if (decoder->tsc_ctc_mult)
2191                 return ctc * decoder->tsc_ctc_mult;
2192         else
2193                 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2194 }
2195
2196 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2197                                            uint32_t ctc,
2198                                            uint32_t fc,
2199                                            uint64_t last_ctc_timestamp,
2200                                            uint64_t ctc_delta,
2201                                            uint32_t last_ctc)
2202 {
2203         /* Number of CTC ticks from last_ctc_timestamp to last_mtc */
2204         uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2205         /*
2206          * Number of CTC ticks from there until current TMA packet. We would
2207          * expect last_mtc_ctc to be before ctc, but the TSC packet can slip
2208          * past an MTC, so a sign-extended value is used.
2209          */
2210         uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2211         /* Total CTC ticks from last_ctc_timestamp to current TMA packet */
2212         uint64_t new_ctc_delta = ctc_delta + delta;
2213         uint64_t expected_tsc;
2214
2215         /*
2216          * Convert CTC ticks to TSC ticks, add the starting point
2217          * (last_ctc_timestamp) and the fast counter from the TMA packet.
2218          */
2219         expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2220
2221         if (intel_pt_enable_logging) {
2222                 intel_pt_log_x64(last_mtc_ctc);
2223                 intel_pt_log_x32(last_ctc);
2224                 intel_pt_log_x64(ctc_delta);
2225                 intel_pt_log_x64(delta);
2226                 intel_pt_log_x32(ctc);
2227                 intel_pt_log_x64(new_ctc_delta);
2228                 intel_pt_log_x64(last_ctc_timestamp);
2229                 intel_pt_log_x32(fc);
2230                 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2231                 intel_pt_log_x64(expected_tsc);
2232         }
2233
2234         return expected_tsc;
2235 }
2236
2237 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2238                                       struct intel_pt_vm_tsc_info *data)
2239 {
2240         uint32_t ctc = data->tma_packet.payload;
2241         uint32_t fc = data->tma_packet.count;
2242
2243         return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2244                                           decoder->ctc_timestamp,
2245                                           data->ctc_delta, data->last_ctc);
2246 }
2247
2248 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2249                                       struct intel_pt_vmcs_info *vmcs_info)
2250 {
2251         uint64_t payload = decoder->packet.payload;
2252
2253         /* VMX adds the TSC Offset, so subtract to get host TSC */
2254         decoder->packet.payload -= vmcs_info->tsc_offset;
2255         /* TSC packet has only 7 bytes */
2256         decoder->packet.payload &= SEVEN_BYTES;
2257
2258         /*
2259          * The buffer is mmapped from the data file, so this also updates the
2260          * data file.
2261          */
2262         if (!decoder->vm_tm_corr_dry_run)
2263                 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2264
2265         intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2266                      "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2267                      payload, decoder->packet.payload, vmcs_info->vmcs,
2268                      vmcs_info->tsc_offset);
2269 }
2270
2271 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2272                                              uint64_t tsc_offset)
2273 {
2274         struct intel_pt_vmcs_info vmcs_info = {
2275                 .vmcs = NO_VMCS,
2276                 .tsc_offset = tsc_offset
2277         };
2278
2279         intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2280 }
2281
2282 static inline bool in_vm(uint64_t pip_payload)
2283 {
2284         return pip_payload & 1;
2285 }
2286
2287 static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2288 {
2289         return pip_packet->payload & 1;
2290 }
2291
2292 static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2293 {
2294         p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,
2295               vmcs_info->vmcs, vmcs_info->tsc_offset);
2296 }
2297
2298 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2299                                     struct intel_pt_vm_tsc_info *data)
2300 {
2301         memset(data, 0, sizeof(*data));
2302         data->ctc_delta = decoder->ctc_delta;
2303         data->last_ctc = decoder->last_ctc;
2304         intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2305         if (data->tsc && !data->psbend)
2306                 p_log("ERROR: PSB without PSBEND");
2307         decoder->in_psb = data->psbend;
2308 }
2309
2310 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2311                                           struct intel_pt_vm_tsc_info *data,
2312                                           struct intel_pt_vmcs_info *vmcs_info,
2313                                           uint64_t host_tsc)
2314 {
2315         if (!decoder->in_psb) {
2316                 /* Can't happen */
2317                 p_log("ERROR: First TSC is not in PSB+");
2318         }
2319
2320         if (data->pip) {
2321                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2322                         if (vmcs_info && vmcs_info->tsc_offset) {
2323                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2324                                 decoder->vm_tm_corr_reliable = true;
2325                         } else {
2326                                 p_log("ERROR: First TSC, unknown TSC Offset");
2327                         }
2328                 } else { /* Host */
2329                         decoder->vm_tm_corr_reliable = true;
2330                 }
2331         } else { /* Host or Guest */
2332                 decoder->vm_tm_corr_reliable = false;
2333                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2334                         /* Assume Host */
2335                 } else {
2336                         /* Assume Guest */
2337                         if (vmcs_info && vmcs_info->tsc_offset)
2338                                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2339                         else
2340                                 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2341                 }
2342         }
2343 }
2344
2345 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2346                                     struct intel_pt_vm_tsc_info *data)
2347 {
2348         struct intel_pt_vmcs_info *vmcs_info;
2349         uint64_t tsc_offset = 0;
2350         uint64_t vmcs;
2351         bool reliable = true;
2352         uint64_t expected_tsc;
2353         uint64_t host_tsc;
2354         uint64_t ref_timestamp;
2355
2356         bool assign = false;
2357         bool assign_reliable = false;
2358
2359         /* Already have 'data' for the in_psb case */
2360         if (!decoder->in_psb) {
2361                 memset(data, 0, sizeof(*data));
2362                 data->ctc_delta = decoder->ctc_delta;
2363                 data->last_ctc = decoder->last_ctc;
2364                 data->max_lookahead = 16;
2365                 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2366                 if (decoder->pge) {
2367                         data->pip = true;
2368                         data->pip_packet.payload = decoder->pip_payload;
2369                 }
2370         }
2371
2372         /* Calculations depend on having TMA packets */
2373         if (!data->tma) {
2374                 p_log("ERROR: TSC without TMA");
2375                 return;
2376         }
2377
2378         vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2379         if (vmcs == NO_VMCS)
2380                 vmcs = 0;
2381
2382         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2383
2384         ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2385         host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2386
2387         if (!decoder->ctc_timestamp) {
2388                 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2389                 return;
2390         }
2391
2392         expected_tsc = intel_pt_expected_tsc(decoder, data);
2393
2394         tsc_offset = host_tsc - expected_tsc;
2395
2396         /* Determine if TSC is from Host or Guest */
2397         if (data->pip) {
2398                 if (pip_in_vm(&data->pip_packet)) { /* Guest */
2399                         if (!vmcs_info) {
2400                                 /* PIP NR=1 without VMCS cannot happen */
2401                                 p_log("ERROR: Missing VMCS");
2402                                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2403                                 decoder->vm_tm_corr_reliable = false;
2404                                 return;
2405                         }
2406                 } else { /* Host */
2407                         decoder->last_reliable_timestamp = host_tsc;
2408                         decoder->vm_tm_corr_reliable = true;
2409                         return;
2410                 }
2411         } else { /* Host or Guest */
2412                 reliable = false; /* Host/Guest is a guess, so not reliable */
2413                 if (decoder->in_psb) {
2414                         if (!tsc_offset)
2415                                 return; /* Zero TSC Offset, assume Host */
2416                         /*
2417                          * TSC packet has only 7 bytes of TSC. We have no
2418                          * information about the Guest's 8th byte, but it
2419                          * doesn't matter because we only need 7 bytes.
2420                          * Here, since the 8th byte is unreliable and
2421                          * irrelevant, compare only 7 byes.
2422                          */
2423                         if (vmcs_info &&
2424                             (tsc_offset & SEVEN_BYTES) ==
2425                             (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2426                                 /* Same TSC Offset as last VMCS, assume Guest */
2427                                 goto guest;
2428                         }
2429                 }
2430                 /*
2431                  * Check if the host_tsc is within the expected range.
2432                  * Note, we could narrow the range more by looking ahead for
2433                  * the next host TSC in the same buffer, but we don't bother to
2434                  * do that because this is probably good enough.
2435                  */
2436                 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2437                         /* Within expected range for Host TSC, assume Host */
2438                         decoder->vm_tm_corr_reliable = false;
2439                         return;
2440                 }
2441         }
2442
2443 guest: /* Assuming Guest */
2444
2445         /* Determine whether to assign TSC Offset */
2446         if (vmcs_info && vmcs_info->vmcs) {
2447                 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2448                         assign = false;
2449                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2450                            decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2451                         /* Continuous tracing, TSC in a PSB is not a time loss */
2452                         assign = true;
2453                         assign_reliable = true;
2454                 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2455                         /*
2456                          * Unlikely to be a time loss TSC in a PSB which is not
2457                          * at the start of a buffer.
2458                          */
2459                         assign = true;
2460                         assign_reliable = false;
2461                 }
2462         }
2463
2464         /* Record VMCS TSC Offset */
2465         if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2466                        vmcs_info->reliable != assign_reliable)) {
2467                 bool print = vmcs_info->tsc_offset != tsc_offset;
2468
2469                 vmcs_info->tsc_offset = tsc_offset;
2470                 vmcs_info->reliable = assign_reliable;
2471                 if (print)
2472                         intel_pt_print_vmcs_info(vmcs_info);
2473         }
2474
2475         /* Determine what TSC Offset to use */
2476         if (vmcs_info && vmcs_info->tsc_offset) {
2477                 if (!vmcs_info->reliable)
2478                         reliable = false;
2479                 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2480         } else {
2481                 reliable = false;
2482                 if (vmcs_info) {
2483                         if (!vmcs_info->error_printed) {
2484                                 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2485                                       vmcs_info->vmcs);
2486                                 vmcs_info->error_printed = true;
2487                         }
2488                 } else {
2489                         if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2490                                 p_log("ERROR: Unknown VMCS");
2491                 }
2492                 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2493         }
2494
2495         decoder->vm_tm_corr_reliable = reliable;
2496 }
2497
2498 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2499 {
2500         uint64_t host_tsc = decoder->packet.payload;
2501         uint64_t guest_tsc = decoder->packet.payload;
2502         struct intel_pt_vmcs_info *vmcs_info;
2503         uint64_t vmcs;
2504
2505         vmcs = decoder->vmcs;
2506         if (vmcs == NO_VMCS)
2507                 vmcs = 0;
2508
2509         vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2510
2511         if (decoder->pge) {
2512                 if (in_vm(decoder->pip_payload)) { /* Guest */
2513                         if (!vmcs_info) {
2514                                 /* PIP NR=1 without VMCS cannot happen */
2515                                 p_log("ERROR: Missing VMCS");
2516                         }
2517                 } else { /* Host */
2518                         return;
2519                 }
2520         } else { /* Host or Guest */
2521                 if (intel_pt_time_in_range(decoder, host_tsc)) {
2522                         /* Within expected range for Host TSC, assume Host */
2523                         return;
2524                 }
2525         }
2526
2527         if (vmcs_info) {
2528                 /* Translate Guest TSC to Host TSC */
2529                 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2530                 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2531                 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2532                              "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",
2533                              guest_tsc, host_tsc, vmcs_info->vmcs,
2534                              vmcs_info->tsc_offset);
2535                 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2536                     intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2537                         p_log("Timestamp out of range");
2538         } else {
2539                 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2540                         p_log("ERROR: Unknown VMCS");
2541                 host_tsc = decoder->timestamp;
2542         }
2543
2544         decoder->packet.payload = host_tsc;
2545
2546         if (!decoder->vm_tm_corr_dry_run)
2547                 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2548 }
2549
2550 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2551 {
2552         struct intel_pt_vm_tsc_info data = { .psbend = false };
2553         bool pge;
2554         int err;
2555
2556         if (decoder->in_psb)
2557                 intel_pt_vm_tm_corr_psb(decoder, &data);
2558
2559         while (1) {
2560                 err = intel_pt_get_next_packet(decoder);
2561                 if (err == -ENOLINK)
2562                         continue;
2563                 if (err)
2564                         break;
2565
2566                 switch (decoder->packet.type) {
2567                 case INTEL_PT_TIP_PGD:
2568                         decoder->pge = false;
2569                         decoder->vm_tm_corr_continuous = false;
2570                         break;
2571
2572                 case INTEL_PT_TNT:
2573                 case INTEL_PT_TIP:
2574                 case INTEL_PT_TIP_PGE:
2575                         decoder->pge = true;
2576                         break;
2577
2578                 case INTEL_PT_OVF:
2579                         decoder->in_psb = false;
2580                         pge = decoder->pge;
2581                         decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2582                         if (pge != decoder->pge)
2583                                 intel_pt_log("Surprising PGE change in OVF!");
2584                         if (!decoder->pge)
2585                                 decoder->vm_tm_corr_continuous = false;
2586                         break;
2587
2588                 case INTEL_PT_FUP:
2589                         if (decoder->in_psb)
2590                                 decoder->pge = true;
2591                         break;
2592
2593                 case INTEL_PT_TRACESTOP:
2594                         decoder->pge = false;
2595                         decoder->vm_tm_corr_continuous = false;
2596                         decoder->have_tma = false;
2597                         break;
2598
2599                 case INTEL_PT_PSB:
2600                         intel_pt_vm_tm_corr_psb(decoder, &data);
2601                         break;
2602
2603                 case INTEL_PT_PIP:
2604                         decoder->pip_payload = decoder->packet.payload;
2605                         break;
2606
2607                 case INTEL_PT_MTC:
2608                         intel_pt_calc_mtc_timestamp(decoder);
2609                         break;
2610
2611                 case INTEL_PT_TSC:
2612                         intel_pt_vm_tm_corr_tsc(decoder, &data);
2613                         intel_pt_calc_tsc_timestamp(decoder);
2614                         decoder->vm_tm_corr_same_buf = true;
2615                         decoder->vm_tm_corr_continuous = decoder->pge;
2616                         break;
2617
2618                 case INTEL_PT_TMA:
2619                         intel_pt_calc_tma(decoder);
2620                         break;
2621
2622                 case INTEL_PT_CYC:
2623                         intel_pt_calc_cyc_timestamp(decoder);
2624                         break;
2625
2626                 case INTEL_PT_CBR:
2627                         intel_pt_calc_cbr(decoder);
2628                         break;
2629
2630                 case INTEL_PT_PSBEND:
2631                         decoder->in_psb = false;
2632                         data.psbend = false;
2633                         break;
2634
2635                 case INTEL_PT_VMCS:
2636                         if (decoder->packet.payload != NO_VMCS)
2637                                 decoder->vmcs = decoder->packet.payload;
2638                         break;
2639
2640                 case INTEL_PT_BBP:
2641                         decoder->blk_type = decoder->packet.payload;
2642                         break;
2643
2644                 case INTEL_PT_BIP:
2645                         if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2646                             decoder->packet.count == 2)
2647                                 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2648                         break;
2649
2650                 case INTEL_PT_BEP:
2651                 case INTEL_PT_BEP_IP:
2652                         decoder->blk_type = 0;
2653                         break;
2654
2655                 case INTEL_PT_MODE_EXEC:
2656                 case INTEL_PT_MODE_TSX:
2657                 case INTEL_PT_MNT:
2658                 case INTEL_PT_PAD:
2659                 case INTEL_PT_PTWRITE_IP:
2660                 case INTEL_PT_PTWRITE:
2661                 case INTEL_PT_MWAIT:
2662                 case INTEL_PT_PWRE:
2663                 case INTEL_PT_EXSTOP_IP:
2664                 case INTEL_PT_EXSTOP:
2665                 case INTEL_PT_PWRX:
2666                 case INTEL_PT_BAD: /* Does not happen */
2667                 default:
2668                         break;
2669                 }
2670         }
2671
2672         return err;
2673 }
2674
2675 #define HOP_PROCESS     0
2676 #define HOP_IGNORE      1
2677 #define HOP_RETURN      2
2678 #define HOP_AGAIN       3
2679
2680 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2681
2682 /* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */
2683 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2684 {
2685         *err = 0;
2686
2687         /* Leap from PSB to PSB, getting ip from FUP within PSB+ */
2688         if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2689                 *err = intel_pt_scan_for_psb(decoder);
2690                 if (*err)
2691                         return HOP_RETURN;
2692         }
2693
2694         switch (decoder->packet.type) {
2695         case INTEL_PT_TNT:
2696                 return HOP_IGNORE;
2697
2698         case INTEL_PT_TIP_PGD:
2699                 decoder->pge = false;
2700                 if (!decoder->packet.count) {
2701                         intel_pt_set_nr(decoder);
2702                         return HOP_IGNORE;
2703                 }
2704                 intel_pt_set_ip(decoder);
2705                 decoder->state.type |= INTEL_PT_TRACE_END;
2706                 decoder->state.from_ip = 0;
2707                 decoder->state.to_ip = decoder->ip;
2708                 intel_pt_update_nr(decoder);
2709                 return HOP_RETURN;
2710
2711         case INTEL_PT_TIP:
2712                 if (!decoder->packet.count) {
2713                         intel_pt_set_nr(decoder);
2714                         return HOP_IGNORE;
2715                 }
2716                 intel_pt_set_ip(decoder);
2717                 decoder->state.type = INTEL_PT_INSTRUCTION;
2718                 decoder->state.from_ip = decoder->ip;
2719                 decoder->state.to_ip = 0;
2720                 intel_pt_update_nr(decoder);
2721                 return HOP_RETURN;
2722
2723         case INTEL_PT_FUP:
2724                 if (!decoder->packet.count)
2725                         return HOP_IGNORE;
2726                 intel_pt_set_ip(decoder);
2727                 if (decoder->set_fup_mwait || decoder->set_fup_pwre)
2728                         *no_tip = true;
2729                 if (!decoder->branch_enable || !decoder->pge)
2730                         *no_tip = true;
2731                 if (*no_tip) {
2732                         decoder->state.type = INTEL_PT_INSTRUCTION;
2733                         decoder->state.from_ip = decoder->ip;
2734                         decoder->state.to_ip = 0;
2735                         intel_pt_fup_event(decoder);
2736                         return HOP_RETURN;
2737                 }
2738                 intel_pt_fup_event(decoder);
2739                 decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;
2740                 *err = intel_pt_walk_fup_tip(decoder);
2741                 if (!*err && decoder->state.to_ip)
2742                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2743                 return HOP_RETURN;
2744
2745         case INTEL_PT_PSB:
2746                 decoder->state.psb_offset = decoder->pos;
2747                 decoder->psb_ip = 0;
2748                 decoder->last_ip = 0;
2749                 decoder->have_last_ip = true;
2750                 *err = intel_pt_walk_psbend(decoder);
2751                 if (*err == -EAGAIN)
2752                         return HOP_AGAIN;
2753                 if (*err)
2754                         return HOP_RETURN;
2755                 decoder->state.type = INTEL_PT_PSB_EVT;
2756                 if (decoder->psb_ip) {
2757                         decoder->state.type |= INTEL_PT_INSTRUCTION;
2758                         decoder->ip = decoder->psb_ip;
2759                 }
2760                 decoder->state.from_ip = decoder->psb_ip;
2761                 decoder->state.to_ip = 0;
2762                 return HOP_RETURN;
2763
2764         case INTEL_PT_BAD:
2765         case INTEL_PT_PAD:
2766         case INTEL_PT_TIP_PGE:
2767         case INTEL_PT_TSC:
2768         case INTEL_PT_TMA:
2769         case INTEL_PT_MODE_EXEC:
2770         case INTEL_PT_MODE_TSX:
2771         case INTEL_PT_MTC:
2772         case INTEL_PT_CYC:
2773         case INTEL_PT_VMCS:
2774         case INTEL_PT_PSBEND:
2775         case INTEL_PT_CBR:
2776         case INTEL_PT_TRACESTOP:
2777         case INTEL_PT_PIP:
2778         case INTEL_PT_OVF:
2779         case INTEL_PT_MNT:
2780         case INTEL_PT_PTWRITE:
2781         case INTEL_PT_PTWRITE_IP:
2782         case INTEL_PT_EXSTOP:
2783         case INTEL_PT_EXSTOP_IP:
2784         case INTEL_PT_MWAIT:
2785         case INTEL_PT_PWRE:
2786         case INTEL_PT_PWRX:
2787         case INTEL_PT_BBP:
2788         case INTEL_PT_BIP:
2789         case INTEL_PT_BEP:
2790         case INTEL_PT_BEP_IP:
2791         default:
2792                 return HOP_PROCESS;
2793         }
2794 }
2795
2796 struct intel_pt_psb_info {
2797         struct intel_pt_pkt fup_packet;
2798         bool fup;
2799         int after_psbend;
2800 };
2801
2802 /* Lookahead and get the FUP packet from PSB+ */
2803 static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2804 {
2805         struct intel_pt_psb_info *data = pkt_info->data;
2806
2807         switch (pkt_info->packet.type) {
2808         case INTEL_PT_PAD:
2809         case INTEL_PT_MNT:
2810         case INTEL_PT_TSC:
2811         case INTEL_PT_TMA:
2812         case INTEL_PT_MODE_EXEC:
2813         case INTEL_PT_MODE_TSX:
2814         case INTEL_PT_MTC:
2815         case INTEL_PT_CYC:
2816         case INTEL_PT_VMCS:
2817         case INTEL_PT_CBR:
2818         case INTEL_PT_PIP:
2819                 if (data->after_psbend) {
2820                         data->after_psbend -= 1;
2821                         if (!data->after_psbend)
2822                                 return 1;
2823                 }
2824                 break;
2825
2826         case INTEL_PT_FUP:
2827                 if (data->after_psbend)
2828                         return 1;
2829                 if (data->fup || pkt_info->packet.count == 0)
2830                         return 1;
2831                 data->fup_packet = pkt_info->packet;
2832                 data->fup = true;
2833                 break;
2834
2835         case INTEL_PT_PSBEND:
2836                 if (!data->fup)
2837                         return 1;
2838                 /* Keep going to check for a TIP.PGE */
2839                 data->after_psbend = 6;
2840                 break;
2841
2842         case INTEL_PT_TIP_PGE:
2843                 /* Ignore FUP in PSB+ if followed by TIP.PGE */
2844                 if (data->after_psbend)
2845                         data->fup = false;
2846                 return 1;
2847
2848         case INTEL_PT_PTWRITE:
2849         case INTEL_PT_PTWRITE_IP:
2850         case INTEL_PT_EXSTOP:
2851         case INTEL_PT_EXSTOP_IP:
2852         case INTEL_PT_MWAIT:
2853         case INTEL_PT_PWRE:
2854         case INTEL_PT_PWRX:
2855         case INTEL_PT_BBP:
2856         case INTEL_PT_BIP:
2857         case INTEL_PT_BEP:
2858         case INTEL_PT_BEP_IP:
2859                 if (data->after_psbend) {
2860                         data->after_psbend -= 1;
2861                         if (!data->after_psbend)
2862                                 return 1;
2863                         break;
2864                 }
2865                 return 1;
2866
2867         case INTEL_PT_OVF:
2868         case INTEL_PT_BAD:
2869         case INTEL_PT_TNT:
2870         case INTEL_PT_TIP_PGD:
2871         case INTEL_PT_TIP:
2872         case INTEL_PT_PSB:
2873         case INTEL_PT_TRACESTOP:
2874         default:
2875                 return 1;
2876         }
2877
2878         return 0;
2879 }
2880
2881 static int intel_pt_psb(struct intel_pt_decoder *decoder)
2882 {
2883         int err;
2884
2885         decoder->last_ip = 0;
2886         decoder->psb_ip = 0;
2887         decoder->have_last_ip = true;
2888         intel_pt_clear_stack(&decoder->stack);
2889         err = intel_pt_walk_psbend(decoder);
2890         if (err)
2891                 return err;
2892         decoder->state.type = INTEL_PT_PSB_EVT;
2893         decoder->state.from_ip = decoder->psb_ip;
2894         decoder->state.to_ip = 0;
2895         return 0;
2896 }
2897
2898 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2899 {
2900         int err;
2901
2902         if (decoder->ip != decoder->last_ip) {
2903                 err = intel_pt_walk_fup(decoder);
2904                 if (!err || err != -EAGAIN)
2905                         return err;
2906         }
2907
2908         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2909         err = intel_pt_psb(decoder);
2910         if (err) {
2911                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2912                 return -ENOENT;
2913         }
2914
2915         return 0;
2916 }
2917
2918 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2919 {
2920         struct intel_pt_psb_info data = { .fup = false };
2921
2922         if (!decoder->branch_enable)
2923                 return false;
2924
2925         intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2926         if (!data.fup)
2927                 return false;
2928
2929         decoder->packet = data.fup_packet;
2930         intel_pt_set_last_ip(decoder);
2931         decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2932
2933         *err = intel_pt_fup_in_psb(decoder);
2934
2935         return true;
2936 }
2937
2938 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2939 {
2940         int last_packet_type = INTEL_PT_PAD;
2941         bool no_tip = false;
2942         int err;
2943
2944         while (1) {
2945                 err = intel_pt_get_next_packet(decoder);
2946                 if (err)
2947                         return err;
2948 next:
2949                 err = 0;
2950                 if (decoder->cyc_threshold) {
2951                         if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2952                                 decoder->sample_cyc = false;
2953                         last_packet_type = decoder->packet.type;
2954                 }
2955
2956                 if (decoder->hop) {
2957                         switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2958                         case HOP_IGNORE:
2959                                 continue;
2960                         case HOP_RETURN:
2961                                 return err;
2962                         case HOP_AGAIN:
2963                                 goto next;
2964                         default:
2965                                 break;
2966                         }
2967                 }
2968
2969                 switch (decoder->packet.type) {
2970                 case INTEL_PT_TNT:
2971                         if (!decoder->packet.count)
2972                                 break;
2973                         decoder->tnt = decoder->packet;
2974                         decoder->pkt_state = INTEL_PT_STATE_TNT;
2975                         err = intel_pt_walk_tnt(decoder);
2976                         if (err == -EAGAIN)
2977                                 break;
2978                         return err;
2979
2980                 case INTEL_PT_TIP_PGD:
2981                         if (decoder->packet.count != 0)
2982                                 intel_pt_set_last_ip(decoder);
2983                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2984                         return intel_pt_walk_tip(decoder);
2985
2986                 case INTEL_PT_TIP_PGE: {
2987                         decoder->pge = true;
2988                         decoder->overflow = false;
2989                         intel_pt_mtc_cyc_cnt_pge(decoder);
2990                         intel_pt_set_nr(decoder);
2991                         if (decoder->packet.count == 0) {
2992                                 intel_pt_log_at("Skipping zero TIP.PGE",
2993                                                 decoder->pos);
2994                                 break;
2995                         }
2996                         intel_pt_set_ip(decoder);
2997                         decoder->state.from_ip = 0;
2998                         decoder->state.to_ip = decoder->ip;
2999                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3000                         /*
3001                          * In hop mode, resample to get the to_ip as an
3002                          * "instruction" sample.
3003                          */
3004                         if (decoder->hop)
3005                                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3006                         return 0;
3007                 }
3008
3009                 case INTEL_PT_OVF:
3010                         return intel_pt_overflow(decoder);
3011
3012                 case INTEL_PT_TIP:
3013                         if (decoder->packet.count != 0)
3014                                 intel_pt_set_last_ip(decoder);
3015                         decoder->pkt_state = INTEL_PT_STATE_TIP;
3016                         return intel_pt_walk_tip(decoder);
3017
3018                 case INTEL_PT_FUP:
3019                         if (decoder->packet.count == 0) {
3020                                 intel_pt_log_at("Skipping zero FUP",
3021                                                 decoder->pos);
3022                                 no_tip = false;
3023                                 break;
3024                         }
3025                         intel_pt_set_last_ip(decoder);
3026                         if (!decoder->branch_enable || !decoder->pge) {
3027                                 decoder->ip = decoder->last_ip;
3028                                 if (intel_pt_fup_event(decoder))
3029                                         return 0;
3030                                 no_tip = false;
3031                                 break;
3032                         }
3033                         if (decoder->set_fup_mwait)
3034                                 no_tip = true;
3035                         if (no_tip)
3036                                 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3037                         else
3038                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
3039                         err = intel_pt_walk_fup(decoder);
3040                         if (err != -EAGAIN)
3041                                 return err;
3042                         if (no_tip) {
3043                                 no_tip = false;
3044                                 break;
3045                         }
3046                         return intel_pt_walk_fup_tip(decoder);
3047
3048                 case INTEL_PT_TRACESTOP:
3049                         decoder->pge = false;
3050                         decoder->continuous_period = false;
3051                         intel_pt_clear_tx_flags(decoder);
3052                         decoder->have_tma = false;
3053                         break;
3054
3055                 case INTEL_PT_PSB:
3056                         decoder->state.psb_offset = decoder->pos;
3057                         decoder->psb_ip = 0;
3058                         if (intel_pt_psb_with_fup(decoder, &err))
3059                                 return err;
3060                         err = intel_pt_psb(decoder);
3061                         if (err == -EAGAIN)
3062                                 goto next;
3063                         return err;
3064
3065                 case INTEL_PT_PIP:
3066                         intel_pt_update_pip(decoder);
3067                         break;
3068
3069                 case INTEL_PT_MTC:
3070                         intel_pt_calc_mtc_timestamp(decoder);
3071                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3072                                 break;
3073                         /*
3074                          * Ensure that there has been an instruction since the
3075                          * last MTC.
3076                          */
3077                         if (!decoder->mtc_insn)
3078                                 break;
3079                         decoder->mtc_insn = false;
3080                         /* Ensure that there is a timestamp */
3081                         if (!decoder->timestamp)
3082                                 break;
3083                         decoder->state.type = INTEL_PT_INSTRUCTION;
3084                         decoder->state.from_ip = decoder->ip;
3085                         decoder->state.to_ip = 0;
3086                         decoder->mtc_insn = false;
3087                         return 0;
3088
3089                 case INTEL_PT_TSC:
3090                         intel_pt_calc_tsc_timestamp(decoder);
3091                         break;
3092
3093                 case INTEL_PT_TMA:
3094                         intel_pt_calc_tma(decoder);
3095                         break;
3096
3097                 case INTEL_PT_CYC:
3098                         intel_pt_calc_cyc_timestamp(decoder);
3099                         break;
3100
3101                 case INTEL_PT_CBR:
3102                         intel_pt_calc_cbr(decoder);
3103                         if (decoder->cbr != decoder->cbr_seen) {
3104                                 decoder->state.type = 0;
3105                                 return 0;
3106                         }
3107                         break;
3108
3109                 case INTEL_PT_MODE_EXEC:
3110                         decoder->exec_mode = decoder->packet.payload;
3111                         break;
3112
3113                 case INTEL_PT_MODE_TSX:
3114                         /* MODE_TSX need not be followed by FUP */
3115                         if (!decoder->pge || decoder->in_psb) {
3116                                 intel_pt_update_in_tx(decoder);
3117                                 break;
3118                         }
3119                         err = intel_pt_mode_tsx(decoder, &no_tip);
3120                         if (err)
3121                                 return err;
3122                         goto next;
3123
3124                 case INTEL_PT_BAD: /* Does not happen */
3125                         return intel_pt_bug(decoder);
3126
3127                 case INTEL_PT_PSBEND:
3128                 case INTEL_PT_VMCS:
3129                 case INTEL_PT_MNT:
3130                 case INTEL_PT_PAD:
3131                         break;
3132
3133                 case INTEL_PT_PTWRITE_IP:
3134                         decoder->fup_ptw_payload = decoder->packet.payload;
3135                         err = intel_pt_get_next_packet(decoder);
3136                         if (err)
3137                                 return err;
3138                         if (decoder->packet.type == INTEL_PT_FUP) {
3139                                 decoder->set_fup_ptw = true;
3140                                 no_tip = true;
3141                         } else {
3142                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3143                                                 decoder->pos);
3144                         }
3145                         goto next;
3146
3147                 case INTEL_PT_PTWRITE:
3148                         decoder->state.type = INTEL_PT_PTW;
3149                         decoder->state.from_ip = decoder->ip;
3150                         decoder->state.to_ip = 0;
3151                         decoder->state.ptw_payload = decoder->packet.payload;
3152                         return 0;
3153
3154                 case INTEL_PT_MWAIT:
3155                         decoder->fup_mwait_payload = decoder->packet.payload;
3156                         decoder->set_fup_mwait = true;
3157                         break;
3158
3159                 case INTEL_PT_PWRE:
3160                         if (decoder->set_fup_mwait) {
3161                                 decoder->fup_pwre_payload =
3162                                                         decoder->packet.payload;
3163                                 decoder->set_fup_pwre = true;
3164                                 break;
3165                         }
3166                         decoder->state.type = INTEL_PT_PWR_ENTRY;
3167                         decoder->state.from_ip = decoder->ip;
3168                         decoder->state.to_ip = 0;
3169                         decoder->state.pwrx_payload = decoder->packet.payload;
3170                         return 0;
3171
3172                 case INTEL_PT_EXSTOP_IP:
3173                         err = intel_pt_get_next_packet(decoder);
3174                         if (err)
3175                                 return err;
3176                         if (decoder->packet.type == INTEL_PT_FUP) {
3177                                 decoder->set_fup_exstop = true;
3178                                 no_tip = true;
3179                         } else {
3180                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3181                                                 decoder->pos);
3182                         }
3183                         goto next;
3184
3185                 case INTEL_PT_EXSTOP:
3186                         decoder->state.type = INTEL_PT_EX_STOP;
3187                         decoder->state.from_ip = decoder->ip;
3188                         decoder->state.to_ip = 0;
3189                         return 0;
3190
3191                 case INTEL_PT_PWRX:
3192                         decoder->state.type = INTEL_PT_PWR_EXIT;
3193                         decoder->state.from_ip = decoder->ip;
3194                         decoder->state.to_ip = 0;
3195                         decoder->state.pwrx_payload = decoder->packet.payload;
3196                         return 0;
3197
3198                 case INTEL_PT_BBP:
3199                         intel_pt_bbp(decoder);
3200                         break;
3201
3202                 case INTEL_PT_BIP:
3203                         intel_pt_bip(decoder);
3204                         break;
3205
3206                 case INTEL_PT_BEP:
3207                         decoder->state.type = INTEL_PT_BLK_ITEMS;
3208                         decoder->state.from_ip = decoder->ip;
3209                         decoder->state.to_ip = 0;
3210                         return 0;
3211
3212                 case INTEL_PT_BEP_IP:
3213                         err = intel_pt_get_next_packet(decoder);
3214                         if (err)
3215                                 return err;
3216                         if (decoder->packet.type == INTEL_PT_FUP) {
3217                                 decoder->set_fup_bep = true;
3218                                 no_tip = true;
3219                         } else {
3220                                 intel_pt_log_at("ERROR: Missing FUP after BEP",
3221                                                 decoder->pos);
3222                         }
3223                         goto next;
3224
3225                 default:
3226                         return intel_pt_bug(decoder);
3227                 }
3228         }
3229 }
3230
3231 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3232 {
3233         return decoder->packet.count &&
3234                (decoder->have_last_ip || decoder->packet.count == 3 ||
3235                 decoder->packet.count == 6);
3236 }
3237
3238 /* Walk PSB+ packets to get in sync. */
3239 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3240 {
3241         int err;
3242
3243         decoder->in_psb = true;
3244
3245         while (1) {
3246                 err = intel_pt_get_next_packet(decoder);
3247                 if (err)
3248                         goto out;
3249
3250                 switch (decoder->packet.type) {
3251                 case INTEL_PT_TIP_PGD:
3252                         decoder->continuous_period = false;
3253                         __fallthrough;
3254                 case INTEL_PT_TIP_PGE:
3255                 case INTEL_PT_TIP:
3256                 case INTEL_PT_PTWRITE:
3257                 case INTEL_PT_PTWRITE_IP:
3258                 case INTEL_PT_EXSTOP:
3259                 case INTEL_PT_EXSTOP_IP:
3260                 case INTEL_PT_MWAIT:
3261                 case INTEL_PT_PWRE:
3262                 case INTEL_PT_PWRX:
3263                 case INTEL_PT_BBP:
3264                 case INTEL_PT_BIP:
3265                 case INTEL_PT_BEP:
3266                 case INTEL_PT_BEP_IP:
3267                         intel_pt_log("ERROR: Unexpected packet\n");
3268                         err = -ENOENT;
3269                         goto out;
3270
3271                 case INTEL_PT_FUP:
3272                         decoder->pge = true;
3273                         if (intel_pt_have_ip(decoder)) {
3274                                 uint64_t current_ip = decoder->ip;
3275
3276                                 intel_pt_set_ip(decoder);
3277                                 decoder->psb_ip = decoder->ip;
3278                                 if (current_ip)
3279                                         intel_pt_log_to("Setting IP",
3280                                                         decoder->ip);
3281                         }
3282                         break;
3283
3284                 case INTEL_PT_MTC:
3285                         intel_pt_calc_mtc_timestamp(decoder);
3286                         break;
3287
3288                 case INTEL_PT_TSC:
3289                         intel_pt_calc_tsc_timestamp(decoder);
3290                         break;
3291
3292                 case INTEL_PT_TMA:
3293                         intel_pt_calc_tma(decoder);
3294                         break;
3295
3296                 case INTEL_PT_CYC:
3297                         intel_pt_calc_cyc_timestamp(decoder);
3298                         break;
3299
3300                 case INTEL_PT_CBR:
3301                         intel_pt_calc_cbr(decoder);
3302                         break;
3303
3304                 case INTEL_PT_PIP:
3305                         intel_pt_set_pip(decoder);
3306                         break;
3307
3308                 case INTEL_PT_MODE_EXEC:
3309                         decoder->exec_mode = decoder->packet.payload;
3310                         break;
3311
3312                 case INTEL_PT_MODE_TSX:
3313                         intel_pt_update_in_tx(decoder);
3314                         break;
3315
3316                 case INTEL_PT_TRACESTOP:
3317                         decoder->pge = false;
3318                         decoder->continuous_period = false;
3319                         intel_pt_clear_tx_flags(decoder);
3320                         __fallthrough;
3321
3322                 case INTEL_PT_TNT:
3323                         decoder->have_tma = false;
3324                         intel_pt_log("ERROR: Unexpected packet\n");
3325                         if (decoder->ip)
3326                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3327                         else
3328                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3329                         err = -ENOENT;
3330                         goto out;
3331
3332                 case INTEL_PT_BAD: /* Does not happen */
3333                         err = intel_pt_bug(decoder);
3334                         goto out;
3335
3336                 case INTEL_PT_OVF:
3337                         err = intel_pt_overflow(decoder);
3338                         goto out;
3339
3340                 case INTEL_PT_PSBEND:
3341                         err = 0;
3342                         goto out;
3343
3344                 case INTEL_PT_PSB:
3345                 case INTEL_PT_VMCS:
3346                 case INTEL_PT_MNT:
3347                 case INTEL_PT_PAD:
3348                 default:
3349                         break;
3350                 }
3351         }
3352 out:
3353         decoder->in_psb = false;
3354
3355         return err;
3356 }
3357
3358 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3359 {
3360         int err;
3361
3362         while (1) {
3363                 err = intel_pt_get_next_packet(decoder);
3364                 if (err)
3365                         return err;
3366
3367                 switch (decoder->packet.type) {
3368                 case INTEL_PT_TIP_PGD:
3369                         decoder->continuous_period = false;
3370                         decoder->pge = false;
3371                         if (intel_pt_have_ip(decoder))
3372                                 intel_pt_set_ip(decoder);
3373                         if (!decoder->ip)
3374                                 break;
3375                         decoder->state.type |= INTEL_PT_TRACE_END;
3376                         return 0;
3377
3378                 case INTEL_PT_TIP_PGE:
3379                         decoder->pge = true;
3380                         intel_pt_mtc_cyc_cnt_pge(decoder);
3381                         if (intel_pt_have_ip(decoder))
3382                                 intel_pt_set_ip(decoder);
3383                         if (!decoder->ip)
3384                                 break;
3385                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3386                         return 0;
3387
3388                 case INTEL_PT_TIP:
3389                         decoder->pge = true;
3390                         if (intel_pt_have_ip(decoder))
3391                                 intel_pt_set_ip(decoder);
3392                         if (!decoder->ip)
3393                                 break;
3394                         return 0;
3395
3396                 case INTEL_PT_FUP:
3397                         if (intel_pt_have_ip(decoder))
3398                                 intel_pt_set_ip(decoder);
3399                         if (decoder->ip)
3400                                 return 0;
3401                         break;
3402
3403                 case INTEL_PT_MTC:
3404                         intel_pt_calc_mtc_timestamp(decoder);
3405                         break;
3406
3407                 case INTEL_PT_TSC:
3408                         intel_pt_calc_tsc_timestamp(decoder);
3409                         break;
3410
3411                 case INTEL_PT_TMA:
3412                         intel_pt_calc_tma(decoder);
3413                         break;
3414
3415                 case INTEL_PT_CYC:
3416                         intel_pt_calc_cyc_timestamp(decoder);
3417                         break;
3418
3419                 case INTEL_PT_CBR:
3420                         intel_pt_calc_cbr(decoder);
3421                         break;
3422
3423                 case INTEL_PT_PIP:
3424                         intel_pt_set_pip(decoder);
3425                         break;
3426
3427                 case INTEL_PT_MODE_EXEC:
3428                         decoder->exec_mode = decoder->packet.payload;
3429                         break;
3430
3431                 case INTEL_PT_MODE_TSX:
3432                         intel_pt_update_in_tx(decoder);
3433                         break;
3434
3435                 case INTEL_PT_OVF:
3436                         return intel_pt_overflow(decoder);
3437
3438                 case INTEL_PT_BAD: /* Does not happen */
3439                         return intel_pt_bug(decoder);
3440
3441                 case INTEL_PT_TRACESTOP:
3442                         decoder->pge = false;
3443                         decoder->continuous_period = false;
3444                         intel_pt_clear_tx_flags(decoder);
3445                         decoder->have_tma = false;
3446                         break;
3447
3448                 case INTEL_PT_PSB:
3449                         decoder->state.psb_offset = decoder->pos;
3450                         decoder->psb_ip = 0;
3451                         decoder->last_ip = 0;
3452                         decoder->have_last_ip = true;
3453                         intel_pt_clear_stack(&decoder->stack);
3454                         err = intel_pt_walk_psb(decoder);
3455                         if (err)
3456                                 return err;
3457                         decoder->state.type = INTEL_PT_PSB_EVT;
3458                         decoder->state.from_ip = decoder->psb_ip;
3459                         decoder->state.to_ip = 0;
3460                         return 0;
3461
3462                 case INTEL_PT_TNT:
3463                 case INTEL_PT_PSBEND:
3464                 case INTEL_PT_VMCS:
3465                 case INTEL_PT_MNT:
3466                 case INTEL_PT_PAD:
3467                 case INTEL_PT_PTWRITE:
3468                 case INTEL_PT_PTWRITE_IP:
3469                 case INTEL_PT_EXSTOP:
3470                 case INTEL_PT_EXSTOP_IP:
3471                 case INTEL_PT_MWAIT:
3472                 case INTEL_PT_PWRE:
3473                 case INTEL_PT_PWRX:
3474                 case INTEL_PT_BBP:
3475                 case INTEL_PT_BIP:
3476                 case INTEL_PT_BEP:
3477                 case INTEL_PT_BEP_IP:
3478                 default:
3479                         break;
3480                 }
3481         }
3482 }
3483
3484 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3485 {
3486         int err;
3487
3488         decoder->set_fup_tx_flags = false;
3489         decoder->set_fup_ptw = false;
3490         decoder->set_fup_mwait = false;
3491         decoder->set_fup_pwre = false;
3492         decoder->set_fup_exstop = false;
3493         decoder->set_fup_bep = false;
3494         decoder->overflow = false;
3495
3496         if (!decoder->branch_enable) {
3497                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3498                 decoder->state.type = 0; /* Do not have a sample */
3499                 return 0;
3500         }
3501
3502         intel_pt_log("Scanning for full IP\n");
3503         err = intel_pt_walk_to_ip(decoder);
3504         if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3505                 return err;
3506
3507         /* In hop mode, resample to get the to_ip as an "instruction" sample */
3508         if (decoder->hop)
3509                 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3510         else
3511                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3512
3513         decoder->state.from_ip = 0;
3514         decoder->state.to_ip = decoder->ip;
3515         intel_pt_log_to("Setting IP", decoder->ip);
3516
3517         return 0;
3518 }
3519
3520 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3521 {
3522         const unsigned char *end = decoder->buf + decoder->len;
3523         size_t i;
3524
3525         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3526                 if (i > decoder->len)
3527                         continue;
3528                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3529                         return i;
3530         }
3531         return 0;
3532 }
3533
3534 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3535 {
3536         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3537         const char *psb = INTEL_PT_PSB_STR;
3538
3539         if (rest_psb > decoder->len ||
3540             memcmp(decoder->buf, psb + part_psb, rest_psb))
3541                 return 0;
3542
3543         return rest_psb;
3544 }
3545
3546 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3547                                   int part_psb)
3548 {
3549         int rest_psb, ret;
3550
3551         decoder->pos += decoder->len;
3552         decoder->len = 0;
3553
3554         ret = intel_pt_get_next_data(decoder, false);
3555         if (ret)
3556                 return ret;
3557
3558         rest_psb = intel_pt_rest_psb(decoder, part_psb);
3559         if (!rest_psb)
3560                 return 0;
3561
3562         decoder->pos -= part_psb;
3563         decoder->next_buf = decoder->buf + rest_psb;
3564         decoder->next_len = decoder->len - rest_psb;
3565         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3566         decoder->buf = decoder->temp_buf;
3567         decoder->len = INTEL_PT_PSB_LEN;
3568
3569         return 0;
3570 }
3571
3572 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3573 {
3574         unsigned char *next;
3575         int ret;
3576
3577         intel_pt_log("Scanning for PSB\n");
3578         while (1) {
3579                 if (!decoder->len) {
3580                         ret = intel_pt_get_next_data(decoder, false);
3581                         if (ret)
3582                                 return ret;
3583                 }
3584
3585                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3586                               INTEL_PT_PSB_LEN);
3587                 if (!next) {
3588                         int part_psb;
3589
3590                         part_psb = intel_pt_part_psb(decoder);
3591                         if (part_psb) {
3592                                 ret = intel_pt_get_split_psb(decoder, part_psb);
3593                                 if (ret)
3594                                         return ret;
3595                         } else {
3596                                 decoder->pos += decoder->len;
3597                                 decoder->len = 0;
3598                         }
3599                         continue;
3600                 }
3601
3602                 decoder->pkt_step = next - decoder->buf;
3603                 return intel_pt_get_next_packet(decoder);
3604         }
3605 }
3606
3607 static int intel_pt_sync(struct intel_pt_decoder *decoder)
3608 {
3609         int err;
3610
3611         decoder->pge = false;
3612         decoder->continuous_period = false;
3613         decoder->have_last_ip = false;
3614         decoder->last_ip = 0;
3615         decoder->psb_ip = 0;
3616         decoder->ip = 0;
3617         intel_pt_clear_stack(&decoder->stack);
3618
3619         err = intel_pt_scan_for_psb(decoder);
3620         if (err)
3621                 return err;
3622
3623         if (decoder->vm_time_correlation) {
3624                 decoder->in_psb = true;
3625                 if (!decoder->timestamp)
3626                         decoder->timestamp = 1;
3627                 decoder->state.type = 0;
3628                 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3629                 return 0;
3630         }
3631
3632         decoder->have_last_ip = true;
3633         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3634
3635         err = intel_pt_walk_psb(decoder);
3636         if (err)
3637                 return err;
3638
3639         decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */
3640         decoder->state.from_ip = decoder->psb_ip;
3641         decoder->state.to_ip = 0;
3642
3643         if (decoder->ip) {
3644                 /*
3645                  * In hop mode, resample to get the PSB FUP ip as an
3646                  * "instruction" sample.
3647                  */
3648                 if (decoder->hop)
3649                         decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3650                 else
3651                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3652         }
3653
3654         return 0;
3655 }
3656
3657 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3658 {
3659         uint64_t est = decoder->sample_insn_cnt << 1;
3660
3661         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3662                 goto out;
3663
3664         est *= decoder->max_non_turbo_ratio;
3665         est /= decoder->cbr;
3666 out:
3667         return decoder->sample_timestamp + est;
3668 }
3669
3670 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3671 {
3672         int err;
3673
3674         do {
3675                 decoder->state.type = INTEL_PT_BRANCH;
3676                 decoder->state.flags = 0;
3677
3678                 switch (decoder->pkt_state) {
3679                 case INTEL_PT_STATE_NO_PSB:
3680                         err = intel_pt_sync(decoder);
3681                         break;
3682                 case INTEL_PT_STATE_NO_IP:
3683                         decoder->have_last_ip = false;
3684                         decoder->last_ip = 0;
3685                         decoder->ip = 0;
3686                         __fallthrough;
3687                 case INTEL_PT_STATE_ERR_RESYNC:
3688                         err = intel_pt_sync_ip(decoder);
3689                         break;
3690                 case INTEL_PT_STATE_IN_SYNC:
3691                         err = intel_pt_walk_trace(decoder);
3692                         break;
3693                 case INTEL_PT_STATE_TNT:
3694                 case INTEL_PT_STATE_TNT_CONT:
3695                         err = intel_pt_walk_tnt(decoder);
3696                         if (err == -EAGAIN)
3697                                 err = intel_pt_walk_trace(decoder);
3698                         break;
3699                 case INTEL_PT_STATE_TIP:
3700                 case INTEL_PT_STATE_TIP_PGD:
3701                         err = intel_pt_walk_tip(decoder);
3702                         break;
3703                 case INTEL_PT_STATE_FUP:
3704                         err = intel_pt_walk_fup(decoder);
3705                         if (err == -EAGAIN)
3706                                 err = intel_pt_walk_fup_tip(decoder);
3707                         break;
3708                 case INTEL_PT_STATE_FUP_NO_TIP:
3709                         err = intel_pt_walk_fup(decoder);
3710                         if (err == -EAGAIN)
3711                                 err = intel_pt_walk_trace(decoder);
3712                         break;
3713                 case INTEL_PT_STATE_FUP_IN_PSB:
3714                         err = intel_pt_fup_in_psb(decoder);
3715                         break;
3716                 case INTEL_PT_STATE_RESAMPLE:
3717                         err = intel_pt_resample(decoder);
3718                         break;
3719                 case INTEL_PT_STATE_VM_TIME_CORRELATION:
3720                         err = intel_pt_vm_time_correlation(decoder);
3721                         break;
3722                 default:
3723                         err = intel_pt_bug(decoder);
3724                         break;
3725                 }
3726         } while (err == -ENOLINK);
3727
3728         if (err) {
3729                 decoder->state.err = intel_pt_ext_err(err);
3730                 if (err != -EOVERFLOW)
3731                         decoder->state.from_ip = decoder->ip;
3732                 intel_pt_update_sample_time(decoder);
3733                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3734                 intel_pt_set_nr(decoder);
3735         } else {
3736                 decoder->state.err = 0;
3737                 if (decoder->cbr != decoder->cbr_seen) {
3738                         decoder->cbr_seen = decoder->cbr;
3739                         if (!decoder->state.type) {
3740                                 decoder->state.from_ip = decoder->ip;
3741                                 decoder->state.to_ip = 0;
3742                         }
3743                         decoder->state.type |= INTEL_PT_CBR_CHG;
3744                         decoder->state.cbr_payload = decoder->cbr_payload;
3745                         decoder->state.cbr = decoder->cbr;
3746                 }
3747                 if (intel_pt_sample_time(decoder->pkt_state)) {
3748                         intel_pt_update_sample_time(decoder);
3749                         if (decoder->sample_cyc) {
3750                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3751                                 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3752                                 decoder->sample_cyc = false;
3753                         }
3754                 }
3755                 /*
3756                  * When using only TSC/MTC to compute cycles, IPC can be
3757                  * sampled as soon as the cycle count changes.
3758                  */
3759                 if (!decoder->have_cyc)
3760                         decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3761         }
3762
3763          /* Let PSB event always have TSC timestamp */
3764         if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3765                 decoder->sample_timestamp = decoder->tsc_timestamp;
3766
3767         decoder->state.from_nr = decoder->nr;
3768         decoder->state.to_nr = decoder->next_nr;
3769         decoder->nr = decoder->next_nr;
3770
3771         decoder->state.timestamp = decoder->sample_timestamp;
3772         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3773         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3774         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3775
3776         return &decoder->state;
3777 }
3778
3779 /**
3780  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
3781  * @buf: pointer to buffer pointer
3782  * @len: size of buffer
3783  *
3784  * Updates the buffer pointer to point to the start of the next PSB packet if
3785  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
3786  * @len is adjusted accordingly.
3787  *
3788  * Return: %true if a PSB packet is found, %false otherwise.
3789  */
3790 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3791 {
3792         unsigned char *next;
3793
3794         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3795         if (next) {
3796                 *len -= next - *buf;
3797                 *buf = next;
3798                 return true;
3799         }
3800         return false;
3801 }
3802
3803 /**
3804  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
3805  *                     packet.
3806  * @buf: pointer to buffer pointer
3807  * @len: size of buffer
3808  *
3809  * Updates the buffer pointer to point to the start of the following PSB packet
3810  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
3811  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
3812  *
3813  * Return: %true if a PSB packet is found, %false otherwise.
3814  */
3815 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3816 {
3817         unsigned char *next;
3818
3819         if (!*len)
3820                 return false;
3821
3822         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3823         if (next) {
3824                 *len -= next - *buf;
3825                 *buf = next;
3826                 return true;
3827         }
3828         return false;
3829 }
3830
3831 /**
3832  * intel_pt_last_psb - find the last PSB packet in a buffer.
3833  * @buf: buffer
3834  * @len: size of buffer
3835  *
3836  * This function finds the last PSB in a buffer.
3837  *
3838  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
3839  */
3840 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3841 {
3842         const char *n = INTEL_PT_PSB_STR;
3843         unsigned char *p;
3844         size_t k;
3845
3846         if (len < INTEL_PT_PSB_LEN)
3847                 return NULL;
3848
3849         k = len - INTEL_PT_PSB_LEN + 1;
3850         while (1) {
3851                 p = memrchr(buf, n[0], k);
3852                 if (!p)
3853                         return NULL;
3854                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3855                         return p;
3856                 k = p - buf;
3857                 if (!k)
3858                         return NULL;
3859         }
3860 }
3861
3862 /**
3863  * intel_pt_next_tsc - find and return next TSC.
3864  * @buf: buffer
3865  * @len: size of buffer
3866  * @tsc: TSC value returned
3867  * @rem: returns remaining size when TSC is found
3868  *
3869  * Find a TSC packet in @buf and return the TSC value.  This function assumes
3870  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
3871  * PSBEND packet is found.
3872  *
3873  * Return: %true if TSC is found, false otherwise.
3874  */
3875 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3876                               size_t *rem)
3877 {
3878         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3879         struct intel_pt_pkt packet;
3880         int ret;
3881
3882         while (len) {
3883                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3884                 if (ret <= 0)
3885                         return false;
3886                 if (packet.type == INTEL_PT_TSC) {
3887                         *tsc = packet.payload;
3888                         *rem = len;
3889                         return true;
3890                 }
3891                 if (packet.type == INTEL_PT_PSBEND)
3892                         return false;
3893                 buf += ret;
3894                 len -= ret;
3895         }
3896         return false;
3897 }
3898
3899 /**
3900  * intel_pt_tsc_cmp - compare 7-byte TSCs.
3901  * @tsc1: first TSC to compare
3902  * @tsc2: second TSC to compare
3903  *
3904  * This function compares 7-byte TSC values allowing for the possibility that
3905  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
3906  * around so for that purpose this function assumes the absolute difference is
3907  * less than half the maximum difference.
3908  *
3909  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
3910  * after @tsc2.
3911  */
3912 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3913 {
3914         const uint64_t halfway = (1ULL << 55);
3915
3916         if (tsc1 == tsc2)
3917                 return 0;
3918
3919         if (tsc1 < tsc2) {
3920                 if (tsc2 - tsc1 < halfway)
3921                         return -1;
3922                 else
3923                         return 1;
3924         } else {
3925                 if (tsc1 - tsc2 < halfway)
3926                         return 1;
3927                 else
3928                         return -1;
3929         }
3930 }
3931
3932 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3933
3934 /**
3935  * adj_for_padding - adjust overlap to account for padding.
3936  * @buf_b: second buffer
3937  * @buf_a: first buffer
3938  * @len_a: size of first buffer
3939  *
3940  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
3941  * accordingly.
3942  *
3943  * Return: A pointer into @buf_b from where non-overlapped data starts
3944  */
3945 static unsigned char *adj_for_padding(unsigned char *buf_b,
3946                                       unsigned char *buf_a, size_t len_a)
3947 {
3948         unsigned char *p = buf_b - MAX_PADDING;
3949         unsigned char *q = buf_a + len_a - MAX_PADDING;
3950         int i;
3951
3952         for (i = MAX_PADDING; i; i--, p++, q++) {
3953                 if (*p != *q)
3954                         break;
3955         }
3956
3957         return p;
3958 }
3959
3960 /**
3961  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
3962  *                             using TSC.
3963  * @buf_a: first buffer
3964  * @len_a: size of first buffer
3965  * @buf_b: second buffer
3966  * @len_b: size of second buffer
3967  * @consecutive: returns true if there is data in buf_b that is consecutive
3968  *               to buf_a
3969  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
3970  *
3971  * If the trace contains TSC we can look at the last TSC of @buf_a and the
3972  * first TSC of @buf_b in order to determine if the buffers overlap, and then
3973  * walk forward in @buf_b until a later TSC is found.  A precondition is that
3974  * @buf_a and @buf_b are positioned at a PSB.
3975  *
3976  * Return: A pointer into @buf_b from where non-overlapped data starts, or
3977  * @buf_b + @len_b if there is no non-overlapped data.
3978  */
3979 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3980                                                 size_t len_a,
3981                                                 unsigned char *buf_b,
3982                                                 size_t len_b, bool *consecutive,
3983                                                 bool ooo_tsc)
3984 {
3985         uint64_t tsc_a, tsc_b;
3986         unsigned char *p;
3987         size_t len, rem_a, rem_b;
3988
3989         p = intel_pt_last_psb(buf_a, len_a);
3990         if (!p)
3991                 return buf_b; /* No PSB in buf_a => no overlap */
3992
3993         len = len_a - (p - buf_a);
3994         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3995                 /* The last PSB+ in buf_a is incomplete, so go back one more */
3996                 len_a -= len;
3997                 p = intel_pt_last_psb(buf_a, len_a);
3998                 if (!p)
3999                         return buf_b; /* No full PSB+ => assume no overlap */
4000                 len = len_a - (p - buf_a);
4001                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
4002                         return buf_b; /* No TSC in buf_a => assume no overlap */
4003         }
4004
4005         while (1) {
4006                 /* Ignore PSB+ with no TSC */
4007                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
4008                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
4009
4010                         /* Same TSC, so buffers are consecutive */
4011                         if (!cmp && rem_b >= rem_a) {
4012                                 unsigned char *start;
4013
4014                                 *consecutive = true;
4015                                 start = buf_b + len_b - (rem_b - rem_a);
4016                                 return adj_for_padding(start, buf_a, len_a);
4017                         }
4018                         if (cmp < 0 && !ooo_tsc)
4019                                 return buf_b; /* tsc_a < tsc_b => no overlap */
4020                 }
4021
4022                 if (!intel_pt_step_psb(&buf_b, &len_b))
4023                         return buf_b + len_b; /* No PSB in buf_b => no data */
4024         }
4025 }
4026
4027 /**
4028  * intel_pt_find_overlap - determine start of non-overlapped trace data.
4029  * @buf_a: first buffer
4030  * @len_a: size of first buffer
4031  * @buf_b: second buffer
4032  * @len_b: size of second buffer
4033  * @have_tsc: can use TSC packets to detect overlap
4034  * @consecutive: returns true if there is data in buf_b that is consecutive
4035  *               to buf_a
4036  * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling
4037  *
4038  * When trace samples or snapshots are recorded there is the possibility that
4039  * the data overlaps.  Note that, for the purposes of decoding, data is only
4040  * useful if it begins with a PSB packet.
4041  *
4042  * Return: A pointer into @buf_b from where non-overlapped data starts, or
4043  * @buf_b + @len_b if there is no non-overlapped data.
4044  */
4045 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4046                                      unsigned char *buf_b, size_t len_b,
4047                                      bool have_tsc, bool *consecutive,
4048                                      bool ooo_tsc)
4049 {
4050         unsigned char *found;
4051
4052         /* Buffer 'b' must start at PSB so throw away everything before that */
4053         if (!intel_pt_next_psb(&buf_b, &len_b))
4054                 return buf_b + len_b; /* No PSB */
4055
4056         if (!intel_pt_next_psb(&buf_a, &len_a))
4057                 return buf_b; /* No overlap */
4058
4059         if (have_tsc) {
4060                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4061                                                   consecutive, ooo_tsc);
4062                 if (found)
4063                         return found;
4064         }
4065
4066         /*
4067          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
4068          * we can ignore the first part of buffer 'a'.
4069          */
4070         while (len_b < len_a) {
4071                 if (!intel_pt_step_psb(&buf_a, &len_a))
4072                         return buf_b; /* No overlap */
4073         }
4074
4075         /* Now len_b >= len_a */
4076         while (1) {
4077                 /* Potential overlap so check the bytes */
4078                 found = memmem(buf_a, len_a, buf_b, len_a);
4079                 if (found) {
4080                         *consecutive = true;
4081                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
4082                 }
4083
4084                 /* Try again at next PSB in buffer 'a' */
4085                 if (!intel_pt_step_psb(&buf_a, &len_a))
4086                         return buf_b; /* No overlap */
4087         }
4088 }
4089
4090 /**
4091  * struct fast_forward_data - data used by intel_pt_ff_cb().
4092  * @timestamp: timestamp to fast forward towards
4093  * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than
4094  *                 the fast forward timestamp.
4095  */
4096 struct fast_forward_data {
4097         uint64_t timestamp;
4098         uint64_t buf_timestamp;
4099 };
4100
4101 /**
4102  * intel_pt_ff_cb - fast forward lookahead callback.
4103  * @buffer: Intel PT trace buffer
4104  * @data: opaque pointer to fast forward data (struct fast_forward_data)
4105  *
4106  * Determine if @buffer trace is past the fast forward timestamp.
4107  *
4108  * Return: 1 (stop lookahead) if @buffer trace is past the fast forward
4109  *         timestamp, and 0 otherwise.
4110  */
4111 static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4112 {
4113         struct fast_forward_data *d = data;
4114         unsigned char *buf;
4115         uint64_t tsc;
4116         size_t rem;
4117         size_t len;
4118
4119         buf = (unsigned char *)buffer->buf;
4120         len = buffer->len;
4121
4122         if (!intel_pt_next_psb(&buf, &len) ||
4123             !intel_pt_next_tsc(buf, len, &tsc, &rem))
4124                 return 0;
4125
4126         tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4127
4128         intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4129                      tsc, buffer->ref_timestamp);
4130
4131         /*
4132          * If the buffer contains a timestamp earlier that the fast forward
4133          * timestamp, then record it, else stop.
4134          */
4135         if (tsc < d->timestamp)
4136                 d->buf_timestamp = buffer->ref_timestamp;
4137         else
4138                 return 1;
4139
4140         return 0;
4141 }
4142
4143 /**
4144  * intel_pt_fast_forward - reposition decoder forwards.
4145  * @decoder: Intel PT decoder
4146  * @timestamp: timestamp to fast forward towards
4147  *
4148  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4149  *
4150  * Return: 0 on success or negative error code on failure.
4151  */
4152 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4153 {
4154         struct fast_forward_data d = { .timestamp = timestamp };
4155         unsigned char *buf;
4156         size_t len;
4157         int err;
4158
4159         intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4160
4161         /* Find buffer timestamp of buffer to fast forward to */
4162         err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4163         if (err < 0)
4164                 return err;
4165
4166         /* Walk to buffer with same buffer timestamp */
4167         if (d.buf_timestamp) {
4168                 do {
4169                         decoder->pos += decoder->len;
4170                         decoder->len = 0;
4171                         err = intel_pt_get_next_data(decoder, true);
4172                         /* -ENOLINK means non-consecutive trace */
4173                         if (err && err != -ENOLINK)
4174                                 return err;
4175                 } while (decoder->buf_timestamp != d.buf_timestamp);
4176         }
4177
4178         if (!decoder->buf)
4179                 return 0;
4180
4181         buf = (unsigned char *)decoder->buf;
4182         len = decoder->len;
4183
4184         if (!intel_pt_next_psb(&buf, &len))
4185                 return 0;
4186
4187         /*
4188          * Walk PSBs while the PSB timestamp is less than the fast forward
4189          * timestamp.
4190          */
4191         do {
4192                 uint64_t tsc;
4193                 size_t rem;
4194
4195                 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4196                         break;
4197                 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4198                 /*
4199                  * A TSC packet can slip past MTC packets but, after fast
4200                  * forward, decoding starts at the TSC timestamp. That means
4201                  * the timestamps may not be exactly the same as the timestamps
4202                  * that would have been decoded without fast forward.
4203                  */
4204                 if (tsc < timestamp) {
4205                         intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4206                         decoder->pos += decoder->len - len;
4207                         decoder->buf = buf;
4208                         decoder->len = len;
4209                         intel_pt_reposition(decoder);
4210                 } else {
4211                         break;
4212                 }
4213         } while (intel_pt_step_psb(&buf, &len));
4214
4215         return 0;
4216 }