tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29
  30 #include "intel-pt-insn-decoder.h"
  31 #include "intel-pt-pkt-decoder.h"
  32 #include "intel-pt-decoder.h"
  33 #include "intel-pt-log.h"
  34
  35 #define INTEL_PT_BLK_SIZE 1024
  36
  37 #define BIT63 (((uint64_t)1 << 63))
  38
  39 #define INTEL_PT_RETURN 1
  40
  41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  42 #define INTEL_PT_MAX_LOOPS 10000
  43
  44 struct intel_pt_blk {
  45         struct intel_pt_blk *prev;
  46         uint64_t ip[INTEL_PT_BLK_SIZE];
  47 };
  48
  49 struct intel_pt_stack {
  50         struct intel_pt_blk *blk;
  51         struct intel_pt_blk *spare;
  52         int pos;
  53 };
  54
  55 enum intel_pt_pkt_state {
  56         INTEL_PT_STATE_NO_PSB,
  57         INTEL_PT_STATE_NO_IP,
  58         INTEL_PT_STATE_ERR_RESYNC,
  59         INTEL_PT_STATE_IN_SYNC,
  60         INTEL_PT_STATE_TNT,
  61         INTEL_PT_STATE_TIP,
  62         INTEL_PT_STATE_TIP_PGD,
  63         INTEL_PT_STATE_FUP,
  64         INTEL_PT_STATE_FUP_NO_TIP,
  65 };
  66
  67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  68 {
  69         switch (pkt_state) {
  70         case INTEL_PT_STATE_NO_PSB:
  71         case INTEL_PT_STATE_NO_IP:
  72         case INTEL_PT_STATE_ERR_RESYNC:
  73         case INTEL_PT_STATE_IN_SYNC:
  74         case INTEL_PT_STATE_TNT:
  75                 return true;
  76         case INTEL_PT_STATE_TIP:
  77         case INTEL_PT_STATE_TIP_PGD:
  78         case INTEL_PT_STATE_FUP:
  79         case INTEL_PT_STATE_FUP_NO_TIP:
  80                 return false;
  81         default:
  82                 return true;
  83         };
  84 }
  85
  86 #ifdef INTEL_PT_STRICT
  87 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  88 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  89 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  90 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  91 #else
  92 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  93 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  94 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  95 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  96 #endif
  97
  98 struct intel_pt_decoder {
  99         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 100         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 101                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 102                          uint64_t max_insn_cnt, void *data);
 103         bool (*pgd_ip)(uint64_t ip, void *data);
 104         void *data;
 105         struct intel_pt_state state;
 106         const unsigned char *buf;
 107         size_t len;
 108         bool return_compression;
 109         bool mtc_insn;
 110         bool pge;
 111         bool have_tma;
 112         bool have_cyc;
 113         bool fixup_last_mtc;
 114         bool have_last_ip;
 115         uint64_t pos;
 116         uint64_t last_ip;
 117         uint64_t ip;
 118         uint64_t cr3;
 119         uint64_t timestamp;
 120         uint64_t tsc_timestamp;
 121         uint64_t ref_timestamp;
 122         uint64_t sample_timestamp;
 123         uint64_t ret_addr;
 124         uint64_t ctc_timestamp;
 125         uint64_t ctc_delta;
 126         uint64_t cycle_cnt;
 127         uint64_t cyc_ref_timestamp;
 128         uint32_t last_mtc;
 129         uint32_t tsc_ctc_ratio_n;
 130         uint32_t tsc_ctc_ratio_d;
 131         uint32_t tsc_ctc_mult;
 132         uint32_t tsc_slip;
 133         uint32_t ctc_rem_mask;
 134         int mtc_shift;
 135         struct intel_pt_stack stack;
 136         enum intel_pt_pkt_state pkt_state;
 137         struct intel_pt_pkt packet;
 138         struct intel_pt_pkt tnt;
 139         int pkt_step;
 140         int pkt_len;
 141         int last_packet_type;
 142         unsigned int cbr;
 143         unsigned int max_non_turbo_ratio;
 144         double max_non_turbo_ratio_fp;
 145         double cbr_cyc_to_tsc;
 146         double calc_cyc_to_tsc;
 147         bool have_calc_cyc_to_tsc;
 148         int exec_mode;
 149         unsigned int insn_bytes;
 150         uint64_t period;
 151         enum intel_pt_period_type period_type;
 152         uint64_t tot_insn_cnt;
 153         uint64_t period_insn_cnt;
 154         uint64_t period_mask;
 155         uint64_t period_ticks;
 156         uint64_t last_masked_timestamp;
 157         bool continuous_period;
 158         bool overflow;
 159         bool set_fup_tx_flags;
 160         unsigned int fup_tx_flags;
 161         unsigned int tx_flags;
 162         uint64_t timestamp_insn_cnt;
 163         uint64_t sample_insn_cnt;
 164         uint64_t stuck_ip;
 165         int no_progress;
 166         int stuck_ip_prd;
 167         int stuck_ip_cnt;
 168         const unsigned char *next_buf;
 169         size_t next_len;
 170         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 171 };
 172
 173 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 174 {
 175         int i;
 176
 177         for (i = 0; x != 1; i++)
 178                 x >>= 1;
 179
 180         return x << i;
 181 }
 182
 183 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 184 {
 185         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 186                 uint64_t period;
 187
 188                 period = intel_pt_lower_power_of_2(decoder->period);
 189                 decoder->period_mask  = ~(period - 1);
 190                 decoder->period_ticks = period;
 191         }
 192 }
 193
 194 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 195 {
 196         if (!d)
 197                 return 0;
 198         return (t / d) * n + ((t % d) * n) / d;
 199 }
 200
 201 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 202 {
 203         struct intel_pt_decoder *decoder;
 204
 205         if (!params->get_trace || !params->walk_insn)
 206                 return NULL;
 207
 208         decoder = zalloc(sizeof(struct intel_pt_decoder));
 209         if (!decoder)
 210                 return NULL;
 211
 212         decoder->get_trace          = params->get_trace;
 213         decoder->walk_insn          = params->walk_insn;
 214         decoder->pgd_ip             = params->pgd_ip;
 215         decoder->data               = params->data;
 216         decoder->return_compression = params->return_compression;
 217
 218         decoder->period             = params->period;
 219         decoder->period_type        = params->period_type;
 220
 221         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 222         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 223
 224         intel_pt_setup_period(decoder);
 225
 226         decoder->mtc_shift = params->mtc_period;
 227         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 228
 229         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 230         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 231
 232         if (!decoder->tsc_ctc_ratio_n)
 233                 decoder->tsc_ctc_ratio_d = 0;
 234
 235         if (decoder->tsc_ctc_ratio_d) {
 236                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 237                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 238                                                 decoder->tsc_ctc_ratio_d;
 239
 240                 /*
 241                  * Allow for timestamps appearing to backwards because a TSC
 242                  * packet has slipped past a MTC packet, so allow 2 MTC ticks
 243                  * or ...
 244                  */
 245                 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 246                                         decoder->tsc_ctc_ratio_n,
 247                                         decoder->tsc_ctc_ratio_d);
 248         }
 249         /* ... or 0x100 paranoia */
 250         if (decoder->tsc_slip < 0x100)
 251                 decoder->tsc_slip = 0x100;
 252
 253         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 254         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 255         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 256         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 257         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 258
 259         return decoder;
 260 }
 261
 262 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 263 {
 264         struct intel_pt_blk *blk = stack->blk;
 265
 266         stack->blk = blk->prev;
 267         if (!stack->spare)
 268                 stack->spare = blk;
 269         else
 270                 free(blk);
 271 }
 272
 273 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 274 {
 275         if (!stack->pos) {
 276                 if (!stack->blk)
 277                         return 0;
 278                 intel_pt_pop_blk(stack);
 279                 if (!stack->blk)
 280                         return 0;
 281                 stack->pos = INTEL_PT_BLK_SIZE;
 282         }
 283         return stack->blk->ip[--stack->pos];
 284 }
 285
 286 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 287 {
 288         struct intel_pt_blk *blk;
 289
 290         if (stack->spare) {
 291                 blk = stack->spare;
 292                 stack->spare = NULL;
 293         } else {
 294                 blk = malloc(sizeof(struct intel_pt_blk));
 295                 if (!blk)
 296                         return -ENOMEM;
 297         }
 298
 299         blk->prev = stack->blk;
 300         stack->blk = blk;
 301         stack->pos = 0;
 302         return 0;
 303 }
 304
 305 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 306 {
 307         int err;
 308
 309         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 310                 err = intel_pt_alloc_blk(stack);
 311                 if (err)
 312                         return err;
 313         }
 314
 315         stack->blk->ip[stack->pos++] = ip;
 316         return 0;
 317 }
 318
 319 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 320 {
 321         while (stack->blk)
 322                 intel_pt_pop_blk(stack);
 323         stack->pos = 0;
 324 }
 325
 326 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 327 {
 328         intel_pt_clear_stack(stack);
 329         zfree(&stack->blk);
 330         zfree(&stack->spare);
 331 }
 332
 333 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 334 {
 335         intel_pt_free_stack(&decoder->stack);
 336         free(decoder);
 337 }
 338
 339 static int intel_pt_ext_err(int code)
 340 {
 341         switch (code) {
 342         case -ENOMEM:
 343                 return INTEL_PT_ERR_NOMEM;
 344         case -ENOSYS:
 345                 return INTEL_PT_ERR_INTERN;
 346         case -EBADMSG:
 347                 return INTEL_PT_ERR_BADPKT;
 348         case -ENODATA:
 349                 return INTEL_PT_ERR_NODATA;
 350         case -EILSEQ:
 351                 return INTEL_PT_ERR_NOINSN;
 352         case -ENOENT:
 353                 return INTEL_PT_ERR_MISMAT;
 354         case -EOVERFLOW:
 355                 return INTEL_PT_ERR_OVR;
 356         case -ENOSPC:
 357                 return INTEL_PT_ERR_LOST;
 358         case -ELOOP:
 359                 return INTEL_PT_ERR_NELOOP;
 360         default:
 361                 return INTEL_PT_ERR_UNK;
 362         }
 363 }
 364
 365 static const char *intel_pt_err_msgs[] = {
 366         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 367         [INTEL_PT_ERR_INTERN] = "Internal error",
 368         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 369         [INTEL_PT_ERR_NODATA] = "No more data",
 370         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 371         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 372         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 373         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 374         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 375         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 376 };
 377
 378 int intel_pt__strerror(int code, char *buf, size_t buflen)
 379 {
 380         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 381                 code = INTEL_PT_ERR_UNK;
 382         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 383         return 0;
 384 }
 385
 386 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 387                                  uint64_t last_ip)
 388 {
 389         uint64_t ip;
 390
 391         switch (packet->count) {
 392         case 1:
 393                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 394                      packet->payload;
 395                 break;
 396         case 2:
 397                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 398                      packet->payload;
 399                 break;
 400         case 3:
 401                 ip = packet->payload;
 402                 /* Sign-extend 6-byte ip */
 403                 if (ip & (uint64_t)0x800000000000ULL)
 404                         ip |= (uint64_t)0xffff000000000000ULL;
 405                 break;
 406         case 4:
 407                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 408                      packet->payload;
 409                 break;
 410         case 6:
 411                 ip = packet->payload;
 412                 break;
 413         default:
 414                 return 0;
 415         }
 416
 417         return ip;
 418 }
 419
 420 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 421 {
 422         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 423         decoder->have_last_ip = true;
 424 }
 425
 426 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 427 {
 428         intel_pt_set_last_ip(decoder);
 429         decoder->ip = decoder->last_ip;
 430 }
 431
 432 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 433 {
 434         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 435                             decoder->buf);
 436 }
 437
 438 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 439 {
 440         intel_pt_log("ERROR: Internal error\n");
 441         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 442         return -ENOSYS;
 443 }
 444
 445 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 446 {
 447         decoder->tx_flags = 0;
 448 }
 449
 450 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 451 {
 452         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 453 }
 454
 455 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 456 {
 457         intel_pt_clear_tx_flags(decoder);
 458         decoder->have_tma = false;
 459         decoder->pkt_len = 1;
 460         decoder->pkt_step = 1;
 461         intel_pt_decoder_log_packet(decoder);
 462         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 463                 intel_pt_log("ERROR: Bad packet\n");
 464                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 465         }
 466         return -EBADMSG;
 467 }
 468
 469 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 470 {
 471         struct intel_pt_buffer buffer = { .buf = 0, };
 472         int ret;
 473
 474         decoder->pkt_step = 0;
 475
 476         intel_pt_log("Getting more data\n");
 477         ret = decoder->get_trace(&buffer, decoder->data);
 478         if (ret)
 479                 return ret;
 480         decoder->buf = buffer.buf;
 481         decoder->len = buffer.len;
 482         if (!decoder->len) {
 483                 intel_pt_log("No more data\n");
 484                 return -ENODATA;
 485         }
 486         if (!buffer.consecutive) {
 487                 decoder->ip = 0;
 488                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 489                 decoder->ref_timestamp = buffer.ref_timestamp;
 490                 decoder->timestamp = 0;
 491                 decoder->have_tma = false;
 492                 decoder->state.trace_nr = buffer.trace_nr;
 493                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 494                              decoder->ref_timestamp);
 495                 return -ENOLINK;
 496         }
 497
 498         return 0;
 499 }
 500
 501 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 502 {
 503         if (!decoder->next_buf)
 504                 return intel_pt_get_data(decoder);
 505
 506         decoder->buf = decoder->next_buf;
 507         decoder->len = decoder->next_len;
 508         decoder->next_buf = 0;
 509         decoder->next_len = 0;
 510         return 0;
 511 }
 512
 513 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 514 {
 515         unsigned char *buf = decoder->temp_buf;
 516         size_t old_len, len, n;
 517         int ret;
 518
 519         old_len = decoder->len;
 520         len = decoder->len;
 521         memcpy(buf, decoder->buf, len);
 522
 523         ret = intel_pt_get_data(decoder);
 524         if (ret) {
 525                 decoder->pos += old_len;
 526                 return ret < 0 ? ret : -EINVAL;
 527         }
 528
 529         n = INTEL_PT_PKT_MAX_SZ - len;
 530         if (n > decoder->len)
 531                 n = decoder->len;
 532         memcpy(buf + len, decoder->buf, n);
 533         len += n;
 534
 535         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 536         if (ret < (int)old_len) {
 537                 decoder->next_buf = decoder->buf;
 538                 decoder->next_len = decoder->len;
 539                 decoder->buf = buf;
 540                 decoder->len = old_len;
 541                 return intel_pt_bad_packet(decoder);
 542         }
 543
 544         decoder->next_buf = decoder->buf + (ret - old_len);
 545         decoder->next_len = decoder->len - (ret - old_len);
 546
 547         decoder->buf = buf;
 548         decoder->len = ret;
 549
 550         return ret;
 551 }
 552
 553 struct intel_pt_pkt_info {
 554         struct intel_pt_decoder   *decoder;
 555         struct intel_pt_pkt       packet;
 556         uint64_t                  pos;
 557         int                       pkt_len;
 558         int                       last_packet_type;
 559         void                      *data;
 560 };
 561
 562 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 563
 564 /* Lookahead packets in current buffer */
 565 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 566                                   intel_pt_pkt_cb_t cb, void *data)
 567 {
 568         struct intel_pt_pkt_info pkt_info;
 569         const unsigned char *buf = decoder->buf;
 570         size_t len = decoder->len;
 571         int ret;
 572
 573         pkt_info.decoder          = decoder;
 574         pkt_info.pos              = decoder->pos;
 575         pkt_info.pkt_len          = decoder->pkt_step;
 576         pkt_info.last_packet_type = decoder->last_packet_type;
 577         pkt_info.data             = data;
 578
 579         while (1) {
 580                 do {
 581                         pkt_info.pos += pkt_info.pkt_len;
 582                         buf          += pkt_info.pkt_len;
 583                         len          -= pkt_info.pkt_len;
 584
 585                         if (!len)
 586                                 return INTEL_PT_NEED_MORE_BYTES;
 587
 588                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 589                         if (!ret)
 590                                 return INTEL_PT_NEED_MORE_BYTES;
 591                         if (ret < 0)
 592                                 return ret;
 593
 594                         pkt_info.pkt_len = ret;
 595                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 596
 597                 ret = cb(&pkt_info);
 598                 if (ret)
 599                         return 0;
 600
 601                 pkt_info.last_packet_type = pkt_info.packet.type;
 602         }
 603 }
 604
 605 struct intel_pt_calc_cyc_to_tsc_info {
 606         uint64_t        cycle_cnt;
 607         unsigned int    cbr;
 608         uint32_t        last_mtc;
 609         uint64_t        ctc_timestamp;
 610         uint64_t        ctc_delta;
 611         uint64_t        tsc_timestamp;
 612         uint64_t        timestamp;
 613         bool            have_tma;
 614         bool            fixup_last_mtc;
 615         bool            from_mtc;
 616         double          cbr_cyc_to_tsc;
 617 };
 618
 619 /*
 620  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 621  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 622  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 623  * packet by copying the missing bits from the current MTC assuming the least
 624  * difference between the two, and that the current MTC comes after last_mtc.
 625  */
 626 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 627                                     uint32_t *last_mtc)
 628 {
 629         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 630         uint32_t mask = ~(first_missing_bit - 1);
 631
 632         *last_mtc |= mtc & mask;
 633         if (*last_mtc >= mtc) {
 634                 *last_mtc -= first_missing_bit;
 635                 *last_mtc &= 0xff;
 636         }
 637 }
 638
 639 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 640 {
 641         struct intel_pt_decoder *decoder = pkt_info->decoder;
 642         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 643         uint64_t timestamp;
 644         double cyc_to_tsc;
 645         unsigned int cbr;
 646         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 647
 648         switch (pkt_info->packet.type) {
 649         case INTEL_PT_TNT:
 650         case INTEL_PT_TIP_PGE:
 651         case INTEL_PT_TIP:
 652         case INTEL_PT_FUP:
 653         case INTEL_PT_PSB:
 654         case INTEL_PT_PIP:
 655         case INTEL_PT_MODE_EXEC:
 656         case INTEL_PT_MODE_TSX:
 657         case INTEL_PT_PSBEND:
 658         case INTEL_PT_PAD:
 659         case INTEL_PT_VMCS:
 660         case INTEL_PT_MNT:
 661                 return 0;
 662
 663         case INTEL_PT_MTC:
 664                 if (!data->have_tma)
 665                         return 0;
 666
 667                 mtc = pkt_info->packet.payload;
 668                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 669                         data->fixup_last_mtc = false;
 670                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 671                                                 &data->last_mtc);
 672                 }
 673                 if (mtc > data->last_mtc)
 674                         mtc_delta = mtc - data->last_mtc;
 675                 else
 676                         mtc_delta = mtc + 256 - data->last_mtc;
 677                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 678                 data->last_mtc = mtc;
 679
 680                 if (decoder->tsc_ctc_mult) {
 681                         timestamp = data->ctc_timestamp +
 682                                 data->ctc_delta * decoder->tsc_ctc_mult;
 683                 } else {
 684                         timestamp = data->ctc_timestamp +
 685                                 multdiv(data->ctc_delta,
 686                                         decoder->tsc_ctc_ratio_n,
 687                                         decoder->tsc_ctc_ratio_d);
 688                 }
 689
 690                 if (timestamp < data->timestamp)
 691                         return 1;
 692
 693                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 694                         data->timestamp = timestamp;
 695                         return 0;
 696                 }
 697
 698                 break;
 699
 700         case INTEL_PT_TSC:
 701                 timestamp = pkt_info->packet.payload |
 702                             (data->timestamp & (0xffULL << 56));
 703                 if (data->from_mtc && timestamp < data->timestamp &&
 704                     data->timestamp - timestamp < decoder->tsc_slip)
 705                         return 1;
 706                 if (timestamp < data->timestamp)
 707                         timestamp += (1ULL << 56);
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         if (data->from_mtc)
 710                                 return 1;
 711                         data->tsc_timestamp = timestamp;
 712                         data->timestamp = timestamp;
 713                         return 0;
 714                 }
 715                 break;
 716
 717         case INTEL_PT_TMA:
 718                 if (data->from_mtc)
 719                         return 1;
 720
 721                 if (!decoder->tsc_ctc_ratio_d)
 722                         return 0;
 723
 724                 ctc = pkt_info->packet.payload;
 725                 fc = pkt_info->packet.count;
 726                 ctc_rem = ctc & decoder->ctc_rem_mask;
 727
 728                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 729
 730                 data->ctc_timestamp = data->tsc_timestamp - fc;
 731                 if (decoder->tsc_ctc_mult) {
 732                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 733                 } else {
 734                         data->ctc_timestamp -=
 735                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 736                                         decoder->tsc_ctc_ratio_d);
 737                 }
 738
 739                 data->ctc_delta = 0;
 740                 data->have_tma = true;
 741                 data->fixup_last_mtc = true;
 742
 743                 return 0;
 744
 745         case INTEL_PT_CYC:
 746                 data->cycle_cnt += pkt_info->packet.payload;
 747                 return 0;
 748
 749         case INTEL_PT_CBR:
 750                 cbr = pkt_info->packet.payload;
 751                 if (data->cbr && data->cbr != cbr)
 752                         return 1;
 753                 data->cbr = cbr;
 754                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 755                 return 0;
 756
 757         case INTEL_PT_TIP_PGD:
 758         case INTEL_PT_TRACESTOP:
 759         case INTEL_PT_OVF:
 760         case INTEL_PT_BAD: /* Does not happen */
 761         default:
 762                 return 1;
 763         }
 764
 765         if (!data->cbr && decoder->cbr) {
 766                 data->cbr = decoder->cbr;
 767                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 768         }
 769
 770         if (!data->cycle_cnt)
 771                 return 1;
 772
 773         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 774
 775         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 776             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 777                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 778                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 779                 return 1;
 780         }
 781
 782         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 783         decoder->have_calc_cyc_to_tsc = true;
 784
 785         if (data->cbr) {
 786                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 787                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 788         } else {
 789                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 790                              cyc_to_tsc, pkt_info->pos);
 791         }
 792
 793         return 1;
 794 }
 795
 796 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 797                                      bool from_mtc)
 798 {
 799         struct intel_pt_calc_cyc_to_tsc_info data = {
 800                 .cycle_cnt      = 0,
 801                 .cbr            = 0,
 802                 .last_mtc       = decoder->last_mtc,
 803                 .ctc_timestamp  = decoder->ctc_timestamp,
 804                 .ctc_delta      = decoder->ctc_delta,
 805                 .tsc_timestamp  = decoder->tsc_timestamp,
 806                 .timestamp      = decoder->timestamp,
 807                 .have_tma       = decoder->have_tma,
 808                 .fixup_last_mtc = decoder->fixup_last_mtc,
 809                 .from_mtc       = from_mtc,
 810                 .cbr_cyc_to_tsc = 0,
 811         };
 812
 813         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 814 }
 815
 816 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 817 {
 818         int ret;
 819
 820         decoder->last_packet_type = decoder->packet.type;
 821
 822         do {
 823                 decoder->pos += decoder->pkt_step;
 824                 decoder->buf += decoder->pkt_step;
 825                 decoder->len -= decoder->pkt_step;
 826
 827                 if (!decoder->len) {
 828                         ret = intel_pt_get_next_data(decoder);
 829                         if (ret)
 830                                 return ret;
 831                 }
 832
 833                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 834                                           &decoder->packet);
 835                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 836                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 837                         ret = intel_pt_get_split_packet(decoder);
 838                         if (ret < 0)
 839                                 return ret;
 840                 }
 841                 if (ret <= 0)
 842                         return intel_pt_bad_packet(decoder);
 843
 844                 decoder->pkt_len = ret;
 845                 decoder->pkt_step = ret;
 846                 intel_pt_decoder_log_packet(decoder);
 847         } while (decoder->packet.type == INTEL_PT_PAD);
 848
 849         return 0;
 850 }
 851
 852 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 853 {
 854         uint64_t timestamp, masked_timestamp;
 855
 856         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 857         masked_timestamp = timestamp & decoder->period_mask;
 858         if (decoder->continuous_period) {
 859                 if (masked_timestamp != decoder->last_masked_timestamp)
 860                         return 1;
 861         } else {
 862                 timestamp += 1;
 863                 masked_timestamp = timestamp & decoder->period_mask;
 864                 if (masked_timestamp != decoder->last_masked_timestamp) {
 865                         decoder->last_masked_timestamp = masked_timestamp;
 866                         decoder->continuous_period = true;
 867                 }
 868         }
 869         return decoder->period_ticks - (timestamp - masked_timestamp);
 870 }
 871
 872 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 873 {
 874         switch (decoder->period_type) {
 875         case INTEL_PT_PERIOD_INSTRUCTIONS:
 876                 return decoder->period - decoder->period_insn_cnt;
 877         case INTEL_PT_PERIOD_TICKS:
 878                 return intel_pt_next_period(decoder);
 879         case INTEL_PT_PERIOD_NONE:
 880         case INTEL_PT_PERIOD_MTC:
 881         default:
 882                 return 0;
 883         }
 884 }
 885
 886 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 887 {
 888         uint64_t timestamp, masked_timestamp;
 889
 890         switch (decoder->period_type) {
 891         case INTEL_PT_PERIOD_INSTRUCTIONS:
 892                 decoder->period_insn_cnt = 0;
 893                 break;
 894         case INTEL_PT_PERIOD_TICKS:
 895                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 896                 masked_timestamp = timestamp & decoder->period_mask;
 897                 decoder->last_masked_timestamp = masked_timestamp;
 898                 break;
 899         case INTEL_PT_PERIOD_NONE:
 900         case INTEL_PT_PERIOD_MTC:
 901         default:
 902                 break;
 903         }
 904
 905         decoder->state.type |= INTEL_PT_INSTRUCTION;
 906 }
 907
 908 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 909                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 910 {
 911         uint64_t max_insn_cnt, insn_cnt = 0;
 912         int err;
 913
 914         if (!decoder->mtc_insn)
 915                 decoder->mtc_insn = true;
 916
 917         max_insn_cnt = intel_pt_next_sample(decoder);
 918
 919         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 920                                  max_insn_cnt, decoder->data);
 921
 922         decoder->tot_insn_cnt += insn_cnt;
 923         decoder->timestamp_insn_cnt += insn_cnt;
 924         decoder->sample_insn_cnt += insn_cnt;
 925         decoder->period_insn_cnt += insn_cnt;
 926
 927         if (err) {
 928                 decoder->no_progress = 0;
 929                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 930                 intel_pt_log_at("ERROR: Failed to get instruction",
 931                                 decoder->ip);
 932                 if (err == -ENOENT)
 933                         return -ENOLINK;
 934                 return -EILSEQ;
 935         }
 936
 937         if (ip && decoder->ip == ip) {
 938                 err = -EAGAIN;
 939                 goto out;
 940         }
 941
 942         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 943                 intel_pt_sample_insn(decoder);
 944
 945         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 946                 decoder->state.type = INTEL_PT_INSTRUCTION;
 947                 decoder->state.from_ip = decoder->ip;
 948                 decoder->state.to_ip = 0;
 949                 decoder->ip += intel_pt_insn->length;
 950                 err = INTEL_PT_RETURN;
 951                 goto out;
 952         }
 953
 954         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 955                 /* Zero-length calls are excluded */
 956                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 957                     intel_pt_insn->rel) {
 958                         err = intel_pt_push(&decoder->stack, decoder->ip +
 959                                             intel_pt_insn->length);
 960                         if (err)
 961                                 goto out;
 962                 }
 963         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 964                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 965         }
 966
 967         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 968                 int cnt = decoder->no_progress++;
 969
 970                 decoder->state.from_ip = decoder->ip;
 971                 decoder->ip += intel_pt_insn->length +
 972                                 intel_pt_insn->rel;
 973                 decoder->state.to_ip = decoder->ip;
 974                 err = INTEL_PT_RETURN;
 975
 976                 /*
 977                  * Check for being stuck in a loop.  This can happen if a
 978                  * decoder error results in the decoder erroneously setting the
 979                  * ip to an address that is itself in an infinite loop that
 980                  * consumes no packets.  When that happens, there must be an
 981                  * unconditional branch.
 982                  */
 983                 if (cnt) {
 984                         if (cnt == 1) {
 985                                 decoder->stuck_ip = decoder->state.to_ip;
 986                                 decoder->stuck_ip_prd = 1;
 987                                 decoder->stuck_ip_cnt = 1;
 988                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 989                                    decoder->state.to_ip == decoder->stuck_ip) {
 990                                 intel_pt_log_at("ERROR: Never-ending loop",
 991                                                 decoder->state.to_ip);
 992                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 993                                 err = -ELOOP;
 994                                 goto out;
 995                         } else if (!--decoder->stuck_ip_cnt) {
 996                                 decoder->stuck_ip_prd += 1;
 997                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 998                                 decoder->stuck_ip = decoder->state.to_ip;
 999                         }
1000                 }
1001                 goto out_no_progress;
1002         }
1003 out:
1004         decoder->no_progress = 0;
1005 out_no_progress:
1006         decoder->state.insn_op = intel_pt_insn->op;
1007         decoder->state.insn_len = intel_pt_insn->length;
1008
1009         if (decoder->tx_flags & INTEL_PT_IN_TX)
1010                 decoder->state.flags |= INTEL_PT_IN_TX;
1011
1012         return err;
1013 }
1014
1015 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1016 {
1017         struct intel_pt_insn intel_pt_insn;
1018         uint64_t ip;
1019         int err;
1020
1021         ip = decoder->last_ip;
1022
1023         while (1) {
1024                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1025                 if (err == INTEL_PT_RETURN)
1026                         return 0;
1027                 if (err == -EAGAIN) {
1028                         if (decoder->set_fup_tx_flags) {
1029                                 decoder->set_fup_tx_flags = false;
1030                                 decoder->tx_flags = decoder->fup_tx_flags;
1031                                 decoder->state.type = INTEL_PT_TRANSACTION;
1032                                 decoder->state.from_ip = decoder->ip;
1033                                 decoder->state.to_ip = 0;
1034                                 decoder->state.flags = decoder->fup_tx_flags;
1035                                 return 0;
1036                         }
1037                         return err;
1038                 }
1039                 decoder->set_fup_tx_flags = false;
1040                 if (err)
1041                         return err;
1042
1043                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1044                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1045                                         decoder->ip);
1046                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1047                         return -ENOENT;
1048                 }
1049
1050                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1051                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1052                                         decoder->ip);
1053                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1054                         return -ENOENT;
1055                 }
1056
1057                 intel_pt_bug(decoder);
1058         }
1059 }
1060
1061 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1062 {
1063         struct intel_pt_insn intel_pt_insn;
1064         int err;
1065
1066         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1067         if (err == INTEL_PT_RETURN &&
1068             decoder->pgd_ip &&
1069             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1070             (decoder->state.type & INTEL_PT_BRANCH) &&
1071             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1072                 /* Unconditional branch leaving filter region */
1073                 decoder->no_progress = 0;
1074                 decoder->pge = false;
1075                 decoder->continuous_period = false;
1076                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1077                 decoder->state.to_ip = 0;
1078                 return 0;
1079         }
1080         if (err == INTEL_PT_RETURN)
1081                 return 0;
1082         if (err)
1083                 return err;
1084
1085         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1086                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1087                         decoder->pge = false;
1088                         decoder->continuous_period = false;
1089                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1090                         decoder->state.from_ip = decoder->ip;
1091                         decoder->state.to_ip = 0;
1092                         if (decoder->packet.count != 0)
1093                                 decoder->ip = decoder->last_ip;
1094                 } else {
1095                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1096                         decoder->state.from_ip = decoder->ip;
1097                         if (decoder->packet.count == 0) {
1098                                 decoder->state.to_ip = 0;
1099                         } else {
1100                                 decoder->state.to_ip = decoder->last_ip;
1101                                 decoder->ip = decoder->last_ip;
1102                         }
1103                 }
1104                 return 0;
1105         }
1106
1107         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1108                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1109                                  intel_pt_insn.rel;
1110
1111                 if (decoder->pgd_ip &&
1112                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1113                     decoder->pgd_ip(to_ip, decoder->data)) {
1114                         /* Conditional branch leaving filter region */
1115                         decoder->pge = false;
1116                         decoder->continuous_period = false;
1117                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1118                         decoder->ip = to_ip;
1119                         decoder->state.from_ip = decoder->ip;
1120                         decoder->state.to_ip = 0;
1121                         return 0;
1122                 }
1123                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1124                                 decoder->ip);
1125                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1126                 return -ENOENT;
1127         }
1128
1129         return intel_pt_bug(decoder);
1130 }
1131
1132 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1133 {
1134         struct intel_pt_insn intel_pt_insn;
1135         int err;
1136
1137         while (1) {
1138                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1139                 if (err == INTEL_PT_RETURN)
1140                         return 0;
1141                 if (err)
1142                         return err;
1143
1144                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1145                         if (!decoder->return_compression) {
1146                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1147                                                 decoder->ip);
1148                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1149                                 return -ENOENT;
1150                         }
1151                         if (!decoder->ret_addr) {
1152                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1153                                                 decoder->ip);
1154                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1155                                 return -ENOENT;
1156                         }
1157                         if (!(decoder->tnt.payload & BIT63)) {
1158                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1159                                                 decoder->ip);
1160                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1161                                 return -ENOENT;
1162                         }
1163                         decoder->tnt.count -= 1;
1164                         if (!decoder->tnt.count)
1165                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1166                         decoder->tnt.payload <<= 1;
1167                         decoder->state.from_ip = decoder->ip;
1168                         decoder->ip = decoder->ret_addr;
1169                         decoder->state.to_ip = decoder->ip;
1170                         return 0;
1171                 }
1172
1173                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1174                         /* Handle deferred TIPs */
1175                         err = intel_pt_get_next_packet(decoder);
1176                         if (err)
1177                                 return err;
1178                         if (decoder->packet.type != INTEL_PT_TIP ||
1179                             decoder->packet.count == 0) {
1180                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1181                                                 decoder->ip);
1182                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1183                                 decoder->pkt_step = 0;
1184                                 return -ENOENT;
1185                         }
1186                         intel_pt_set_last_ip(decoder);
1187                         decoder->state.from_ip = decoder->ip;
1188                         decoder->state.to_ip = decoder->last_ip;
1189                         decoder->ip = decoder->last_ip;
1190                         return 0;
1191                 }
1192
1193                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1194                         decoder->tnt.count -= 1;
1195                         if (!decoder->tnt.count)
1196                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1197                         if (decoder->tnt.payload & BIT63) {
1198                                 decoder->tnt.payload <<= 1;
1199                                 decoder->state.from_ip = decoder->ip;
1200                                 decoder->ip += intel_pt_insn.length +
1201                                                intel_pt_insn.rel;
1202                                 decoder->state.to_ip = decoder->ip;
1203                                 return 0;
1204                         }
1205                         /* Instruction sample for a non-taken branch */
1206                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1207                                 decoder->tnt.payload <<= 1;
1208                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1209                                 decoder->state.from_ip = decoder->ip;
1210                                 decoder->state.to_ip = 0;
1211                                 decoder->ip += intel_pt_insn.length;
1212                                 return 0;
1213                         }
1214                         decoder->ip += intel_pt_insn.length;
1215                         if (!decoder->tnt.count)
1216                                 return -EAGAIN;
1217                         decoder->tnt.payload <<= 1;
1218                         continue;
1219                 }
1220
1221                 return intel_pt_bug(decoder);
1222         }
1223 }
1224
1225 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1226 {
1227         unsigned int fup_tx_flags;
1228         int err;
1229
1230         fup_tx_flags = decoder->packet.payload &
1231                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1232         err = intel_pt_get_next_packet(decoder);
1233         if (err)
1234                 return err;
1235         if (decoder->packet.type == INTEL_PT_FUP) {
1236                 decoder->fup_tx_flags = fup_tx_flags;
1237                 decoder->set_fup_tx_flags = true;
1238                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1239                         *no_tip = true;
1240         } else {
1241                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1242                                 decoder->pos);
1243                 intel_pt_update_in_tx(decoder);
1244         }
1245         return 0;
1246 }
1247
1248 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1249 {
1250         uint64_t timestamp;
1251
1252         decoder->have_tma = false;
1253
1254         if (decoder->ref_timestamp) {
1255                 timestamp = decoder->packet.payload |
1256                             (decoder->ref_timestamp & (0xffULL << 56));
1257                 if (timestamp < decoder->ref_timestamp) {
1258                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1259                                 timestamp += (1ULL << 56);
1260                 } else {
1261                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1262                                 timestamp -= (1ULL << 56);
1263                 }
1264                 decoder->tsc_timestamp = timestamp;
1265                 decoder->timestamp = timestamp;
1266                 decoder->ref_timestamp = 0;
1267                 decoder->timestamp_insn_cnt = 0;
1268         } else if (decoder->timestamp) {
1269                 timestamp = decoder->packet.payload |
1270                             (decoder->timestamp & (0xffULL << 56));
1271                 decoder->tsc_timestamp = timestamp;
1272                 if (timestamp < decoder->timestamp &&
1273                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1274                         intel_pt_log_to("Suppressing backwards timestamp",
1275                                         timestamp);
1276                         timestamp = decoder->timestamp;
1277                 }
1278                 if (timestamp < decoder->timestamp) {
1279                         intel_pt_log_to("Wraparound timestamp", timestamp);
1280                         timestamp += (1ULL << 56);
1281                         decoder->tsc_timestamp = timestamp;
1282                 }
1283                 decoder->timestamp = timestamp;
1284                 decoder->timestamp_insn_cnt = 0;
1285         }
1286
1287         if (decoder->last_packet_type == INTEL_PT_CYC) {
1288                 decoder->cyc_ref_timestamp = decoder->timestamp;
1289                 decoder->cycle_cnt = 0;
1290                 decoder->have_calc_cyc_to_tsc = false;
1291                 intel_pt_calc_cyc_to_tsc(decoder, false);
1292         }
1293
1294         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1295 }
1296
1297 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1298 {
1299         intel_pt_log("ERROR: Buffer overflow\n");
1300         intel_pt_clear_tx_flags(decoder);
1301         decoder->have_tma = false;
1302         decoder->cbr = 0;
1303         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1304         decoder->overflow = true;
1305         return -EOVERFLOW;
1306 }
1307
1308 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1309 {
1310         uint32_t ctc = decoder->packet.payload;
1311         uint32_t fc = decoder->packet.count;
1312         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1313
1314         if (!decoder->tsc_ctc_ratio_d)
1315                 return;
1316
1317         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1318         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1319         if (decoder->tsc_ctc_mult) {
1320                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1321         } else {
1322                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1323                                                   decoder->tsc_ctc_ratio_n,
1324                                                   decoder->tsc_ctc_ratio_d);
1325         }
1326         decoder->ctc_delta = 0;
1327         decoder->have_tma = true;
1328         decoder->fixup_last_mtc = true;
1329         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1330                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1331 }
1332
1333 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1334 {
1335         uint64_t timestamp;
1336         uint32_t mtc, mtc_delta;
1337
1338         if (!decoder->have_tma)
1339                 return;
1340
1341         mtc = decoder->packet.payload;
1342
1343         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1344                 decoder->fixup_last_mtc = false;
1345                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1346                                         &decoder->last_mtc);
1347         }
1348
1349         if (mtc > decoder->last_mtc)
1350                 mtc_delta = mtc - decoder->last_mtc;
1351         else
1352                 mtc_delta = mtc + 256 - decoder->last_mtc;
1353
1354         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1355
1356         if (decoder->tsc_ctc_mult) {
1357                 timestamp = decoder->ctc_timestamp +
1358                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1359         } else {
1360                 timestamp = decoder->ctc_timestamp +
1361                             multdiv(decoder->ctc_delta,
1362                                     decoder->tsc_ctc_ratio_n,
1363                                     decoder->tsc_ctc_ratio_d);
1364         }
1365
1366         if (timestamp < decoder->timestamp)
1367                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1368                              timestamp, decoder->timestamp);
1369         else
1370                 decoder->timestamp = timestamp;
1371
1372         decoder->timestamp_insn_cnt = 0;
1373         decoder->last_mtc = mtc;
1374
1375         if (decoder->last_packet_type == INTEL_PT_CYC) {
1376                 decoder->cyc_ref_timestamp = decoder->timestamp;
1377                 decoder->cycle_cnt = 0;
1378                 decoder->have_calc_cyc_to_tsc = false;
1379                 intel_pt_calc_cyc_to_tsc(decoder, true);
1380         }
1381 }
1382
1383 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1384 {
1385         unsigned int cbr = decoder->packet.payload;
1386
1387         if (decoder->cbr == cbr)
1388                 return;
1389
1390         decoder->cbr = cbr;
1391         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1392 }
1393
1394 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1395 {
1396         uint64_t timestamp = decoder->cyc_ref_timestamp;
1397
1398         decoder->have_cyc = true;
1399
1400         decoder->cycle_cnt += decoder->packet.payload;
1401
1402         if (!decoder->cyc_ref_timestamp)
1403                 return;
1404
1405         if (decoder->have_calc_cyc_to_tsc)
1406                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1407         else if (decoder->cbr)
1408                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1409         else
1410                 return;
1411
1412         if (timestamp < decoder->timestamp)
1413                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1414                              timestamp, decoder->timestamp);
1415         else
1416                 decoder->timestamp = timestamp;
1417
1418         decoder->timestamp_insn_cnt = 0;
1419 }
1420
1421 /* Walk PSB+ packets when already in sync. */
1422 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1423 {
1424         int err;
1425
1426         while (1) {
1427                 err = intel_pt_get_next_packet(decoder);
1428                 if (err)
1429                         return err;
1430
1431                 switch (decoder->packet.type) {
1432                 case INTEL_PT_PSBEND:
1433                         return 0;
1434
1435                 case INTEL_PT_TIP_PGD:
1436                 case INTEL_PT_TIP_PGE:
1437                 case INTEL_PT_TIP:
1438                 case INTEL_PT_TNT:
1439                 case INTEL_PT_TRACESTOP:
1440                 case INTEL_PT_BAD:
1441                 case INTEL_PT_PSB:
1442                         decoder->have_tma = false;
1443                         intel_pt_log("ERROR: Unexpected packet\n");
1444                         return -EAGAIN;
1445
1446                 case INTEL_PT_OVF:
1447                         return intel_pt_overflow(decoder);
1448
1449                 case INTEL_PT_TSC:
1450                         intel_pt_calc_tsc_timestamp(decoder);
1451                         break;
1452
1453                 case INTEL_PT_TMA:
1454                         intel_pt_calc_tma(decoder);
1455                         break;
1456
1457                 case INTEL_PT_CBR:
1458                         intel_pt_calc_cbr(decoder);
1459                         break;
1460
1461                 case INTEL_PT_MODE_EXEC:
1462                         decoder->exec_mode = decoder->packet.payload;
1463                         break;
1464
1465                 case INTEL_PT_PIP:
1466                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1467                         break;
1468
1469                 case INTEL_PT_FUP:
1470                         decoder->pge = true;
1471                         if (decoder->packet.count)
1472                                 intel_pt_set_last_ip(decoder);
1473                         break;
1474
1475                 case INTEL_PT_MODE_TSX:
1476                         intel_pt_update_in_tx(decoder);
1477                         break;
1478
1479                 case INTEL_PT_MTC:
1480                         intel_pt_calc_mtc_timestamp(decoder);
1481                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1482                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1483                         break;
1484
1485                 case INTEL_PT_CYC:
1486                 case INTEL_PT_VMCS:
1487                 case INTEL_PT_MNT:
1488                 case INTEL_PT_PAD:
1489                 default:
1490                         break;
1491                 }
1492         }
1493 }
1494
1495 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1496 {
1497         int err;
1498
1499         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1500                 decoder->tx_flags = 0;
1501                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1502                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1503         } else {
1504                 decoder->state.flags |= INTEL_PT_ASYNC;
1505         }
1506
1507         while (1) {
1508                 err = intel_pt_get_next_packet(decoder);
1509                 if (err)
1510                         return err;
1511
1512                 switch (decoder->packet.type) {
1513                 case INTEL_PT_TNT:
1514                 case INTEL_PT_FUP:
1515                 case INTEL_PT_TRACESTOP:
1516                 case INTEL_PT_PSB:
1517                 case INTEL_PT_TSC:
1518                 case INTEL_PT_TMA:
1519                 case INTEL_PT_CBR:
1520                 case INTEL_PT_MODE_TSX:
1521                 case INTEL_PT_BAD:
1522                 case INTEL_PT_PSBEND:
1523                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1524                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1525                         return -ENOENT;
1526
1527                 case INTEL_PT_OVF:
1528                         return intel_pt_overflow(decoder);
1529
1530                 case INTEL_PT_TIP_PGD:
1531                         decoder->state.from_ip = decoder->ip;
1532                         decoder->state.to_ip = 0;
1533                         if (decoder->packet.count != 0) {
1534                                 intel_pt_set_ip(decoder);
1535                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1536                                              decoder->ip);
1537                         }
1538                         decoder->pge = false;
1539                         decoder->continuous_period = false;
1540                         return 0;
1541
1542                 case INTEL_PT_TIP_PGE:
1543                         decoder->pge = true;
1544                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1545                                      decoder->ip);
1546                         decoder->state.from_ip = 0;
1547                         if (decoder->packet.count == 0) {
1548                                 decoder->state.to_ip = 0;
1549                         } else {
1550                                 intel_pt_set_ip(decoder);
1551                                 decoder->state.to_ip = decoder->ip;
1552                         }
1553                         return 0;
1554
1555                 case INTEL_PT_TIP:
1556                         decoder->state.from_ip = decoder->ip;
1557                         if (decoder->packet.count == 0) {
1558                                 decoder->state.to_ip = 0;
1559                         } else {
1560                                 intel_pt_set_ip(decoder);
1561                                 decoder->state.to_ip = decoder->ip;
1562                         }
1563                         return 0;
1564
1565                 case INTEL_PT_PIP:
1566                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1567                         break;
1568
1569                 case INTEL_PT_MTC:
1570                         intel_pt_calc_mtc_timestamp(decoder);
1571                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1572                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1573                         break;
1574
1575                 case INTEL_PT_CYC:
1576                         intel_pt_calc_cyc_timestamp(decoder);
1577                         break;
1578
1579                 case INTEL_PT_MODE_EXEC:
1580                         decoder->exec_mode = decoder->packet.payload;
1581                         break;
1582
1583                 case INTEL_PT_VMCS:
1584                 case INTEL_PT_MNT:
1585                 case INTEL_PT_PAD:
1586                         break;
1587
1588                 default:
1589                         return intel_pt_bug(decoder);
1590                 }
1591         }
1592 }
1593
1594 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1595 {
1596         bool no_tip = false;
1597         int err;
1598
1599         while (1) {
1600                 err = intel_pt_get_next_packet(decoder);
1601                 if (err)
1602                         return err;
1603 next:
1604                 switch (decoder->packet.type) {
1605                 case INTEL_PT_TNT:
1606                         if (!decoder->packet.count)
1607                                 break;
1608                         decoder->tnt = decoder->packet;
1609                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1610                         err = intel_pt_walk_tnt(decoder);
1611                         if (err == -EAGAIN)
1612                                 break;
1613                         return err;
1614
1615                 case INTEL_PT_TIP_PGD:
1616                         if (decoder->packet.count != 0)
1617                                 intel_pt_set_last_ip(decoder);
1618                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1619                         return intel_pt_walk_tip(decoder);
1620
1621                 case INTEL_PT_TIP_PGE: {
1622                         decoder->pge = true;
1623                         if (decoder->packet.count == 0) {
1624                                 intel_pt_log_at("Skipping zero TIP.PGE",
1625                                                 decoder->pos);
1626                                 break;
1627                         }
1628                         intel_pt_set_ip(decoder);
1629                         decoder->state.from_ip = 0;
1630                         decoder->state.to_ip = decoder->ip;
1631                         return 0;
1632                 }
1633
1634                 case INTEL_PT_OVF:
1635                         return intel_pt_overflow(decoder);
1636
1637                 case INTEL_PT_TIP:
1638                         if (decoder->packet.count != 0)
1639                                 intel_pt_set_last_ip(decoder);
1640                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1641                         return intel_pt_walk_tip(decoder);
1642
1643                 case INTEL_PT_FUP:
1644                         if (decoder->packet.count == 0) {
1645                                 intel_pt_log_at("Skipping zero FUP",
1646                                                 decoder->pos);
1647                                 no_tip = false;
1648                                 break;
1649                         }
1650                         intel_pt_set_last_ip(decoder);
1651                         err = intel_pt_walk_fup(decoder);
1652                         if (err != -EAGAIN) {
1653                                 if (err)
1654                                         return err;
1655                                 if (no_tip)
1656                                         decoder->pkt_state =
1657                                                 INTEL_PT_STATE_FUP_NO_TIP;
1658                                 else
1659                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1660                                 return 0;
1661                         }
1662                         if (no_tip) {
1663                                 no_tip = false;
1664                                 break;
1665                         }
1666                         return intel_pt_walk_fup_tip(decoder);
1667
1668                 case INTEL_PT_TRACESTOP:
1669                         decoder->pge = false;
1670                         decoder->continuous_period = false;
1671                         intel_pt_clear_tx_flags(decoder);
1672                         decoder->have_tma = false;
1673                         break;
1674
1675                 case INTEL_PT_PSB:
1676                         decoder->last_ip = 0;
1677                         decoder->have_last_ip = true;
1678                         intel_pt_clear_stack(&decoder->stack);
1679                         err = intel_pt_walk_psbend(decoder);
1680                         if (err == -EAGAIN)
1681                                 goto next;
1682                         if (err)
1683                                 return err;
1684                         break;
1685
1686                 case INTEL_PT_PIP:
1687                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1688                         break;
1689
1690                 case INTEL_PT_MTC:
1691                         intel_pt_calc_mtc_timestamp(decoder);
1692                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1693                                 break;
1694                         /*
1695                          * Ensure that there has been an instruction since the
1696                          * last MTC.
1697                          */
1698                         if (!decoder->mtc_insn)
1699                                 break;
1700                         decoder->mtc_insn = false;
1701                         /* Ensure that there is a timestamp */
1702                         if (!decoder->timestamp)
1703                                 break;
1704                         decoder->state.type = INTEL_PT_INSTRUCTION;
1705                         decoder->state.from_ip = decoder->ip;
1706                         decoder->state.to_ip = 0;
1707                         decoder->mtc_insn = false;
1708                         return 0;
1709
1710                 case INTEL_PT_TSC:
1711                         intel_pt_calc_tsc_timestamp(decoder);
1712                         break;
1713
1714                 case INTEL_PT_TMA:
1715                         intel_pt_calc_tma(decoder);
1716                         break;
1717
1718                 case INTEL_PT_CYC:
1719                         intel_pt_calc_cyc_timestamp(decoder);
1720                         break;
1721
1722                 case INTEL_PT_CBR:
1723                         intel_pt_calc_cbr(decoder);
1724                         break;
1725
1726                 case INTEL_PT_MODE_EXEC:
1727                         decoder->exec_mode = decoder->packet.payload;
1728                         break;
1729
1730                 case INTEL_PT_MODE_TSX:
1731                         /* MODE_TSX need not be followed by FUP */
1732                         if (!decoder->pge) {
1733                                 intel_pt_update_in_tx(decoder);
1734                                 break;
1735                         }
1736                         err = intel_pt_mode_tsx(decoder, &no_tip);
1737                         if (err)
1738                                 return err;
1739                         goto next;
1740
1741                 case INTEL_PT_BAD: /* Does not happen */
1742                         return intel_pt_bug(decoder);
1743
1744                 case INTEL_PT_PSBEND:
1745                 case INTEL_PT_VMCS:
1746                 case INTEL_PT_MNT:
1747                 case INTEL_PT_PAD:
1748                         break;
1749
1750                 default:
1751                         return intel_pt_bug(decoder);
1752                 }
1753         }
1754 }
1755
1756 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1757 {
1758         return decoder->packet.count &&
1759                (decoder->have_last_ip || decoder->packet.count == 3 ||
1760                 decoder->packet.count == 6);
1761 }
1762
1763 /* Walk PSB+ packets to get in sync. */
1764 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1765 {
1766         int err;
1767
1768         while (1) {
1769                 err = intel_pt_get_next_packet(decoder);
1770                 if (err)
1771                         return err;
1772
1773                 switch (decoder->packet.type) {
1774                 case INTEL_PT_TIP_PGD:
1775                         decoder->continuous_period = false;
1776                         __fallthrough;
1777                 case INTEL_PT_TIP_PGE:
1778                 case INTEL_PT_TIP:
1779                         intel_pt_log("ERROR: Unexpected packet\n");
1780                         return -ENOENT;
1781
1782                 case INTEL_PT_FUP:
1783                         decoder->pge = true;
1784                         if (intel_pt_have_ip(decoder)) {
1785                                 uint64_t current_ip = decoder->ip;
1786
1787                                 intel_pt_set_ip(decoder);
1788                                 if (current_ip)
1789                                         intel_pt_log_to("Setting IP",
1790                                                         decoder->ip);
1791                         }
1792                         break;
1793
1794                 case INTEL_PT_MTC:
1795                         intel_pt_calc_mtc_timestamp(decoder);
1796                         break;
1797
1798                 case INTEL_PT_TSC:
1799                         intel_pt_calc_tsc_timestamp(decoder);
1800                         break;
1801
1802                 case INTEL_PT_TMA:
1803                         intel_pt_calc_tma(decoder);
1804                         break;
1805
1806                 case INTEL_PT_CYC:
1807                         intel_pt_calc_cyc_timestamp(decoder);
1808                         break;
1809
1810                 case INTEL_PT_CBR:
1811                         intel_pt_calc_cbr(decoder);
1812                         break;
1813
1814                 case INTEL_PT_PIP:
1815                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1816                         break;
1817
1818                 case INTEL_PT_MODE_EXEC:
1819                         decoder->exec_mode = decoder->packet.payload;
1820                         break;
1821
1822                 case INTEL_PT_MODE_TSX:
1823                         intel_pt_update_in_tx(decoder);
1824                         break;
1825
1826                 case INTEL_PT_TRACESTOP:
1827                         decoder->pge = false;
1828                         decoder->continuous_period = false;
1829                         intel_pt_clear_tx_flags(decoder);
1830                         __fallthrough;
1831
1832                 case INTEL_PT_TNT:
1833                         decoder->have_tma = false;
1834                         intel_pt_log("ERROR: Unexpected packet\n");
1835                         if (decoder->ip)
1836                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1837                         else
1838                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1839                         return -ENOENT;
1840
1841                 case INTEL_PT_BAD: /* Does not happen */
1842                         return intel_pt_bug(decoder);
1843
1844                 case INTEL_PT_OVF:
1845                         return intel_pt_overflow(decoder);
1846
1847                 case INTEL_PT_PSBEND:
1848                         return 0;
1849
1850                 case INTEL_PT_PSB:
1851                 case INTEL_PT_VMCS:
1852                 case INTEL_PT_MNT:
1853                 case INTEL_PT_PAD:
1854                 default:
1855                         break;
1856                 }
1857         }
1858 }
1859
1860 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1861 {
1862         int err;
1863
1864         while (1) {
1865                 err = intel_pt_get_next_packet(decoder);
1866                 if (err)
1867                         return err;
1868
1869                 switch (decoder->packet.type) {
1870                 case INTEL_PT_TIP_PGD:
1871                         decoder->continuous_period = false;
1872                         __fallthrough;
1873                 case INTEL_PT_TIP_PGE:
1874                 case INTEL_PT_TIP:
1875                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1876                         if (intel_pt_have_ip(decoder))
1877                                 intel_pt_set_ip(decoder);
1878                         if (decoder->ip)
1879                                 return 0;
1880                         break;
1881
1882                 case INTEL_PT_FUP:
1883                         if (decoder->overflow) {
1884                                 if (intel_pt_have_ip(decoder))
1885                                         intel_pt_set_ip(decoder);
1886                                 if (decoder->ip)
1887                                         return 0;
1888                         }
1889                         if (decoder->packet.count && decoder->have_last_ip)
1890                                 intel_pt_set_last_ip(decoder);
1891                         break;
1892
1893                 case INTEL_PT_MTC:
1894                         intel_pt_calc_mtc_timestamp(decoder);
1895                         break;
1896
1897                 case INTEL_PT_TSC:
1898                         intel_pt_calc_tsc_timestamp(decoder);
1899                         break;
1900
1901                 case INTEL_PT_TMA:
1902                         intel_pt_calc_tma(decoder);
1903                         break;
1904
1905                 case INTEL_PT_CYC:
1906                         intel_pt_calc_cyc_timestamp(decoder);
1907                         break;
1908
1909                 case INTEL_PT_CBR:
1910                         intel_pt_calc_cbr(decoder);
1911                         break;
1912
1913                 case INTEL_PT_PIP:
1914                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1915                         break;
1916
1917                 case INTEL_PT_MODE_EXEC:
1918                         decoder->exec_mode = decoder->packet.payload;
1919                         break;
1920
1921                 case INTEL_PT_MODE_TSX:
1922                         intel_pt_update_in_tx(decoder);
1923                         break;
1924
1925                 case INTEL_PT_OVF:
1926                         return intel_pt_overflow(decoder);
1927
1928                 case INTEL_PT_BAD: /* Does not happen */
1929                         return intel_pt_bug(decoder);
1930
1931                 case INTEL_PT_TRACESTOP:
1932                         decoder->pge = false;
1933                         decoder->continuous_period = false;
1934                         intel_pt_clear_tx_flags(decoder);
1935                         decoder->have_tma = false;
1936                         break;
1937
1938                 case INTEL_PT_PSB:
1939                         decoder->last_ip = 0;
1940                         decoder->have_last_ip = true;
1941                         intel_pt_clear_stack(&decoder->stack);
1942                         err = intel_pt_walk_psb(decoder);
1943                         if (err)
1944                                 return err;
1945                         if (decoder->ip) {
1946                                 /* Do not have a sample */
1947                                 decoder->state.type = 0;
1948                                 return 0;
1949                         }
1950                         break;
1951
1952                 case INTEL_PT_TNT:
1953                 case INTEL_PT_PSBEND:
1954                 case INTEL_PT_VMCS:
1955                 case INTEL_PT_MNT:
1956                 case INTEL_PT_PAD:
1957                 default:
1958                         break;
1959                 }
1960         }
1961 }
1962
1963 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1964 {
1965         int err;
1966
1967         intel_pt_log("Scanning for full IP\n");
1968         err = intel_pt_walk_to_ip(decoder);
1969         if (err)
1970                 return err;
1971
1972         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1973         decoder->overflow = false;
1974
1975         decoder->state.from_ip = 0;
1976         decoder->state.to_ip = decoder->ip;
1977         intel_pt_log_to("Setting IP", decoder->ip);
1978
1979         return 0;
1980 }
1981
1982 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1983 {
1984         const unsigned char *end = decoder->buf + decoder->len;
1985         size_t i;
1986
1987         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1988                 if (i > decoder->len)
1989                         continue;
1990                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1991                         return i;
1992         }
1993         return 0;
1994 }
1995
1996 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1997 {
1998         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1999         const char *psb = INTEL_PT_PSB_STR;
2000
2001         if (rest_psb > decoder->len ||
2002             memcmp(decoder->buf, psb + part_psb, rest_psb))
2003                 return 0;
2004
2005         return rest_psb;
2006 }
2007
2008 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2009                                   int part_psb)
2010 {
2011         int rest_psb, ret;
2012
2013         decoder->pos += decoder->len;
2014         decoder->len = 0;
2015
2016         ret = intel_pt_get_next_data(decoder);
2017         if (ret)
2018                 return ret;
2019
2020         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2021         if (!rest_psb)
2022                 return 0;
2023
2024         decoder->pos -= part_psb;
2025         decoder->next_buf = decoder->buf + rest_psb;
2026         decoder->next_len = decoder->len - rest_psb;
2027         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2028         decoder->buf = decoder->temp_buf;
2029         decoder->len = INTEL_PT_PSB_LEN;
2030
2031         return 0;
2032 }
2033
2034 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2035 {
2036         unsigned char *next;
2037         int ret;
2038
2039         intel_pt_log("Scanning for PSB\n");
2040         while (1) {
2041                 if (!decoder->len) {
2042                         ret = intel_pt_get_next_data(decoder);
2043                         if (ret)
2044                                 return ret;
2045                 }
2046
2047                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2048                               INTEL_PT_PSB_LEN);
2049                 if (!next) {
2050                         int part_psb;
2051
2052                         part_psb = intel_pt_part_psb(decoder);
2053                         if (part_psb) {
2054                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2055                                 if (ret)
2056                                         return ret;
2057                         } else {
2058                                 decoder->pos += decoder->len;
2059                                 decoder->len = 0;
2060                         }
2061                         continue;
2062                 }
2063
2064                 decoder->pkt_step = next - decoder->buf;
2065                 return intel_pt_get_next_packet(decoder);
2066         }
2067 }
2068
2069 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2070 {
2071         int err;
2072
2073         decoder->pge = false;
2074         decoder->continuous_period = false;
2075         decoder->have_last_ip = false;
2076         decoder->last_ip = 0;
2077         decoder->ip = 0;
2078         intel_pt_clear_stack(&decoder->stack);
2079
2080         err = intel_pt_scan_for_psb(decoder);
2081         if (err)
2082                 return err;
2083
2084         decoder->have_last_ip = true;
2085         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2086
2087         err = intel_pt_walk_psb(decoder);
2088         if (err)
2089                 return err;
2090
2091         if (decoder->ip) {
2092                 decoder->state.type = 0; /* Do not have a sample */
2093                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2094         } else {
2095                 return intel_pt_sync_ip(decoder);
2096         }
2097
2098         return 0;
2099 }
2100
2101 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2102 {
2103         uint64_t est = decoder->sample_insn_cnt << 1;
2104
2105         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2106                 goto out;
2107
2108         est *= decoder->max_non_turbo_ratio;
2109         est /= decoder->cbr;
2110 out:
2111         return decoder->sample_timestamp + est;
2112 }
2113
2114 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2115 {
2116         int err;
2117
2118         do {
2119                 decoder->state.type = INTEL_PT_BRANCH;
2120                 decoder->state.flags = 0;
2121
2122                 switch (decoder->pkt_state) {
2123                 case INTEL_PT_STATE_NO_PSB:
2124                         err = intel_pt_sync(decoder);
2125                         break;
2126                 case INTEL_PT_STATE_NO_IP:
2127                         decoder->have_last_ip = false;
2128                         decoder->last_ip = 0;
2129                         decoder->ip = 0;
2130                         /* Fall through */
2131                 case INTEL_PT_STATE_ERR_RESYNC:
2132                         err = intel_pt_sync_ip(decoder);
2133                         break;
2134                 case INTEL_PT_STATE_IN_SYNC:
2135                         err = intel_pt_walk_trace(decoder);
2136                         break;
2137                 case INTEL_PT_STATE_TNT:
2138                         err = intel_pt_walk_tnt(decoder);
2139                         if (err == -EAGAIN)
2140                                 err = intel_pt_walk_trace(decoder);
2141                         break;
2142                 case INTEL_PT_STATE_TIP:
2143                 case INTEL_PT_STATE_TIP_PGD:
2144                         err = intel_pt_walk_tip(decoder);
2145                         break;
2146                 case INTEL_PT_STATE_FUP:
2147                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2148                         err = intel_pt_walk_fup(decoder);
2149                         if (err == -EAGAIN)
2150                                 err = intel_pt_walk_fup_tip(decoder);
2151                         else if (!err)
2152                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2153                         break;
2154                 case INTEL_PT_STATE_FUP_NO_TIP:
2155                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2156                         err = intel_pt_walk_fup(decoder);
2157                         if (err == -EAGAIN)
2158                                 err = intel_pt_walk_trace(decoder);
2159                         break;
2160                 default:
2161                         err = intel_pt_bug(decoder);
2162                         break;
2163                 }
2164         } while (err == -ENOLINK);
2165
2166         if (err) {
2167                 decoder->state.err = intel_pt_ext_err(err);
2168                 decoder->state.from_ip = decoder->ip;
2169                 decoder->sample_timestamp = decoder->timestamp;
2170                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2171         } else {
2172                 decoder->state.err = 0;
2173                 if (intel_pt_sample_time(decoder->pkt_state)) {
2174                         decoder->sample_timestamp = decoder->timestamp;
2175                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2176                 }
2177         }
2178
2179         decoder->state.timestamp = decoder->sample_timestamp;
2180         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2181         decoder->state.cr3 = decoder->cr3;
2182         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2183
2184         return &decoder->state;
2185 }
2186
2187 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2188 {
2189         if (len < INTEL_PT_PSB_LEN)
2190                 return false;
2191         return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2192                       INTEL_PT_PSB_LEN);
2193 }
2194
2195 /**
2196  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2197  * @buf: pointer to buffer pointer
2198  * @len: size of buffer
2199  *
2200  * Updates the buffer pointer to point to the start of the next PSB packet if
2201  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2202  * @len is adjusted accordingly.
2203  *
2204  * Return: %true if a PSB packet is found, %false otherwise.
2205  */
2206 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2207 {
2208         unsigned char *next;
2209
2210         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2211         if (next) {
2212                 *len -= next - *buf;
2213                 *buf = next;
2214                 return true;
2215         }
2216         return false;
2217 }
2218
2219 /**
2220  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2221  *                     packet.
2222  * @buf: pointer to buffer pointer
2223  * @len: size of buffer
2224  *
2225  * Updates the buffer pointer to point to the start of the following PSB packet
2226  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2227  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2228  *
2229  * Return: %true if a PSB packet is found, %false otherwise.
2230  */
2231 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2232 {
2233         unsigned char *next;
2234
2235         if (!*len)
2236                 return false;
2237
2238         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2239         if (next) {
2240                 *len -= next - *buf;
2241                 *buf = next;
2242                 return true;
2243         }
2244         return false;
2245 }
2246
2247 /**
2248  * intel_pt_last_psb - find the last PSB packet in a buffer.
2249  * @buf: buffer
2250  * @len: size of buffer
2251  *
2252  * This function finds the last PSB in a buffer.
2253  *
2254  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2255  */
2256 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2257 {
2258         const char *n = INTEL_PT_PSB_STR;
2259         unsigned char *p;
2260         size_t k;
2261
2262         if (len < INTEL_PT_PSB_LEN)
2263                 return NULL;
2264
2265         k = len - INTEL_PT_PSB_LEN + 1;
2266         while (1) {
2267                 p = memrchr(buf, n[0], k);
2268                 if (!p)
2269                         return NULL;
2270                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2271                         return p;
2272                 k = p - buf;
2273                 if (!k)
2274                         return NULL;
2275         }
2276 }
2277
2278 /**
2279  * intel_pt_next_tsc - find and return next TSC.
2280  * @buf: buffer
2281  * @len: size of buffer
2282  * @tsc: TSC value returned
2283  *
2284  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2285  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2286  * PSBEND packet is found.
2287  *
2288  * Return: %true if TSC is found, false otherwise.
2289  */
2290 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2291 {
2292         struct intel_pt_pkt packet;
2293         int ret;
2294
2295         while (len) {
2296                 ret = intel_pt_get_packet(buf, len, &packet);
2297                 if (ret <= 0)
2298                         return false;
2299                 if (packet.type == INTEL_PT_TSC) {
2300                         *tsc = packet.payload;
2301                         return true;
2302                 }
2303                 if (packet.type == INTEL_PT_PSBEND)
2304                         return false;
2305                 buf += ret;
2306                 len -= ret;
2307         }
2308         return false;
2309 }
2310
2311 /**
2312  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2313  * @tsc1: first TSC to compare
2314  * @tsc2: second TSC to compare
2315  *
2316  * This function compares 7-byte TSC values allowing for the possibility that
2317  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2318  * around so for that purpose this function assumes the absolute difference is
2319  * less than half the maximum difference.
2320  *
2321  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2322  * after @tsc2.
2323  */
2324 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2325 {
2326         const uint64_t halfway = (1ULL << 55);
2327
2328         if (tsc1 == tsc2)
2329                 return 0;
2330
2331         if (tsc1 < tsc2) {
2332                 if (tsc2 - tsc1 < halfway)
2333                         return -1;
2334                 else
2335                         return 1;
2336         } else {
2337                 if (tsc1 - tsc2 < halfway)
2338                         return 1;
2339                 else
2340                         return -1;
2341         }
2342 }
2343
2344 /**
2345  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2346  *                             using TSC.
2347  * @buf_a: first buffer
2348  * @len_a: size of first buffer
2349  * @buf_b: second buffer
2350  * @len_b: size of second buffer
2351  *
2352  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2353  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2354  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2355  * @buf_a and @buf_b are positioned at a PSB.
2356  *
2357  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2358  * @buf_b + @len_b if there is no non-overlapped data.
2359  */
2360 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2361                                                 size_t len_a,
2362                                                 unsigned char *buf_b,
2363                                                 size_t len_b)
2364 {
2365         uint64_t tsc_a, tsc_b;
2366         unsigned char *p;
2367         size_t len;
2368
2369         p = intel_pt_last_psb(buf_a, len_a);
2370         if (!p)
2371                 return buf_b; /* No PSB in buf_a => no overlap */
2372
2373         len = len_a - (p - buf_a);
2374         if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2375                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2376                 len_a -= len;
2377                 p = intel_pt_last_psb(buf_a, len_a);
2378                 if (!p)
2379                         return buf_b; /* No full PSB+ => assume no overlap */
2380                 len = len_a - (p - buf_a);
2381                 if (!intel_pt_next_tsc(p, len, &tsc_a))
2382                         return buf_b; /* No TSC in buf_a => assume no overlap */
2383         }
2384
2385         while (1) {
2386                 /* Ignore PSB+ with no TSC */
2387                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2388                     intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2389                         return buf_b; /* tsc_a < tsc_b => no overlap */
2390
2391                 if (!intel_pt_step_psb(&buf_b, &len_b))
2392                         return buf_b + len_b; /* No PSB in buf_b => no data */
2393         }
2394 }
2395
2396 /**
2397  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2398  * @buf_a: first buffer
2399  * @len_a: size of first buffer
2400  * @buf_b: second buffer
2401  * @len_b: size of second buffer
2402  * @have_tsc: can use TSC packets to detect overlap
2403  *
2404  * When trace samples or snapshots are recorded there is the possibility that
2405  * the data overlaps.  Note that, for the purposes of decoding, data is only
2406  * useful if it begins with a PSB packet.
2407  *
2408  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2409  * @buf_b + @len_b if there is no non-overlapped data.
2410  */
2411 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2412                                      unsigned char *buf_b, size_t len_b,
2413                                      bool have_tsc)
2414 {
2415         unsigned char *found;
2416
2417         /* Buffer 'b' must start at PSB so throw away everything before that */
2418         if (!intel_pt_next_psb(&buf_b, &len_b))
2419                 return buf_b + len_b; /* No PSB */
2420
2421         if (!intel_pt_next_psb(&buf_a, &len_a))
2422                 return buf_b; /* No overlap */
2423
2424         if (have_tsc) {
2425                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2426                 if (found)
2427                         return found;
2428         }
2429
2430         /*
2431          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2432          * we can ignore the first part of buffer 'a'.
2433          */
2434         while (len_b < len_a) {
2435                 if (!intel_pt_step_psb(&buf_a, &len_a))
2436                         return buf_b; /* No overlap */
2437         }
2438
2439         /* Now len_b >= len_a */
2440         if (len_b > len_a) {
2441                 /* The leftover buffer 'b' must start at a PSB */
2442                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2443                         if (!intel_pt_step_psb(&buf_a, &len_a))
2444                                 return buf_b; /* No overlap */
2445                 }
2446         }
2447
2448         while (1) {
2449                 /* Potential overlap so check the bytes */
2450                 found = memmem(buf_a, len_a, buf_b, len_a);
2451                 if (found)
2452                         return buf_b + len_a;
2453
2454                 /* Try again at next PSB in buffer 'a' */
2455                 if (!intel_pt_step_psb(&buf_a, &len_a))
2456                         return buf_b; /* No overlap */
2457
2458                 /* The leftover buffer 'b' must start at a PSB */
2459                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2460                         if (!intel_pt_step_psb(&buf_a, &len_a))
2461                                 return buf_b; /* No overlap */
2462                 }
2463         }
2464 }