tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29
  30 #include "intel-pt-insn-decoder.h"
  31 #include "intel-pt-pkt-decoder.h"
  32 #include "intel-pt-decoder.h"
  33 #include "intel-pt-log.h"
  34
  35 #define INTEL_PT_BLK_SIZE 1024
  36
  37 #define BIT63 (((uint64_t)1 << 63))
  38
  39 #define INTEL_PT_RETURN 1
  40
  41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  42 #define INTEL_PT_MAX_LOOPS 10000
  43
  44 struct intel_pt_blk {
  45         struct intel_pt_blk *prev;
  46         uint64_t ip[INTEL_PT_BLK_SIZE];
  47 };
  48
  49 struct intel_pt_stack {
  50         struct intel_pt_blk *blk;
  51         struct intel_pt_blk *spare;
  52         int pos;
  53 };
  54
  55 enum intel_pt_pkt_state {
  56         INTEL_PT_STATE_NO_PSB,
  57         INTEL_PT_STATE_NO_IP,
  58         INTEL_PT_STATE_ERR_RESYNC,
  59         INTEL_PT_STATE_IN_SYNC,
  60         INTEL_PT_STATE_TNT,
  61         INTEL_PT_STATE_TIP,
  62         INTEL_PT_STATE_TIP_PGD,
  63         INTEL_PT_STATE_FUP,
  64         INTEL_PT_STATE_FUP_NO_TIP,
  65 };
  66
  67 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  68 {
  69         switch (pkt_state) {
  70         case INTEL_PT_STATE_NO_PSB:
  71         case INTEL_PT_STATE_NO_IP:
  72         case INTEL_PT_STATE_ERR_RESYNC:
  73         case INTEL_PT_STATE_IN_SYNC:
  74         case INTEL_PT_STATE_TNT:
  75                 return true;
  76         case INTEL_PT_STATE_TIP:
  77         case INTEL_PT_STATE_TIP_PGD:
  78         case INTEL_PT_STATE_FUP:
  79         case INTEL_PT_STATE_FUP_NO_TIP:
  80                 return false;
  81         default:
  82                 return true;
  83         };
  84 }
  85
  86 #ifdef INTEL_PT_STRICT
  87 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  88 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  89 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  90 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  91 #else
  92 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  93 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  94 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  95 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  96 #endif
  97
  98 struct intel_pt_decoder {
  99         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 100         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 101                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 102                          uint64_t max_insn_cnt, void *data);
 103         bool (*pgd_ip)(uint64_t ip, void *data);
 104         void *data;
 105         struct intel_pt_state state;
 106         const unsigned char *buf;
 107         size_t len;
 108         bool return_compression;
 109         bool mtc_insn;
 110         bool pge;
 111         bool have_tma;
 112         bool have_cyc;
 113         bool fixup_last_mtc;
 114         uint64_t pos;
 115         uint64_t last_ip;
 116         uint64_t ip;
 117         uint64_t cr3;
 118         uint64_t timestamp;
 119         uint64_t tsc_timestamp;
 120         uint64_t ref_timestamp;
 121         uint64_t sample_timestamp;
 122         uint64_t ret_addr;
 123         uint64_t ctc_timestamp;
 124         uint64_t ctc_delta;
 125         uint64_t cycle_cnt;
 126         uint64_t cyc_ref_timestamp;
 127         uint32_t last_mtc;
 128         uint32_t tsc_ctc_ratio_n;
 129         uint32_t tsc_ctc_ratio_d;
 130         uint32_t tsc_ctc_mult;
 131         uint32_t tsc_slip;
 132         uint32_t ctc_rem_mask;
 133         int mtc_shift;
 134         struct intel_pt_stack stack;
 135         enum intel_pt_pkt_state pkt_state;
 136         struct intel_pt_pkt packet;
 137         struct intel_pt_pkt tnt;
 138         int pkt_step;
 139         int pkt_len;
 140         int last_packet_type;
 141         unsigned int cbr;
 142         unsigned int max_non_turbo_ratio;
 143         double max_non_turbo_ratio_fp;
 144         double cbr_cyc_to_tsc;
 145         double calc_cyc_to_tsc;
 146         bool have_calc_cyc_to_tsc;
 147         int exec_mode;
 148         unsigned int insn_bytes;
 149         uint64_t period;
 150         enum intel_pt_period_type period_type;
 151         uint64_t tot_insn_cnt;
 152         uint64_t period_insn_cnt;
 153         uint64_t period_mask;
 154         uint64_t period_ticks;
 155         uint64_t last_masked_timestamp;
 156         bool continuous_period;
 157         bool overflow;
 158         bool set_fup_tx_flags;
 159         unsigned int fup_tx_flags;
 160         unsigned int tx_flags;
 161         uint64_t timestamp_insn_cnt;
 162         uint64_t sample_insn_cnt;
 163         uint64_t stuck_ip;
 164         int no_progress;
 165         int stuck_ip_prd;
 166         int stuck_ip_cnt;
 167         const unsigned char *next_buf;
 168         size_t next_len;
 169         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 170 };
 171
 172 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 173 {
 174         int i;
 175
 176         for (i = 0; x != 1; i++)
 177                 x >>= 1;
 178
 179         return x << i;
 180 }
 181
 182 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 183 {
 184         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 185                 uint64_t period;
 186
 187                 period = intel_pt_lower_power_of_2(decoder->period);
 188                 decoder->period_mask  = ~(period - 1);
 189                 decoder->period_ticks = period;
 190         }
 191 }
 192
 193 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 194 {
 195         if (!d)
 196                 return 0;
 197         return (t / d) * n + ((t % d) * n) / d;
 198 }
 199
 200 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 201 {
 202         struct intel_pt_decoder *decoder;
 203
 204         if (!params->get_trace || !params->walk_insn)
 205                 return NULL;
 206
 207         decoder = zalloc(sizeof(struct intel_pt_decoder));
 208         if (!decoder)
 209                 return NULL;
 210
 211         decoder->get_trace          = params->get_trace;
 212         decoder->walk_insn          = params->walk_insn;
 213         decoder->pgd_ip             = params->pgd_ip;
 214         decoder->data               = params->data;
 215         decoder->return_compression = params->return_compression;
 216
 217         decoder->period             = params->period;
 218         decoder->period_type        = params->period_type;
 219
 220         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 221         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 222
 223         intel_pt_setup_period(decoder);
 224
 225         decoder->mtc_shift = params->mtc_period;
 226         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 227
 228         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 229         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 230
 231         if (!decoder->tsc_ctc_ratio_n)
 232                 decoder->tsc_ctc_ratio_d = 0;
 233
 234         if (decoder->tsc_ctc_ratio_d) {
 235                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 236                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 237                                                 decoder->tsc_ctc_ratio_d;
 238
 239                 /*
 240                  * Allow for timestamps appearing to backwards because a TSC
 241                  * packet has slipped past a MTC packet, so allow 2 MTC ticks
 242                  * or ...
 243                  */
 244                 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
 245                                         decoder->tsc_ctc_ratio_n,
 246                                         decoder->tsc_ctc_ratio_d);
 247         }
 248         /* ... or 0x100 paranoia */
 249         if (decoder->tsc_slip < 0x100)
 250                 decoder->tsc_slip = 0x100;
 251
 252         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 253         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 254         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 255         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 256         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 257
 258         return decoder;
 259 }
 260
 261 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 262 {
 263         struct intel_pt_blk *blk = stack->blk;
 264
 265         stack->blk = blk->prev;
 266         if (!stack->spare)
 267                 stack->spare = blk;
 268         else
 269                 free(blk);
 270 }
 271
 272 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 273 {
 274         if (!stack->pos) {
 275                 if (!stack->blk)
 276                         return 0;
 277                 intel_pt_pop_blk(stack);
 278                 if (!stack->blk)
 279                         return 0;
 280                 stack->pos = INTEL_PT_BLK_SIZE;
 281         }
 282         return stack->blk->ip[--stack->pos];
 283 }
 284
 285 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 286 {
 287         struct intel_pt_blk *blk;
 288
 289         if (stack->spare) {
 290                 blk = stack->spare;
 291                 stack->spare = NULL;
 292         } else {
 293                 blk = malloc(sizeof(struct intel_pt_blk));
 294                 if (!blk)
 295                         return -ENOMEM;
 296         }
 297
 298         blk->prev = stack->blk;
 299         stack->blk = blk;
 300         stack->pos = 0;
 301         return 0;
 302 }
 303
 304 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 305 {
 306         int err;
 307
 308         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 309                 err = intel_pt_alloc_blk(stack);
 310                 if (err)
 311                         return err;
 312         }
 313
 314         stack->blk->ip[stack->pos++] = ip;
 315         return 0;
 316 }
 317
 318 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 319 {
 320         while (stack->blk)
 321                 intel_pt_pop_blk(stack);
 322         stack->pos = 0;
 323 }
 324
 325 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 326 {
 327         intel_pt_clear_stack(stack);
 328         zfree(&stack->blk);
 329         zfree(&stack->spare);
 330 }
 331
 332 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 333 {
 334         intel_pt_free_stack(&decoder->stack);
 335         free(decoder);
 336 }
 337
 338 static int intel_pt_ext_err(int code)
 339 {
 340         switch (code) {
 341         case -ENOMEM:
 342                 return INTEL_PT_ERR_NOMEM;
 343         case -ENOSYS:
 344                 return INTEL_PT_ERR_INTERN;
 345         case -EBADMSG:
 346                 return INTEL_PT_ERR_BADPKT;
 347         case -ENODATA:
 348                 return INTEL_PT_ERR_NODATA;
 349         case -EILSEQ:
 350                 return INTEL_PT_ERR_NOINSN;
 351         case -ENOENT:
 352                 return INTEL_PT_ERR_MISMAT;
 353         case -EOVERFLOW:
 354                 return INTEL_PT_ERR_OVR;
 355         case -ENOSPC:
 356                 return INTEL_PT_ERR_LOST;
 357         case -ELOOP:
 358                 return INTEL_PT_ERR_NELOOP;
 359         default:
 360                 return INTEL_PT_ERR_UNK;
 361         }
 362 }
 363
 364 static const char *intel_pt_err_msgs[] = {
 365         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 366         [INTEL_PT_ERR_INTERN] = "Internal error",
 367         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 368         [INTEL_PT_ERR_NODATA] = "No more data",
 369         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 370         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 371         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 372         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 373         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 374         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 375 };
 376
 377 int intel_pt__strerror(int code, char *buf, size_t buflen)
 378 {
 379         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 380                 code = INTEL_PT_ERR_UNK;
 381         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 382         return 0;
 383 }
 384
 385 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 386                                  uint64_t last_ip)
 387 {
 388         uint64_t ip;
 389
 390         switch (packet->count) {
 391         case 1:
 392                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 393                      packet->payload;
 394                 break;
 395         case 2:
 396                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 397                      packet->payload;
 398                 break;
 399         case 3:
 400                 ip = packet->payload;
 401                 /* Sign-extend 6-byte ip */
 402                 if (ip & (uint64_t)0x800000000000ULL)
 403                         ip |= (uint64_t)0xffff000000000000ULL;
 404                 break;
 405         case 4:
 406                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 407                      packet->payload;
 408                 break;
 409         case 6:
 410                 ip = packet->payload;
 411                 break;
 412         default:
 413                 return 0;
 414         }
 415
 416         return ip;
 417 }
 418
 419 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 420 {
 421         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 422 }
 423
 424 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 425 {
 426         intel_pt_set_last_ip(decoder);
 427         decoder->ip = decoder->last_ip;
 428 }
 429
 430 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 431 {
 432         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 433                             decoder->buf);
 434 }
 435
 436 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 437 {
 438         intel_pt_log("ERROR: Internal error\n");
 439         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 440         return -ENOSYS;
 441 }
 442
 443 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 444 {
 445         decoder->tx_flags = 0;
 446 }
 447
 448 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 449 {
 450         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 451 }
 452
 453 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 454 {
 455         intel_pt_clear_tx_flags(decoder);
 456         decoder->have_tma = false;
 457         decoder->pkt_len = 1;
 458         decoder->pkt_step = 1;
 459         intel_pt_decoder_log_packet(decoder);
 460         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 461                 intel_pt_log("ERROR: Bad packet\n");
 462                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 463         }
 464         return -EBADMSG;
 465 }
 466
 467 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 468 {
 469         struct intel_pt_buffer buffer = { .buf = 0, };
 470         int ret;
 471
 472         decoder->pkt_step = 0;
 473
 474         intel_pt_log("Getting more data\n");
 475         ret = decoder->get_trace(&buffer, decoder->data);
 476         if (ret)
 477                 return ret;
 478         decoder->buf = buffer.buf;
 479         decoder->len = buffer.len;
 480         if (!decoder->len) {
 481                 intel_pt_log("No more data\n");
 482                 return -ENODATA;
 483         }
 484         if (!buffer.consecutive) {
 485                 decoder->ip = 0;
 486                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 487                 decoder->ref_timestamp = buffer.ref_timestamp;
 488                 decoder->timestamp = 0;
 489                 decoder->have_tma = false;
 490                 decoder->state.trace_nr = buffer.trace_nr;
 491                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 492                              decoder->ref_timestamp);
 493                 return -ENOLINK;
 494         }
 495
 496         return 0;
 497 }
 498
 499 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 500 {
 501         if (!decoder->next_buf)
 502                 return intel_pt_get_data(decoder);
 503
 504         decoder->buf = decoder->next_buf;
 505         decoder->len = decoder->next_len;
 506         decoder->next_buf = 0;
 507         decoder->next_len = 0;
 508         return 0;
 509 }
 510
 511 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 512 {
 513         unsigned char *buf = decoder->temp_buf;
 514         size_t old_len, len, n;
 515         int ret;
 516
 517         old_len = decoder->len;
 518         len = decoder->len;
 519         memcpy(buf, decoder->buf, len);
 520
 521         ret = intel_pt_get_data(decoder);
 522         if (ret) {
 523                 decoder->pos += old_len;
 524                 return ret < 0 ? ret : -EINVAL;
 525         }
 526
 527         n = INTEL_PT_PKT_MAX_SZ - len;
 528         if (n > decoder->len)
 529                 n = decoder->len;
 530         memcpy(buf + len, decoder->buf, n);
 531         len += n;
 532
 533         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 534         if (ret < (int)old_len) {
 535                 decoder->next_buf = decoder->buf;
 536                 decoder->next_len = decoder->len;
 537                 decoder->buf = buf;
 538                 decoder->len = old_len;
 539                 return intel_pt_bad_packet(decoder);
 540         }
 541
 542         decoder->next_buf = decoder->buf + (ret - old_len);
 543         decoder->next_len = decoder->len - (ret - old_len);
 544
 545         decoder->buf = buf;
 546         decoder->len = ret;
 547
 548         return ret;
 549 }
 550
 551 struct intel_pt_pkt_info {
 552         struct intel_pt_decoder   *decoder;
 553         struct intel_pt_pkt       packet;
 554         uint64_t                  pos;
 555         int                       pkt_len;
 556         int                       last_packet_type;
 557         void                      *data;
 558 };
 559
 560 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 561
 562 /* Lookahead packets in current buffer */
 563 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 564                                   intel_pt_pkt_cb_t cb, void *data)
 565 {
 566         struct intel_pt_pkt_info pkt_info;
 567         const unsigned char *buf = decoder->buf;
 568         size_t len = decoder->len;
 569         int ret;
 570
 571         pkt_info.decoder          = decoder;
 572         pkt_info.pos              = decoder->pos;
 573         pkt_info.pkt_len          = decoder->pkt_step;
 574         pkt_info.last_packet_type = decoder->last_packet_type;
 575         pkt_info.data             = data;
 576
 577         while (1) {
 578                 do {
 579                         pkt_info.pos += pkt_info.pkt_len;
 580                         buf          += pkt_info.pkt_len;
 581                         len          -= pkt_info.pkt_len;
 582
 583                         if (!len)
 584                                 return INTEL_PT_NEED_MORE_BYTES;
 585
 586                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 587                         if (!ret)
 588                                 return INTEL_PT_NEED_MORE_BYTES;
 589                         if (ret < 0)
 590                                 return ret;
 591
 592                         pkt_info.pkt_len = ret;
 593                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 594
 595                 ret = cb(&pkt_info);
 596                 if (ret)
 597                         return 0;
 598
 599                 pkt_info.last_packet_type = pkt_info.packet.type;
 600         }
 601 }
 602
 603 struct intel_pt_calc_cyc_to_tsc_info {
 604         uint64_t        cycle_cnt;
 605         unsigned int    cbr;
 606         uint32_t        last_mtc;
 607         uint64_t        ctc_timestamp;
 608         uint64_t        ctc_delta;
 609         uint64_t        tsc_timestamp;
 610         uint64_t        timestamp;
 611         bool            have_tma;
 612         bool            fixup_last_mtc;
 613         bool            from_mtc;
 614         double          cbr_cyc_to_tsc;
 615 };
 616
 617 /*
 618  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 619  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 620  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 621  * packet by copying the missing bits from the current MTC assuming the least
 622  * difference between the two, and that the current MTC comes after last_mtc.
 623  */
 624 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 625                                     uint32_t *last_mtc)
 626 {
 627         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 628         uint32_t mask = ~(first_missing_bit - 1);
 629
 630         *last_mtc |= mtc & mask;
 631         if (*last_mtc >= mtc) {
 632                 *last_mtc -= first_missing_bit;
 633                 *last_mtc &= 0xff;
 634         }
 635 }
 636
 637 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 638 {
 639         struct intel_pt_decoder *decoder = pkt_info->decoder;
 640         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 641         uint64_t timestamp;
 642         double cyc_to_tsc;
 643         unsigned int cbr;
 644         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 645
 646         switch (pkt_info->packet.type) {
 647         case INTEL_PT_TNT:
 648         case INTEL_PT_TIP_PGE:
 649         case INTEL_PT_TIP:
 650         case INTEL_PT_FUP:
 651         case INTEL_PT_PSB:
 652         case INTEL_PT_PIP:
 653         case INTEL_PT_MODE_EXEC:
 654         case INTEL_PT_MODE_TSX:
 655         case INTEL_PT_PSBEND:
 656         case INTEL_PT_PAD:
 657         case INTEL_PT_VMCS:
 658         case INTEL_PT_MNT:
 659                 return 0;
 660
 661         case INTEL_PT_MTC:
 662                 if (!data->have_tma)
 663                         return 0;
 664
 665                 mtc = pkt_info->packet.payload;
 666                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 667                         data->fixup_last_mtc = false;
 668                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 669                                                 &data->last_mtc);
 670                 }
 671                 if (mtc > data->last_mtc)
 672                         mtc_delta = mtc - data->last_mtc;
 673                 else
 674                         mtc_delta = mtc + 256 - data->last_mtc;
 675                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 676                 data->last_mtc = mtc;
 677
 678                 if (decoder->tsc_ctc_mult) {
 679                         timestamp = data->ctc_timestamp +
 680                                 data->ctc_delta * decoder->tsc_ctc_mult;
 681                 } else {
 682                         timestamp = data->ctc_timestamp +
 683                                 multdiv(data->ctc_delta,
 684                                         decoder->tsc_ctc_ratio_n,
 685                                         decoder->tsc_ctc_ratio_d);
 686                 }
 687
 688                 if (timestamp < data->timestamp)
 689                         return 1;
 690
 691                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 692                         data->timestamp = timestamp;
 693                         return 0;
 694                 }
 695
 696                 break;
 697
 698         case INTEL_PT_TSC:
 699                 timestamp = pkt_info->packet.payload |
 700                             (data->timestamp & (0xffULL << 56));
 701                 if (data->from_mtc && timestamp < data->timestamp &&
 702                     data->timestamp - timestamp < decoder->tsc_slip)
 703                         return 1;
 704                 if (timestamp < data->timestamp)
 705                         timestamp += (1ULL << 56);
 706                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 707                         if (data->from_mtc)
 708                                 return 1;
 709                         data->tsc_timestamp = timestamp;
 710                         data->timestamp = timestamp;
 711                         return 0;
 712                 }
 713                 break;
 714
 715         case INTEL_PT_TMA:
 716                 if (data->from_mtc)
 717                         return 1;
 718
 719                 if (!decoder->tsc_ctc_ratio_d)
 720                         return 0;
 721
 722                 ctc = pkt_info->packet.payload;
 723                 fc = pkt_info->packet.count;
 724                 ctc_rem = ctc & decoder->ctc_rem_mask;
 725
 726                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 727
 728                 data->ctc_timestamp = data->tsc_timestamp - fc;
 729                 if (decoder->tsc_ctc_mult) {
 730                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 731                 } else {
 732                         data->ctc_timestamp -=
 733                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 734                                         decoder->tsc_ctc_ratio_d);
 735                 }
 736
 737                 data->ctc_delta = 0;
 738                 data->have_tma = true;
 739                 data->fixup_last_mtc = true;
 740
 741                 return 0;
 742
 743         case INTEL_PT_CYC:
 744                 data->cycle_cnt += pkt_info->packet.payload;
 745                 return 0;
 746
 747         case INTEL_PT_CBR:
 748                 cbr = pkt_info->packet.payload;
 749                 if (data->cbr && data->cbr != cbr)
 750                         return 1;
 751                 data->cbr = cbr;
 752                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 753                 return 0;
 754
 755         case INTEL_PT_TIP_PGD:
 756         case INTEL_PT_TRACESTOP:
 757         case INTEL_PT_OVF:
 758         case INTEL_PT_BAD: /* Does not happen */
 759         default:
 760                 return 1;
 761         }
 762
 763         if (!data->cbr && decoder->cbr) {
 764                 data->cbr = decoder->cbr;
 765                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 766         }
 767
 768         if (!data->cycle_cnt)
 769                 return 1;
 770
 771         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 772
 773         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 774             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 775                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 776                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 777                 return 1;
 778         }
 779
 780         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 781         decoder->have_calc_cyc_to_tsc = true;
 782
 783         if (data->cbr) {
 784                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 785                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 786         } else {
 787                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 788                              cyc_to_tsc, pkt_info->pos);
 789         }
 790
 791         return 1;
 792 }
 793
 794 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 795                                      bool from_mtc)
 796 {
 797         struct intel_pt_calc_cyc_to_tsc_info data = {
 798                 .cycle_cnt      = 0,
 799                 .cbr            = 0,
 800                 .last_mtc       = decoder->last_mtc,
 801                 .ctc_timestamp  = decoder->ctc_timestamp,
 802                 .ctc_delta      = decoder->ctc_delta,
 803                 .tsc_timestamp  = decoder->tsc_timestamp,
 804                 .timestamp      = decoder->timestamp,
 805                 .have_tma       = decoder->have_tma,
 806                 .fixup_last_mtc = decoder->fixup_last_mtc,
 807                 .from_mtc       = from_mtc,
 808                 .cbr_cyc_to_tsc = 0,
 809         };
 810
 811         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 812 }
 813
 814 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 815 {
 816         int ret;
 817
 818         decoder->last_packet_type = decoder->packet.type;
 819
 820         do {
 821                 decoder->pos += decoder->pkt_step;
 822                 decoder->buf += decoder->pkt_step;
 823                 decoder->len -= decoder->pkt_step;
 824
 825                 if (!decoder->len) {
 826                         ret = intel_pt_get_next_data(decoder);
 827                         if (ret)
 828                                 return ret;
 829                 }
 830
 831                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 832                                           &decoder->packet);
 833                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 834                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 835                         ret = intel_pt_get_split_packet(decoder);
 836                         if (ret < 0)
 837                                 return ret;
 838                 }
 839                 if (ret <= 0)
 840                         return intel_pt_bad_packet(decoder);
 841
 842                 decoder->pkt_len = ret;
 843                 decoder->pkt_step = ret;
 844                 intel_pt_decoder_log_packet(decoder);
 845         } while (decoder->packet.type == INTEL_PT_PAD);
 846
 847         return 0;
 848 }
 849
 850 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 851 {
 852         uint64_t timestamp, masked_timestamp;
 853
 854         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 855         masked_timestamp = timestamp & decoder->period_mask;
 856         if (decoder->continuous_period) {
 857                 if (masked_timestamp != decoder->last_masked_timestamp)
 858                         return 1;
 859         } else {
 860                 timestamp += 1;
 861                 masked_timestamp = timestamp & decoder->period_mask;
 862                 if (masked_timestamp != decoder->last_masked_timestamp) {
 863                         decoder->last_masked_timestamp = masked_timestamp;
 864                         decoder->continuous_period = true;
 865                 }
 866         }
 867         return decoder->period_ticks - (timestamp - masked_timestamp);
 868 }
 869
 870 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 871 {
 872         switch (decoder->period_type) {
 873         case INTEL_PT_PERIOD_INSTRUCTIONS:
 874                 return decoder->period - decoder->period_insn_cnt;
 875         case INTEL_PT_PERIOD_TICKS:
 876                 return intel_pt_next_period(decoder);
 877         case INTEL_PT_PERIOD_NONE:
 878         case INTEL_PT_PERIOD_MTC:
 879         default:
 880                 return 0;
 881         }
 882 }
 883
 884 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 885 {
 886         uint64_t timestamp, masked_timestamp;
 887
 888         switch (decoder->period_type) {
 889         case INTEL_PT_PERIOD_INSTRUCTIONS:
 890                 decoder->period_insn_cnt = 0;
 891                 break;
 892         case INTEL_PT_PERIOD_TICKS:
 893                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 894                 masked_timestamp = timestamp & decoder->period_mask;
 895                 decoder->last_masked_timestamp = masked_timestamp;
 896                 break;
 897         case INTEL_PT_PERIOD_NONE:
 898         case INTEL_PT_PERIOD_MTC:
 899         default:
 900                 break;
 901         }
 902
 903         decoder->state.type |= INTEL_PT_INSTRUCTION;
 904 }
 905
 906 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 907                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 908 {
 909         uint64_t max_insn_cnt, insn_cnt = 0;
 910         int err;
 911
 912         if (!decoder->mtc_insn)
 913                 decoder->mtc_insn = true;
 914
 915         max_insn_cnt = intel_pt_next_sample(decoder);
 916
 917         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 918                                  max_insn_cnt, decoder->data);
 919
 920         decoder->tot_insn_cnt += insn_cnt;
 921         decoder->timestamp_insn_cnt += insn_cnt;
 922         decoder->sample_insn_cnt += insn_cnt;
 923         decoder->period_insn_cnt += insn_cnt;
 924
 925         if (err) {
 926                 decoder->no_progress = 0;
 927                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 928                 intel_pt_log_at("ERROR: Failed to get instruction",
 929                                 decoder->ip);
 930                 if (err == -ENOENT)
 931                         return -ENOLINK;
 932                 return -EILSEQ;
 933         }
 934
 935         if (ip && decoder->ip == ip) {
 936                 err = -EAGAIN;
 937                 goto out;
 938         }
 939
 940         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 941                 intel_pt_sample_insn(decoder);
 942
 943         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 944                 decoder->state.type = INTEL_PT_INSTRUCTION;
 945                 decoder->state.from_ip = decoder->ip;
 946                 decoder->state.to_ip = 0;
 947                 decoder->ip += intel_pt_insn->length;
 948                 err = INTEL_PT_RETURN;
 949                 goto out;
 950         }
 951
 952         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 953                 /* Zero-length calls are excluded */
 954                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 955                     intel_pt_insn->rel) {
 956                         err = intel_pt_push(&decoder->stack, decoder->ip +
 957                                             intel_pt_insn->length);
 958                         if (err)
 959                                 goto out;
 960                 }
 961         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 962                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 963         }
 964
 965         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 966                 int cnt = decoder->no_progress++;
 967
 968                 decoder->state.from_ip = decoder->ip;
 969                 decoder->ip += intel_pt_insn->length +
 970                                 intel_pt_insn->rel;
 971                 decoder->state.to_ip = decoder->ip;
 972                 err = INTEL_PT_RETURN;
 973
 974                 /*
 975                  * Check for being stuck in a loop.  This can happen if a
 976                  * decoder error results in the decoder erroneously setting the
 977                  * ip to an address that is itself in an infinite loop that
 978                  * consumes no packets.  When that happens, there must be an
 979                  * unconditional branch.
 980                  */
 981                 if (cnt) {
 982                         if (cnt == 1) {
 983                                 decoder->stuck_ip = decoder->state.to_ip;
 984                                 decoder->stuck_ip_prd = 1;
 985                                 decoder->stuck_ip_cnt = 1;
 986                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 987                                    decoder->state.to_ip == decoder->stuck_ip) {
 988                                 intel_pt_log_at("ERROR: Never-ending loop",
 989                                                 decoder->state.to_ip);
 990                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
 991                                 err = -ELOOP;
 992                                 goto out;
 993                         } else if (!--decoder->stuck_ip_cnt) {
 994                                 decoder->stuck_ip_prd += 1;
 995                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
 996                                 decoder->stuck_ip = decoder->state.to_ip;
 997                         }
 998                 }
 999                 goto out_no_progress;
1000         }
1001 out:
1002         decoder->no_progress = 0;
1003 out_no_progress:
1004         decoder->state.insn_op = intel_pt_insn->op;
1005         decoder->state.insn_len = intel_pt_insn->length;
1006
1007         if (decoder->tx_flags & INTEL_PT_IN_TX)
1008                 decoder->state.flags |= INTEL_PT_IN_TX;
1009
1010         return err;
1011 }
1012
1013 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1014 {
1015         struct intel_pt_insn intel_pt_insn;
1016         uint64_t ip;
1017         int err;
1018
1019         ip = decoder->last_ip;
1020
1021         while (1) {
1022                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1023                 if (err == INTEL_PT_RETURN)
1024                         return 0;
1025                 if (err == -EAGAIN) {
1026                         if (decoder->set_fup_tx_flags) {
1027                                 decoder->set_fup_tx_flags = false;
1028                                 decoder->tx_flags = decoder->fup_tx_flags;
1029                                 decoder->state.type = INTEL_PT_TRANSACTION;
1030                                 decoder->state.from_ip = decoder->ip;
1031                                 decoder->state.to_ip = 0;
1032                                 decoder->state.flags = decoder->fup_tx_flags;
1033                                 return 0;
1034                         }
1035                         return err;
1036                 }
1037                 decoder->set_fup_tx_flags = false;
1038                 if (err)
1039                         return err;
1040
1041                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1042                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1043                                         decoder->ip);
1044                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1045                         return -ENOENT;
1046                 }
1047
1048                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1049                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1050                                         decoder->ip);
1051                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1052                         return -ENOENT;
1053                 }
1054
1055                 intel_pt_bug(decoder);
1056         }
1057 }
1058
1059 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1060 {
1061         struct intel_pt_insn intel_pt_insn;
1062         int err;
1063
1064         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1065         if (err == INTEL_PT_RETURN &&
1066             decoder->pgd_ip &&
1067             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1068             (decoder->state.type & INTEL_PT_BRANCH) &&
1069             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1070                 /* Unconditional branch leaving filter region */
1071                 decoder->no_progress = 0;
1072                 decoder->pge = false;
1073                 decoder->continuous_period = false;
1074                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1075                 decoder->state.to_ip = 0;
1076                 return 0;
1077         }
1078         if (err == INTEL_PT_RETURN)
1079                 return 0;
1080         if (err)
1081                 return err;
1082
1083         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1084                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1085                         decoder->pge = false;
1086                         decoder->continuous_period = false;
1087                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1088                         decoder->state.from_ip = decoder->ip;
1089                         decoder->state.to_ip = 0;
1090                         if (decoder->packet.count != 0)
1091                                 decoder->ip = decoder->last_ip;
1092                 } else {
1093                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1094                         decoder->state.from_ip = decoder->ip;
1095                         if (decoder->packet.count == 0) {
1096                                 decoder->state.to_ip = 0;
1097                         } else {
1098                                 decoder->state.to_ip = decoder->last_ip;
1099                                 decoder->ip = decoder->last_ip;
1100                         }
1101                 }
1102                 return 0;
1103         }
1104
1105         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1106                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1107                                  intel_pt_insn.rel;
1108
1109                 if (decoder->pgd_ip &&
1110                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1111                     decoder->pgd_ip(to_ip, decoder->data)) {
1112                         /* Conditional branch leaving filter region */
1113                         decoder->pge = false;
1114                         decoder->continuous_period = false;
1115                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1116                         decoder->ip = to_ip;
1117                         decoder->state.from_ip = decoder->ip;
1118                         decoder->state.to_ip = 0;
1119                         return 0;
1120                 }
1121                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1122                                 decoder->ip);
1123                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1124                 return -ENOENT;
1125         }
1126
1127         return intel_pt_bug(decoder);
1128 }
1129
1130 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1131 {
1132         struct intel_pt_insn intel_pt_insn;
1133         int err;
1134
1135         while (1) {
1136                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1137                 if (err == INTEL_PT_RETURN)
1138                         return 0;
1139                 if (err)
1140                         return err;
1141
1142                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1143                         if (!decoder->return_compression) {
1144                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1145                                                 decoder->ip);
1146                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1147                                 return -ENOENT;
1148                         }
1149                         if (!decoder->ret_addr) {
1150                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1151                                                 decoder->ip);
1152                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1153                                 return -ENOENT;
1154                         }
1155                         if (!(decoder->tnt.payload & BIT63)) {
1156                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1157                                                 decoder->ip);
1158                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1159                                 return -ENOENT;
1160                         }
1161                         decoder->tnt.count -= 1;
1162                         if (!decoder->tnt.count)
1163                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1164                         decoder->tnt.payload <<= 1;
1165                         decoder->state.from_ip = decoder->ip;
1166                         decoder->ip = decoder->ret_addr;
1167                         decoder->state.to_ip = decoder->ip;
1168                         return 0;
1169                 }
1170
1171                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1172                         /* Handle deferred TIPs */
1173                         err = intel_pt_get_next_packet(decoder);
1174                         if (err)
1175                                 return err;
1176                         if (decoder->packet.type != INTEL_PT_TIP ||
1177                             decoder->packet.count == 0) {
1178                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1179                                                 decoder->ip);
1180                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1181                                 decoder->pkt_step = 0;
1182                                 return -ENOENT;
1183                         }
1184                         intel_pt_set_last_ip(decoder);
1185                         decoder->state.from_ip = decoder->ip;
1186                         decoder->state.to_ip = decoder->last_ip;
1187                         decoder->ip = decoder->last_ip;
1188                         return 0;
1189                 }
1190
1191                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1192                         decoder->tnt.count -= 1;
1193                         if (!decoder->tnt.count)
1194                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1195                         if (decoder->tnt.payload & BIT63) {
1196                                 decoder->tnt.payload <<= 1;
1197                                 decoder->state.from_ip = decoder->ip;
1198                                 decoder->ip += intel_pt_insn.length +
1199                                                intel_pt_insn.rel;
1200                                 decoder->state.to_ip = decoder->ip;
1201                                 return 0;
1202                         }
1203                         /* Instruction sample for a non-taken branch */
1204                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1205                                 decoder->tnt.payload <<= 1;
1206                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1207                                 decoder->state.from_ip = decoder->ip;
1208                                 decoder->state.to_ip = 0;
1209                                 decoder->ip += intel_pt_insn.length;
1210                                 return 0;
1211                         }
1212                         decoder->ip += intel_pt_insn.length;
1213                         if (!decoder->tnt.count)
1214                                 return -EAGAIN;
1215                         decoder->tnt.payload <<= 1;
1216                         continue;
1217                 }
1218
1219                 return intel_pt_bug(decoder);
1220         }
1221 }
1222
1223 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1224 {
1225         unsigned int fup_tx_flags;
1226         int err;
1227
1228         fup_tx_flags = decoder->packet.payload &
1229                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1230         err = intel_pt_get_next_packet(decoder);
1231         if (err)
1232                 return err;
1233         if (decoder->packet.type == INTEL_PT_FUP) {
1234                 decoder->fup_tx_flags = fup_tx_flags;
1235                 decoder->set_fup_tx_flags = true;
1236                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1237                         *no_tip = true;
1238         } else {
1239                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1240                                 decoder->pos);
1241                 intel_pt_update_in_tx(decoder);
1242         }
1243         return 0;
1244 }
1245
1246 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1247 {
1248         uint64_t timestamp;
1249
1250         decoder->have_tma = false;
1251
1252         if (decoder->ref_timestamp) {
1253                 timestamp = decoder->packet.payload |
1254                             (decoder->ref_timestamp & (0xffULL << 56));
1255                 if (timestamp < decoder->ref_timestamp) {
1256                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1257                                 timestamp += (1ULL << 56);
1258                 } else {
1259                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1260                                 timestamp -= (1ULL << 56);
1261                 }
1262                 decoder->tsc_timestamp = timestamp;
1263                 decoder->timestamp = timestamp;
1264                 decoder->ref_timestamp = 0;
1265                 decoder->timestamp_insn_cnt = 0;
1266         } else if (decoder->timestamp) {
1267                 timestamp = decoder->packet.payload |
1268                             (decoder->timestamp & (0xffULL << 56));
1269                 decoder->tsc_timestamp = timestamp;
1270                 if (timestamp < decoder->timestamp &&
1271                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1272                         intel_pt_log_to("Suppressing backwards timestamp",
1273                                         timestamp);
1274                         timestamp = decoder->timestamp;
1275                 }
1276                 if (timestamp < decoder->timestamp) {
1277                         intel_pt_log_to("Wraparound timestamp", timestamp);
1278                         timestamp += (1ULL << 56);
1279                         decoder->tsc_timestamp = timestamp;
1280                 }
1281                 decoder->timestamp = timestamp;
1282                 decoder->timestamp_insn_cnt = 0;
1283         }
1284
1285         if (decoder->last_packet_type == INTEL_PT_CYC) {
1286                 decoder->cyc_ref_timestamp = decoder->timestamp;
1287                 decoder->cycle_cnt = 0;
1288                 decoder->have_calc_cyc_to_tsc = false;
1289                 intel_pt_calc_cyc_to_tsc(decoder, false);
1290         }
1291
1292         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1293 }
1294
1295 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1296 {
1297         intel_pt_log("ERROR: Buffer overflow\n");
1298         intel_pt_clear_tx_flags(decoder);
1299         decoder->have_tma = false;
1300         decoder->cbr = 0;
1301         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1302         decoder->overflow = true;
1303         return -EOVERFLOW;
1304 }
1305
1306 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1307 {
1308         uint32_t ctc = decoder->packet.payload;
1309         uint32_t fc = decoder->packet.count;
1310         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1311
1312         if (!decoder->tsc_ctc_ratio_d)
1313                 return;
1314
1315         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1316         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1317         if (decoder->tsc_ctc_mult) {
1318                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1319         } else {
1320                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1321                                                   decoder->tsc_ctc_ratio_n,
1322                                                   decoder->tsc_ctc_ratio_d);
1323         }
1324         decoder->ctc_delta = 0;
1325         decoder->have_tma = true;
1326         decoder->fixup_last_mtc = true;
1327         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1328                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1329 }
1330
1331 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1332 {
1333         uint64_t timestamp;
1334         uint32_t mtc, mtc_delta;
1335
1336         if (!decoder->have_tma)
1337                 return;
1338
1339         mtc = decoder->packet.payload;
1340
1341         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1342                 decoder->fixup_last_mtc = false;
1343                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1344                                         &decoder->last_mtc);
1345         }
1346
1347         if (mtc > decoder->last_mtc)
1348                 mtc_delta = mtc - decoder->last_mtc;
1349         else
1350                 mtc_delta = mtc + 256 - decoder->last_mtc;
1351
1352         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1353
1354         if (decoder->tsc_ctc_mult) {
1355                 timestamp = decoder->ctc_timestamp +
1356                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1357         } else {
1358                 timestamp = decoder->ctc_timestamp +
1359                             multdiv(decoder->ctc_delta,
1360                                     decoder->tsc_ctc_ratio_n,
1361                                     decoder->tsc_ctc_ratio_d);
1362         }
1363
1364         if (timestamp < decoder->timestamp)
1365                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1366                              timestamp, decoder->timestamp);
1367         else
1368                 decoder->timestamp = timestamp;
1369
1370         decoder->timestamp_insn_cnt = 0;
1371         decoder->last_mtc = mtc;
1372
1373         if (decoder->last_packet_type == INTEL_PT_CYC) {
1374                 decoder->cyc_ref_timestamp = decoder->timestamp;
1375                 decoder->cycle_cnt = 0;
1376                 decoder->have_calc_cyc_to_tsc = false;
1377                 intel_pt_calc_cyc_to_tsc(decoder, true);
1378         }
1379 }
1380
1381 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1382 {
1383         unsigned int cbr = decoder->packet.payload;
1384
1385         if (decoder->cbr == cbr)
1386                 return;
1387
1388         decoder->cbr = cbr;
1389         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1390 }
1391
1392 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1393 {
1394         uint64_t timestamp = decoder->cyc_ref_timestamp;
1395
1396         decoder->have_cyc = true;
1397
1398         decoder->cycle_cnt += decoder->packet.payload;
1399
1400         if (!decoder->cyc_ref_timestamp)
1401                 return;
1402
1403         if (decoder->have_calc_cyc_to_tsc)
1404                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1405         else if (decoder->cbr)
1406                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1407         else
1408                 return;
1409
1410         if (timestamp < decoder->timestamp)
1411                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1412                              timestamp, decoder->timestamp);
1413         else
1414                 decoder->timestamp = timestamp;
1415
1416         decoder->timestamp_insn_cnt = 0;
1417 }
1418
1419 /* Walk PSB+ packets when already in sync. */
1420 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1421 {
1422         int err;
1423
1424         while (1) {
1425                 err = intel_pt_get_next_packet(decoder);
1426                 if (err)
1427                         return err;
1428
1429                 switch (decoder->packet.type) {
1430                 case INTEL_PT_PSBEND:
1431                         return 0;
1432
1433                 case INTEL_PT_TIP_PGD:
1434                 case INTEL_PT_TIP_PGE:
1435                 case INTEL_PT_TIP:
1436                 case INTEL_PT_TNT:
1437                 case INTEL_PT_TRACESTOP:
1438                 case INTEL_PT_BAD:
1439                 case INTEL_PT_PSB:
1440                         decoder->have_tma = false;
1441                         intel_pt_log("ERROR: Unexpected packet\n");
1442                         return -EAGAIN;
1443
1444                 case INTEL_PT_OVF:
1445                         return intel_pt_overflow(decoder);
1446
1447                 case INTEL_PT_TSC:
1448                         intel_pt_calc_tsc_timestamp(decoder);
1449                         break;
1450
1451                 case INTEL_PT_TMA:
1452                         intel_pt_calc_tma(decoder);
1453                         break;
1454
1455                 case INTEL_PT_CBR:
1456                         intel_pt_calc_cbr(decoder);
1457                         break;
1458
1459                 case INTEL_PT_MODE_EXEC:
1460                         decoder->exec_mode = decoder->packet.payload;
1461                         break;
1462
1463                 case INTEL_PT_PIP:
1464                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1465                         break;
1466
1467                 case INTEL_PT_FUP:
1468                         decoder->pge = true;
1469                         intel_pt_set_last_ip(decoder);
1470                         break;
1471
1472                 case INTEL_PT_MODE_TSX:
1473                         intel_pt_update_in_tx(decoder);
1474                         break;
1475
1476                 case INTEL_PT_MTC:
1477                         intel_pt_calc_mtc_timestamp(decoder);
1478                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1479                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1480                         break;
1481
1482                 case INTEL_PT_CYC:
1483                 case INTEL_PT_VMCS:
1484                 case INTEL_PT_MNT:
1485                 case INTEL_PT_PAD:
1486                 default:
1487                         break;
1488                 }
1489         }
1490 }
1491
1492 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1493 {
1494         int err;
1495
1496         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1497                 decoder->tx_flags = 0;
1498                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1499                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1500         } else {
1501                 decoder->state.flags |= INTEL_PT_ASYNC;
1502         }
1503
1504         while (1) {
1505                 err = intel_pt_get_next_packet(decoder);
1506                 if (err)
1507                         return err;
1508
1509                 switch (decoder->packet.type) {
1510                 case INTEL_PT_TNT:
1511                 case INTEL_PT_FUP:
1512                 case INTEL_PT_TRACESTOP:
1513                 case INTEL_PT_PSB:
1514                 case INTEL_PT_TSC:
1515                 case INTEL_PT_TMA:
1516                 case INTEL_PT_CBR:
1517                 case INTEL_PT_MODE_TSX:
1518                 case INTEL_PT_BAD:
1519                 case INTEL_PT_PSBEND:
1520                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1521                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1522                         return -ENOENT;
1523
1524                 case INTEL_PT_OVF:
1525                         return intel_pt_overflow(decoder);
1526
1527                 case INTEL_PT_TIP_PGD:
1528                         decoder->state.from_ip = decoder->ip;
1529                         decoder->state.to_ip = 0;
1530                         if (decoder->packet.count != 0) {
1531                                 intel_pt_set_ip(decoder);
1532                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1533                                              decoder->ip);
1534                         }
1535                         decoder->pge = false;
1536                         decoder->continuous_period = false;
1537                         return 0;
1538
1539                 case INTEL_PT_TIP_PGE:
1540                         decoder->pge = true;
1541                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1542                                      decoder->ip);
1543                         decoder->state.from_ip = 0;
1544                         if (decoder->packet.count == 0) {
1545                                 decoder->state.to_ip = 0;
1546                         } else {
1547                                 intel_pt_set_ip(decoder);
1548                                 decoder->state.to_ip = decoder->ip;
1549                         }
1550                         return 0;
1551
1552                 case INTEL_PT_TIP:
1553                         decoder->state.from_ip = decoder->ip;
1554                         if (decoder->packet.count == 0) {
1555                                 decoder->state.to_ip = 0;
1556                         } else {
1557                                 intel_pt_set_ip(decoder);
1558                                 decoder->state.to_ip = decoder->ip;
1559                         }
1560                         return 0;
1561
1562                 case INTEL_PT_PIP:
1563                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1564                         break;
1565
1566                 case INTEL_PT_MTC:
1567                         intel_pt_calc_mtc_timestamp(decoder);
1568                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1569                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1570                         break;
1571
1572                 case INTEL_PT_CYC:
1573                         intel_pt_calc_cyc_timestamp(decoder);
1574                         break;
1575
1576                 case INTEL_PT_MODE_EXEC:
1577                         decoder->exec_mode = decoder->packet.payload;
1578                         break;
1579
1580                 case INTEL_PT_VMCS:
1581                 case INTEL_PT_MNT:
1582                 case INTEL_PT_PAD:
1583                         break;
1584
1585                 default:
1586                         return intel_pt_bug(decoder);
1587                 }
1588         }
1589 }
1590
1591 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1592 {
1593         bool no_tip = false;
1594         int err;
1595
1596         while (1) {
1597                 err = intel_pt_get_next_packet(decoder);
1598                 if (err)
1599                         return err;
1600 next:
1601                 switch (decoder->packet.type) {
1602                 case INTEL_PT_TNT:
1603                         if (!decoder->packet.count)
1604                                 break;
1605                         decoder->tnt = decoder->packet;
1606                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1607                         err = intel_pt_walk_tnt(decoder);
1608                         if (err == -EAGAIN)
1609                                 break;
1610                         return err;
1611
1612                 case INTEL_PT_TIP_PGD:
1613                         if (decoder->packet.count != 0)
1614                                 intel_pt_set_last_ip(decoder);
1615                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1616                         return intel_pt_walk_tip(decoder);
1617
1618                 case INTEL_PT_TIP_PGE: {
1619                         decoder->pge = true;
1620                         if (decoder->packet.count == 0) {
1621                                 intel_pt_log_at("Skipping zero TIP.PGE",
1622                                                 decoder->pos);
1623                                 break;
1624                         }
1625                         intel_pt_set_ip(decoder);
1626                         decoder->state.from_ip = 0;
1627                         decoder->state.to_ip = decoder->ip;
1628                         return 0;
1629                 }
1630
1631                 case INTEL_PT_OVF:
1632                         return intel_pt_overflow(decoder);
1633
1634                 case INTEL_PT_TIP:
1635                         if (decoder->packet.count != 0)
1636                                 intel_pt_set_last_ip(decoder);
1637                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1638                         return intel_pt_walk_tip(decoder);
1639
1640                 case INTEL_PT_FUP:
1641                         if (decoder->packet.count == 0) {
1642                                 intel_pt_log_at("Skipping zero FUP",
1643                                                 decoder->pos);
1644                                 no_tip = false;
1645                                 break;
1646                         }
1647                         intel_pt_set_last_ip(decoder);
1648                         err = intel_pt_walk_fup(decoder);
1649                         if (err != -EAGAIN) {
1650                                 if (err)
1651                                         return err;
1652                                 if (no_tip)
1653                                         decoder->pkt_state =
1654                                                 INTEL_PT_STATE_FUP_NO_TIP;
1655                                 else
1656                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1657                                 return 0;
1658                         }
1659                         if (no_tip) {
1660                                 no_tip = false;
1661                                 break;
1662                         }
1663                         return intel_pt_walk_fup_tip(decoder);
1664
1665                 case INTEL_PT_TRACESTOP:
1666                         decoder->pge = false;
1667                         decoder->continuous_period = false;
1668                         intel_pt_clear_tx_flags(decoder);
1669                         decoder->have_tma = false;
1670                         break;
1671
1672                 case INTEL_PT_PSB:
1673                         intel_pt_clear_stack(&decoder->stack);
1674                         err = intel_pt_walk_psbend(decoder);
1675                         if (err == -EAGAIN)
1676                                 goto next;
1677                         if (err)
1678                                 return err;
1679                         break;
1680
1681                 case INTEL_PT_PIP:
1682                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1683                         break;
1684
1685                 case INTEL_PT_MTC:
1686                         intel_pt_calc_mtc_timestamp(decoder);
1687                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1688                                 break;
1689                         /*
1690                          * Ensure that there has been an instruction since the
1691                          * last MTC.
1692                          */
1693                         if (!decoder->mtc_insn)
1694                                 break;
1695                         decoder->mtc_insn = false;
1696                         /* Ensure that there is a timestamp */
1697                         if (!decoder->timestamp)
1698                                 break;
1699                         decoder->state.type = INTEL_PT_INSTRUCTION;
1700                         decoder->state.from_ip = decoder->ip;
1701                         decoder->state.to_ip = 0;
1702                         decoder->mtc_insn = false;
1703                         return 0;
1704
1705                 case INTEL_PT_TSC:
1706                         intel_pt_calc_tsc_timestamp(decoder);
1707                         break;
1708
1709                 case INTEL_PT_TMA:
1710                         intel_pt_calc_tma(decoder);
1711                         break;
1712
1713                 case INTEL_PT_CYC:
1714                         intel_pt_calc_cyc_timestamp(decoder);
1715                         break;
1716
1717                 case INTEL_PT_CBR:
1718                         intel_pt_calc_cbr(decoder);
1719                         break;
1720
1721                 case INTEL_PT_MODE_EXEC:
1722                         decoder->exec_mode = decoder->packet.payload;
1723                         break;
1724
1725                 case INTEL_PT_MODE_TSX:
1726                         /* MODE_TSX need not be followed by FUP */
1727                         if (!decoder->pge) {
1728                                 intel_pt_update_in_tx(decoder);
1729                                 break;
1730                         }
1731                         err = intel_pt_mode_tsx(decoder, &no_tip);
1732                         if (err)
1733                                 return err;
1734                         goto next;
1735
1736                 case INTEL_PT_BAD: /* Does not happen */
1737                         return intel_pt_bug(decoder);
1738
1739                 case INTEL_PT_PSBEND:
1740                 case INTEL_PT_VMCS:
1741                 case INTEL_PT_MNT:
1742                 case INTEL_PT_PAD:
1743                         break;
1744
1745                 default:
1746                         return intel_pt_bug(decoder);
1747                 }
1748         }
1749 }
1750
1751 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1752 {
1753         return decoder->last_ip || decoder->packet.count == 0 ||
1754                decoder->packet.count == 3 || decoder->packet.count == 6;
1755 }
1756
1757 /* Walk PSB+ packets to get in sync. */
1758 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1759 {
1760         int err;
1761
1762         while (1) {
1763                 err = intel_pt_get_next_packet(decoder);
1764                 if (err)
1765                         return err;
1766
1767                 switch (decoder->packet.type) {
1768                 case INTEL_PT_TIP_PGD:
1769                         decoder->continuous_period = false;
1770                         __fallthrough;
1771                 case INTEL_PT_TIP_PGE:
1772                 case INTEL_PT_TIP:
1773                         intel_pt_log("ERROR: Unexpected packet\n");
1774                         return -ENOENT;
1775
1776                 case INTEL_PT_FUP:
1777                         decoder->pge = true;
1778                         if (intel_pt_have_ip(decoder)) {
1779                                 uint64_t current_ip = decoder->ip;
1780
1781                                 intel_pt_set_ip(decoder);
1782                                 if (current_ip)
1783                                         intel_pt_log_to("Setting IP",
1784                                                         decoder->ip);
1785                         }
1786                         break;
1787
1788                 case INTEL_PT_MTC:
1789                         intel_pt_calc_mtc_timestamp(decoder);
1790                         break;
1791
1792                 case INTEL_PT_TSC:
1793                         intel_pt_calc_tsc_timestamp(decoder);
1794                         break;
1795
1796                 case INTEL_PT_TMA:
1797                         intel_pt_calc_tma(decoder);
1798                         break;
1799
1800                 case INTEL_PT_CYC:
1801                         intel_pt_calc_cyc_timestamp(decoder);
1802                         break;
1803
1804                 case INTEL_PT_CBR:
1805                         intel_pt_calc_cbr(decoder);
1806                         break;
1807
1808                 case INTEL_PT_PIP:
1809                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1810                         break;
1811
1812                 case INTEL_PT_MODE_EXEC:
1813                         decoder->exec_mode = decoder->packet.payload;
1814                         break;
1815
1816                 case INTEL_PT_MODE_TSX:
1817                         intel_pt_update_in_tx(decoder);
1818                         break;
1819
1820                 case INTEL_PT_TRACESTOP:
1821                         decoder->pge = false;
1822                         decoder->continuous_period = false;
1823                         intel_pt_clear_tx_flags(decoder);
1824                         __fallthrough;
1825
1826                 case INTEL_PT_TNT:
1827                         decoder->have_tma = false;
1828                         intel_pt_log("ERROR: Unexpected packet\n");
1829                         if (decoder->ip)
1830                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1831                         else
1832                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1833                         return -ENOENT;
1834
1835                 case INTEL_PT_BAD: /* Does not happen */
1836                         return intel_pt_bug(decoder);
1837
1838                 case INTEL_PT_OVF:
1839                         return intel_pt_overflow(decoder);
1840
1841                 case INTEL_PT_PSBEND:
1842                         return 0;
1843
1844                 case INTEL_PT_PSB:
1845                 case INTEL_PT_VMCS:
1846                 case INTEL_PT_MNT:
1847                 case INTEL_PT_PAD:
1848                 default:
1849                         break;
1850                 }
1851         }
1852 }
1853
1854 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1855 {
1856         int err;
1857
1858         while (1) {
1859                 err = intel_pt_get_next_packet(decoder);
1860                 if (err)
1861                         return err;
1862
1863                 switch (decoder->packet.type) {
1864                 case INTEL_PT_TIP_PGD:
1865                         decoder->continuous_period = false;
1866                         __fallthrough;
1867                 case INTEL_PT_TIP_PGE:
1868                 case INTEL_PT_TIP:
1869                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1870                         if (intel_pt_have_ip(decoder))
1871                                 intel_pt_set_ip(decoder);
1872                         if (decoder->ip)
1873                                 return 0;
1874                         break;
1875
1876                 case INTEL_PT_FUP:
1877                         if (decoder->overflow) {
1878                                 if (intel_pt_have_ip(decoder))
1879                                         intel_pt_set_ip(decoder);
1880                                 if (decoder->ip)
1881                                         return 0;
1882                         }
1883                         if (decoder->packet.count)
1884                                 intel_pt_set_last_ip(decoder);
1885                         break;
1886
1887                 case INTEL_PT_MTC:
1888                         intel_pt_calc_mtc_timestamp(decoder);
1889                         break;
1890
1891                 case INTEL_PT_TSC:
1892                         intel_pt_calc_tsc_timestamp(decoder);
1893                         break;
1894
1895                 case INTEL_PT_TMA:
1896                         intel_pt_calc_tma(decoder);
1897                         break;
1898
1899                 case INTEL_PT_CYC:
1900                         intel_pt_calc_cyc_timestamp(decoder);
1901                         break;
1902
1903                 case INTEL_PT_CBR:
1904                         intel_pt_calc_cbr(decoder);
1905                         break;
1906
1907                 case INTEL_PT_PIP:
1908                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1909                         break;
1910
1911                 case INTEL_PT_MODE_EXEC:
1912                         decoder->exec_mode = decoder->packet.payload;
1913                         break;
1914
1915                 case INTEL_PT_MODE_TSX:
1916                         intel_pt_update_in_tx(decoder);
1917                         break;
1918
1919                 case INTEL_PT_OVF:
1920                         return intel_pt_overflow(decoder);
1921
1922                 case INTEL_PT_BAD: /* Does not happen */
1923                         return intel_pt_bug(decoder);
1924
1925                 case INTEL_PT_TRACESTOP:
1926                         decoder->pge = false;
1927                         decoder->continuous_period = false;
1928                         intel_pt_clear_tx_flags(decoder);
1929                         decoder->have_tma = false;
1930                         break;
1931
1932                 case INTEL_PT_PSB:
1933                         intel_pt_clear_stack(&decoder->stack);
1934                         err = intel_pt_walk_psb(decoder);
1935                         if (err)
1936                                 return err;
1937                         if (decoder->ip) {
1938                                 /* Do not have a sample */
1939                                 decoder->state.type = 0;
1940                                 return 0;
1941                         }
1942                         break;
1943
1944                 case INTEL_PT_TNT:
1945                 case INTEL_PT_PSBEND:
1946                 case INTEL_PT_VMCS:
1947                 case INTEL_PT_MNT:
1948                 case INTEL_PT_PAD:
1949                 default:
1950                         break;
1951                 }
1952         }
1953 }
1954
1955 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1956 {
1957         int err;
1958
1959         intel_pt_log("Scanning for full IP\n");
1960         err = intel_pt_walk_to_ip(decoder);
1961         if (err)
1962                 return err;
1963
1964         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1965         decoder->overflow = false;
1966
1967         decoder->state.from_ip = 0;
1968         decoder->state.to_ip = decoder->ip;
1969         intel_pt_log_to("Setting IP", decoder->ip);
1970
1971         return 0;
1972 }
1973
1974 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1975 {
1976         const unsigned char *end = decoder->buf + decoder->len;
1977         size_t i;
1978
1979         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1980                 if (i > decoder->len)
1981                         continue;
1982                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1983                         return i;
1984         }
1985         return 0;
1986 }
1987
1988 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1989 {
1990         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1991         const char *psb = INTEL_PT_PSB_STR;
1992
1993         if (rest_psb > decoder->len ||
1994             memcmp(decoder->buf, psb + part_psb, rest_psb))
1995                 return 0;
1996
1997         return rest_psb;
1998 }
1999
2000 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2001                                   int part_psb)
2002 {
2003         int rest_psb, ret;
2004
2005         decoder->pos += decoder->len;
2006         decoder->len = 0;
2007
2008         ret = intel_pt_get_next_data(decoder);
2009         if (ret)
2010                 return ret;
2011
2012         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2013         if (!rest_psb)
2014                 return 0;
2015
2016         decoder->pos -= part_psb;
2017         decoder->next_buf = decoder->buf + rest_psb;
2018         decoder->next_len = decoder->len - rest_psb;
2019         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2020         decoder->buf = decoder->temp_buf;
2021         decoder->len = INTEL_PT_PSB_LEN;
2022
2023         return 0;
2024 }
2025
2026 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2027 {
2028         unsigned char *next;
2029         int ret;
2030
2031         intel_pt_log("Scanning for PSB\n");
2032         while (1) {
2033                 if (!decoder->len) {
2034                         ret = intel_pt_get_next_data(decoder);
2035                         if (ret)
2036                                 return ret;
2037                 }
2038
2039                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2040                               INTEL_PT_PSB_LEN);
2041                 if (!next) {
2042                         int part_psb;
2043
2044                         part_psb = intel_pt_part_psb(decoder);
2045                         if (part_psb) {
2046                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2047                                 if (ret)
2048                                         return ret;
2049                         } else {
2050                                 decoder->pos += decoder->len;
2051                                 decoder->len = 0;
2052                         }
2053                         continue;
2054                 }
2055
2056                 decoder->pkt_step = next - decoder->buf;
2057                 return intel_pt_get_next_packet(decoder);
2058         }
2059 }
2060
2061 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2062 {
2063         int err;
2064
2065         decoder->pge = false;
2066         decoder->continuous_period = false;
2067         decoder->last_ip = 0;
2068         decoder->ip = 0;
2069         intel_pt_clear_stack(&decoder->stack);
2070
2071         err = intel_pt_scan_for_psb(decoder);
2072         if (err)
2073                 return err;
2074
2075         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2076
2077         err = intel_pt_walk_psb(decoder);
2078         if (err)
2079                 return err;
2080
2081         if (decoder->ip) {
2082                 decoder->state.type = 0; /* Do not have a sample */
2083                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2084         } else {
2085                 return intel_pt_sync_ip(decoder);
2086         }
2087
2088         return 0;
2089 }
2090
2091 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2092 {
2093         uint64_t est = decoder->sample_insn_cnt << 1;
2094
2095         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2096                 goto out;
2097
2098         est *= decoder->max_non_turbo_ratio;
2099         est /= decoder->cbr;
2100 out:
2101         return decoder->sample_timestamp + est;
2102 }
2103
2104 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2105 {
2106         int err;
2107
2108         do {
2109                 decoder->state.type = INTEL_PT_BRANCH;
2110                 decoder->state.flags = 0;
2111
2112                 switch (decoder->pkt_state) {
2113                 case INTEL_PT_STATE_NO_PSB:
2114                         err = intel_pt_sync(decoder);
2115                         break;
2116                 case INTEL_PT_STATE_NO_IP:
2117                         decoder->last_ip = 0;
2118                         /* Fall through */
2119                 case INTEL_PT_STATE_ERR_RESYNC:
2120                         err = intel_pt_sync_ip(decoder);
2121                         break;
2122                 case INTEL_PT_STATE_IN_SYNC:
2123                         err = intel_pt_walk_trace(decoder);
2124                         break;
2125                 case INTEL_PT_STATE_TNT:
2126                         err = intel_pt_walk_tnt(decoder);
2127                         if (err == -EAGAIN)
2128                                 err = intel_pt_walk_trace(decoder);
2129                         break;
2130                 case INTEL_PT_STATE_TIP:
2131                 case INTEL_PT_STATE_TIP_PGD:
2132                         err = intel_pt_walk_tip(decoder);
2133                         break;
2134                 case INTEL_PT_STATE_FUP:
2135                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2136                         err = intel_pt_walk_fup(decoder);
2137                         if (err == -EAGAIN)
2138                                 err = intel_pt_walk_fup_tip(decoder);
2139                         else if (!err)
2140                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2141                         break;
2142                 case INTEL_PT_STATE_FUP_NO_TIP:
2143                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2144                         err = intel_pt_walk_fup(decoder);
2145                         if (err == -EAGAIN)
2146                                 err = intel_pt_walk_trace(decoder);
2147                         break;
2148                 default:
2149                         err = intel_pt_bug(decoder);
2150                         break;
2151                 }
2152         } while (err == -ENOLINK);
2153
2154         if (err) {
2155                 decoder->state.err = intel_pt_ext_err(err);
2156                 decoder->state.from_ip = decoder->ip;
2157                 decoder->sample_timestamp = decoder->timestamp;
2158                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2159         } else {
2160                 decoder->state.err = 0;
2161                 if (intel_pt_sample_time(decoder->pkt_state)) {
2162                         decoder->sample_timestamp = decoder->timestamp;
2163                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2164                 }
2165         }
2166
2167         decoder->state.timestamp = decoder->sample_timestamp;
2168         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2169         decoder->state.cr3 = decoder->cr3;
2170         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2171
2172         return &decoder->state;
2173 }
2174
2175 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2176 {
2177         if (len < INTEL_PT_PSB_LEN)
2178                 return false;
2179         return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2180                       INTEL_PT_PSB_LEN);
2181 }
2182
2183 /**
2184  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2185  * @buf: pointer to buffer pointer
2186  * @len: size of buffer
2187  *
2188  * Updates the buffer pointer to point to the start of the next PSB packet if
2189  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2190  * @len is adjusted accordingly.
2191  *
2192  * Return: %true if a PSB packet is found, %false otherwise.
2193  */
2194 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2195 {
2196         unsigned char *next;
2197
2198         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2199         if (next) {
2200                 *len -= next - *buf;
2201                 *buf = next;
2202                 return true;
2203         }
2204         return false;
2205 }
2206
2207 /**
2208  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2209  *                     packet.
2210  * @buf: pointer to buffer pointer
2211  * @len: size of buffer
2212  *
2213  * Updates the buffer pointer to point to the start of the following PSB packet
2214  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2215  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2216  *
2217  * Return: %true if a PSB packet is found, %false otherwise.
2218  */
2219 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2220 {
2221         unsigned char *next;
2222
2223         if (!*len)
2224                 return false;
2225
2226         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2227         if (next) {
2228                 *len -= next - *buf;
2229                 *buf = next;
2230                 return true;
2231         }
2232         return false;
2233 }
2234
2235 /**
2236  * intel_pt_last_psb - find the last PSB packet in a buffer.
2237  * @buf: buffer
2238  * @len: size of buffer
2239  *
2240  * This function finds the last PSB in a buffer.
2241  *
2242  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2243  */
2244 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2245 {
2246         const char *n = INTEL_PT_PSB_STR;
2247         unsigned char *p;
2248         size_t k;
2249
2250         if (len < INTEL_PT_PSB_LEN)
2251                 return NULL;
2252
2253         k = len - INTEL_PT_PSB_LEN + 1;
2254         while (1) {
2255                 p = memrchr(buf, n[0], k);
2256                 if (!p)
2257                         return NULL;
2258                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2259                         return p;
2260                 k = p - buf;
2261                 if (!k)
2262                         return NULL;
2263         }
2264 }
2265
2266 /**
2267  * intel_pt_next_tsc - find and return next TSC.
2268  * @buf: buffer
2269  * @len: size of buffer
2270  * @tsc: TSC value returned
2271  *
2272  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2273  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2274  * PSBEND packet is found.
2275  *
2276  * Return: %true if TSC is found, false otherwise.
2277  */
2278 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2279 {
2280         struct intel_pt_pkt packet;
2281         int ret;
2282
2283         while (len) {
2284                 ret = intel_pt_get_packet(buf, len, &packet);
2285                 if (ret <= 0)
2286                         return false;
2287                 if (packet.type == INTEL_PT_TSC) {
2288                         *tsc = packet.payload;
2289                         return true;
2290                 }
2291                 if (packet.type == INTEL_PT_PSBEND)
2292                         return false;
2293                 buf += ret;
2294                 len -= ret;
2295         }
2296         return false;
2297 }
2298
2299 /**
2300  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2301  * @tsc1: first TSC to compare
2302  * @tsc2: second TSC to compare
2303  *
2304  * This function compares 7-byte TSC values allowing for the possibility that
2305  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2306  * around so for that purpose this function assumes the absolute difference is
2307  * less than half the maximum difference.
2308  *
2309  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2310  * after @tsc2.
2311  */
2312 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2313 {
2314         const uint64_t halfway = (1ULL << 55);
2315
2316         if (tsc1 == tsc2)
2317                 return 0;
2318
2319         if (tsc1 < tsc2) {
2320                 if (tsc2 - tsc1 < halfway)
2321                         return -1;
2322                 else
2323                         return 1;
2324         } else {
2325                 if (tsc1 - tsc2 < halfway)
2326                         return 1;
2327                 else
2328                         return -1;
2329         }
2330 }
2331
2332 /**
2333  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2334  *                             using TSC.
2335  * @buf_a: first buffer
2336  * @len_a: size of first buffer
2337  * @buf_b: second buffer
2338  * @len_b: size of second buffer
2339  *
2340  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2341  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2342  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2343  * @buf_a and @buf_b are positioned at a PSB.
2344  *
2345  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2346  * @buf_b + @len_b if there is no non-overlapped data.
2347  */
2348 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2349                                                 size_t len_a,
2350                                                 unsigned char *buf_b,
2351                                                 size_t len_b)
2352 {
2353         uint64_t tsc_a, tsc_b;
2354         unsigned char *p;
2355         size_t len;
2356
2357         p = intel_pt_last_psb(buf_a, len_a);
2358         if (!p)
2359                 return buf_b; /* No PSB in buf_a => no overlap */
2360
2361         len = len_a - (p - buf_a);
2362         if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2363                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2364                 len_a -= len;
2365                 p = intel_pt_last_psb(buf_a, len_a);
2366                 if (!p)
2367                         return buf_b; /* No full PSB+ => assume no overlap */
2368                 len = len_a - (p - buf_a);
2369                 if (!intel_pt_next_tsc(p, len, &tsc_a))
2370                         return buf_b; /* No TSC in buf_a => assume no overlap */
2371         }
2372
2373         while (1) {
2374                 /* Ignore PSB+ with no TSC */
2375                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2376                     intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2377                         return buf_b; /* tsc_a < tsc_b => no overlap */
2378
2379                 if (!intel_pt_step_psb(&buf_b, &len_b))
2380                         return buf_b + len_b; /* No PSB in buf_b => no data */
2381         }
2382 }
2383
2384 /**
2385  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2386  * @buf_a: first buffer
2387  * @len_a: size of first buffer
2388  * @buf_b: second buffer
2389  * @len_b: size of second buffer
2390  * @have_tsc: can use TSC packets to detect overlap
2391  *
2392  * When trace samples or snapshots are recorded there is the possibility that
2393  * the data overlaps.  Note that, for the purposes of decoding, data is only
2394  * useful if it begins with a PSB packet.
2395  *
2396  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2397  * @buf_b + @len_b if there is no non-overlapped data.
2398  */
2399 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2400                                      unsigned char *buf_b, size_t len_b,
2401                                      bool have_tsc)
2402 {
2403         unsigned char *found;
2404
2405         /* Buffer 'b' must start at PSB so throw away everything before that */
2406         if (!intel_pt_next_psb(&buf_b, &len_b))
2407                 return buf_b + len_b; /* No PSB */
2408
2409         if (!intel_pt_next_psb(&buf_a, &len_a))
2410                 return buf_b; /* No overlap */
2411
2412         if (have_tsc) {
2413                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2414                 if (found)
2415                         return found;
2416         }
2417
2418         /*
2419          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2420          * we can ignore the first part of buffer 'a'.
2421          */
2422         while (len_b < len_a) {
2423                 if (!intel_pt_step_psb(&buf_a, &len_a))
2424                         return buf_b; /* No overlap */
2425         }
2426
2427         /* Now len_b >= len_a */
2428         if (len_b > len_a) {
2429                 /* The leftover buffer 'b' must start at a PSB */
2430                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2431                         if (!intel_pt_step_psb(&buf_a, &len_a))
2432                                 return buf_b; /* No overlap */
2433                 }
2434         }
2435
2436         while (1) {
2437                 /* Potential overlap so check the bytes */
2438                 found = memmem(buf_a, len_a, buf_b, len_a);
2439                 if (found)
2440                         return buf_b + len_a;
2441
2442                 /* Try again at next PSB in buffer 'a' */
2443                 if (!intel_pt_step_psb(&buf_a, &len_a))
2444                         return buf_b; /* No overlap */
2445
2446                 /* The leftover buffer 'b' must start at a PSB */
2447                 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2448                         if (!intel_pt_step_psb(&buf_a, &len_a))
2449                                 return buf_b; /* No overlap */
2450                 }
2451         }
2452 }