tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42 #include "config.h"
  43
  44 #include "intel-pt-decoder/intel-pt-log.h"
  45 #include "intel-pt-decoder/intel-pt-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  47 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  48
  49 #define MAX_TIMESTAMP (~0ULL)
  50
  51 struct intel_pt {
  52         struct auxtrace auxtrace;
  53         struct auxtrace_queues queues;
  54         struct auxtrace_heap heap;
  55         u32 auxtrace_type;
  56         struct perf_session *session;
  57         struct machine *machine;
  58         struct perf_evsel *switch_evsel;
  59         struct thread *unknown_thread;
  60         bool timeless_decoding;
  61         bool sampling_mode;
  62         bool snapshot_mode;
  63         bool per_cpu_mmaps;
  64         bool have_tsc;
  65         bool data_queued;
  66         bool est_tsc;
  67         bool sync_switch;
  68         bool mispred_all;
  69         int have_sched_switch;
  70         u32 pmu_type;
  71         u64 kernel_start;
  72         u64 switch_ip;
  73         u64 ptss_ip;
  74
  75         struct perf_tsc_conversion tc;
  76         bool cap_user_time_zero;
  77
  78         struct itrace_synth_opts synth_opts;
  79
  80         bool sample_instructions;
  81         u64 instructions_sample_type;
  82         u64 instructions_sample_period;
  83         u64 instructions_id;
  84
  85         bool sample_branches;
  86         u32 branches_filter;
  87         u64 branches_sample_type;
  88         u64 branches_id;
  89
  90         bool sample_transactions;
  91         u64 transactions_sample_type;
  92         u64 transactions_id;
  93
  94         bool synth_needs_swap;
  95
  96         u64 tsc_bit;
  97         u64 mtc_bit;
  98         u64 mtc_freq_bits;
  99         u32 tsc_ctc_ratio_n;
 100         u32 tsc_ctc_ratio_d;
 101         u64 cyc_bit;
 102         u64 noretcomp_bit;
 103         unsigned max_non_turbo_ratio;
 104
 105         unsigned long num_events;
 106
 107         char *filter;
 108 };
 109
 110 enum switch_state {
 111         INTEL_PT_SS_NOT_TRACING,
 112         INTEL_PT_SS_UNKNOWN,
 113         INTEL_PT_SS_TRACING,
 114         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 115         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 116 };
 117
 118 struct intel_pt_queue {
 119         struct intel_pt *pt;
 120         unsigned int queue_nr;
 121         struct auxtrace_buffer *buffer;
 122         void *decoder;
 123         const struct intel_pt_state *state;
 124         struct ip_callchain *chain;
 125         struct branch_stack *last_branch;
 126         struct branch_stack *last_branch_rb;
 127         size_t last_branch_pos;
 128         union perf_event *event_buf;
 129         bool on_heap;
 130         bool stop;
 131         bool step_through_buffers;
 132         bool use_buffer_pid_tid;
 133         pid_t pid, tid;
 134         int cpu;
 135         int switch_state;
 136         pid_t next_tid;
 137         struct thread *thread;
 138         bool exclude_kernel;
 139         bool have_sample;
 140         u64 time;
 141         u64 timestamp;
 142         u32 flags;
 143         u16 insn_len;
 144         u64 last_insn_cnt;
 145 };
 146
 147 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 148                           unsigned char *buf, size_t len)
 149 {
 150         struct intel_pt_pkt packet;
 151         size_t pos = 0;
 152         int ret, pkt_len, i;
 153         char desc[INTEL_PT_PKT_DESC_MAX];
 154         const char *color = PERF_COLOR_BLUE;
 155
 156         color_fprintf(stdout, color,
 157                       ". ... Intel Processor Trace data: size %zu bytes\n",
 158                       len);
 159
 160         while (len) {
 161                 ret = intel_pt_get_packet(buf, len, &packet);
 162                 if (ret > 0)
 163                         pkt_len = ret;
 164                 else
 165                         pkt_len = 1;
 166                 printf(".");
 167                 color_fprintf(stdout, color, "  %08x: ", pos);
 168                 for (i = 0; i < pkt_len; i++)
 169                         color_fprintf(stdout, color, " %02x", buf[i]);
 170                 for (; i < 16; i++)
 171                         color_fprintf(stdout, color, "   ");
 172                 if (ret > 0) {
 173                         ret = intel_pt_pkt_desc(&packet, desc,
 174                                                 INTEL_PT_PKT_DESC_MAX);
 175                         if (ret > 0)
 176                                 color_fprintf(stdout, color, " %s\n", desc);
 177                 } else {
 178                         color_fprintf(stdout, color, " Bad packet!\n");
 179                 }
 180                 pos += pkt_len;
 181                 buf += pkt_len;
 182                 len -= pkt_len;
 183         }
 184 }
 185
 186 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 187                                 size_t len)
 188 {
 189         printf(".\n");
 190         intel_pt_dump(pt, buf, len);
 191 }
 192
 193 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 194                                    struct auxtrace_buffer *b)
 195 {
 196         void *start;
 197
 198         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 199                                       pt->have_tsc);
 200         if (!start)
 201                 return -EINVAL;
 202         b->use_size = b->data + b->size - start;
 203         b->use_data = start;
 204         return 0;
 205 }
 206
 207 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 208                                         struct auxtrace_queue *queue,
 209                                         struct auxtrace_buffer *buffer)
 210 {
 211         if (queue->cpu == -1 && buffer->cpu != -1)
 212                 ptq->cpu = buffer->cpu;
 213
 214         ptq->pid = buffer->pid;
 215         ptq->tid = buffer->tid;
 216
 217         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 218                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 219
 220         thread__zput(ptq->thread);
 221
 222         if (ptq->tid != -1) {
 223                 if (ptq->pid != -1)
 224                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 225                                                               ptq->pid,
 226                                                               ptq->tid);
 227                 else
 228                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 229                                                            ptq->tid);
 230         }
 231 }
 232
 233 /* This function assumes data is processed sequentially only */
 234 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 235 {
 236         struct intel_pt_queue *ptq = data;
 237         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 238         struct auxtrace_queue *queue;
 239
 240         if (ptq->stop) {
 241                 b->len = 0;
 242                 return 0;
 243         }
 244
 245         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 246 next:
 247         buffer = auxtrace_buffer__next(queue, buffer);
 248         if (!buffer) {
 249                 if (old_buffer)
 250                         auxtrace_buffer__drop_data(old_buffer);
 251                 b->len = 0;
 252                 return 0;
 253         }
 254
 255         ptq->buffer = buffer;
 256
 257         if (!buffer->data) {
 258                 int fd = perf_data_file__fd(ptq->pt->session->file);
 259
 260                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 261                 if (!buffer->data)
 262                         return -ENOMEM;
 263         }
 264
 265         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 266             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 267                 return -ENOMEM;
 268
 269         if (buffer->use_data) {
 270                 b->len = buffer->use_size;
 271                 b->buf = buffer->use_data;
 272         } else {
 273                 b->len = buffer->size;
 274                 b->buf = buffer->data;
 275         }
 276         b->ref_timestamp = buffer->reference;
 277
 278         /*
 279          * If in snapshot mode and the buffer has no usable data, get next
 280          * buffer and again check overlap against old_buffer.
 281          */
 282         if (ptq->pt->snapshot_mode && !b->len)
 283                 goto next;
 284
 285         if (old_buffer)
 286                 auxtrace_buffer__drop_data(old_buffer);
 287
 288         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 289                                                       !buffer->consecutive)) {
 290                 b->consecutive = false;
 291                 b->trace_nr = buffer->buffer_nr + 1;
 292         } else {
 293                 b->consecutive = true;
 294         }
 295
 296         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 297                                         ptq->tid != buffer->tid))
 298                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 299
 300         if (ptq->step_through_buffers)
 301                 ptq->stop = true;
 302
 303         if (!b->len)
 304                 return intel_pt_get_trace(b, data);
 305
 306         return 0;
 307 }
 308
 309 struct intel_pt_cache_entry {
 310         struct auxtrace_cache_entry     entry;
 311         u64                             insn_cnt;
 312         u64                             byte_cnt;
 313         enum intel_pt_insn_op           op;
 314         enum intel_pt_insn_branch       branch;
 315         int                             length;
 316         int32_t                         rel;
 317 };
 318
 319 static int intel_pt_config_div(const char *var, const char *value, void *data)
 320 {
 321         int *d = data;
 322         long val;
 323
 324         if (!strcmp(var, "intel-pt.cache-divisor")) {
 325                 val = strtol(value, NULL, 0);
 326                 if (val > 0 && val <= INT_MAX)
 327                         *d = val;
 328         }
 329
 330         return 0;
 331 }
 332
 333 static int intel_pt_cache_divisor(void)
 334 {
 335         static int d;
 336
 337         if (d)
 338                 return d;
 339
 340         perf_config(intel_pt_config_div, &d);
 341
 342         if (!d)
 343                 d = 64;
 344
 345         return d;
 346 }
 347
 348 static unsigned int intel_pt_cache_size(struct dso *dso,
 349                                         struct machine *machine)
 350 {
 351         off_t size;
 352
 353         size = dso__data_size(dso, machine);
 354         size /= intel_pt_cache_divisor();
 355         if (size < 1000)
 356                 return 10;
 357         if (size > (1 << 21))
 358                 return 21;
 359         return 32 - __builtin_clz(size);
 360 }
 361
 362 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 363                                              struct machine *machine)
 364 {
 365         struct auxtrace_cache *c;
 366         unsigned int bits;
 367
 368         if (dso->auxtrace_cache)
 369                 return dso->auxtrace_cache;
 370
 371         bits = intel_pt_cache_size(dso, machine);
 372
 373         /* Ignoring cache creation failure */
 374         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 375
 376         dso->auxtrace_cache = c;
 377
 378         return c;
 379 }
 380
 381 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 382                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 383                               struct intel_pt_insn *intel_pt_insn)
 384 {
 385         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 386         struct intel_pt_cache_entry *e;
 387         int err;
 388
 389         if (!c)
 390                 return -ENOMEM;
 391
 392         e = auxtrace_cache__alloc_entry(c);
 393         if (!e)
 394                 return -ENOMEM;
 395
 396         e->insn_cnt = insn_cnt;
 397         e->byte_cnt = byte_cnt;
 398         e->op = intel_pt_insn->op;
 399         e->branch = intel_pt_insn->branch;
 400         e->length = intel_pt_insn->length;
 401         e->rel = intel_pt_insn->rel;
 402
 403         err = auxtrace_cache__add(c, offset, &e->entry);
 404         if (err)
 405                 auxtrace_cache__free_entry(c, e);
 406
 407         return err;
 408 }
 409
 410 static struct intel_pt_cache_entry *
 411 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 412 {
 413         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 414
 415         if (!c)
 416                 return NULL;
 417
 418         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 419 }
 420
 421 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 422                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 423                                    uint64_t to_ip, uint64_t max_insn_cnt,
 424                                    void *data)
 425 {
 426         struct intel_pt_queue *ptq = data;
 427         struct machine *machine = ptq->pt->machine;
 428         struct thread *thread;
 429         struct addr_location al;
 430         unsigned char buf[1024];
 431         size_t bufsz;
 432         ssize_t len;
 433         int x86_64;
 434         u8 cpumode;
 435         u64 offset, start_offset, start_ip;
 436         u64 insn_cnt = 0;
 437         bool one_map = true;
 438
 439         if (to_ip && *ip == to_ip)
 440                 goto out_no_cache;
 441
 442         bufsz = intel_pt_insn_max_size();
 443
 444         if (*ip >= ptq->pt->kernel_start)
 445                 cpumode = PERF_RECORD_MISC_KERNEL;
 446         else
 447                 cpumode = PERF_RECORD_MISC_USER;
 448
 449         thread = ptq->thread;
 450         if (!thread) {
 451                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 452                         return -EINVAL;
 453                 thread = ptq->pt->unknown_thread;
 454         }
 455
 456         while (1) {
 457                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 458                 if (!al.map || !al.map->dso)
 459                         return -EINVAL;
 460
 461                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 462                     dso__data_status_seen(al.map->dso,
 463                                           DSO_DATA_STATUS_SEEN_ITRACE))
 464                         return -ENOENT;
 465
 466                 offset = al.map->map_ip(al.map, *ip);
 467
 468                 if (!to_ip && one_map) {
 469                         struct intel_pt_cache_entry *e;
 470
 471                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 472                         if (e &&
 473                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 474                                 *insn_cnt_ptr = e->insn_cnt;
 475                                 *ip += e->byte_cnt;
 476                                 intel_pt_insn->op = e->op;
 477                                 intel_pt_insn->branch = e->branch;
 478                                 intel_pt_insn->length = e->length;
 479                                 intel_pt_insn->rel = e->rel;
 480                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 481                                 return 0;
 482                         }
 483                 }
 484
 485                 start_offset = offset;
 486                 start_ip = *ip;
 487
 488                 /* Load maps to ensure dso->is_64_bit has been updated */
 489                 map__load(al.map);
 490
 491                 x86_64 = al.map->dso->is_64_bit;
 492
 493                 while (1) {
 494                         len = dso__data_read_offset(al.map->dso, machine,
 495                                                     offset, buf, bufsz);
 496                         if (len <= 0)
 497                                 return -EINVAL;
 498
 499                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 500                                 return -EINVAL;
 501
 502                         intel_pt_log_insn(intel_pt_insn, *ip);
 503
 504                         insn_cnt += 1;
 505
 506                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 507                                 goto out;
 508
 509                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 510                                 goto out_no_cache;
 511
 512                         *ip += intel_pt_insn->length;
 513
 514                         if (to_ip && *ip == to_ip)
 515                                 goto out_no_cache;
 516
 517                         if (*ip >= al.map->end)
 518                                 break;
 519
 520                         offset += intel_pt_insn->length;
 521                 }
 522                 one_map = false;
 523         }
 524 out:
 525         *insn_cnt_ptr = insn_cnt;
 526
 527         if (!one_map)
 528                 goto out_no_cache;
 529
 530         /*
 531          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 532          * entries.
 533          */
 534         if (to_ip) {
 535                 struct intel_pt_cache_entry *e;
 536
 537                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 538                 if (e)
 539                         return 0;
 540         }
 541
 542         /* Ignore cache errors */
 543         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 544                            *ip - start_ip, intel_pt_insn);
 545
 546         return 0;
 547
 548 out_no_cache:
 549         *insn_cnt_ptr = insn_cnt;
 550         return 0;
 551 }
 552
 553 static bool intel_pt_get_config(struct intel_pt *pt,
 554                                 struct perf_event_attr *attr, u64 *config)
 555 {
 556         if (attr->type == pt->pmu_type) {
 557                 if (config)
 558                         *config = attr->config;
 559                 return true;
 560         }
 561
 562         return false;
 563 }
 564
 565 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 566 {
 567         struct perf_evsel *evsel;
 568
 569         evlist__for_each_entry(pt->session->evlist, evsel) {
 570                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 571                     !evsel->attr.exclude_kernel)
 572                         return false;
 573         }
 574         return true;
 575 }
 576
 577 static bool intel_pt_return_compression(struct intel_pt *pt)
 578 {
 579         struct perf_evsel *evsel;
 580         u64 config;
 581
 582         if (!pt->noretcomp_bit)
 583                 return true;
 584
 585         evlist__for_each_entry(pt->session->evlist, evsel) {
 586                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 587                     (config & pt->noretcomp_bit))
 588                         return false;
 589         }
 590         return true;
 591 }
 592
 593 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 594 {
 595         struct perf_evsel *evsel;
 596         unsigned int shift;
 597         u64 config;
 598
 599         if (!pt->mtc_freq_bits)
 600                 return 0;
 601
 602         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 603                 config >>= 1;
 604
 605         evlist__for_each_entry(pt->session->evlist, evsel) {
 606                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 607                         return (config & pt->mtc_freq_bits) >> shift;
 608         }
 609         return 0;
 610 }
 611
 612 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 613 {
 614         struct perf_evsel *evsel;
 615         bool timeless_decoding = true;
 616         u64 config;
 617
 618         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 619                 return true;
 620
 621         evlist__for_each_entry(pt->session->evlist, evsel) {
 622                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 623                         return true;
 624                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 625                         if (config & pt->tsc_bit)
 626                                 timeless_decoding = false;
 627                         else
 628                                 return true;
 629                 }
 630         }
 631         return timeless_decoding;
 632 }
 633
 634 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 635 {
 636         struct perf_evsel *evsel;
 637
 638         evlist__for_each_entry(pt->session->evlist, evsel) {
 639                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 640                     !evsel->attr.exclude_kernel)
 641                         return true;
 642         }
 643         return false;
 644 }
 645
 646 static bool intel_pt_have_tsc(struct intel_pt *pt)
 647 {
 648         struct perf_evsel *evsel;
 649         bool have_tsc = false;
 650         u64 config;
 651
 652         if (!pt->tsc_bit)
 653                 return false;
 654
 655         evlist__for_each_entry(pt->session->evlist, evsel) {
 656                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 657                         if (config & pt->tsc_bit)
 658                                 have_tsc = true;
 659                         else
 660                                 return false;
 661                 }
 662         }
 663         return have_tsc;
 664 }
 665
 666 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 667 {
 668         u64 quot, rem;
 669
 670         quot = ns / pt->tc.time_mult;
 671         rem  = ns % pt->tc.time_mult;
 672         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 673                 pt->tc.time_mult;
 674 }
 675
 676 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 677                                                    unsigned int queue_nr)
 678 {
 679         struct intel_pt_params params = { .get_trace = 0, };
 680         struct intel_pt_queue *ptq;
 681
 682         ptq = zalloc(sizeof(struct intel_pt_queue));
 683         if (!ptq)
 684                 return NULL;
 685
 686         if (pt->synth_opts.callchain) {
 687                 size_t sz = sizeof(struct ip_callchain);
 688
 689                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 690                 ptq->chain = zalloc(sz);
 691                 if (!ptq->chain)
 692                         goto out_free;
 693         }
 694
 695         if (pt->synth_opts.last_branch) {
 696                 size_t sz = sizeof(struct branch_stack);
 697
 698                 sz += pt->synth_opts.last_branch_sz *
 699                       sizeof(struct branch_entry);
 700                 ptq->last_branch = zalloc(sz);
 701                 if (!ptq->last_branch)
 702                         goto out_free;
 703                 ptq->last_branch_rb = zalloc(sz);
 704                 if (!ptq->last_branch_rb)
 705                         goto out_free;
 706         }
 707
 708         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 709         if (!ptq->event_buf)
 710                 goto out_free;
 711
 712         ptq->pt = pt;
 713         ptq->queue_nr = queue_nr;
 714         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 715         ptq->pid = -1;
 716         ptq->tid = -1;
 717         ptq->cpu = -1;
 718         ptq->next_tid = -1;
 719
 720         params.get_trace = intel_pt_get_trace;
 721         params.walk_insn = intel_pt_walk_next_insn;
 722         params.data = ptq;
 723         params.return_compression = intel_pt_return_compression(pt);
 724         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 725         params.mtc_period = intel_pt_mtc_period(pt);
 726         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 727         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 728
 729         if (pt->synth_opts.instructions) {
 730                 if (pt->synth_opts.period) {
 731                         switch (pt->synth_opts.period_type) {
 732                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 733                                 params.period_type =
 734                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 735                                 params.period = pt->synth_opts.period;
 736                                 break;
 737                         case PERF_ITRACE_PERIOD_TICKS:
 738                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 739                                 params.period = pt->synth_opts.period;
 740                                 break;
 741                         case PERF_ITRACE_PERIOD_NANOSECS:
 742                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 743                                 params.period = intel_pt_ns_to_ticks(pt,
 744                                                         pt->synth_opts.period);
 745                                 break;
 746                         default:
 747                                 break;
 748                         }
 749                 }
 750
 751                 if (!params.period) {
 752                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 753                         params.period = 1;
 754                 }
 755         }
 756
 757         ptq->decoder = intel_pt_decoder_new(&params);
 758         if (!ptq->decoder)
 759                 goto out_free;
 760
 761         return ptq;
 762
 763 out_free:
 764         zfree(&ptq->event_buf);
 765         zfree(&ptq->last_branch);
 766         zfree(&ptq->last_branch_rb);
 767         zfree(&ptq->chain);
 768         free(ptq);
 769         return NULL;
 770 }
 771
 772 static void intel_pt_free_queue(void *priv)
 773 {
 774         struct intel_pt_queue *ptq = priv;
 775
 776         if (!ptq)
 777                 return;
 778         thread__zput(ptq->thread);
 779         intel_pt_decoder_free(ptq->decoder);
 780         zfree(&ptq->event_buf);
 781         zfree(&ptq->last_branch);
 782         zfree(&ptq->last_branch_rb);
 783         zfree(&ptq->chain);
 784         free(ptq);
 785 }
 786
 787 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 788                                      struct auxtrace_queue *queue)
 789 {
 790         struct intel_pt_queue *ptq = queue->priv;
 791
 792         if (queue->tid == -1 || pt->have_sched_switch) {
 793                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 794                 thread__zput(ptq->thread);
 795         }
 796
 797         if (!ptq->thread && ptq->tid != -1)
 798                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 799
 800         if (ptq->thread) {
 801                 ptq->pid = ptq->thread->pid_;
 802                 if (queue->cpu == -1)
 803                         ptq->cpu = ptq->thread->cpu;
 804         }
 805 }
 806
 807 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 808 {
 809         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 810                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 811         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 812                 if (ptq->state->to_ip)
 813                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 814                                      PERF_IP_FLAG_ASYNC |
 815                                      PERF_IP_FLAG_INTERRUPT;
 816                 else
 817                         ptq->flags = PERF_IP_FLAG_BRANCH |
 818                                      PERF_IP_FLAG_TRACE_END;
 819                 ptq->insn_len = 0;
 820         } else {
 821                 if (ptq->state->from_ip)
 822                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 823                 else
 824                         ptq->flags = PERF_IP_FLAG_BRANCH |
 825                                      PERF_IP_FLAG_TRACE_BEGIN;
 826                 if (ptq->state->flags & INTEL_PT_IN_TX)
 827                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 828                 ptq->insn_len = ptq->state->insn_len;
 829         }
 830 }
 831
 832 static int intel_pt_setup_queue(struct intel_pt *pt,
 833                                 struct auxtrace_queue *queue,
 834                                 unsigned int queue_nr)
 835 {
 836         struct intel_pt_queue *ptq = queue->priv;
 837
 838         if (list_empty(&queue->head))
 839                 return 0;
 840
 841         if (!ptq) {
 842                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 843                 if (!ptq)
 844                         return -ENOMEM;
 845                 queue->priv = ptq;
 846
 847                 if (queue->cpu != -1)
 848                         ptq->cpu = queue->cpu;
 849                 ptq->tid = queue->tid;
 850
 851                 if (pt->sampling_mode) {
 852                         if (pt->timeless_decoding)
 853                                 ptq->step_through_buffers = true;
 854                         if (pt->timeless_decoding || !pt->have_sched_switch)
 855                                 ptq->use_buffer_pid_tid = true;
 856                 }
 857         }
 858
 859         if (!ptq->on_heap &&
 860             (!pt->sync_switch ||
 861              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 862                 const struct intel_pt_state *state;
 863                 int ret;
 864
 865                 if (pt->timeless_decoding)
 866                         return 0;
 867
 868                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 869                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 870                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 871                 while (1) {
 872                         state = intel_pt_decode(ptq->decoder);
 873                         if (state->err) {
 874                                 if (state->err == INTEL_PT_ERR_NODATA) {
 875                                         intel_pt_log("queue %u has no timestamp\n",
 876                                                      queue_nr);
 877                                         return 0;
 878                                 }
 879                                 continue;
 880                         }
 881                         if (state->timestamp)
 882                                 break;
 883                 }
 884
 885                 ptq->timestamp = state->timestamp;
 886                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 887                              queue_nr, ptq->timestamp);
 888                 ptq->state = state;
 889                 ptq->have_sample = true;
 890                 intel_pt_sample_flags(ptq);
 891                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 892                 if (ret)
 893                         return ret;
 894                 ptq->on_heap = true;
 895         }
 896
 897         return 0;
 898 }
 899
 900 static int intel_pt_setup_queues(struct intel_pt *pt)
 901 {
 902         unsigned int i;
 903         int ret;
 904
 905         for (i = 0; i < pt->queues.nr_queues; i++) {
 906                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 907                 if (ret)
 908                         return ret;
 909         }
 910         return 0;
 911 }
 912
 913 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 914 {
 915         struct branch_stack *bs_src = ptq->last_branch_rb;
 916         struct branch_stack *bs_dst = ptq->last_branch;
 917         size_t nr = 0;
 918
 919         bs_dst->nr = bs_src->nr;
 920
 921         if (!bs_src->nr)
 922                 return;
 923
 924         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
 925         memcpy(&bs_dst->entries[0],
 926                &bs_src->entries[ptq->last_branch_pos],
 927                sizeof(struct branch_entry) * nr);
 928
 929         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
 930                 memcpy(&bs_dst->entries[nr],
 931                        &bs_src->entries[0],
 932                        sizeof(struct branch_entry) * ptq->last_branch_pos);
 933         }
 934 }
 935
 936 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
 937 {
 938         ptq->last_branch_pos = 0;
 939         ptq->last_branch_rb->nr = 0;
 940 }
 941
 942 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
 943 {
 944         const struct intel_pt_state *state = ptq->state;
 945         struct branch_stack *bs = ptq->last_branch_rb;
 946         struct branch_entry *be;
 947
 948         if (!ptq->last_branch_pos)
 949                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
 950
 951         ptq->last_branch_pos -= 1;
 952
 953         be              = &bs->entries[ptq->last_branch_pos];
 954         be->from        = state->from_ip;
 955         be->to          = state->to_ip;
 956         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
 957         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
 958         /* No support for mispredict */
 959         be->flags.mispred = ptq->pt->mispred_all;
 960
 961         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
 962                 bs->nr += 1;
 963 }
 964
 965 static int intel_pt_inject_event(union perf_event *event,
 966                                  struct perf_sample *sample, u64 type,
 967                                  bool swapped)
 968 {
 969         event->header.size = perf_event__sample_event_size(sample, type, 0);
 970         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
 971 }
 972
 973 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 974 {
 975         int ret;
 976         struct intel_pt *pt = ptq->pt;
 977         union perf_event *event = ptq->event_buf;
 978         struct perf_sample sample = { .ip = 0, };
 979         struct dummy_branch_stack {
 980                 u64                     nr;
 981                 struct branch_entry     entries;
 982         } dummy_bs;
 983
 984         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
 985                 return 0;
 986
 987         if (pt->synth_opts.initial_skip &&
 988             pt->num_events++ < pt->synth_opts.initial_skip)
 989                 return 0;
 990
 991         event->sample.header.type = PERF_RECORD_SAMPLE;
 992         event->sample.header.misc = PERF_RECORD_MISC_USER;
 993         event->sample.header.size = sizeof(struct perf_event_header);
 994
 995         if (!pt->timeless_decoding)
 996                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 997
 998         sample.cpumode = PERF_RECORD_MISC_USER;
 999         sample.ip = ptq->state->from_ip;
1000         sample.pid = ptq->pid;
1001         sample.tid = ptq->tid;
1002         sample.addr = ptq->state->to_ip;
1003         sample.id = ptq->pt->branches_id;
1004         sample.stream_id = ptq->pt->branches_id;
1005         sample.period = 1;
1006         sample.cpu = ptq->cpu;
1007         sample.flags = ptq->flags;
1008         sample.insn_len = ptq->insn_len;
1009
1010         /*
1011          * perf report cannot handle events without a branch stack when using
1012          * SORT_MODE__BRANCH so make a dummy one.
1013          */
1014         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1015                 dummy_bs = (struct dummy_branch_stack){
1016                         .nr = 1,
1017                         .entries = {
1018                                 .from = sample.ip,
1019                                 .to = sample.addr,
1020                         },
1021                 };
1022                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1023         }
1024
1025         if (pt->synth_opts.inject) {
1026                 ret = intel_pt_inject_event(event, &sample,
1027                                             pt->branches_sample_type,
1028                                             pt->synth_needs_swap);
1029                 if (ret)
1030                         return ret;
1031         }
1032
1033         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1034         if (ret)
1035                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1036                        ret);
1037
1038         return ret;
1039 }
1040
1041 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1042 {
1043         int ret;
1044         struct intel_pt *pt = ptq->pt;
1045         union perf_event *event = ptq->event_buf;
1046         struct perf_sample sample = { .ip = 0, };
1047
1048         if (pt->synth_opts.initial_skip &&
1049             pt->num_events++ < pt->synth_opts.initial_skip)
1050                 return 0;
1051
1052         event->sample.header.type = PERF_RECORD_SAMPLE;
1053         event->sample.header.misc = PERF_RECORD_MISC_USER;
1054         event->sample.header.size = sizeof(struct perf_event_header);
1055
1056         if (!pt->timeless_decoding)
1057                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1058
1059         sample.cpumode = PERF_RECORD_MISC_USER;
1060         sample.ip = ptq->state->from_ip;
1061         sample.pid = ptq->pid;
1062         sample.tid = ptq->tid;
1063         sample.addr = ptq->state->to_ip;
1064         sample.id = ptq->pt->instructions_id;
1065         sample.stream_id = ptq->pt->instructions_id;
1066         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1067         sample.cpu = ptq->cpu;
1068         sample.flags = ptq->flags;
1069         sample.insn_len = ptq->insn_len;
1070
1071         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1072
1073         if (pt->synth_opts.callchain) {
1074                 thread_stack__sample(ptq->thread, ptq->chain,
1075                                      pt->synth_opts.callchain_sz, sample.ip);
1076                 sample.callchain = ptq->chain;
1077         }
1078
1079         if (pt->synth_opts.last_branch) {
1080                 intel_pt_copy_last_branch_rb(ptq);
1081                 sample.branch_stack = ptq->last_branch;
1082         }
1083
1084         if (pt->synth_opts.inject) {
1085                 ret = intel_pt_inject_event(event, &sample,
1086                                             pt->instructions_sample_type,
1087                                             pt->synth_needs_swap);
1088                 if (ret)
1089                         return ret;
1090         }
1091
1092         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1093         if (ret)
1094                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1095                        ret);
1096
1097         if (pt->synth_opts.last_branch)
1098                 intel_pt_reset_last_branch_rb(ptq);
1099
1100         return ret;
1101 }
1102
1103 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1104 {
1105         int ret;
1106         struct intel_pt *pt = ptq->pt;
1107         union perf_event *event = ptq->event_buf;
1108         struct perf_sample sample = { .ip = 0, };
1109
1110         if (pt->synth_opts.initial_skip &&
1111             pt->num_events++ < pt->synth_opts.initial_skip)
1112                 return 0;
1113
1114         event->sample.header.type = PERF_RECORD_SAMPLE;
1115         event->sample.header.misc = PERF_RECORD_MISC_USER;
1116         event->sample.header.size = sizeof(struct perf_event_header);
1117
1118         if (!pt->timeless_decoding)
1119                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1120
1121         sample.cpumode = PERF_RECORD_MISC_USER;
1122         sample.ip = ptq->state->from_ip;
1123         sample.pid = ptq->pid;
1124         sample.tid = ptq->tid;
1125         sample.addr = ptq->state->to_ip;
1126         sample.id = ptq->pt->transactions_id;
1127         sample.stream_id = ptq->pt->transactions_id;
1128         sample.period = 1;
1129         sample.cpu = ptq->cpu;
1130         sample.flags = ptq->flags;
1131         sample.insn_len = ptq->insn_len;
1132
1133         if (pt->synth_opts.callchain) {
1134                 thread_stack__sample(ptq->thread, ptq->chain,
1135                                      pt->synth_opts.callchain_sz, sample.ip);
1136                 sample.callchain = ptq->chain;
1137         }
1138
1139         if (pt->synth_opts.last_branch) {
1140                 intel_pt_copy_last_branch_rb(ptq);
1141                 sample.branch_stack = ptq->last_branch;
1142         }
1143
1144         if (pt->synth_opts.inject) {
1145                 ret = intel_pt_inject_event(event, &sample,
1146                                             pt->transactions_sample_type,
1147                                             pt->synth_needs_swap);
1148                 if (ret)
1149                         return ret;
1150         }
1151
1152         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1153         if (ret)
1154                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1155                        ret);
1156
1157         if (pt->synth_opts.last_branch)
1158                 intel_pt_reset_last_branch_rb(ptq);
1159
1160         return ret;
1161 }
1162
1163 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1164                                 pid_t pid, pid_t tid, u64 ip)
1165 {
1166         union perf_event event;
1167         char msg[MAX_AUXTRACE_ERROR_MSG];
1168         int err;
1169
1170         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1171
1172         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1173                              code, cpu, pid, tid, ip, msg);
1174
1175         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1176         if (err)
1177                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1178                        err);
1179
1180         return err;
1181 }
1182
1183 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1184 {
1185         struct auxtrace_queue *queue;
1186         pid_t tid = ptq->next_tid;
1187         int err;
1188
1189         if (tid == -1)
1190                 return 0;
1191
1192         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1193
1194         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1195
1196         queue = &pt->queues.queue_array[ptq->queue_nr];
1197         intel_pt_set_pid_tid_cpu(pt, queue);
1198
1199         ptq->next_tid = -1;
1200
1201         return err;
1202 }
1203
1204 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1205 {
1206         struct intel_pt *pt = ptq->pt;
1207
1208         return ip == pt->switch_ip &&
1209                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1210                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1211                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1212 }
1213
1214 static int intel_pt_sample(struct intel_pt_queue *ptq)
1215 {
1216         const struct intel_pt_state *state = ptq->state;
1217         struct intel_pt *pt = ptq->pt;
1218         int err;
1219
1220         if (!ptq->have_sample)
1221                 return 0;
1222
1223         ptq->have_sample = false;
1224
1225         if (pt->sample_instructions &&
1226             (state->type & INTEL_PT_INSTRUCTION) &&
1227             (!pt->synth_opts.initial_skip ||
1228              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1229                 err = intel_pt_synth_instruction_sample(ptq);
1230                 if (err)
1231                         return err;
1232         }
1233
1234         if (pt->sample_transactions &&
1235             (state->type & INTEL_PT_TRANSACTION) &&
1236             (!pt->synth_opts.initial_skip ||
1237              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1238                 err = intel_pt_synth_transaction_sample(ptq);
1239                 if (err)
1240                         return err;
1241         }
1242
1243         if (!(state->type & INTEL_PT_BRANCH))
1244                 return 0;
1245
1246         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1247                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1248                                     state->to_ip, ptq->insn_len,
1249                                     state->trace_nr);
1250         else
1251                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1252
1253         if (pt->sample_branches) {
1254                 err = intel_pt_synth_branch_sample(ptq);
1255                 if (err)
1256                         return err;
1257         }
1258
1259         if (pt->synth_opts.last_branch)
1260                 intel_pt_update_last_branch_rb(ptq);
1261
1262         if (!pt->sync_switch)
1263                 return 0;
1264
1265         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1266                 switch (ptq->switch_state) {
1267                 case INTEL_PT_SS_UNKNOWN:
1268                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1269                         err = intel_pt_next_tid(pt, ptq);
1270                         if (err)
1271                                 return err;
1272                         ptq->switch_state = INTEL_PT_SS_TRACING;
1273                         break;
1274                 default:
1275                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1276                         return 1;
1277                 }
1278         } else if (!state->to_ip) {
1279                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1280         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1281                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1282         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1283                    state->to_ip == pt->ptss_ip &&
1284                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1285                 ptq->switch_state = INTEL_PT_SS_TRACING;
1286         }
1287
1288         return 0;
1289 }
1290
1291 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1292 {
1293         struct machine *machine = pt->machine;
1294         struct map *map;
1295         struct symbol *sym, *start;
1296         u64 ip, switch_ip = 0;
1297         const char *ptss;
1298
1299         if (ptss_ip)
1300                 *ptss_ip = 0;
1301
1302         map = machine__kernel_map(machine);
1303         if (!map)
1304                 return 0;
1305
1306         if (map__load(map))
1307                 return 0;
1308
1309         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1310
1311         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1312                 if (sym->binding == STB_GLOBAL &&
1313                     !strcmp(sym->name, "__switch_to")) {
1314                         ip = map->unmap_ip(map, sym->start);
1315                         if (ip >= map->start && ip < map->end) {
1316                                 switch_ip = ip;
1317                                 break;
1318                         }
1319                 }
1320         }
1321
1322         if (!switch_ip || !ptss_ip)
1323                 return 0;
1324
1325         if (pt->have_sched_switch == 1)
1326                 ptss = "perf_trace_sched_switch";
1327         else
1328                 ptss = "__perf_event_task_sched_out";
1329
1330         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1331                 if (!strcmp(sym->name, ptss)) {
1332                         ip = map->unmap_ip(map, sym->start);
1333                         if (ip >= map->start && ip < map->end) {
1334                                 *ptss_ip = ip;
1335                                 break;
1336                         }
1337                 }
1338         }
1339
1340         return switch_ip;
1341 }
1342
1343 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1344 {
1345         const struct intel_pt_state *state = ptq->state;
1346         struct intel_pt *pt = ptq->pt;
1347         int err;
1348
1349         if (!pt->kernel_start) {
1350                 pt->kernel_start = machine__kernel_start(pt->machine);
1351                 if (pt->per_cpu_mmaps &&
1352                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1353                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1354                     !pt->sampling_mode) {
1355                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1356                         if (pt->switch_ip) {
1357                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1358                                              pt->switch_ip, pt->ptss_ip);
1359                                 pt->sync_switch = true;
1360                         }
1361                 }
1362         }
1363
1364         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1365                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1366         while (1) {
1367                 err = intel_pt_sample(ptq);
1368                 if (err)
1369                         return err;
1370
1371                 state = intel_pt_decode(ptq->decoder);
1372                 if (state->err) {
1373                         if (state->err == INTEL_PT_ERR_NODATA)
1374                                 return 1;
1375                         if (pt->sync_switch &&
1376                             state->from_ip >= pt->kernel_start) {
1377                                 pt->sync_switch = false;
1378                                 intel_pt_next_tid(pt, ptq);
1379                         }
1380                         if (pt->synth_opts.errors) {
1381                                 err = intel_pt_synth_error(pt, state->err,
1382                                                            ptq->cpu, ptq->pid,
1383                                                            ptq->tid,
1384                                                            state->from_ip);
1385                                 if (err)
1386                                         return err;
1387                         }
1388                         continue;
1389                 }
1390
1391                 ptq->state = state;
1392                 ptq->have_sample = true;
1393                 intel_pt_sample_flags(ptq);
1394
1395                 /* Use estimated TSC upon return to user space */
1396                 if (pt->est_tsc &&
1397                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1398                     state->to_ip && state->to_ip < pt->kernel_start) {
1399                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1400                                      state->timestamp, state->est_timestamp);
1401                         ptq->timestamp = state->est_timestamp;
1402                 /* Use estimated TSC in unknown switch state */
1403                 } else if (pt->sync_switch &&
1404                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1405                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1406                            ptq->next_tid == -1) {
1407                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1408                                      state->timestamp, state->est_timestamp);
1409                         ptq->timestamp = state->est_timestamp;
1410                 } else if (state->timestamp > ptq->timestamp) {
1411                         ptq->timestamp = state->timestamp;
1412                 }
1413
1414                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1415                         *timestamp = ptq->timestamp;
1416                         return 0;
1417                 }
1418         }
1419         return 0;
1420 }
1421
1422 static inline int intel_pt_update_queues(struct intel_pt *pt)
1423 {
1424         if (pt->queues.new_data) {
1425                 pt->queues.new_data = false;
1426                 return intel_pt_setup_queues(pt);
1427         }
1428         return 0;
1429 }
1430
1431 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1432 {
1433         unsigned int queue_nr;
1434         u64 ts;
1435         int ret;
1436
1437         while (1) {
1438                 struct auxtrace_queue *queue;
1439                 struct intel_pt_queue *ptq;
1440
1441                 if (!pt->heap.heap_cnt)
1442                         return 0;
1443
1444                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1445                         return 0;
1446
1447                 queue_nr = pt->heap.heap_array[0].queue_nr;
1448                 queue = &pt->queues.queue_array[queue_nr];
1449                 ptq = queue->priv;
1450
1451                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1452                              queue_nr, pt->heap.heap_array[0].ordinal,
1453                              timestamp);
1454
1455                 auxtrace_heap__pop(&pt->heap);
1456
1457                 if (pt->heap.heap_cnt) {
1458                         ts = pt->heap.heap_array[0].ordinal + 1;
1459                         if (ts > timestamp)
1460                                 ts = timestamp;
1461                 } else {
1462                         ts = timestamp;
1463                 }
1464
1465                 intel_pt_set_pid_tid_cpu(pt, queue);
1466
1467                 ret = intel_pt_run_decoder(ptq, &ts);
1468
1469                 if (ret < 0) {
1470                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1471                         return ret;
1472                 }
1473
1474                 if (!ret) {
1475                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1476                         if (ret < 0)
1477                                 return ret;
1478                 } else {
1479                         ptq->on_heap = false;
1480                 }
1481         }
1482
1483         return 0;
1484 }
1485
1486 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1487                                             u64 time_)
1488 {
1489         struct auxtrace_queues *queues = &pt->queues;
1490         unsigned int i;
1491         u64 ts = 0;
1492
1493         for (i = 0; i < queues->nr_queues; i++) {
1494                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1495                 struct intel_pt_queue *ptq = queue->priv;
1496
1497                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1498                         ptq->time = time_;
1499                         intel_pt_set_pid_tid_cpu(pt, queue);
1500                         intel_pt_run_decoder(ptq, &ts);
1501                 }
1502         }
1503         return 0;
1504 }
1505
1506 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1507 {
1508         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1509                                     sample->pid, sample->tid, 0);
1510 }
1511
1512 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1513 {
1514         unsigned i, j;
1515
1516         if (cpu < 0 || !pt->queues.nr_queues)
1517                 return NULL;
1518
1519         if ((unsigned)cpu >= pt->queues.nr_queues)
1520                 i = pt->queues.nr_queues - 1;
1521         else
1522                 i = cpu;
1523
1524         if (pt->queues.queue_array[i].cpu == cpu)
1525                 return pt->queues.queue_array[i].priv;
1526
1527         for (j = 0; i > 0; j++) {
1528                 if (pt->queues.queue_array[--i].cpu == cpu)
1529                         return pt->queues.queue_array[i].priv;
1530         }
1531
1532         for (; j < pt->queues.nr_queues; j++) {
1533                 if (pt->queues.queue_array[j].cpu == cpu)
1534                         return pt->queues.queue_array[j].priv;
1535         }
1536
1537         return NULL;
1538 }
1539
1540 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1541                                 u64 timestamp)
1542 {
1543         struct intel_pt_queue *ptq;
1544         int err;
1545
1546         if (!pt->sync_switch)
1547                 return 1;
1548
1549         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1550         if (!ptq)
1551                 return 1;
1552
1553         switch (ptq->switch_state) {
1554         case INTEL_PT_SS_NOT_TRACING:
1555                 ptq->next_tid = -1;
1556                 break;
1557         case INTEL_PT_SS_UNKNOWN:
1558         case INTEL_PT_SS_TRACING:
1559                 ptq->next_tid = tid;
1560                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1561                 return 0;
1562         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1563                 if (!ptq->on_heap) {
1564                         ptq->timestamp = perf_time_to_tsc(timestamp,
1565                                                           &pt->tc);
1566                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1567                                                  ptq->timestamp);
1568                         if (err)
1569                                 return err;
1570                         ptq->on_heap = true;
1571                 }
1572                 ptq->switch_state = INTEL_PT_SS_TRACING;
1573                 break;
1574         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1575                 ptq->next_tid = tid;
1576                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1577                 break;
1578         default:
1579                 break;
1580         }
1581
1582         return 1;
1583 }
1584
1585 static int intel_pt_process_switch(struct intel_pt *pt,
1586                                    struct perf_sample *sample)
1587 {
1588         struct perf_evsel *evsel;
1589         pid_t tid;
1590         int cpu, ret;
1591
1592         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1593         if (evsel != pt->switch_evsel)
1594                 return 0;
1595
1596         tid = perf_evsel__intval(evsel, sample, "next_pid");
1597         cpu = sample->cpu;
1598
1599         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1600                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1601                      &pt->tc));
1602
1603         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1604         if (ret <= 0)
1605                 return ret;
1606
1607         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1608 }
1609
1610 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1611                                    struct perf_sample *sample)
1612 {
1613         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1614         pid_t pid, tid;
1615         int cpu, ret;
1616
1617         cpu = sample->cpu;
1618
1619         if (pt->have_sched_switch == 3) {
1620                 if (!out)
1621                         return 0;
1622                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1623                         pr_err("Expecting CPU-wide context switch event\n");
1624                         return -EINVAL;
1625                 }
1626                 pid = event->context_switch.next_prev_pid;
1627                 tid = event->context_switch.next_prev_tid;
1628         } else {
1629                 if (out)
1630                         return 0;
1631                 pid = sample->pid;
1632                 tid = sample->tid;
1633         }
1634
1635         if (tid == -1) {
1636                 pr_err("context_switch event has no tid\n");
1637                 return -EINVAL;
1638         }
1639
1640         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1641                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1642                      &pt->tc));
1643
1644         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1645         if (ret <= 0)
1646                 return ret;
1647
1648         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1649 }
1650
1651 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1652                                          union perf_event *event,
1653                                          struct perf_sample *sample)
1654 {
1655         if (!pt->per_cpu_mmaps)
1656                 return 0;
1657
1658         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1659                      sample->cpu, event->itrace_start.pid,
1660                      event->itrace_start.tid, sample->time,
1661                      perf_time_to_tsc(sample->time, &pt->tc));
1662
1663         return machine__set_current_tid(pt->machine, sample->cpu,
1664                                         event->itrace_start.pid,
1665                                         event->itrace_start.tid);
1666 }
1667
1668 static int intel_pt_process_event(struct perf_session *session,
1669                                   union perf_event *event,
1670                                   struct perf_sample *sample,
1671                                   struct perf_tool *tool)
1672 {
1673         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1674                                            auxtrace);
1675         u64 timestamp;
1676         int err = 0;
1677
1678         if (dump_trace)
1679                 return 0;
1680
1681         if (!tool->ordered_events) {
1682                 pr_err("Intel Processor Trace requires ordered events\n");
1683                 return -EINVAL;
1684         }
1685
1686         if (sample->time && sample->time != (u64)-1)
1687                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1688         else
1689                 timestamp = 0;
1690
1691         if (timestamp || pt->timeless_decoding) {
1692                 err = intel_pt_update_queues(pt);
1693                 if (err)
1694                         return err;
1695         }
1696
1697         if (pt->timeless_decoding) {
1698                 if (event->header.type == PERF_RECORD_EXIT) {
1699                         err = intel_pt_process_timeless_queues(pt,
1700                                                                event->fork.tid,
1701                                                                sample->time);
1702                 }
1703         } else if (timestamp) {
1704                 err = intel_pt_process_queues(pt, timestamp);
1705         }
1706         if (err)
1707                 return err;
1708
1709         if (event->header.type == PERF_RECORD_AUX &&
1710             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1711             pt->synth_opts.errors) {
1712                 err = intel_pt_lost(pt, sample);
1713                 if (err)
1714                         return err;
1715         }
1716
1717         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1718                 err = intel_pt_process_switch(pt, sample);
1719         else if (event->header.type == PERF_RECORD_ITRACE_START)
1720                 err = intel_pt_process_itrace_start(pt, event, sample);
1721         else if (event->header.type == PERF_RECORD_SWITCH ||
1722                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1723                 err = intel_pt_context_switch(pt, event, sample);
1724
1725         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1726                      perf_event__name(event->header.type), event->header.type,
1727                      sample->cpu, sample->time, timestamp);
1728
1729         return err;
1730 }
1731
1732 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1733 {
1734         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1735                                            auxtrace);
1736         int ret;
1737
1738         if (dump_trace)
1739                 return 0;
1740
1741         if (!tool->ordered_events)
1742                 return -EINVAL;
1743
1744         ret = intel_pt_update_queues(pt);
1745         if (ret < 0)
1746                 return ret;
1747
1748         if (pt->timeless_decoding)
1749                 return intel_pt_process_timeless_queues(pt, -1,
1750                                                         MAX_TIMESTAMP - 1);
1751
1752         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1753 }
1754
1755 static void intel_pt_free_events(struct perf_session *session)
1756 {
1757         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1758                                            auxtrace);
1759         struct auxtrace_queues *queues = &pt->queues;
1760         unsigned int i;
1761
1762         for (i = 0; i < queues->nr_queues; i++) {
1763                 intel_pt_free_queue(queues->queue_array[i].priv);
1764                 queues->queue_array[i].priv = NULL;
1765         }
1766         intel_pt_log_disable();
1767         auxtrace_queues__free(queues);
1768 }
1769
1770 static void intel_pt_free(struct perf_session *session)
1771 {
1772         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1773                                            auxtrace);
1774
1775         auxtrace_heap__free(&pt->heap);
1776         intel_pt_free_events(session);
1777         session->auxtrace = NULL;
1778         thread__put(pt->unknown_thread);
1779         zfree(&pt->filter);
1780         free(pt);
1781 }
1782
1783 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1784                                            union perf_event *event,
1785                                            struct perf_tool *tool __maybe_unused)
1786 {
1787         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1788                                            auxtrace);
1789
1790         if (pt->sampling_mode)
1791                 return 0;
1792
1793         if (!pt->data_queued) {
1794                 struct auxtrace_buffer *buffer;
1795                 off_t data_offset;
1796                 int fd = perf_data_file__fd(session->file);
1797                 int err;
1798
1799                 if (perf_data_file__is_pipe(session->file)) {
1800                         data_offset = 0;
1801                 } else {
1802                         data_offset = lseek(fd, 0, SEEK_CUR);
1803                         if (data_offset == -1)
1804                                 return -errno;
1805                 }
1806
1807                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1808                                                  data_offset, &buffer);
1809                 if (err)
1810                         return err;
1811
1812                 /* Dump here now we have copied a piped trace out of the pipe */
1813                 if (dump_trace) {
1814                         if (auxtrace_buffer__get_data(buffer, fd)) {
1815                                 intel_pt_dump_event(pt, buffer->data,
1816                                                     buffer->size);
1817                                 auxtrace_buffer__put_data(buffer);
1818                         }
1819                 }
1820         }
1821
1822         return 0;
1823 }
1824
1825 struct intel_pt_synth {
1826         struct perf_tool dummy_tool;
1827         struct perf_session *session;
1828 };
1829
1830 static int intel_pt_event_synth(struct perf_tool *tool,
1831                                 union perf_event *event,
1832                                 struct perf_sample *sample __maybe_unused,
1833                                 struct machine *machine __maybe_unused)
1834 {
1835         struct intel_pt_synth *intel_pt_synth =
1836                         container_of(tool, struct intel_pt_synth, dummy_tool);
1837
1838         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1839                                                  NULL);
1840 }
1841
1842 static int intel_pt_synth_event(struct perf_session *session,
1843                                 struct perf_event_attr *attr, u64 id)
1844 {
1845         struct intel_pt_synth intel_pt_synth;
1846
1847         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1848         intel_pt_synth.session = session;
1849
1850         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1851                                            &id, intel_pt_event_synth);
1852 }
1853
1854 static int intel_pt_synth_events(struct intel_pt *pt,
1855                                  struct perf_session *session)
1856 {
1857         struct perf_evlist *evlist = session->evlist;
1858         struct perf_evsel *evsel;
1859         struct perf_event_attr attr;
1860         bool found = false;
1861         u64 id;
1862         int err;
1863
1864         evlist__for_each_entry(evlist, evsel) {
1865                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1866                         found = true;
1867                         break;
1868                 }
1869         }
1870
1871         if (!found) {
1872                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1873                 return 0;
1874         }
1875
1876         memset(&attr, 0, sizeof(struct perf_event_attr));
1877         attr.size = sizeof(struct perf_event_attr);
1878         attr.type = PERF_TYPE_HARDWARE;
1879         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1880         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1881                             PERF_SAMPLE_PERIOD;
1882         if (pt->timeless_decoding)
1883                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1884         else
1885                 attr.sample_type |= PERF_SAMPLE_TIME;
1886         if (!pt->per_cpu_mmaps)
1887                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1888         attr.exclude_user = evsel->attr.exclude_user;
1889         attr.exclude_kernel = evsel->attr.exclude_kernel;
1890         attr.exclude_hv = evsel->attr.exclude_hv;
1891         attr.exclude_host = evsel->attr.exclude_host;
1892         attr.exclude_guest = evsel->attr.exclude_guest;
1893         attr.sample_id_all = evsel->attr.sample_id_all;
1894         attr.read_format = evsel->attr.read_format;
1895
1896         id = evsel->id[0] + 1000000000;
1897         if (!id)
1898                 id = 1;
1899
1900         if (pt->synth_opts.instructions) {
1901                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1902                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1903                         attr.sample_period =
1904                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1905                 else
1906                         attr.sample_period = pt->synth_opts.period;
1907                 pt->instructions_sample_period = attr.sample_period;
1908                 if (pt->synth_opts.callchain)
1909                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1910                 if (pt->synth_opts.last_branch)
1911                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1912                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1913                          id, (u64)attr.sample_type);
1914                 err = intel_pt_synth_event(session, &attr, id);
1915                 if (err) {
1916                         pr_err("%s: failed to synthesize 'instructions' event type\n",
1917                                __func__);
1918                         return err;
1919                 }
1920                 pt->sample_instructions = true;
1921                 pt->instructions_sample_type = attr.sample_type;
1922                 pt->instructions_id = id;
1923                 id += 1;
1924         }
1925
1926         if (pt->synth_opts.transactions) {
1927                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1928                 attr.sample_period = 1;
1929                 if (pt->synth_opts.callchain)
1930                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1931                 if (pt->synth_opts.last_branch)
1932                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1933                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1934                          id, (u64)attr.sample_type);
1935                 err = intel_pt_synth_event(session, &attr, id);
1936                 if (err) {
1937                         pr_err("%s: failed to synthesize 'transactions' event type\n",
1938                                __func__);
1939                         return err;
1940                 }
1941                 pt->sample_transactions = true;
1942                 pt->transactions_id = id;
1943                 id += 1;
1944                 evlist__for_each_entry(evlist, evsel) {
1945                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
1946                                 if (evsel->name)
1947                                         zfree(&evsel->name);
1948                                 evsel->name = strdup("transactions");
1949                                 break;
1950                         }
1951                 }
1952         }
1953
1954         if (pt->synth_opts.branches) {
1955                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1956                 attr.sample_period = 1;
1957                 attr.sample_type |= PERF_SAMPLE_ADDR;
1958                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1959                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
1960                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1961                          id, (u64)attr.sample_type);
1962                 err = intel_pt_synth_event(session, &attr, id);
1963                 if (err) {
1964                         pr_err("%s: failed to synthesize 'branches' event type\n",
1965                                __func__);
1966                         return err;
1967                 }
1968                 pt->sample_branches = true;
1969                 pt->branches_sample_type = attr.sample_type;
1970                 pt->branches_id = id;
1971         }
1972
1973         pt->synth_needs_swap = evsel->needs_swap;
1974
1975         return 0;
1976 }
1977
1978 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1979 {
1980         struct perf_evsel *evsel;
1981
1982         evlist__for_each_entry_reverse(evlist, evsel) {
1983                 const char *name = perf_evsel__name(evsel);
1984
1985                 if (!strcmp(name, "sched:sched_switch"))
1986                         return evsel;
1987         }
1988
1989         return NULL;
1990 }
1991
1992 static bool intel_pt_find_switch(struct perf_evlist *evlist)
1993 {
1994         struct perf_evsel *evsel;
1995
1996         evlist__for_each_entry(evlist, evsel) {
1997                 if (evsel->attr.context_switch)
1998                         return true;
1999         }
2000
2001         return false;
2002 }
2003
2004 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2005 {
2006         struct intel_pt *pt = data;
2007
2008         if (!strcmp(var, "intel-pt.mispred-all"))
2009                 pt->mispred_all = perf_config_bool(var, value);
2010
2011         return 0;
2012 }
2013
2014 static const char * const intel_pt_info_fmts[] = {
2015         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2016         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2017         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2018         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2019         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2020         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2021         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2022         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2023         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2024         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2025         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2026         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2027         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2028         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2029         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2030         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2031 };
2032
2033 static void intel_pt_print_info(u64 *arr, int start, int finish)
2034 {
2035         int i;
2036
2037         if (!dump_trace)
2038                 return;
2039
2040         for (i = start; i <= finish; i++)
2041                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2042 }
2043
2044 static void intel_pt_print_info_str(const char *name, const char *str)
2045 {
2046         if (!dump_trace)
2047                 return;
2048
2049         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2050 }
2051
2052 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2053 {
2054         return auxtrace_info->header.size >=
2055                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2056 }
2057
2058 int intel_pt_process_auxtrace_info(union perf_event *event,
2059                                    struct perf_session *session)
2060 {
2061         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2062         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2063         struct intel_pt *pt;
2064         void *info_end;
2065         u64 *info;
2066         int err;
2067
2068         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2069                                         min_sz)
2070                 return -EINVAL;
2071
2072         pt = zalloc(sizeof(struct intel_pt));
2073         if (!pt)
2074                 return -ENOMEM;
2075
2076         perf_config(intel_pt_perf_config, pt);
2077
2078         err = auxtrace_queues__init(&pt->queues);
2079         if (err)
2080                 goto err_free;
2081
2082         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2083
2084         pt->session = session;
2085         pt->machine = &session->machines.host; /* No kvm support */
2086         pt->auxtrace_type = auxtrace_info->type;
2087         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2088         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2089         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2090         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2091         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2092         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2093         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2094         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2095         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2096         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2097         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2098                             INTEL_PT_PER_CPU_MMAPS);
2099
2100         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2101                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2102                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2103                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2104                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2105                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2106                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2107                                     INTEL_PT_CYC_BIT);
2108         }
2109
2110         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2111                 pt->max_non_turbo_ratio =
2112                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2113                 intel_pt_print_info(&auxtrace_info->priv[0],
2114                                     INTEL_PT_MAX_NONTURBO_RATIO,
2115                                     INTEL_PT_MAX_NONTURBO_RATIO);
2116         }
2117
2118         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2119         info_end = (void *)info + auxtrace_info->header.size;
2120
2121         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2122                 size_t len;
2123
2124                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2125                 intel_pt_print_info(&auxtrace_info->priv[0],
2126                                     INTEL_PT_FILTER_STR_LEN,
2127                                     INTEL_PT_FILTER_STR_LEN);
2128                 if (len) {
2129                         const char *filter = (const char *)info;
2130
2131                         len = roundup(len + 1, 8);
2132                         info += len >> 3;
2133                         if ((void *)info > info_end) {
2134                                 pr_err("%s: bad filter string length\n", __func__);
2135                                 err = -EINVAL;
2136                                 goto err_free_queues;
2137                         }
2138                         pt->filter = memdup(filter, len);
2139                         if (!pt->filter) {
2140                                 err = -ENOMEM;
2141                                 goto err_free_queues;
2142                         }
2143                         if (session->header.needs_swap)
2144                                 mem_bswap_64(pt->filter, len);
2145                         if (pt->filter[len - 1]) {
2146                                 pr_err("%s: filter string not null terminated\n", __func__);
2147                                 err = -EINVAL;
2148                                 goto err_free_queues;
2149                         }
2150                 }
2151                 intel_pt_print_info_str("Filter string", pt->filter);
2152         }
2153
2154         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2155         pt->have_tsc = intel_pt_have_tsc(pt);
2156         pt->sampling_mode = false;
2157         pt->est_tsc = !pt->timeless_decoding;
2158
2159         pt->unknown_thread = thread__new(999999999, 999999999);
2160         if (!pt->unknown_thread) {
2161                 err = -ENOMEM;
2162                 goto err_free_queues;
2163         }
2164
2165         /*
2166          * Since this thread will not be kept in any rbtree not in a
2167          * list, initialize its list node so that at thread__put() the
2168          * current thread lifetime assuption is kept and we don't segfault
2169          * at list_del_init().
2170          */
2171         INIT_LIST_HEAD(&pt->unknown_thread->node);
2172
2173         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2174         if (err)
2175                 goto err_delete_thread;
2176         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2177                 err = -ENOMEM;
2178                 goto err_delete_thread;
2179         }
2180
2181         pt->auxtrace.process_event = intel_pt_process_event;
2182         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2183         pt->auxtrace.flush_events = intel_pt_flush;
2184         pt->auxtrace.free_events = intel_pt_free_events;
2185         pt->auxtrace.free = intel_pt_free;
2186         session->auxtrace = &pt->auxtrace;
2187
2188         if (dump_trace)
2189                 return 0;
2190
2191         if (pt->have_sched_switch == 1) {
2192                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2193                 if (!pt->switch_evsel) {
2194                         pr_err("%s: missing sched_switch event\n", __func__);
2195                         err = -EINVAL;
2196                         goto err_delete_thread;
2197                 }
2198         } else if (pt->have_sched_switch == 2 &&
2199                    !intel_pt_find_switch(session->evlist)) {
2200                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2201                 err = -EINVAL;
2202                 goto err_delete_thread;
2203         }
2204
2205         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2206                 pt->synth_opts = *session->itrace_synth_opts;
2207         } else {
2208                 itrace_synth_opts__set_default(&pt->synth_opts);
2209                 if (use_browser != -1) {
2210                         pt->synth_opts.branches = false;
2211                         pt->synth_opts.callchain = true;
2212                 }
2213                 if (session->itrace_synth_opts)
2214                         pt->synth_opts.thread_stack =
2215                                 session->itrace_synth_opts->thread_stack;
2216         }
2217
2218         if (pt->synth_opts.log)
2219                 intel_pt_log_enable();
2220
2221         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2222         if (pt->tc.time_mult) {
2223                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2224
2225                 if (!pt->max_non_turbo_ratio)
2226                         pt->max_non_turbo_ratio =
2227                                         (tsc_freq + 50000000) / 100000000;
2228                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2229                 intel_pt_log("Maximum non-turbo ratio %u\n",
2230                              pt->max_non_turbo_ratio);
2231         }
2232
2233         if (pt->synth_opts.calls)
2234                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2235                                        PERF_IP_FLAG_TRACE_END;
2236         if (pt->synth_opts.returns)
2237                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2238                                        PERF_IP_FLAG_TRACE_BEGIN;
2239
2240         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2241                 symbol_conf.use_callchain = true;
2242                 if (callchain_register_param(&callchain_param) < 0) {
2243                         symbol_conf.use_callchain = false;
2244                         pt->synth_opts.callchain = false;
2245                 }
2246         }
2247
2248         err = intel_pt_synth_events(pt, session);
2249         if (err)
2250                 goto err_delete_thread;
2251
2252         err = auxtrace_queues__process_index(&pt->queues, session);
2253         if (err)
2254                 goto err_delete_thread;
2255
2256         if (pt->queues.populated)
2257                 pt->data_queued = true;
2258
2259         if (pt->timeless_decoding)
2260                 pr_debug2("Intel PT decoding without timestamps\n");
2261
2262         return 0;
2263
2264 err_delete_thread:
2265         thread__zput(pt->unknown_thread);
2266 err_free_queues:
2267         intel_pt_log_disable();
2268         auxtrace_queues__free(&pt->queues);
2269         session->auxtrace = NULL;
2270 err_free:
2271         zfree(&pt->filter);
2272         free(pt);
2273         return err;
2274 }