tools/perf/util/intel-pt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_pt.c: Intel Processor Trace support
   4  * Copyright (c) 2013-2015, Intel Corporation.
   5  */
   6
   7 #include <inttypes.h>
   8 #include <stdio.h>
   9 #include <stdbool.h>
  10 #include <errno.h>
  11 #include <linux/kernel.h>
  12 #include <linux/string.h>
  13 #include <linux/types.h>
  14 #include <linux/zalloc.h>
  15
  16 #include "session.h"
  17 #include "machine.h"
  18 #include "memswap.h"
  19 #include "sort.h"
  20 #include "tool.h"
  21 #include "event.h"
  22 #include "evlist.h"
  23 #include "evsel.h"
  24 #include "map.h"
  25 #include "color.h"
  26 #include "thread.h"
  27 #include "thread-stack.h"
  28 #include "symbol.h"
  29 #include "callchain.h"
  30 #include "dso.h"
  31 #include "debug.h"
  32 #include "auxtrace.h"
  33 #include "tsc.h"
  34 #include "intel-pt.h"
  35 #include "config.h"
  36 #include "util/synthetic-events.h"
  37 #include "time-utils.h"
  38
  39 #include "../arch/x86/include/uapi/asm/perf_regs.h"
  40
  41 #include "intel-pt-decoder/intel-pt-log.h"
  42 #include "intel-pt-decoder/intel-pt-decoder.h"
  43 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  44 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  45
  46 #define MAX_TIMESTAMP (~0ULL)
  47
  48 struct range {
  49         u64 start;
  50         u64 end;
  51 };
  52
  53 struct intel_pt {
  54         struct auxtrace auxtrace;
  55         struct auxtrace_queues queues;
  56         struct auxtrace_heap heap;
  57         u32 auxtrace_type;
  58         struct perf_session *session;
  59         struct machine *machine;
  60         struct evsel *switch_evsel;
  61         struct thread *unknown_thread;
  62         bool timeless_decoding;
  63         bool sampling_mode;
  64         bool snapshot_mode;
  65         bool per_cpu_mmaps;
  66         bool have_tsc;
  67         bool data_queued;
  68         bool est_tsc;
  69         bool sync_switch;
  70         bool mispred_all;
  71         int have_sched_switch;
  72         u32 pmu_type;
  73         u64 kernel_start;
  74         u64 switch_ip;
  75         u64 ptss_ip;
  76
  77         struct perf_tsc_conversion tc;
  78         bool cap_user_time_zero;
  79
  80         struct itrace_synth_opts synth_opts;
  81
  82         bool sample_instructions;
  83         u64 instructions_sample_type;
  84         u64 instructions_id;
  85
  86         bool sample_branches;
  87         u32 branches_filter;
  88         u64 branches_sample_type;
  89         u64 branches_id;
  90
  91         bool sample_transactions;
  92         u64 transactions_sample_type;
  93         u64 transactions_id;
  94
  95         bool sample_ptwrites;
  96         u64 ptwrites_sample_type;
  97         u64 ptwrites_id;
  98
  99         bool sample_pwr_events;
 100         u64 pwr_events_sample_type;
 101         u64 mwait_id;
 102         u64 pwre_id;
 103         u64 exstop_id;
 104         u64 pwrx_id;
 105         u64 cbr_id;
 106
 107         bool sample_pebs;
 108         struct evsel *pebs_evsel;
 109
 110         u64 tsc_bit;
 111         u64 mtc_bit;
 112         u64 mtc_freq_bits;
 113         u32 tsc_ctc_ratio_n;
 114         u32 tsc_ctc_ratio_d;
 115         u64 cyc_bit;
 116         u64 noretcomp_bit;
 117         unsigned max_non_turbo_ratio;
 118         unsigned cbr2khz;
 119
 120         unsigned long num_events;
 121
 122         char *filter;
 123         struct addr_filters filts;
 124
 125         struct range *time_ranges;
 126         unsigned int range_cnt;
 127 };
 128
 129 enum switch_state {
 130         INTEL_PT_SS_NOT_TRACING,
 131         INTEL_PT_SS_UNKNOWN,
 132         INTEL_PT_SS_TRACING,
 133         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 134         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 135 };
 136
 137 struct intel_pt_queue {
 138         struct intel_pt *pt;
 139         unsigned int queue_nr;
 140         struct auxtrace_buffer *buffer;
 141         struct auxtrace_buffer *old_buffer;
 142         void *decoder;
 143         const struct intel_pt_state *state;
 144         struct ip_callchain *chain;
 145         struct branch_stack *last_branch;
 146         struct branch_stack *last_branch_rb;
 147         size_t last_branch_pos;
 148         union perf_event *event_buf;
 149         bool on_heap;
 150         bool stop;
 151         bool step_through_buffers;
 152         bool use_buffer_pid_tid;
 153         bool sync_switch;
 154         pid_t pid, tid;
 155         int cpu;
 156         int switch_state;
 157         pid_t next_tid;
 158         struct thread *thread;
 159         bool exclude_kernel;
 160         bool have_sample;
 161         u64 time;
 162         u64 timestamp;
 163         u64 sel_timestamp;
 164         bool sel_start;
 165         unsigned int sel_idx;
 166         u32 flags;
 167         u16 insn_len;
 168         u64 last_insn_cnt;
 169         u64 ipc_insn_cnt;
 170         u64 ipc_cyc_cnt;
 171         u64 last_in_insn_cnt;
 172         u64 last_in_cyc_cnt;
 173         u64 last_br_insn_cnt;
 174         u64 last_br_cyc_cnt;
 175         unsigned int cbr_seen;
 176         char insn[INTEL_PT_INSN_BUF_SZ];
 177 };
 178
 179 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 180                           unsigned char *buf, size_t len)
 181 {
 182         struct intel_pt_pkt packet;
 183         size_t pos = 0;
 184         int ret, pkt_len, i;
 185         char desc[INTEL_PT_PKT_DESC_MAX];
 186         const char *color = PERF_COLOR_BLUE;
 187         enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
 188
 189         color_fprintf(stdout, color,
 190                       ". ... Intel Processor Trace data: size %zu bytes\n",
 191                       len);
 192
 193         while (len) {
 194                 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
 195                 if (ret > 0)
 196                         pkt_len = ret;
 197                 else
 198                         pkt_len = 1;
 199                 printf(".");
 200                 color_fprintf(stdout, color, "  %08x: ", pos);
 201                 for (i = 0; i < pkt_len; i++)
 202                         color_fprintf(stdout, color, " %02x", buf[i]);
 203                 for (; i < 16; i++)
 204                         color_fprintf(stdout, color, "   ");
 205                 if (ret > 0) {
 206                         ret = intel_pt_pkt_desc(&packet, desc,
 207                                                 INTEL_PT_PKT_DESC_MAX);
 208                         if (ret > 0)
 209                                 color_fprintf(stdout, color, " %s\n", desc);
 210                 } else {
 211                         color_fprintf(stdout, color, " Bad packet!\n");
 212                 }
 213                 pos += pkt_len;
 214                 buf += pkt_len;
 215                 len -= pkt_len;
 216         }
 217 }
 218
 219 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 220                                 size_t len)
 221 {
 222         printf(".\n");
 223         intel_pt_dump(pt, buf, len);
 224 }
 225
 226 static void intel_pt_log_event(union perf_event *event)
 227 {
 228         FILE *f = intel_pt_log_fp();
 229
 230         if (!intel_pt_enable_logging || !f)
 231                 return;
 232
 233         perf_event__fprintf(event, f);
 234 }
 235
 236 static void intel_pt_dump_sample(struct perf_session *session,
 237                                  struct perf_sample *sample)
 238 {
 239         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
 240                                            auxtrace);
 241
 242         printf("\n");
 243         intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);
 244 }
 245
 246 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 247                                    struct auxtrace_buffer *b)
 248 {
 249         bool consecutive = false;
 250         void *start;
 251
 252         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 253                                       pt->have_tsc, &consecutive);
 254         if (!start)
 255                 return -EINVAL;
 256         b->use_size = b->data + b->size - start;
 257         b->use_data = start;
 258         if (b->use_size && consecutive)
 259                 b->consecutive = true;
 260         return 0;
 261 }
 262
 263 static int intel_pt_get_buffer(struct intel_pt_queue *ptq,
 264                                struct auxtrace_buffer *buffer,
 265                                struct auxtrace_buffer *old_buffer,
 266                                struct intel_pt_buffer *b)
 267 {
 268         bool might_overlap;
 269
 270         if (!buffer->data) {
 271                 int fd = perf_data__fd(ptq->pt->session->data);
 272
 273                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 274                 if (!buffer->data)
 275                         return -ENOMEM;
 276         }
 277
 278         might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
 279         if (might_overlap && !buffer->consecutive && old_buffer &&
 280             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 281                 return -ENOMEM;
 282
 283         if (buffer->use_data) {
 284                 b->len = buffer->use_size;
 285                 b->buf = buffer->use_data;
 286         } else {
 287                 b->len = buffer->size;
 288                 b->buf = buffer->data;
 289         }
 290         b->ref_timestamp = buffer->reference;
 291
 292         if (!old_buffer || (might_overlap && !buffer->consecutive)) {
 293                 b->consecutive = false;
 294                 b->trace_nr = buffer->buffer_nr + 1;
 295         } else {
 296                 b->consecutive = true;
 297         }
 298
 299         return 0;
 300 }
 301
 302 /* Do not drop buffers with references - refer intel_pt_get_trace() */
 303 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq,
 304                                            struct auxtrace_buffer *buffer)
 305 {
 306         if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer)
 307                 return;
 308
 309         auxtrace_buffer__drop_data(buffer);
 310 }
 311
 312 /* Must be serialized with respect to intel_pt_get_trace() */
 313 static int intel_pt_lookahead(void *data, intel_pt_lookahead_cb_t cb,
 314                               void *cb_data)
 315 {
 316         struct intel_pt_queue *ptq = data;
 317         struct auxtrace_buffer *buffer = ptq->buffer;
 318         struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 319         struct auxtrace_queue *queue;
 320         int err = 0;
 321
 322         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 323
 324         while (1) {
 325                 struct intel_pt_buffer b = { .len = 0 };
 326
 327                 buffer = auxtrace_buffer__next(queue, buffer);
 328                 if (!buffer)
 329                         break;
 330
 331                 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b);
 332                 if (err)
 333                         break;
 334
 335                 if (b.len) {
 336                         intel_pt_lookahead_drop_buffer(ptq, old_buffer);
 337                         old_buffer = buffer;
 338                 } else {
 339                         intel_pt_lookahead_drop_buffer(ptq, buffer);
 340                         continue;
 341                 }
 342
 343                 err = cb(&b, cb_data);
 344                 if (err)
 345                         break;
 346         }
 347
 348         if (buffer != old_buffer)
 349                 intel_pt_lookahead_drop_buffer(ptq, buffer);
 350         intel_pt_lookahead_drop_buffer(ptq, old_buffer);
 351
 352         return err;
 353 }
 354
 355 /*
 356  * This function assumes data is processed sequentially only.
 357  * Must be serialized with respect to intel_pt_lookahead()
 358  */
 359 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 360 {
 361         struct intel_pt_queue *ptq = data;
 362         struct auxtrace_buffer *buffer = ptq->buffer;
 363         struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 364         struct auxtrace_queue *queue;
 365         int err;
 366
 367         if (ptq->stop) {
 368                 b->len = 0;
 369                 return 0;
 370         }
 371
 372         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 373
 374         buffer = auxtrace_buffer__next(queue, buffer);
 375         if (!buffer) {
 376                 if (old_buffer)
 377                         auxtrace_buffer__drop_data(old_buffer);
 378                 b->len = 0;
 379                 return 0;
 380         }
 381
 382         ptq->buffer = buffer;
 383
 384         err = intel_pt_get_buffer(ptq, buffer, old_buffer, b);
 385         if (err)
 386                 return err;
 387
 388         if (ptq->step_through_buffers)
 389                 ptq->stop = true;
 390
 391         if (b->len) {
 392                 if (old_buffer)
 393                         auxtrace_buffer__drop_data(old_buffer);
 394                 ptq->old_buffer = buffer;
 395         } else {
 396                 auxtrace_buffer__drop_data(buffer);
 397                 return intel_pt_get_trace(b, data);
 398         }
 399
 400         return 0;
 401 }
 402
 403 struct intel_pt_cache_entry {
 404         struct auxtrace_cache_entry     entry;
 405         u64                             insn_cnt;
 406         u64                             byte_cnt;
 407         enum intel_pt_insn_op           op;
 408         enum intel_pt_insn_branch       branch;
 409         int                             length;
 410         int32_t                         rel;
 411         char                            insn[INTEL_PT_INSN_BUF_SZ];
 412 };
 413
 414 static int intel_pt_config_div(const char *var, const char *value, void *data)
 415 {
 416         int *d = data;
 417         long val;
 418
 419         if (!strcmp(var, "intel-pt.cache-divisor")) {
 420                 val = strtol(value, NULL, 0);
 421                 if (val > 0 && val <= INT_MAX)
 422                         *d = val;
 423         }
 424
 425         return 0;
 426 }
 427
 428 static int intel_pt_cache_divisor(void)
 429 {
 430         static int d;
 431
 432         if (d)
 433                 return d;
 434
 435         perf_config(intel_pt_config_div, &d);
 436
 437         if (!d)
 438                 d = 64;
 439
 440         return d;
 441 }
 442
 443 static unsigned int intel_pt_cache_size(struct dso *dso,
 444                                         struct machine *machine)
 445 {
 446         off_t size;
 447
 448         size = dso__data_size(dso, machine);
 449         size /= intel_pt_cache_divisor();
 450         if (size < 1000)
 451                 return 10;
 452         if (size > (1 << 21))
 453                 return 21;
 454         return 32 - __builtin_clz(size);
 455 }
 456
 457 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 458                                              struct machine *machine)
 459 {
 460         struct auxtrace_cache *c;
 461         unsigned int bits;
 462
 463         if (dso->auxtrace_cache)
 464                 return dso->auxtrace_cache;
 465
 466         bits = intel_pt_cache_size(dso, machine);
 467
 468         /* Ignoring cache creation failure */
 469         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 470
 471         dso->auxtrace_cache = c;
 472
 473         return c;
 474 }
 475
 476 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 477                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 478                               struct intel_pt_insn *intel_pt_insn)
 479 {
 480         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 481         struct intel_pt_cache_entry *e;
 482         int err;
 483
 484         if (!c)
 485                 return -ENOMEM;
 486
 487         e = auxtrace_cache__alloc_entry(c);
 488         if (!e)
 489                 return -ENOMEM;
 490
 491         e->insn_cnt = insn_cnt;
 492         e->byte_cnt = byte_cnt;
 493         e->op = intel_pt_insn->op;
 494         e->branch = intel_pt_insn->branch;
 495         e->length = intel_pt_insn->length;
 496         e->rel = intel_pt_insn->rel;
 497         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 498
 499         err = auxtrace_cache__add(c, offset, &e->entry);
 500         if (err)
 501                 auxtrace_cache__free_entry(c, e);
 502
 503         return err;
 504 }
 505
 506 static struct intel_pt_cache_entry *
 507 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 508 {
 509         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 510
 511         if (!c)
 512                 return NULL;
 513
 514         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 515 }
 516
 517 static inline u8 intel_pt_cpumode(struct intel_pt *pt, uint64_t ip)
 518 {
 519         return ip >= pt->kernel_start ?
 520                PERF_RECORD_MISC_KERNEL :
 521                PERF_RECORD_MISC_USER;
 522 }
 523
 524 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 525                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 526                                    uint64_t to_ip, uint64_t max_insn_cnt,
 527                                    void *data)
 528 {
 529         struct intel_pt_queue *ptq = data;
 530         struct machine *machine = ptq->pt->machine;
 531         struct thread *thread;
 532         struct addr_location al;
 533         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 534         ssize_t len;
 535         int x86_64;
 536         u8 cpumode;
 537         u64 offset, start_offset, start_ip;
 538         u64 insn_cnt = 0;
 539         bool one_map = true;
 540
 541         intel_pt_insn->length = 0;
 542
 543         if (to_ip && *ip == to_ip)
 544                 goto out_no_cache;
 545
 546         cpumode = intel_pt_cpumode(ptq->pt, *ip);
 547
 548         thread = ptq->thread;
 549         if (!thread) {
 550                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 551                         return -EINVAL;
 552                 thread = ptq->pt->unknown_thread;
 553         }
 554
 555         while (1) {
 556                 if (!thread__find_map(thread, cpumode, *ip, &al) || !al.map->dso)
 557                         return -EINVAL;
 558
 559                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 560                     dso__data_status_seen(al.map->dso,
 561                                           DSO_DATA_STATUS_SEEN_ITRACE))
 562                         return -ENOENT;
 563
 564                 offset = al.map->map_ip(al.map, *ip);
 565
 566                 if (!to_ip && one_map) {
 567                         struct intel_pt_cache_entry *e;
 568
 569                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 570                         if (e &&
 571                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 572                                 *insn_cnt_ptr = e->insn_cnt;
 573                                 *ip += e->byte_cnt;
 574                                 intel_pt_insn->op = e->op;
 575                                 intel_pt_insn->branch = e->branch;
 576                                 intel_pt_insn->length = e->length;
 577                                 intel_pt_insn->rel = e->rel;
 578                                 memcpy(intel_pt_insn->buf, e->insn,
 579                                        INTEL_PT_INSN_BUF_SZ);
 580                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 581                                 return 0;
 582                         }
 583                 }
 584
 585                 start_offset = offset;
 586                 start_ip = *ip;
 587
 588                 /* Load maps to ensure dso->is_64_bit has been updated */
 589                 map__load(al.map);
 590
 591                 x86_64 = al.map->dso->is_64_bit;
 592
 593                 while (1) {
 594                         len = dso__data_read_offset(al.map->dso, machine,
 595                                                     offset, buf,
 596                                                     INTEL_PT_INSN_BUF_SZ);
 597                         if (len <= 0)
 598                                 return -EINVAL;
 599
 600                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 601                                 return -EINVAL;
 602
 603                         intel_pt_log_insn(intel_pt_insn, *ip);
 604
 605                         insn_cnt += 1;
 606
 607                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 608                                 goto out;
 609
 610                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 611                                 goto out_no_cache;
 612
 613                         *ip += intel_pt_insn->length;
 614
 615                         if (to_ip && *ip == to_ip)
 616                                 goto out_no_cache;
 617
 618                         if (*ip >= al.map->end)
 619                                 break;
 620
 621                         offset += intel_pt_insn->length;
 622                 }
 623                 one_map = false;
 624         }
 625 out:
 626         *insn_cnt_ptr = insn_cnt;
 627
 628         if (!one_map)
 629                 goto out_no_cache;
 630
 631         /*
 632          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 633          * entries.
 634          */
 635         if (to_ip) {
 636                 struct intel_pt_cache_entry *e;
 637
 638                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 639                 if (e)
 640                         return 0;
 641         }
 642
 643         /* Ignore cache errors */
 644         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 645                            *ip - start_ip, intel_pt_insn);
 646
 647         return 0;
 648
 649 out_no_cache:
 650         *insn_cnt_ptr = insn_cnt;
 651         return 0;
 652 }
 653
 654 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 655                                   uint64_t offset, const char *filename)
 656 {
 657         struct addr_filter *filt;
 658         bool have_filter   = false;
 659         bool hit_tracestop = false;
 660         bool hit_filter    = false;
 661
 662         list_for_each_entry(filt, &pt->filts.head, list) {
 663                 if (filt->start)
 664                         have_filter = true;
 665
 666                 if ((filename && !filt->filename) ||
 667                     (!filename && filt->filename) ||
 668                     (filename && strcmp(filename, filt->filename)))
 669                         continue;
 670
 671                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 672                         continue;
 673
 674                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 675                              ip, offset, filename ? filename : "[kernel]",
 676                              filt->start ? "filter" : "stop",
 677                              filt->addr, filt->size);
 678
 679                 if (filt->start)
 680                         hit_filter = true;
 681                 else
 682                         hit_tracestop = true;
 683         }
 684
 685         if (!hit_tracestop && !hit_filter)
 686                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 687                              ip, offset, filename ? filename : "[kernel]");
 688
 689         return hit_tracestop || (have_filter && !hit_filter);
 690 }
 691
 692 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 693 {
 694         struct intel_pt_queue *ptq = data;
 695         struct thread *thread;
 696         struct addr_location al;
 697         u8 cpumode;
 698         u64 offset;
 699
 700         if (ip >= ptq->pt->kernel_start)
 701                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 702
 703         cpumode = PERF_RECORD_MISC_USER;
 704
 705         thread = ptq->thread;
 706         if (!thread)
 707                 return -EINVAL;
 708
 709         if (!thread__find_map(thread, cpumode, ip, &al) || !al.map->dso)
 710                 return -EINVAL;
 711
 712         offset = al.map->map_ip(al.map, ip);
 713
 714         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 715                                      al.map->dso->long_name);
 716 }
 717
 718 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 719 {
 720         return __intel_pt_pgd_ip(ip, data) > 0;
 721 }
 722
 723 static bool intel_pt_get_config(struct intel_pt *pt,
 724                                 struct perf_event_attr *attr, u64 *config)
 725 {
 726         if (attr->type == pt->pmu_type) {
 727                 if (config)
 728                         *config = attr->config;
 729                 return true;
 730         }
 731
 732         return false;
 733 }
 734
 735 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 736 {
 737         struct evsel *evsel;
 738
 739         evlist__for_each_entry(pt->session->evlist, evsel) {
 740                 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
 741                     !evsel->core.attr.exclude_kernel)
 742                         return false;
 743         }
 744         return true;
 745 }
 746
 747 static bool intel_pt_return_compression(struct intel_pt *pt)
 748 {
 749         struct evsel *evsel;
 750         u64 config;
 751
 752         if (!pt->noretcomp_bit)
 753                 return true;
 754
 755         evlist__for_each_entry(pt->session->evlist, evsel) {
 756                 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
 757                     (config & pt->noretcomp_bit))
 758                         return false;
 759         }
 760         return true;
 761 }
 762
 763 static bool intel_pt_branch_enable(struct intel_pt *pt)
 764 {
 765         struct evsel *evsel;
 766         u64 config;
 767
 768         evlist__for_each_entry(pt->session->evlist, evsel) {
 769                 if (intel_pt_get_config(pt, &evsel->core.attr, &config) &&
 770                     (config & 1) && !(config & 0x2000))
 771                         return false;
 772         }
 773         return true;
 774 }
 775
 776 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 777 {
 778         struct evsel *evsel;
 779         unsigned int shift;
 780         u64 config;
 781
 782         if (!pt->mtc_freq_bits)
 783                 return 0;
 784
 785         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 786                 config >>= 1;
 787
 788         evlist__for_each_entry(pt->session->evlist, evsel) {
 789                 if (intel_pt_get_config(pt, &evsel->core.attr, &config))
 790                         return (config & pt->mtc_freq_bits) >> shift;
 791         }
 792         return 0;
 793 }
 794
 795 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 796 {
 797         struct evsel *evsel;
 798         bool timeless_decoding = true;
 799         u64 config;
 800
 801         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 802                 return true;
 803
 804         evlist__for_each_entry(pt->session->evlist, evsel) {
 805                 if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME))
 806                         return true;
 807                 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
 808                         if (config & pt->tsc_bit)
 809                                 timeless_decoding = false;
 810                         else
 811                                 return true;
 812                 }
 813         }
 814         return timeless_decoding;
 815 }
 816
 817 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 818 {
 819         struct evsel *evsel;
 820
 821         evlist__for_each_entry(pt->session->evlist, evsel) {
 822                 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) &&
 823                     !evsel->core.attr.exclude_kernel)
 824                         return true;
 825         }
 826         return false;
 827 }
 828
 829 static bool intel_pt_have_tsc(struct intel_pt *pt)
 830 {
 831         struct evsel *evsel;
 832         bool have_tsc = false;
 833         u64 config;
 834
 835         if (!pt->tsc_bit)
 836                 return false;
 837
 838         evlist__for_each_entry(pt->session->evlist, evsel) {
 839                 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) {
 840                         if (config & pt->tsc_bit)
 841                                 have_tsc = true;
 842                         else
 843                                 return false;
 844                 }
 845         }
 846         return have_tsc;
 847 }
 848
 849 static bool intel_pt_sampling_mode(struct intel_pt *pt)
 850 {
 851         struct evsel *evsel;
 852
 853         evlist__for_each_entry(pt->session->evlist, evsel) {
 854                 if ((evsel->core.attr.sample_type & PERF_SAMPLE_AUX) &&
 855                     evsel->core.attr.aux_sample_size)
 856                         return true;
 857         }
 858         return false;
 859 }
 860
 861 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 862 {
 863         u64 quot, rem;
 864
 865         quot = ns / pt->tc.time_mult;
 866         rem  = ns % pt->tc.time_mult;
 867         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 868                 pt->tc.time_mult;
 869 }
 870
 871 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 872                                                    unsigned int queue_nr)
 873 {
 874         struct intel_pt_params params = { .get_trace = 0, };
 875         struct perf_env *env = pt->machine->env;
 876         struct intel_pt_queue *ptq;
 877
 878         ptq = zalloc(sizeof(struct intel_pt_queue));
 879         if (!ptq)
 880                 return NULL;
 881
 882         if (pt->synth_opts.callchain) {
 883                 size_t sz = sizeof(struct ip_callchain);
 884
 885                 /* Add 1 to callchain_sz for callchain context */
 886                 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
 887                 ptq->chain = zalloc(sz);
 888                 if (!ptq->chain)
 889                         goto out_free;
 890         }
 891
 892         if (pt->synth_opts.last_branch) {
 893                 size_t sz = sizeof(struct branch_stack);
 894
 895                 sz += pt->synth_opts.last_branch_sz *
 896                       sizeof(struct branch_entry);
 897                 ptq->last_branch = zalloc(sz);
 898                 if (!ptq->last_branch)
 899                         goto out_free;
 900                 ptq->last_branch_rb = zalloc(sz);
 901                 if (!ptq->last_branch_rb)
 902                         goto out_free;
 903         }
 904
 905         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 906         if (!ptq->event_buf)
 907                 goto out_free;
 908
 909         ptq->pt = pt;
 910         ptq->queue_nr = queue_nr;
 911         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 912         ptq->pid = -1;
 913         ptq->tid = -1;
 914         ptq->cpu = -1;
 915         ptq->next_tid = -1;
 916
 917         params.get_trace = intel_pt_get_trace;
 918         params.walk_insn = intel_pt_walk_next_insn;
 919         params.lookahead = intel_pt_lookahead;
 920         params.data = ptq;
 921         params.return_compression = intel_pt_return_compression(pt);
 922         params.branch_enable = intel_pt_branch_enable(pt);
 923         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 924         params.mtc_period = intel_pt_mtc_period(pt);
 925         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 926         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 927
 928         if (pt->filts.cnt > 0)
 929                 params.pgd_ip = intel_pt_pgd_ip;
 930
 931         if (pt->synth_opts.instructions) {
 932                 if (pt->synth_opts.period) {
 933                         switch (pt->synth_opts.period_type) {
 934                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 935                                 params.period_type =
 936                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 937                                 params.period = pt->synth_opts.period;
 938                                 break;
 939                         case PERF_ITRACE_PERIOD_TICKS:
 940                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 941                                 params.period = pt->synth_opts.period;
 942                                 break;
 943                         case PERF_ITRACE_PERIOD_NANOSECS:
 944                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 945                                 params.period = intel_pt_ns_to_ticks(pt,
 946                                                         pt->synth_opts.period);
 947                                 break;
 948                         default:
 949                                 break;
 950                         }
 951                 }
 952
 953                 if (!params.period) {
 954                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 955                         params.period = 1;
 956                 }
 957         }
 958
 959         if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
 960                 params.flags |= INTEL_PT_FUP_WITH_NLIP;
 961
 962         ptq->decoder = intel_pt_decoder_new(&params);
 963         if (!ptq->decoder)
 964                 goto out_free;
 965
 966         return ptq;
 967
 968 out_free:
 969         zfree(&ptq->event_buf);
 970         zfree(&ptq->last_branch);
 971         zfree(&ptq->last_branch_rb);
 972         zfree(&ptq->chain);
 973         free(ptq);
 974         return NULL;
 975 }
 976
 977 static void intel_pt_free_queue(void *priv)
 978 {
 979         struct intel_pt_queue *ptq = priv;
 980
 981         if (!ptq)
 982                 return;
 983         thread__zput(ptq->thread);
 984         intel_pt_decoder_free(ptq->decoder);
 985         zfree(&ptq->event_buf);
 986         zfree(&ptq->last_branch);
 987         zfree(&ptq->last_branch_rb);
 988         zfree(&ptq->chain);
 989         free(ptq);
 990 }
 991
 992 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 993                                      struct auxtrace_queue *queue)
 994 {
 995         struct intel_pt_queue *ptq = queue->priv;
 996
 997         if (queue->tid == -1 || pt->have_sched_switch) {
 998                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 999                 thread__zput(ptq->thread);
1000         }
1001
1002         if (!ptq->thread && ptq->tid != -1)
1003                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
1004
1005         if (ptq->thread) {
1006                 ptq->pid = ptq->thread->pid_;
1007                 if (queue->cpu == -1)
1008                         ptq->cpu = ptq->thread->cpu;
1009         }
1010 }
1011
1012 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
1013 {
1014         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
1015                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
1016         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
1017                 if (ptq->state->to_ip)
1018                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
1019                                      PERF_IP_FLAG_ASYNC |
1020                                      PERF_IP_FLAG_INTERRUPT;
1021                 else
1022                         ptq->flags = PERF_IP_FLAG_BRANCH |
1023                                      PERF_IP_FLAG_TRACE_END;
1024                 ptq->insn_len = 0;
1025         } else {
1026                 if (ptq->state->from_ip)
1027                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
1028                 else
1029                         ptq->flags = PERF_IP_FLAG_BRANCH |
1030                                      PERF_IP_FLAG_TRACE_BEGIN;
1031                 if (ptq->state->flags & INTEL_PT_IN_TX)
1032                         ptq->flags |= PERF_IP_FLAG_IN_TX;
1033                 ptq->insn_len = ptq->state->insn_len;
1034                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
1035         }
1036
1037         if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
1038                 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;
1039         if (ptq->state->type & INTEL_PT_TRACE_END)
1040                 ptq->flags |= PERF_IP_FLAG_TRACE_END;
1041 }
1042
1043 static void intel_pt_setup_time_range(struct intel_pt *pt,
1044                                       struct intel_pt_queue *ptq)
1045 {
1046         if (!pt->range_cnt)
1047                 return;
1048
1049         ptq->sel_timestamp = pt->time_ranges[0].start;
1050         ptq->sel_idx = 0;
1051
1052         if (ptq->sel_timestamp) {
1053                 ptq->sel_start = true;
1054         } else {
1055                 ptq->sel_timestamp = pt->time_ranges[0].end;
1056                 ptq->sel_start = false;
1057         }
1058 }
1059
1060 static int intel_pt_setup_queue(struct intel_pt *pt,
1061                                 struct auxtrace_queue *queue,
1062                                 unsigned int queue_nr)
1063 {
1064         struct intel_pt_queue *ptq = queue->priv;
1065
1066         if (list_empty(&queue->head))
1067                 return 0;
1068
1069         if (!ptq) {
1070                 ptq = intel_pt_alloc_queue(pt, queue_nr);
1071                 if (!ptq)
1072                         return -ENOMEM;
1073                 queue->priv = ptq;
1074
1075                 if (queue->cpu != -1)
1076                         ptq->cpu = queue->cpu;
1077                 ptq->tid = queue->tid;
1078
1079                 ptq->cbr_seen = UINT_MAX;
1080
1081                 if (pt->sampling_mode && !pt->snapshot_mode &&
1082                     pt->timeless_decoding)
1083                         ptq->step_through_buffers = true;
1084
1085                 ptq->sync_switch = pt->sync_switch;
1086
1087                 intel_pt_setup_time_range(pt, ptq);
1088         }
1089
1090         if (!ptq->on_heap &&
1091             (!ptq->sync_switch ||
1092              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
1093                 const struct intel_pt_state *state;
1094                 int ret;
1095
1096                 if (pt->timeless_decoding)
1097                         return 0;
1098
1099                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
1100                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1101                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1102
1103                 if (ptq->sel_start && ptq->sel_timestamp) {
1104                         ret = intel_pt_fast_forward(ptq->decoder,
1105                                                     ptq->sel_timestamp);
1106                         if (ret)
1107                                 return ret;
1108                 }
1109
1110                 while (1) {
1111                         state = intel_pt_decode(ptq->decoder);
1112                         if (state->err) {
1113                                 if (state->err == INTEL_PT_ERR_NODATA) {
1114                                         intel_pt_log("queue %u has no timestamp\n",
1115                                                      queue_nr);
1116                                         return 0;
1117                                 }
1118                                 continue;
1119                         }
1120                         if (state->timestamp)
1121                                 break;
1122                 }
1123
1124                 ptq->timestamp = state->timestamp;
1125                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
1126                              queue_nr, ptq->timestamp);
1127                 ptq->state = state;
1128                 ptq->have_sample = true;
1129                 if (ptq->sel_start && ptq->sel_timestamp &&
1130                     ptq->timestamp < ptq->sel_timestamp)
1131                         ptq->have_sample = false;
1132                 intel_pt_sample_flags(ptq);
1133                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
1134                 if (ret)
1135                         return ret;
1136                 ptq->on_heap = true;
1137         }
1138
1139         return 0;
1140 }
1141
1142 static int intel_pt_setup_queues(struct intel_pt *pt)
1143 {
1144         unsigned int i;
1145         int ret;
1146
1147         for (i = 0; i < pt->queues.nr_queues; i++) {
1148                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1149                 if (ret)
1150                         return ret;
1151         }
1152         return 0;
1153 }
1154
1155 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1156 {
1157         struct branch_stack *bs_src = ptq->last_branch_rb;
1158         struct branch_stack *bs_dst = ptq->last_branch;
1159         size_t nr = 0;
1160
1161         bs_dst->nr = bs_src->nr;
1162
1163         if (!bs_src->nr)
1164                 return;
1165
1166         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1167         memcpy(&bs_dst->entries[0],
1168                &bs_src->entries[ptq->last_branch_pos],
1169                sizeof(struct branch_entry) * nr);
1170
1171         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1172                 memcpy(&bs_dst->entries[nr],
1173                        &bs_src->entries[0],
1174                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1175         }
1176 }
1177
1178 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1179 {
1180         ptq->last_branch_pos = 0;
1181         ptq->last_branch_rb->nr = 0;
1182 }
1183
1184 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1185 {
1186         const struct intel_pt_state *state = ptq->state;
1187         struct branch_stack *bs = ptq->last_branch_rb;
1188         struct branch_entry *be;
1189
1190         if (!ptq->last_branch_pos)
1191                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1192
1193         ptq->last_branch_pos -= 1;
1194
1195         be              = &bs->entries[ptq->last_branch_pos];
1196         be->from        = state->from_ip;
1197         be->to          = state->to_ip;
1198         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1199         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1200         /* No support for mispredict */
1201         be->flags.mispred = ptq->pt->mispred_all;
1202
1203         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1204                 bs->nr += 1;
1205 }
1206
1207 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1208 {
1209         return pt->synth_opts.initial_skip &&
1210                pt->num_events++ < pt->synth_opts.initial_skip;
1211 }
1212
1213 /*
1214  * Cannot count CBR as skipped because it won't go away until cbr == cbr_seen.
1215  * Also ensure CBR is first non-skipped event by allowing for 4 more samples
1216  * from this decoder state.
1217  */
1218 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt)
1219 {
1220         return pt->synth_opts.initial_skip &&
1221                pt->num_events + 4 < pt->synth_opts.initial_skip;
1222 }
1223
1224 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq,
1225                                    union perf_event *event,
1226                                    struct perf_sample *sample)
1227 {
1228         event->sample.header.type = PERF_RECORD_SAMPLE;
1229         event->sample.header.size = sizeof(struct perf_event_header);
1230
1231         sample->pid = ptq->pid;
1232         sample->tid = ptq->tid;
1233         sample->cpu = ptq->cpu;
1234         sample->insn_len = ptq->insn_len;
1235         memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1236 }
1237
1238 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1239                                    struct intel_pt_queue *ptq,
1240                                    union perf_event *event,
1241                                    struct perf_sample *sample)
1242 {
1243         intel_pt_prep_a_sample(ptq, event, sample);
1244
1245         if (!pt->timeless_decoding)
1246                 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1247
1248         sample->ip = ptq->state->from_ip;
1249         sample->cpumode = intel_pt_cpumode(pt, sample->ip);
1250         sample->addr = ptq->state->to_ip;
1251         sample->period = 1;
1252         sample->flags = ptq->flags;
1253
1254         event->sample.header.misc = sample->cpumode;
1255 }
1256
1257 static int intel_pt_inject_event(union perf_event *event,
1258                                  struct perf_sample *sample, u64 type)
1259 {
1260         event->header.size = perf_event__sample_event_size(sample, type, 0);
1261         return perf_event__synthesize_sample(event, type, 0, sample);
1262 }
1263
1264 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1265                                       union perf_event *event,
1266                                       struct perf_sample *sample, u64 type)
1267 {
1268         if (!pt->synth_opts.inject)
1269                 return 0;
1270
1271         return intel_pt_inject_event(event, sample, type);
1272 }
1273
1274 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1275                                           union perf_event *event,
1276                                           struct perf_sample *sample, u64 type)
1277 {
1278         int ret;
1279
1280         ret = intel_pt_opt_inject(pt, event, sample, type);
1281         if (ret)
1282                 return ret;
1283
1284         ret = perf_session__deliver_synth_event(pt->session, event, sample);
1285         if (ret)
1286                 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1287
1288         return ret;
1289 }
1290
1291 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1292 {
1293         struct intel_pt *pt = ptq->pt;
1294         union perf_event *event = ptq->event_buf;
1295         struct perf_sample sample = { .ip = 0, };
1296         struct dummy_branch_stack {
1297                 u64                     nr;
1298                 struct branch_entry     entries;
1299         } dummy_bs;
1300
1301         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1302                 return 0;
1303
1304         if (intel_pt_skip_event(pt))
1305                 return 0;
1306
1307         intel_pt_prep_b_sample(pt, ptq, event, &sample);
1308
1309         sample.id = ptq->pt->branches_id;
1310         sample.stream_id = ptq->pt->branches_id;
1311
1312         /*
1313          * perf report cannot handle events without a branch stack when using
1314          * SORT_MODE__BRANCH so make a dummy one.
1315          */
1316         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1317                 dummy_bs = (struct dummy_branch_stack){
1318                         .nr = 1,
1319                         .entries = {
1320                                 .from = sample.ip,
1321                                 .to = sample.addr,
1322                         },
1323                 };
1324                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1325         }
1326
1327         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
1328         if (sample.cyc_cnt) {
1329                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
1330                 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
1331                 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
1332         }
1333
1334         return intel_pt_deliver_synth_b_event(pt, event, &sample,
1335                                               pt->branches_sample_type);
1336 }
1337
1338 static void intel_pt_prep_sample(struct intel_pt *pt,
1339                                  struct intel_pt_queue *ptq,
1340                                  union perf_event *event,
1341                                  struct perf_sample *sample)
1342 {
1343         intel_pt_prep_b_sample(pt, ptq, event, sample);
1344
1345         if (pt->synth_opts.callchain) {
1346                 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1347                                      pt->synth_opts.callchain_sz + 1,
1348                                      sample->ip, pt->kernel_start);
1349                 sample->callchain = ptq->chain;
1350         }
1351
1352         if (pt->synth_opts.last_branch) {
1353                 intel_pt_copy_last_branch_rb(ptq);
1354                 sample->branch_stack = ptq->last_branch;
1355         }
1356 }
1357
1358 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1359                                                struct intel_pt_queue *ptq,
1360                                                union perf_event *event,
1361                                                struct perf_sample *sample,
1362                                                u64 type)
1363 {
1364         int ret;
1365
1366         ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1367
1368         if (pt->synth_opts.last_branch)
1369                 intel_pt_reset_last_branch_rb(ptq);
1370
1371         return ret;
1372 }
1373
1374 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1375 {
1376         struct intel_pt *pt = ptq->pt;
1377         union perf_event *event = ptq->event_buf;
1378         struct perf_sample sample = { .ip = 0, };
1379
1380         if (intel_pt_skip_event(pt))
1381                 return 0;
1382
1383         intel_pt_prep_sample(pt, ptq, event, &sample);
1384
1385         sample.id = ptq->pt->instructions_id;
1386         sample.stream_id = ptq->pt->instructions_id;
1387         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1388
1389         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
1390         if (sample.cyc_cnt) {
1391                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
1392                 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
1393                 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
1394         }
1395
1396         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1397
1398         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1399                                             pt->instructions_sample_type);
1400 }
1401
1402 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1403 {
1404         struct intel_pt *pt = ptq->pt;
1405         union perf_event *event = ptq->event_buf;
1406         struct perf_sample sample = { .ip = 0, };
1407
1408         if (intel_pt_skip_event(pt))
1409                 return 0;
1410
1411         intel_pt_prep_sample(pt, ptq, event, &sample);
1412
1413         sample.id = ptq->pt->transactions_id;
1414         sample.stream_id = ptq->pt->transactions_id;
1415
1416         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1417                                             pt->transactions_sample_type);
1418 }
1419
1420 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1421                                    struct intel_pt_queue *ptq,
1422                                    union perf_event *event,
1423                                    struct perf_sample *sample)
1424 {
1425         intel_pt_prep_sample(pt, ptq, event, sample);
1426
1427         /*
1428          * Zero IP is used to mean "trace start" but that is not the case for
1429          * power or PTWRITE events with no IP, so clear the flags.
1430          */
1431         if (!sample->ip)
1432                 sample->flags = 0;
1433 }
1434
1435 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1436 {
1437         struct intel_pt *pt = ptq->pt;
1438         union perf_event *event = ptq->event_buf;
1439         struct perf_sample sample = { .ip = 0, };
1440         struct perf_synth_intel_ptwrite raw;
1441
1442         if (intel_pt_skip_event(pt))
1443                 return 0;
1444
1445         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1446
1447         sample.id = ptq->pt->ptwrites_id;
1448         sample.stream_id = ptq->pt->ptwrites_id;
1449
1450         raw.flags = 0;
1451         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1452         raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1453
1454         sample.raw_size = perf_synth__raw_size(raw);
1455         sample.raw_data = perf_synth__raw_data(&raw);
1456
1457         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1458                                             pt->ptwrites_sample_type);
1459 }
1460
1461 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1462 {
1463         struct intel_pt *pt = ptq->pt;
1464         union perf_event *event = ptq->event_buf;
1465         struct perf_sample sample = { .ip = 0, };
1466         struct perf_synth_intel_cbr raw;
1467         u32 flags;
1468
1469         if (intel_pt_skip_cbr_event(pt))
1470                 return 0;
1471
1472         ptq->cbr_seen = ptq->state->cbr;
1473
1474         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1475
1476         sample.id = ptq->pt->cbr_id;
1477         sample.stream_id = ptq->pt->cbr_id;
1478
1479         flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1480         raw.flags = cpu_to_le32(flags);
1481         raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1482         raw.reserved3 = 0;
1483
1484         sample.raw_size = perf_synth__raw_size(raw);
1485         sample.raw_data = perf_synth__raw_data(&raw);
1486
1487         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1488                                             pt->pwr_events_sample_type);
1489 }
1490
1491 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1492 {
1493         struct intel_pt *pt = ptq->pt;
1494         union perf_event *event = ptq->event_buf;
1495         struct perf_sample sample = { .ip = 0, };
1496         struct perf_synth_intel_mwait raw;
1497
1498         if (intel_pt_skip_event(pt))
1499                 return 0;
1500
1501         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1502
1503         sample.id = ptq->pt->mwait_id;
1504         sample.stream_id = ptq->pt->mwait_id;
1505
1506         raw.reserved = 0;
1507         raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1508
1509         sample.raw_size = perf_synth__raw_size(raw);
1510         sample.raw_data = perf_synth__raw_data(&raw);
1511
1512         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1513                                             pt->pwr_events_sample_type);
1514 }
1515
1516 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1517 {
1518         struct intel_pt *pt = ptq->pt;
1519         union perf_event *event = ptq->event_buf;
1520         struct perf_sample sample = { .ip = 0, };
1521         struct perf_synth_intel_pwre raw;
1522
1523         if (intel_pt_skip_event(pt))
1524                 return 0;
1525
1526         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1527
1528         sample.id = ptq->pt->pwre_id;
1529         sample.stream_id = ptq->pt->pwre_id;
1530
1531         raw.reserved = 0;
1532         raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1533
1534         sample.raw_size = perf_synth__raw_size(raw);
1535         sample.raw_data = perf_synth__raw_data(&raw);
1536
1537         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1538                                             pt->pwr_events_sample_type);
1539 }
1540
1541 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1542 {
1543         struct intel_pt *pt = ptq->pt;
1544         union perf_event *event = ptq->event_buf;
1545         struct perf_sample sample = { .ip = 0, };
1546         struct perf_synth_intel_exstop raw;
1547
1548         if (intel_pt_skip_event(pt))
1549                 return 0;
1550
1551         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1552
1553         sample.id = ptq->pt->exstop_id;
1554         sample.stream_id = ptq->pt->exstop_id;
1555
1556         raw.flags = 0;
1557         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1558
1559         sample.raw_size = perf_synth__raw_size(raw);
1560         sample.raw_data = perf_synth__raw_data(&raw);
1561
1562         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1563                                             pt->pwr_events_sample_type);
1564 }
1565
1566 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1567 {
1568         struct intel_pt *pt = ptq->pt;
1569         union perf_event *event = ptq->event_buf;
1570         struct perf_sample sample = { .ip = 0, };
1571         struct perf_synth_intel_pwrx raw;
1572
1573         if (intel_pt_skip_event(pt))
1574                 return 0;
1575
1576         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1577
1578         sample.id = ptq->pt->pwrx_id;
1579         sample.stream_id = ptq->pt->pwrx_id;
1580
1581         raw.reserved = 0;
1582         raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1583
1584         sample.raw_size = perf_synth__raw_size(raw);
1585         sample.raw_data = perf_synth__raw_data(&raw);
1586
1587         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1588                                             pt->pwr_events_sample_type);
1589 }
1590
1591 /*
1592  * PEBS gp_regs array indexes plus 1 so that 0 means not present. Refer
1593  * intel_pt_add_gp_regs().
1594  */
1595 static const int pebs_gp_regs[] = {
1596         [PERF_REG_X86_FLAGS]    = 1,
1597         [PERF_REG_X86_IP]       = 2,
1598         [PERF_REG_X86_AX]       = 3,
1599         [PERF_REG_X86_CX]       = 4,
1600         [PERF_REG_X86_DX]       = 5,
1601         [PERF_REG_X86_BX]       = 6,
1602         [PERF_REG_X86_SP]       = 7,
1603         [PERF_REG_X86_BP]       = 8,
1604         [PERF_REG_X86_SI]       = 9,
1605         [PERF_REG_X86_DI]       = 10,
1606         [PERF_REG_X86_R8]       = 11,
1607         [PERF_REG_X86_R9]       = 12,
1608         [PERF_REG_X86_R10]      = 13,
1609         [PERF_REG_X86_R11]      = 14,
1610         [PERF_REG_X86_R12]      = 15,
1611         [PERF_REG_X86_R13]      = 16,
1612         [PERF_REG_X86_R14]      = 17,
1613         [PERF_REG_X86_R15]      = 18,
1614 };
1615
1616 static u64 *intel_pt_add_gp_regs(struct regs_dump *intr_regs, u64 *pos,
1617                                  const struct intel_pt_blk_items *items,
1618                                  u64 regs_mask)
1619 {
1620         const u64 *gp_regs = items->val[INTEL_PT_GP_REGS_POS];
1621         u32 mask = items->mask[INTEL_PT_GP_REGS_POS];
1622         u32 bit;
1623         int i;
1624
1625         for (i = 0, bit = 1; i < PERF_REG_X86_64_MAX; i++, bit <<= 1) {
1626                 /* Get the PEBS gp_regs array index */
1627                 int n = pebs_gp_regs[i] - 1;
1628
1629                 if (n < 0)
1630                         continue;
1631                 /*
1632                  * Add only registers that were requested (i.e. 'regs_mask') and
1633                  * that were provided (i.e. 'mask'), and update the resulting
1634                  * mask (i.e. 'intr_regs->mask') accordingly.
1635                  */
1636                 if (mask & 1 << n && regs_mask & bit) {
1637                         intr_regs->mask |= bit;
1638                         *pos++ = gp_regs[n];
1639                 }
1640         }
1641
1642         return pos;
1643 }
1644
1645 #ifndef PERF_REG_X86_XMM0
1646 #define PERF_REG_X86_XMM0 32
1647 #endif
1648
1649 static void intel_pt_add_xmm(struct regs_dump *intr_regs, u64 *pos,
1650                              const struct intel_pt_blk_items *items,
1651                              u64 regs_mask)
1652 {
1653         u32 mask = items->has_xmm & (regs_mask >> PERF_REG_X86_XMM0);
1654         const u64 *xmm = items->xmm;
1655
1656         /*
1657          * If there are any XMM registers, then there should be all of them.
1658          * Nevertheless, follow the logic to add only registers that were
1659          * requested (i.e. 'regs_mask') and that were provided (i.e. 'mask'),
1660          * and update the resulting mask (i.e. 'intr_regs->mask') accordingly.
1661          */
1662         intr_regs->mask |= (u64)mask << PERF_REG_X86_XMM0;
1663
1664         for (; mask; mask >>= 1, xmm++) {
1665                 if (mask & 1)
1666                         *pos++ = *xmm;
1667         }
1668 }
1669
1670 #define LBR_INFO_MISPRED        (1ULL << 63)
1671 #define LBR_INFO_IN_TX          (1ULL << 62)
1672 #define LBR_INFO_ABORT          (1ULL << 61)
1673 #define LBR_INFO_CYCLES         0xffff
1674
1675 /* Refer kernel's intel_pmu_store_pebs_lbrs() */
1676 static u64 intel_pt_lbr_flags(u64 info)
1677 {
1678         union {
1679                 struct branch_flags flags;
1680                 u64 result;
1681         } u = {
1682                 .flags = {
1683                         .mispred        = !!(info & LBR_INFO_MISPRED),
1684                         .predicted      = !(info & LBR_INFO_MISPRED),
1685                         .in_tx          = !!(info & LBR_INFO_IN_TX),
1686                         .abort          = !!(info & LBR_INFO_ABORT),
1687                         .cycles         = info & LBR_INFO_CYCLES,
1688                 }
1689         };
1690
1691         return u.result;
1692 }
1693
1694 static void intel_pt_add_lbrs(struct branch_stack *br_stack,
1695                               const struct intel_pt_blk_items *items)
1696 {
1697         u64 *to;
1698         int i;
1699
1700         br_stack->nr = 0;
1701
1702         to = &br_stack->entries[0].from;
1703
1704         for (i = INTEL_PT_LBR_0_POS; i <= INTEL_PT_LBR_2_POS; i++) {
1705                 u32 mask = items->mask[i];
1706                 const u64 *from = items->val[i];
1707
1708                 for (; mask; mask >>= 3, from += 3) {
1709                         if ((mask & 7) == 7) {
1710                                 *to++ = from[0];
1711                                 *to++ = from[1];
1712                                 *to++ = intel_pt_lbr_flags(from[2]);
1713                                 br_stack->nr += 1;
1714                         }
1715                 }
1716         }
1717 }
1718
1719 /* INTEL_PT_LBR_0, INTEL_PT_LBR_1 and INTEL_PT_LBR_2 */
1720 #define LBRS_MAX (INTEL_PT_BLK_ITEM_ID_CNT * 3)
1721
1722 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq)
1723 {
1724         const struct intel_pt_blk_items *items = &ptq->state->items;
1725         struct perf_sample sample = { .ip = 0, };
1726         union perf_event *event = ptq->event_buf;
1727         struct intel_pt *pt = ptq->pt;
1728         struct evsel *evsel = pt->pebs_evsel;
1729         u64 sample_type = evsel->core.attr.sample_type;
1730         u64 id = evsel->core.id[0];
1731         u8 cpumode;
1732
1733         if (intel_pt_skip_event(pt))
1734                 return 0;
1735
1736         intel_pt_prep_a_sample(ptq, event, &sample);
1737
1738         sample.id = id;
1739         sample.stream_id = id;
1740
1741         if (!evsel->core.attr.freq)
1742                 sample.period = evsel->core.attr.sample_period;
1743
1744         /* No support for non-zero CS base */
1745         if (items->has_ip)
1746                 sample.ip = items->ip;
1747         else if (items->has_rip)
1748                 sample.ip = items->rip;
1749         else
1750                 sample.ip = ptq->state->from_ip;
1751
1752         /* No support for guest mode at this time */
1753         cpumode = sample.ip < ptq->pt->kernel_start ?
1754                   PERF_RECORD_MISC_USER :
1755                   PERF_RECORD_MISC_KERNEL;
1756
1757         event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;
1758
1759         sample.cpumode = cpumode;
1760
1761         if (sample_type & PERF_SAMPLE_TIME) {
1762                 u64 timestamp = 0;
1763
1764                 if (items->has_timestamp)
1765                         timestamp = items->timestamp;
1766                 else if (!pt->timeless_decoding)
1767                         timestamp = ptq->timestamp;
1768                 if (timestamp)
1769                         sample.time = tsc_to_perf_time(timestamp, &pt->tc);
1770         }
1771
1772         if (sample_type & PERF_SAMPLE_CALLCHAIN &&
1773             pt->synth_opts.callchain) {
1774                 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1775                                      pt->synth_opts.callchain_sz, sample.ip,
1776                                      pt->kernel_start);
1777                 sample.callchain = ptq->chain;
1778         }
1779
1780         if (sample_type & PERF_SAMPLE_REGS_INTR &&
1781             items->mask[INTEL_PT_GP_REGS_POS]) {
1782                 u64 regs[sizeof(sample.intr_regs.mask)];
1783                 u64 regs_mask = evsel->core.attr.sample_regs_intr;
1784                 u64 *pos;
1785
1786                 sample.intr_regs.abi = items->is_32_bit ?
1787                                        PERF_SAMPLE_REGS_ABI_32 :
1788                                        PERF_SAMPLE_REGS_ABI_64;
1789                 sample.intr_regs.regs = regs;
1790
1791                 pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);
1792
1793                 intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);
1794         }
1795
1796         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
1797                 struct {
1798                         struct branch_stack br_stack;
1799                         struct branch_entry entries[LBRS_MAX];
1800                 } br;
1801
1802                 if (items->mask[INTEL_PT_LBR_0_POS] ||
1803                     items->mask[INTEL_PT_LBR_1_POS] ||
1804                     items->mask[INTEL_PT_LBR_2_POS]) {
1805                         intel_pt_add_lbrs(&br.br_stack, items);
1806                         sample.branch_stack = &br.br_stack;
1807                 } else if (pt->synth_opts.last_branch) {
1808                         intel_pt_copy_last_branch_rb(ptq);
1809                         sample.branch_stack = ptq->last_branch;
1810                 } else {
1811                         br.br_stack.nr = 0;
1812                         sample.branch_stack = &br.br_stack;
1813                 }
1814         }
1815
1816         if (sample_type & PERF_SAMPLE_ADDR && items->has_mem_access_address)
1817                 sample.addr = items->mem_access_address;
1818
1819         if (sample_type & PERF_SAMPLE_WEIGHT) {
1820                 /*
1821                  * Refer kernel's setup_pebs_adaptive_sample_data() and
1822                  * intel_hsw_weight().
1823                  */
1824                 if (items->has_mem_access_latency)
1825                         sample.weight = items->mem_access_latency;
1826                 if (!sample.weight && items->has_tsx_aux_info) {
1827                         /* Cycles last block */
1828                         sample.weight = (u32)items->tsx_aux_info;
1829                 }
1830         }
1831
1832         if (sample_type & PERF_SAMPLE_TRANSACTION && items->has_tsx_aux_info) {
1833                 u64 ax = items->has_rax ? items->rax : 0;
1834                 /* Refer kernel's intel_hsw_transaction() */
1835                 u64 txn = (u8)(items->tsx_aux_info >> 32);
1836
1837                 /* For RTM XABORTs also log the abort code from AX */
1838                 if (txn & PERF_TXN_TRANSACTION && ax & 1)
1839                         txn |= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1840                 sample.transaction = txn;
1841         }
1842
1843         return intel_pt_deliver_synth_event(pt, ptq, event, &sample, sample_type);
1844 }
1845
1846 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1847                                 pid_t pid, pid_t tid, u64 ip, u64 timestamp)
1848 {
1849         union perf_event event;
1850         char msg[MAX_AUXTRACE_ERROR_MSG];
1851         int err;
1852
1853         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1854
1855         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1856                              code, cpu, pid, tid, ip, msg, timestamp);
1857
1858         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1859         if (err)
1860                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1861                        err);
1862
1863         return err;
1864 }
1865
1866 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
1867                                  const struct intel_pt_state *state)
1868 {
1869         struct intel_pt *pt = ptq->pt;
1870         u64 tm = ptq->timestamp;
1871
1872         tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
1873
1874         return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
1875                                     ptq->tid, state->from_ip, tm);
1876 }
1877
1878 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1879 {
1880         struct auxtrace_queue *queue;
1881         pid_t tid = ptq->next_tid;
1882         int err;
1883
1884         if (tid == -1)
1885                 return 0;
1886
1887         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1888
1889         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1890
1891         queue = &pt->queues.queue_array[ptq->queue_nr];
1892         intel_pt_set_pid_tid_cpu(pt, queue);
1893
1894         ptq->next_tid = -1;
1895
1896         return err;
1897 }
1898
1899 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1900 {
1901         struct intel_pt *pt = ptq->pt;
1902
1903         return ip == pt->switch_ip &&
1904                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1905                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1906                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1907 }
1908
1909 #define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1910                           INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT)
1911
1912 static int intel_pt_sample(struct intel_pt_queue *ptq)
1913 {
1914         const struct intel_pt_state *state = ptq->state;
1915         struct intel_pt *pt = ptq->pt;
1916         int err;
1917
1918         if (!ptq->have_sample)
1919                 return 0;
1920
1921         ptq->have_sample = false;
1922
1923         if (ptq->state->tot_cyc_cnt > ptq->ipc_cyc_cnt) {
1924                 /*
1925                  * Cycle count and instruction count only go together to create
1926                  * a valid IPC ratio when the cycle count changes.
1927                  */
1928                 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
1929                 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
1930         }
1931
1932         /*
1933          * Do PEBS first to allow for the possibility that the PEBS timestamp
1934          * precedes the current timestamp.
1935          */
1936         if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) {
1937                 err = intel_pt_synth_pebs_sample(ptq);
1938                 if (err)
1939                         return err;
1940         }
1941
1942         if (pt->sample_pwr_events) {
1943                 if (ptq->state->cbr != ptq->cbr_seen) {
1944                         err = intel_pt_synth_cbr_sample(ptq);
1945                         if (err)
1946                                 return err;
1947                 }
1948                 if (state->type & INTEL_PT_PWR_EVT) {
1949                         if (state->type & INTEL_PT_MWAIT_OP) {
1950                                 err = intel_pt_synth_mwait_sample(ptq);
1951                                 if (err)
1952                                         return err;
1953                         }
1954                         if (state->type & INTEL_PT_PWR_ENTRY) {
1955                                 err = intel_pt_synth_pwre_sample(ptq);
1956                                 if (err)
1957                                         return err;
1958                         }
1959                         if (state->type & INTEL_PT_EX_STOP) {
1960                                 err = intel_pt_synth_exstop_sample(ptq);
1961                                 if (err)
1962                                         return err;
1963                         }
1964                         if (state->type & INTEL_PT_PWR_EXIT) {
1965                                 err = intel_pt_synth_pwrx_sample(ptq);
1966                                 if (err)
1967                                         return err;
1968                         }
1969                 }
1970         }
1971
1972         if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1973                 err = intel_pt_synth_instruction_sample(ptq);
1974                 if (err)
1975                         return err;
1976         }
1977
1978         if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1979                 err = intel_pt_synth_transaction_sample(ptq);
1980                 if (err)
1981                         return err;
1982         }
1983
1984         if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1985                 err = intel_pt_synth_ptwrite_sample(ptq);
1986                 if (err)
1987                         return err;
1988         }
1989
1990         if (!(state->type & INTEL_PT_BRANCH))
1991                 return 0;
1992
1993         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1994                 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, state->from_ip,
1995                                     state->to_ip, ptq->insn_len,
1996                                     state->trace_nr);
1997         else
1998                 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
1999
2000         if (pt->sample_branches) {
2001                 err = intel_pt_synth_branch_sample(ptq);
2002                 if (err)
2003                         return err;
2004         }
2005
2006         if (pt->synth_opts.last_branch)
2007                 intel_pt_update_last_branch_rb(ptq);
2008
2009         if (!ptq->sync_switch)
2010                 return 0;
2011
2012         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
2013                 switch (ptq->switch_state) {
2014                 case INTEL_PT_SS_NOT_TRACING:
2015                 case INTEL_PT_SS_UNKNOWN:
2016                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2017                         err = intel_pt_next_tid(pt, ptq);
2018                         if (err)
2019                                 return err;
2020                         ptq->switch_state = INTEL_PT_SS_TRACING;
2021                         break;
2022                 default:
2023                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
2024                         return 1;
2025                 }
2026         } else if (!state->to_ip) {
2027                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2028         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
2029                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2030         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2031                    state->to_ip == pt->ptss_ip &&
2032                    (ptq->flags & PERF_IP_FLAG_CALL)) {
2033                 ptq->switch_state = INTEL_PT_SS_TRACING;
2034         }
2035
2036         return 0;
2037 }
2038
2039 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
2040 {
2041         struct machine *machine = pt->machine;
2042         struct map *map;
2043         struct symbol *sym, *start;
2044         u64 ip, switch_ip = 0;
2045         const char *ptss;
2046
2047         if (ptss_ip)
2048                 *ptss_ip = 0;
2049
2050         map = machine__kernel_map(machine);
2051         if (!map)
2052                 return 0;
2053
2054         if (map__load(map))
2055                 return 0;
2056
2057         start = dso__first_symbol(map->dso);
2058
2059         for (sym = start; sym; sym = dso__next_symbol(sym)) {
2060                 if (sym->binding == STB_GLOBAL &&
2061                     !strcmp(sym->name, "__switch_to")) {
2062                         ip = map->unmap_ip(map, sym->start);
2063                         if (ip >= map->start && ip < map->end) {
2064                                 switch_ip = ip;
2065                                 break;
2066                         }
2067                 }
2068         }
2069
2070         if (!switch_ip || !ptss_ip)
2071                 return 0;
2072
2073         if (pt->have_sched_switch == 1)
2074                 ptss = "perf_trace_sched_switch";
2075         else
2076                 ptss = "__perf_event_task_sched_out";
2077
2078         for (sym = start; sym; sym = dso__next_symbol(sym)) {
2079                 if (!strcmp(sym->name, ptss)) {
2080                         ip = map->unmap_ip(map, sym->start);
2081                         if (ip >= map->start && ip < map->end) {
2082                                 *ptss_ip = ip;
2083                                 break;
2084                         }
2085                 }
2086         }
2087
2088         return switch_ip;
2089 }
2090
2091 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
2092 {
2093         unsigned int i;
2094
2095         pt->sync_switch = true;
2096
2097         for (i = 0; i < pt->queues.nr_queues; i++) {
2098                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2099                 struct intel_pt_queue *ptq = queue->priv;
2100
2101                 if (ptq)
2102                         ptq->sync_switch = true;
2103         }
2104 }
2105
2106 /*
2107  * To filter against time ranges, it is only necessary to look at the next start
2108  * or end time.
2109  */
2110 static bool intel_pt_next_time(struct intel_pt_queue *ptq)
2111 {
2112         struct intel_pt *pt = ptq->pt;
2113
2114         if (ptq->sel_start) {
2115                 /* Next time is an end time */
2116                 ptq->sel_start = false;
2117                 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end;
2118                 return true;
2119         } else if (ptq->sel_idx + 1 < pt->range_cnt) {
2120                 /* Next time is a start time */
2121                 ptq->sel_start = true;
2122                 ptq->sel_idx += 1;
2123                 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start;
2124                 return true;
2125         }
2126
2127         /* No next time */
2128         return false;
2129 }
2130
2131 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp)
2132 {
2133         int err;
2134
2135         while (1) {
2136                 if (ptq->sel_start) {
2137                         if (ptq->timestamp >= ptq->sel_timestamp) {
2138                                 /* After start time, so consider next time */
2139                                 intel_pt_next_time(ptq);
2140                                 if (!ptq->sel_timestamp) {
2141                                         /* No end time */
2142                                         return 0;
2143                                 }
2144                                 /* Check against end time */
2145                                 continue;
2146                         }
2147                         /* Before start time, so fast forward */
2148                         ptq->have_sample = false;
2149                         if (ptq->sel_timestamp > *ff_timestamp) {
2150                                 if (ptq->sync_switch) {
2151                                         intel_pt_next_tid(ptq->pt, ptq);
2152                                         ptq->switch_state = INTEL_PT_SS_UNKNOWN;
2153                                 }
2154                                 *ff_timestamp = ptq->sel_timestamp;
2155                                 err = intel_pt_fast_forward(ptq->decoder,
2156                                                             ptq->sel_timestamp);
2157                                 if (err)
2158                                         return err;
2159                         }
2160                         return 0;
2161                 } else if (ptq->timestamp > ptq->sel_timestamp) {
2162                         /* After end time, so consider next time */
2163                         if (!intel_pt_next_time(ptq)) {
2164                                 /* No next time range, so stop decoding */
2165                                 ptq->have_sample = false;
2166                                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
2167                                 return 1;
2168                         }
2169                         /* Check against next start time */
2170                         continue;
2171                 } else {
2172                         /* Before end time */
2173                         return 0;
2174                 }
2175         }
2176 }
2177
2178 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
2179 {
2180         const struct intel_pt_state *state = ptq->state;
2181         struct intel_pt *pt = ptq->pt;
2182         u64 ff_timestamp = 0;
2183         int err;
2184
2185         if (!pt->kernel_start) {
2186                 pt->kernel_start = machine__kernel_start(pt->machine);
2187                 if (pt->per_cpu_mmaps &&
2188                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
2189                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
2190                     !pt->sampling_mode) {
2191                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
2192                         if (pt->switch_ip) {
2193                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
2194                                              pt->switch_ip, pt->ptss_ip);
2195                                 intel_pt_enable_sync_switch(pt);
2196                         }
2197                 }
2198         }
2199
2200         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
2201                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
2202         while (1) {
2203                 err = intel_pt_sample(ptq);
2204                 if (err)
2205                         return err;
2206
2207                 state = intel_pt_decode(ptq->decoder);
2208                 if (state->err) {
2209                         if (state->err == INTEL_PT_ERR_NODATA)
2210                                 return 1;
2211                         if (ptq->sync_switch &&
2212                             state->from_ip >= pt->kernel_start) {
2213                                 ptq->sync_switch = false;
2214                                 intel_pt_next_tid(pt, ptq);
2215                         }
2216                         if (pt->synth_opts.errors) {
2217                                 err = intel_ptq_synth_error(ptq, state);
2218                                 if (err)
2219                                         return err;
2220                         }
2221                         continue;
2222                 }
2223
2224                 ptq->state = state;
2225                 ptq->have_sample = true;
2226                 intel_pt_sample_flags(ptq);
2227
2228                 /* Use estimated TSC upon return to user space */
2229                 if (pt->est_tsc &&
2230                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
2231                     state->to_ip && state->to_ip < pt->kernel_start) {
2232                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
2233                                      state->timestamp, state->est_timestamp);
2234                         ptq->timestamp = state->est_timestamp;
2235                 /* Use estimated TSC in unknown switch state */
2236                 } else if (ptq->sync_switch &&
2237                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
2238                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
2239                            ptq->next_tid == -1) {
2240                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
2241                                      state->timestamp, state->est_timestamp);
2242                         ptq->timestamp = state->est_timestamp;
2243                 } else if (state->timestamp > ptq->timestamp) {
2244                         ptq->timestamp = state->timestamp;
2245                 }
2246
2247                 if (ptq->sel_timestamp) {
2248                         err = intel_pt_time_filter(ptq, &ff_timestamp);
2249                         if (err)
2250                                 return err;
2251                 }
2252
2253                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
2254                         *timestamp = ptq->timestamp;
2255                         return 0;
2256                 }
2257         }
2258         return 0;
2259 }
2260
2261 static inline int intel_pt_update_queues(struct intel_pt *pt)
2262 {
2263         if (pt->queues.new_data) {
2264                 pt->queues.new_data = false;
2265                 return intel_pt_setup_queues(pt);
2266         }
2267         return 0;
2268 }
2269
2270 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
2271 {
2272         unsigned int queue_nr;
2273         u64 ts;
2274         int ret;
2275
2276         while (1) {
2277                 struct auxtrace_queue *queue;
2278                 struct intel_pt_queue *ptq;
2279
2280                 if (!pt->heap.heap_cnt)
2281                         return 0;
2282
2283                 if (pt->heap.heap_array[0].ordinal >= timestamp)
2284                         return 0;
2285
2286                 queue_nr = pt->heap.heap_array[0].queue_nr;
2287                 queue = &pt->queues.queue_array[queue_nr];
2288                 ptq = queue->priv;
2289
2290                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
2291                              queue_nr, pt->heap.heap_array[0].ordinal,
2292                              timestamp);
2293
2294                 auxtrace_heap__pop(&pt->heap);
2295
2296                 if (pt->heap.heap_cnt) {
2297                         ts = pt->heap.heap_array[0].ordinal + 1;
2298                         if (ts > timestamp)
2299                                 ts = timestamp;
2300                 } else {
2301                         ts = timestamp;
2302                 }
2303
2304                 intel_pt_set_pid_tid_cpu(pt, queue);
2305
2306                 ret = intel_pt_run_decoder(ptq, &ts);
2307
2308                 if (ret < 0) {
2309                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
2310                         return ret;
2311                 }
2312
2313                 if (!ret) {
2314                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
2315                         if (ret < 0)
2316                                 return ret;
2317                 } else {
2318                         ptq->on_heap = false;
2319                 }
2320         }
2321
2322         return 0;
2323 }
2324
2325 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
2326                                             u64 time_)
2327 {
2328         struct auxtrace_queues *queues = &pt->queues;
2329         unsigned int i;
2330         u64 ts = 0;
2331
2332         for (i = 0; i < queues->nr_queues; i++) {
2333                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
2334                 struct intel_pt_queue *ptq = queue->priv;
2335
2336                 if (ptq && (tid == -1 || ptq->tid == tid)) {
2337                         ptq->time = time_;
2338                         intel_pt_set_pid_tid_cpu(pt, queue);
2339                         intel_pt_run_decoder(ptq, &ts);
2340                 }
2341         }
2342         return 0;
2343 }
2344
2345 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq,
2346                                             struct auxtrace_queue *queue,
2347                                             struct perf_sample *sample)
2348 {
2349         struct machine *m = ptq->pt->machine;
2350
2351         ptq->pid = sample->pid;
2352         ptq->tid = sample->tid;
2353         ptq->cpu = queue->cpu;
2354
2355         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
2356                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
2357
2358         thread__zput(ptq->thread);
2359
2360         if (ptq->tid == -1)
2361                 return;
2362
2363         if (ptq->pid == -1) {
2364                 ptq->thread = machine__find_thread(m, -1, ptq->tid);
2365                 if (ptq->thread)
2366                         ptq->pid = ptq->thread->pid_;
2367                 return;
2368         }
2369
2370         ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid);
2371 }
2372
2373 static int intel_pt_process_timeless_sample(struct intel_pt *pt,
2374                                             struct perf_sample *sample)
2375 {
2376         struct auxtrace_queue *queue;
2377         struct intel_pt_queue *ptq;
2378         u64 ts = 0;
2379
2380         queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);
2381         if (!queue)
2382                 return -EINVAL;
2383
2384         ptq = queue->priv;
2385         if (!ptq)
2386                 return 0;
2387
2388         ptq->stop = false;
2389         ptq->time = sample->time;
2390         intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);
2391         intel_pt_run_decoder(ptq, &ts);
2392         return 0;
2393 }
2394
2395 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
2396 {
2397         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
2398                                     sample->pid, sample->tid, 0, sample->time);
2399 }
2400
2401 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
2402 {
2403         unsigned i, j;
2404
2405         if (cpu < 0 || !pt->queues.nr_queues)
2406                 return NULL;
2407
2408         if ((unsigned)cpu >= pt->queues.nr_queues)
2409                 i = pt->queues.nr_queues - 1;
2410         else
2411                 i = cpu;
2412
2413         if (pt->queues.queue_array[i].cpu == cpu)
2414                 return pt->queues.queue_array[i].priv;
2415
2416         for (j = 0; i > 0; j++) {
2417                 if (pt->queues.queue_array[--i].cpu == cpu)
2418                         return pt->queues.queue_array[i].priv;
2419         }
2420
2421         for (; j < pt->queues.nr_queues; j++) {
2422                 if (pt->queues.queue_array[j].cpu == cpu)
2423                         return pt->queues.queue_array[j].priv;
2424         }
2425
2426         return NULL;
2427 }
2428
2429 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
2430                                 u64 timestamp)
2431 {
2432         struct intel_pt_queue *ptq;
2433         int err;
2434
2435         if (!pt->sync_switch)
2436                 return 1;
2437
2438         ptq = intel_pt_cpu_to_ptq(pt, cpu);
2439         if (!ptq || !ptq->sync_switch)
2440                 return 1;
2441
2442         switch (ptq->switch_state) {
2443         case INTEL_PT_SS_NOT_TRACING:
2444                 break;
2445         case INTEL_PT_SS_UNKNOWN:
2446         case INTEL_PT_SS_TRACING:
2447                 ptq->next_tid = tid;
2448                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
2449                 return 0;
2450         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
2451                 if (!ptq->on_heap) {
2452                         ptq->timestamp = perf_time_to_tsc(timestamp,
2453                                                           &pt->tc);
2454                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
2455                                                  ptq->timestamp);
2456                         if (err)
2457                                 return err;
2458                         ptq->on_heap = true;
2459                 }
2460                 ptq->switch_state = INTEL_PT_SS_TRACING;
2461                 break;
2462         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2463                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
2464                 break;
2465         default:
2466                 break;
2467         }
2468
2469         ptq->next_tid = -1;
2470
2471         return 1;
2472 }
2473
2474 static int intel_pt_process_switch(struct intel_pt *pt,
2475                                    struct perf_sample *sample)
2476 {
2477         struct evsel *evsel;
2478         pid_t tid;
2479         int cpu, ret;
2480
2481         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
2482         if (evsel != pt->switch_evsel)
2483                 return 0;
2484
2485         tid = perf_evsel__intval(evsel, sample, "next_pid");
2486         cpu = sample->cpu;
2487
2488         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2489                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
2490                      &pt->tc));
2491
2492         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2493         if (ret <= 0)
2494                 return ret;
2495
2496         return machine__set_current_tid(pt->machine, cpu, -1, tid);
2497 }
2498
2499 static int intel_pt_context_switch_in(struct intel_pt *pt,
2500                                       struct perf_sample *sample)
2501 {
2502         pid_t pid = sample->pid;
2503         pid_t tid = sample->tid;
2504         int cpu = sample->cpu;
2505
2506         if (pt->sync_switch) {
2507                 struct intel_pt_queue *ptq;
2508
2509                 ptq = intel_pt_cpu_to_ptq(pt, cpu);
2510                 if (ptq && ptq->sync_switch) {
2511                         ptq->next_tid = -1;
2512                         switch (ptq->switch_state) {
2513                         case INTEL_PT_SS_NOT_TRACING:
2514                         case INTEL_PT_SS_UNKNOWN:
2515                         case INTEL_PT_SS_TRACING:
2516                                 break;
2517                         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
2518                         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
2519                                 ptq->switch_state = INTEL_PT_SS_TRACING;
2520                                 break;
2521                         default:
2522                                 break;
2523                         }
2524                 }
2525         }
2526
2527         /*
2528          * If the current tid has not been updated yet, ensure it is now that
2529          * a "switch in" event has occurred.
2530          */
2531         if (machine__get_current_tid(pt->machine, cpu) == tid)
2532                 return 0;
2533
2534         return machine__set_current_tid(pt->machine, cpu, pid, tid);
2535 }
2536
2537 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
2538                                    struct perf_sample *sample)
2539 {
2540         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2541         pid_t pid, tid;
2542         int cpu, ret;
2543
2544         cpu = sample->cpu;
2545
2546         if (pt->have_sched_switch == 3) {
2547                 if (!out)
2548                         return intel_pt_context_switch_in(pt, sample);
2549                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
2550                         pr_err("Expecting CPU-wide context switch event\n");
2551                         return -EINVAL;
2552                 }
2553                 pid = event->context_switch.next_prev_pid;
2554                 tid = event->context_switch.next_prev_tid;
2555         } else {
2556                 if (out)
2557                         return 0;
2558                 pid = sample->pid;
2559                 tid = sample->tid;
2560         }
2561
2562         if (tid == -1) {
2563                 pr_err("context_switch event has no tid\n");
2564                 return -EINVAL;
2565         }
2566
2567         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2568                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
2569                      &pt->tc));
2570
2571         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2572         if (ret <= 0)
2573                 return ret;
2574
2575         return machine__set_current_tid(pt->machine, cpu, pid, tid);
2576 }
2577
2578 static int intel_pt_process_itrace_start(struct intel_pt *pt,
2579                                          union perf_event *event,
2580                                          struct perf_sample *sample)
2581 {
2582         if (!pt->per_cpu_mmaps)
2583                 return 0;
2584
2585         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2586                      sample->cpu, event->itrace_start.pid,
2587                      event->itrace_start.tid, sample->time,
2588                      perf_time_to_tsc(sample->time, &pt->tc));
2589
2590         return machine__set_current_tid(pt->machine, sample->cpu,
2591                                         event->itrace_start.pid,
2592                                         event->itrace_start.tid);
2593 }
2594
2595 static int intel_pt_process_event(struct perf_session *session,
2596                                   union perf_event *event,
2597                                   struct perf_sample *sample,
2598                                   struct perf_tool *tool)
2599 {
2600         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2601                                            auxtrace);
2602         u64 timestamp;
2603         int err = 0;
2604
2605         if (dump_trace)
2606                 return 0;
2607
2608         if (!tool->ordered_events) {
2609                 pr_err("Intel Processor Trace requires ordered events\n");
2610                 return -EINVAL;
2611         }
2612
2613         if (sample->time && sample->time != (u64)-1)
2614                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2615         else
2616                 timestamp = 0;
2617
2618         if (timestamp || pt->timeless_decoding) {
2619                 err = intel_pt_update_queues(pt);
2620                 if (err)
2621                         return err;
2622         }
2623
2624         if (pt->timeless_decoding) {
2625                 if (pt->sampling_mode) {
2626                         if (sample->aux_sample.size)
2627                                 err = intel_pt_process_timeless_sample(pt,
2628                                                                        sample);
2629                 } else if (event->header.type == PERF_RECORD_EXIT) {
2630                         err = intel_pt_process_timeless_queues(pt,
2631                                                                event->fork.tid,
2632                                                                sample->time);
2633                 }
2634         } else if (timestamp) {
2635                 err = intel_pt_process_queues(pt, timestamp);
2636         }
2637         if (err)
2638                 return err;
2639
2640         if (event->header.type == PERF_RECORD_AUX &&
2641             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
2642             pt->synth_opts.errors) {
2643                 err = intel_pt_lost(pt, sample);
2644                 if (err)
2645                         return err;
2646         }
2647
2648         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2649                 err = intel_pt_process_switch(pt, sample);
2650         else if (event->header.type == PERF_RECORD_ITRACE_START)
2651                 err = intel_pt_process_itrace_start(pt, event, sample);
2652         else if (event->header.type == PERF_RECORD_SWITCH ||
2653                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2654                 err = intel_pt_context_switch(pt, event, sample);
2655
2656         intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",
2657                      event->header.type, sample->cpu, sample->time, timestamp);
2658         intel_pt_log_event(event);
2659
2660         return err;
2661 }
2662
2663 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2664 {
2665         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2666                                            auxtrace);
2667         int ret;
2668
2669         if (dump_trace)
2670                 return 0;
2671
2672         if (!tool->ordered_events)
2673                 return -EINVAL;
2674
2675         ret = intel_pt_update_queues(pt);
2676         if (ret < 0)
2677                 return ret;
2678
2679         if (pt->timeless_decoding)
2680                 return intel_pt_process_timeless_queues(pt, -1,
2681                                                         MAX_TIMESTAMP - 1);
2682
2683         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2684 }
2685
2686 static void intel_pt_free_events(struct perf_session *session)
2687 {
2688         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2689                                            auxtrace);
2690         struct auxtrace_queues *queues = &pt->queues;
2691         unsigned int i;
2692
2693         for (i = 0; i < queues->nr_queues; i++) {
2694                 intel_pt_free_queue(queues->queue_array[i].priv);
2695                 queues->queue_array[i].priv = NULL;
2696         }
2697         intel_pt_log_disable();
2698         auxtrace_queues__free(queues);
2699 }
2700
2701 static void intel_pt_free(struct perf_session *session)
2702 {
2703         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2704                                            auxtrace);
2705
2706         auxtrace_heap__free(&pt->heap);
2707         intel_pt_free_events(session);
2708         session->auxtrace = NULL;
2709         thread__put(pt->unknown_thread);
2710         addr_filters__exit(&pt->filts);
2711         zfree(&pt->filter);
2712         zfree(&pt->time_ranges);
2713         free(pt);
2714 }
2715
2716 static int intel_pt_process_auxtrace_event(struct perf_session *session,
2717                                            union perf_event *event,
2718                                            struct perf_tool *tool __maybe_unused)
2719 {
2720         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2721                                            auxtrace);
2722
2723         if (!pt->data_queued) {
2724                 struct auxtrace_buffer *buffer;
2725                 off_t data_offset;
2726                 int fd = perf_data__fd(session->data);
2727                 int err;
2728
2729                 if (perf_data__is_pipe(session->data)) {
2730                         data_offset = 0;
2731                 } else {
2732                         data_offset = lseek(fd, 0, SEEK_CUR);
2733                         if (data_offset == -1)
2734                                 return -errno;
2735                 }
2736
2737                 err = auxtrace_queues__add_event(&pt->queues, session, event,
2738                                                  data_offset, &buffer);
2739                 if (err)
2740                         return err;
2741
2742                 /* Dump here now we have copied a piped trace out of the pipe */
2743                 if (dump_trace) {
2744                         if (auxtrace_buffer__get_data(buffer, fd)) {
2745                                 intel_pt_dump_event(pt, buffer->data,
2746                                                     buffer->size);
2747                                 auxtrace_buffer__put_data(buffer);
2748                         }
2749                 }
2750         }
2751
2752         return 0;
2753 }
2754
2755 static int intel_pt_queue_data(struct perf_session *session,
2756                                struct perf_sample *sample,
2757                                union perf_event *event, u64 data_offset)
2758 {
2759         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2760                                            auxtrace);
2761         u64 timestamp;
2762
2763         if (event) {
2764                 return auxtrace_queues__add_event(&pt->queues, session, event,
2765                                                   data_offset, NULL);
2766         }
2767
2768         if (sample->time && sample->time != (u64)-1)
2769                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2770         else
2771                 timestamp = 0;
2772
2773         return auxtrace_queues__add_sample(&pt->queues, session, sample,
2774                                            data_offset, timestamp);
2775 }
2776
2777 struct intel_pt_synth {
2778         struct perf_tool dummy_tool;
2779         struct perf_session *session;
2780 };
2781
2782 static int intel_pt_event_synth(struct perf_tool *tool,
2783                                 union perf_event *event,
2784                                 struct perf_sample *sample __maybe_unused,
2785                                 struct machine *machine __maybe_unused)
2786 {
2787         struct intel_pt_synth *intel_pt_synth =
2788                         container_of(tool, struct intel_pt_synth, dummy_tool);
2789
2790         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2791                                                  NULL);
2792 }
2793
2794 static int intel_pt_synth_event(struct perf_session *session, const char *name,
2795                                 struct perf_event_attr *attr, u64 id)
2796 {
2797         struct intel_pt_synth intel_pt_synth;
2798         int err;
2799
2800         pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2801                  name, id, (u64)attr->sample_type);
2802
2803         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2804         intel_pt_synth.session = session;
2805
2806         err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2807                                           &id, intel_pt_event_synth);
2808         if (err)
2809                 pr_err("%s: failed to synthesize '%s' event type\n",
2810                        __func__, name);
2811
2812         return err;
2813 }
2814
2815 static void intel_pt_set_event_name(struct evlist *evlist, u64 id,
2816                                     const char *name)
2817 {
2818         struct evsel *evsel;
2819
2820         evlist__for_each_entry(evlist, evsel) {
2821                 if (evsel->core.id && evsel->core.id[0] == id) {
2822                         if (evsel->name)
2823                                 zfree(&evsel->name);
2824                         evsel->name = strdup(name);
2825                         break;
2826                 }
2827         }
2828 }
2829
2830 static struct evsel *intel_pt_evsel(struct intel_pt *pt,
2831                                          struct evlist *evlist)
2832 {
2833         struct evsel *evsel;
2834
2835         evlist__for_each_entry(evlist, evsel) {
2836                 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids)
2837                         return evsel;
2838         }
2839
2840         return NULL;
2841 }
2842
2843 static int intel_pt_synth_events(struct intel_pt *pt,
2844                                  struct perf_session *session)
2845 {
2846         struct evlist *evlist = session->evlist;
2847         struct evsel *evsel = intel_pt_evsel(pt, evlist);
2848         struct perf_event_attr attr;
2849         u64 id;
2850         int err;
2851
2852         if (!evsel) {
2853                 pr_debug("There are no selected events with Intel Processor Trace data\n");
2854                 return 0;
2855         }
2856
2857         memset(&attr, 0, sizeof(struct perf_event_attr));
2858         attr.size = sizeof(struct perf_event_attr);
2859         attr.type = PERF_TYPE_HARDWARE;
2860         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
2861         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2862                             PERF_SAMPLE_PERIOD;
2863         if (pt->timeless_decoding)
2864                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2865         else
2866                 attr.sample_type |= PERF_SAMPLE_TIME;
2867         if (!pt->per_cpu_mmaps)
2868                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2869         attr.exclude_user = evsel->core.attr.exclude_user;
2870         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
2871         attr.exclude_hv = evsel->core.attr.exclude_hv;
2872         attr.exclude_host = evsel->core.attr.exclude_host;
2873         attr.exclude_guest = evsel->core.attr.exclude_guest;
2874         attr.sample_id_all = evsel->core.attr.sample_id_all;
2875         attr.read_format = evsel->core.attr.read_format;
2876
2877         id = evsel->core.id[0] + 1000000000;
2878         if (!id)
2879                 id = 1;
2880
2881         if (pt->synth_opts.branches) {
2882                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2883                 attr.sample_period = 1;
2884                 attr.sample_type |= PERF_SAMPLE_ADDR;
2885                 err = intel_pt_synth_event(session, "branches", &attr, id);
2886                 if (err)
2887                         return err;
2888                 pt->sample_branches = true;
2889                 pt->branches_sample_type = attr.sample_type;
2890                 pt->branches_id = id;
2891                 id += 1;
2892                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2893         }
2894
2895         if (pt->synth_opts.callchain)
2896                 attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2897         if (pt->synth_opts.last_branch)
2898                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2899
2900         if (pt->synth_opts.instructions) {
2901                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2902                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2903                         attr.sample_period =
2904                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2905                 else
2906                         attr.sample_period = pt->synth_opts.period;
2907                 err = intel_pt_synth_event(session, "instructions", &attr, id);
2908                 if (err)
2909                         return err;
2910                 pt->sample_instructions = true;
2911                 pt->instructions_sample_type = attr.sample_type;
2912                 pt->instructions_id = id;
2913                 id += 1;
2914         }
2915
2916         attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2917         attr.sample_period = 1;
2918
2919         if (pt->synth_opts.transactions) {
2920                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2921                 err = intel_pt_synth_event(session, "transactions", &attr, id);
2922                 if (err)
2923                         return err;
2924                 pt->sample_transactions = true;
2925                 pt->transactions_sample_type = attr.sample_type;
2926                 pt->transactions_id = id;
2927                 intel_pt_set_event_name(evlist, id, "transactions");
2928                 id += 1;
2929         }
2930
2931         attr.type = PERF_TYPE_SYNTH;
2932         attr.sample_type |= PERF_SAMPLE_RAW;
2933
2934         if (pt->synth_opts.ptwrites) {
2935                 attr.config = PERF_SYNTH_INTEL_PTWRITE;
2936                 err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2937                 if (err)
2938                         return err;
2939                 pt->sample_ptwrites = true;
2940                 pt->ptwrites_sample_type = attr.sample_type;
2941                 pt->ptwrites_id = id;
2942                 intel_pt_set_event_name(evlist, id, "ptwrite");
2943                 id += 1;
2944         }
2945
2946         if (pt->synth_opts.pwr_events) {
2947                 pt->sample_pwr_events = true;
2948                 pt->pwr_events_sample_type = attr.sample_type;
2949
2950                 attr.config = PERF_SYNTH_INTEL_CBR;
2951                 err = intel_pt_synth_event(session, "cbr", &attr, id);
2952                 if (err)
2953                         return err;
2954                 pt->cbr_id = id;
2955                 intel_pt_set_event_name(evlist, id, "cbr");
2956                 id += 1;
2957         }
2958
2959         if (pt->synth_opts.pwr_events && (evsel->core.attr.config & 0x10)) {
2960                 attr.config = PERF_SYNTH_INTEL_MWAIT;
2961                 err = intel_pt_synth_event(session, "mwait", &attr, id);
2962                 if (err)
2963                         return err;
2964                 pt->mwait_id = id;
2965                 intel_pt_set_event_name(evlist, id, "mwait");
2966                 id += 1;
2967
2968                 attr.config = PERF_SYNTH_INTEL_PWRE;
2969                 err = intel_pt_synth_event(session, "pwre", &attr, id);
2970                 if (err)
2971                         return err;
2972                 pt->pwre_id = id;
2973                 intel_pt_set_event_name(evlist, id, "pwre");
2974                 id += 1;
2975
2976                 attr.config = PERF_SYNTH_INTEL_EXSTOP;
2977                 err = intel_pt_synth_event(session, "exstop", &attr, id);
2978                 if (err)
2979                         return err;
2980                 pt->exstop_id = id;
2981                 intel_pt_set_event_name(evlist, id, "exstop");
2982                 id += 1;
2983
2984                 attr.config = PERF_SYNTH_INTEL_PWRX;
2985                 err = intel_pt_synth_event(session, "pwrx", &attr, id);
2986                 if (err)
2987                         return err;
2988                 pt->pwrx_id = id;
2989                 intel_pt_set_event_name(evlist, id, "pwrx");
2990                 id += 1;
2991         }
2992
2993         return 0;
2994 }
2995
2996 static void intel_pt_setup_pebs_events(struct intel_pt *pt)
2997 {
2998         struct evsel *evsel;
2999
3000         if (!pt->synth_opts.other_events)
3001                 return;
3002
3003         evlist__for_each_entry(pt->session->evlist, evsel) {
3004                 if (evsel->core.attr.aux_output && evsel->core.id) {
3005                         pt->sample_pebs = true;
3006                         pt->pebs_evsel = evsel;
3007                         return;
3008                 }
3009         }
3010 }
3011
3012 static struct evsel *intel_pt_find_sched_switch(struct evlist *evlist)
3013 {
3014         struct evsel *evsel;
3015
3016         evlist__for_each_entry_reverse(evlist, evsel) {
3017                 const char *name = perf_evsel__name(evsel);
3018
3019                 if (!strcmp(name, "sched:sched_switch"))
3020                         return evsel;
3021         }
3022
3023         return NULL;
3024 }
3025
3026 static bool intel_pt_find_switch(struct evlist *evlist)
3027 {
3028         struct evsel *evsel;
3029
3030         evlist__for_each_entry(evlist, evsel) {
3031                 if (evsel->core.attr.context_switch)
3032                         return true;
3033         }
3034
3035         return false;
3036 }
3037
3038 static int intel_pt_perf_config(const char *var, const char *value, void *data)
3039 {
3040         struct intel_pt *pt = data;
3041
3042         if (!strcmp(var, "intel-pt.mispred-all"))
3043                 pt->mispred_all = perf_config_bool(var, value);
3044
3045         return 0;
3046 }
3047
3048 /* Find least TSC which converts to ns or later */
3049 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt)
3050 {
3051         u64 tsc, tm;
3052
3053         tsc = perf_time_to_tsc(ns, &pt->tc);
3054
3055         while (1) {
3056                 tm = tsc_to_perf_time(tsc, &pt->tc);
3057                 if (tm < ns)
3058                         break;
3059                 tsc -= 1;
3060         }
3061
3062         while (tm < ns)
3063                 tm = tsc_to_perf_time(++tsc, &pt->tc);
3064
3065         return tsc;
3066 }
3067
3068 /* Find greatest TSC which converts to ns or earlier */
3069 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt)
3070 {
3071         u64 tsc, tm;
3072
3073         tsc = perf_time_to_tsc(ns, &pt->tc);
3074
3075         while (1) {
3076                 tm = tsc_to_perf_time(tsc, &pt->tc);
3077                 if (tm > ns)
3078                         break;
3079                 tsc += 1;
3080         }
3081
3082         while (tm > ns)
3083                 tm = tsc_to_perf_time(--tsc, &pt->tc);
3084
3085         return tsc;
3086 }
3087
3088 static int intel_pt_setup_time_ranges(struct intel_pt *pt,
3089                                       struct itrace_synth_opts *opts)
3090 {
3091         struct perf_time_interval *p = opts->ptime_range;
3092         int n = opts->range_num;
3093         int i;
3094
3095         if (!n || !p || pt->timeless_decoding)
3096                 return 0;
3097
3098         pt->time_ranges = calloc(n, sizeof(struct range));
3099         if (!pt->time_ranges)
3100                 return -ENOMEM;
3101
3102         pt->range_cnt = n;
3103
3104         intel_pt_log("%s: %u range(s)\n", __func__, n);
3105
3106         for (i = 0; i < n; i++) {
3107                 struct range *r = &pt->time_ranges[i];
3108                 u64 ts = p[i].start;
3109                 u64 te = p[i].end;
3110
3111                 /*
3112                  * Take care to ensure the TSC range matches the perf-time range
3113                  * when converted back to perf-time.
3114                  */
3115                 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0;
3116                 r->end   = te ? intel_pt_tsc_end(te, pt) : 0;
3117
3118                 intel_pt_log("range %d: perf time interval: %"PRIu64" to %"PRIu64"\n",
3119                              i, ts, te);
3120                 intel_pt_log("range %d: TSC time interval: %#"PRIx64" to %#"PRIx64"\n",
3121                              i, r->start, r->end);
3122         }
3123
3124         return 0;
3125 }
3126
3127 static const char * const intel_pt_info_fmts[] = {
3128         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
3129         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
3130         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
3131         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
3132         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
3133         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
3134         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
3135         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
3136         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
3137         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
3138         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
3139         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
3140         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
3141         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
3142         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
3143         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
3144 };
3145
3146 static void intel_pt_print_info(__u64 *arr, int start, int finish)
3147 {
3148         int i;
3149
3150         if (!dump_trace)
3151                 return;
3152
3153         for (i = start; i <= finish; i++)
3154                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
3155 }
3156
3157 static void intel_pt_print_info_str(const char *name, const char *str)
3158 {
3159         if (!dump_trace)
3160                 return;
3161
3162         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
3163 }
3164
3165 static bool intel_pt_has(struct perf_record_auxtrace_info *auxtrace_info, int pos)
3166 {
3167         return auxtrace_info->header.size >=
3168                 sizeof(struct perf_record_auxtrace_info) + (sizeof(u64) * (pos + 1));
3169 }
3170
3171 int intel_pt_process_auxtrace_info(union perf_event *event,
3172                                    struct perf_session *session)
3173 {
3174         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3175         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
3176         struct intel_pt *pt;
3177         void *info_end;
3178         __u64 *info;
3179         int err;
3180
3181         if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) +
3182                                         min_sz)
3183                 return -EINVAL;
3184
3185         pt = zalloc(sizeof(struct intel_pt));
3186         if (!pt)
3187                 return -ENOMEM;
3188
3189         addr_filters__init(&pt->filts);
3190
3191         err = perf_config(intel_pt_perf_config, pt);
3192         if (err)
3193                 goto err_free;
3194
3195         err = auxtrace_queues__init(&pt->queues);
3196         if (err)
3197                 goto err_free;
3198
3199         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
3200
3201         pt->session = session;
3202         pt->machine = &session->machines.host; /* No kvm support */
3203         pt->auxtrace_type = auxtrace_info->type;
3204         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
3205         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
3206         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
3207         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
3208         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
3209         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
3210         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
3211         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
3212         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
3213         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
3214         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
3215                             INTEL_PT_PER_CPU_MMAPS);
3216
3217         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
3218                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
3219                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
3220                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
3221                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
3222                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
3223                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
3224                                     INTEL_PT_CYC_BIT);
3225         }
3226
3227         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
3228                 pt->max_non_turbo_ratio =
3229                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
3230                 intel_pt_print_info(&auxtrace_info->priv[0],
3231                                     INTEL_PT_MAX_NONTURBO_RATIO,
3232                                     INTEL_PT_MAX_NONTURBO_RATIO);
3233         }
3234
3235         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
3236         info_end = (void *)info + auxtrace_info->header.size;
3237
3238         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
3239                 size_t len;
3240
3241                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
3242                 intel_pt_print_info(&auxtrace_info->priv[0],
3243                                     INTEL_PT_FILTER_STR_LEN,
3244                                     INTEL_PT_FILTER_STR_LEN);
3245                 if (len) {
3246                         const char *filter = (const char *)info;
3247
3248                         len = roundup(len + 1, 8);
3249                         info += len >> 3;
3250                         if ((void *)info > info_end) {
3251                                 pr_err("%s: bad filter string length\n", __func__);
3252                                 err = -EINVAL;
3253                                 goto err_free_queues;
3254                         }
3255                         pt->filter = memdup(filter, len);
3256                         if (!pt->filter) {
3257                                 err = -ENOMEM;
3258                                 goto err_free_queues;
3259                         }
3260                         if (session->header.needs_swap)
3261                                 mem_bswap_64(pt->filter, len);
3262                         if (pt->filter[len - 1]) {
3263                                 pr_err("%s: filter string not null terminated\n", __func__);
3264                                 err = -EINVAL;
3265                                 goto err_free_queues;
3266                         }
3267                         err = addr_filters__parse_bare_filter(&pt->filts,
3268                                                               filter);
3269                         if (err)
3270                                 goto err_free_queues;
3271                 }
3272                 intel_pt_print_info_str("Filter string", pt->filter);
3273         }
3274
3275         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
3276         if (pt->timeless_decoding && !pt->tc.time_mult)
3277                 pt->tc.time_mult = 1;
3278         pt->have_tsc = intel_pt_have_tsc(pt);
3279         pt->sampling_mode = intel_pt_sampling_mode(pt);
3280         pt->est_tsc = !pt->timeless_decoding;
3281
3282         pt->unknown_thread = thread__new(999999999, 999999999);
3283         if (!pt->unknown_thread) {
3284                 err = -ENOMEM;
3285                 goto err_free_queues;
3286         }
3287
3288         /*
3289          * Since this thread will not be kept in any rbtree not in a
3290          * list, initialize its list node so that at thread__put() the
3291          * current thread lifetime assuption is kept and we don't segfault
3292          * at list_del_init().
3293          */
3294         INIT_LIST_HEAD(&pt->unknown_thread->node);
3295
3296         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
3297         if (err)
3298                 goto err_delete_thread;
3299         if (thread__init_maps(pt->unknown_thread, pt->machine)) {
3300                 err = -ENOMEM;
3301                 goto err_delete_thread;
3302         }
3303
3304         pt->auxtrace.process_event = intel_pt_process_event;
3305         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
3306         pt->auxtrace.queue_data = intel_pt_queue_data;
3307         pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample;
3308         pt->auxtrace.flush_events = intel_pt_flush;
3309         pt->auxtrace.free_events = intel_pt_free_events;
3310         pt->auxtrace.free = intel_pt_free;
3311         session->auxtrace = &pt->auxtrace;
3312
3313         if (dump_trace)
3314                 return 0;
3315
3316         if (pt->have_sched_switch == 1) {
3317                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
3318                 if (!pt->switch_evsel) {
3319                         pr_err("%s: missing sched_switch event\n", __func__);
3320                         err = -EINVAL;
3321                         goto err_delete_thread;
3322                 }
3323         } else if (pt->have_sched_switch == 2 &&
3324                    !intel_pt_find_switch(session->evlist)) {
3325                 pr_err("%s: missing context_switch attribute flag\n", __func__);
3326                 err = -EINVAL;
3327                 goto err_delete_thread;
3328         }
3329
3330         if (session->itrace_synth_opts->set) {
3331                 pt->synth_opts = *session->itrace_synth_opts;
3332         } else {
3333                 itrace_synth_opts__set_default(&pt->synth_opts,
3334                                 session->itrace_synth_opts->default_no_sample);
3335                 if (!session->itrace_synth_opts->default_no_sample &&
3336                     !session->itrace_synth_opts->inject) {
3337                         pt->synth_opts.branches = false;
3338                         pt->synth_opts.callchain = true;
3339                 }
3340                 pt->synth_opts.thread_stack =
3341                                 session->itrace_synth_opts->thread_stack;
3342         }
3343
3344         if (pt->synth_opts.log)
3345                 intel_pt_log_enable();
3346
3347         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
3348         if (pt->tc.time_mult) {
3349                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
3350
3351                 if (!pt->max_non_turbo_ratio)
3352                         pt->max_non_turbo_ratio =
3353                                         (tsc_freq + 50000000) / 100000000;
3354                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
3355                 intel_pt_log("Maximum non-turbo ratio %u\n",
3356                              pt->max_non_turbo_ratio);
3357                 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
3358         }
3359
3360         err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts);
3361         if (err)
3362                 goto err_delete_thread;
3363
3364         if (pt->synth_opts.calls)
3365                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
3366                                        PERF_IP_FLAG_TRACE_END;
3367         if (pt->synth_opts.returns)
3368                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
3369                                        PERF_IP_FLAG_TRACE_BEGIN;
3370
3371         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
3372                 symbol_conf.use_callchain = true;
3373                 if (callchain_register_param(&callchain_param) < 0) {
3374                         symbol_conf.use_callchain = false;
3375                         pt->synth_opts.callchain = false;
3376                 }
3377         }
3378
3379         err = intel_pt_synth_events(pt, session);
3380         if (err)
3381                 goto err_delete_thread;
3382
3383         intel_pt_setup_pebs_events(pt);
3384
3385         if (pt->sampling_mode || list_empty(&session->auxtrace_index))
3386                 err = auxtrace_queue_data(session, true, true);
3387         else
3388                 err = auxtrace_queues__process_index(&pt->queues, session);
3389         if (err)
3390                 goto err_delete_thread;
3391
3392         if (pt->queues.populated)
3393                 pt->data_queued = true;
3394
3395         if (pt->timeless_decoding)
3396                 pr_debug2("Intel PT decoding without timestamps\n");
3397
3398         return 0;
3399
3400 err_delete_thread:
3401         thread__zput(pt->unknown_thread);
3402 err_free_queues:
3403         intel_pt_log_disable();
3404         auxtrace_queues__free(&pt->queues);
3405         session->auxtrace = NULL;
3406 err_free:
3407         addr_filters__exit(&pt->filts);
3408         zfree(&pt->filter);
3409         zfree(&pt->time_ranges);
3410         free(pt);
3411         return err;
3412 }