tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "tool.h"
  26 #include "event.h"
  27 #include "evlist.h"
  28 #include "evsel.h"
  29 #include "map.h"
  30 #include "color.h"
  31 #include "util.h"
  32 #include "thread.h"
  33 #include "thread-stack.h"
  34 #include "symbol.h"
  35 #include "callchain.h"
  36 #include "dso.h"
  37 #include "debug.h"
  38 #include "auxtrace.h"
  39 #include "tsc.h"
  40 #include "intel-pt.h"
  41
  42 #include "intel-pt-decoder/intel-pt-log.h"
  43 #include "intel-pt-decoder/intel-pt-decoder.h"
  44 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  45 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  46
  47 #define MAX_TIMESTAMP (~0ULL)
  48
  49 struct intel_pt {
  50         struct auxtrace auxtrace;
  51         struct auxtrace_queues queues;
  52         struct auxtrace_heap heap;
  53         u32 auxtrace_type;
  54         struct perf_session *session;
  55         struct machine *machine;
  56         struct perf_evsel *switch_evsel;
  57         struct thread *unknown_thread;
  58         bool timeless_decoding;
  59         bool sampling_mode;
  60         bool snapshot_mode;
  61         bool per_cpu_mmaps;
  62         bool have_tsc;
  63         bool data_queued;
  64         bool est_tsc;
  65         bool sync_switch;
  66         int have_sched_switch;
  67         u32 pmu_type;
  68         u64 kernel_start;
  69         u64 switch_ip;
  70         u64 ptss_ip;
  71
  72         struct perf_tsc_conversion tc;
  73         bool cap_user_time_zero;
  74
  75         struct itrace_synth_opts synth_opts;
  76
  77         bool sample_instructions;
  78         u64 instructions_sample_type;
  79         u64 instructions_sample_period;
  80         u64 instructions_id;
  81
  82         bool sample_branches;
  83         u32 branches_filter;
  84         u64 branches_sample_type;
  85         u64 branches_id;
  86
  87         bool sample_transactions;
  88         u64 transactions_sample_type;
  89         u64 transactions_id;
  90
  91         bool synth_needs_swap;
  92
  93         u64 tsc_bit;
  94         u64 mtc_bit;
  95         u64 mtc_freq_bits;
  96         u32 tsc_ctc_ratio_n;
  97         u32 tsc_ctc_ratio_d;
  98         u64 cyc_bit;
  99         u64 noretcomp_bit;
 100         unsigned max_non_turbo_ratio;
 101 };
 102
 103 enum switch_state {
 104         INTEL_PT_SS_NOT_TRACING,
 105         INTEL_PT_SS_UNKNOWN,
 106         INTEL_PT_SS_TRACING,
 107         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 108         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 109 };
 110
 111 struct intel_pt_queue {
 112         struct intel_pt *pt;
 113         unsigned int queue_nr;
 114         struct auxtrace_buffer *buffer;
 115         void *decoder;
 116         const struct intel_pt_state *state;
 117         struct ip_callchain *chain;
 118         union perf_event *event_buf;
 119         bool on_heap;
 120         bool stop;
 121         bool step_through_buffers;
 122         bool use_buffer_pid_tid;
 123         pid_t pid, tid;
 124         int cpu;
 125         int switch_state;
 126         pid_t next_tid;
 127         struct thread *thread;
 128         bool exclude_kernel;
 129         bool have_sample;
 130         u64 time;
 131         u64 timestamp;
 132         u32 flags;
 133         u16 insn_len;
 134         u64 last_insn_cnt;
 135 };
 136
 137 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 138                           unsigned char *buf, size_t len)
 139 {
 140         struct intel_pt_pkt packet;
 141         size_t pos = 0;
 142         int ret, pkt_len, i;
 143         char desc[INTEL_PT_PKT_DESC_MAX];
 144         const char *color = PERF_COLOR_BLUE;
 145
 146         color_fprintf(stdout, color,
 147                       ". ... Intel Processor Trace data: size %zu bytes\n",
 148                       len);
 149
 150         while (len) {
 151                 ret = intel_pt_get_packet(buf, len, &packet);
 152                 if (ret > 0)
 153                         pkt_len = ret;
 154                 else
 155                         pkt_len = 1;
 156                 printf(".");
 157                 color_fprintf(stdout, color, "  %08x: ", pos);
 158                 for (i = 0; i < pkt_len; i++)
 159                         color_fprintf(stdout, color, " %02x", buf[i]);
 160                 for (; i < 16; i++)
 161                         color_fprintf(stdout, color, "   ");
 162                 if (ret > 0) {
 163                         ret = intel_pt_pkt_desc(&packet, desc,
 164                                                 INTEL_PT_PKT_DESC_MAX);
 165                         if (ret > 0)
 166                                 color_fprintf(stdout, color, " %s\n", desc);
 167                 } else {
 168                         color_fprintf(stdout, color, " Bad packet!\n");
 169                 }
 170                 pos += pkt_len;
 171                 buf += pkt_len;
 172                 len -= pkt_len;
 173         }
 174 }
 175
 176 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 177                                 size_t len)
 178 {
 179         printf(".\n");
 180         intel_pt_dump(pt, buf, len);
 181 }
 182
 183 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 184                                    struct auxtrace_buffer *b)
 185 {
 186         void *start;
 187
 188         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 189                                       pt->have_tsc);
 190         if (!start)
 191                 return -EINVAL;
 192         b->use_size = b->data + b->size - start;
 193         b->use_data = start;
 194         return 0;
 195 }
 196
 197 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 198                                         struct auxtrace_queue *queue,
 199                                         struct auxtrace_buffer *buffer)
 200 {
 201         if (queue->cpu == -1 && buffer->cpu != -1)
 202                 ptq->cpu = buffer->cpu;
 203
 204         ptq->pid = buffer->pid;
 205         ptq->tid = buffer->tid;
 206
 207         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 208                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 209
 210         thread__zput(ptq->thread);
 211
 212         if (ptq->tid != -1) {
 213                 if (ptq->pid != -1)
 214                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 215                                                               ptq->pid,
 216                                                               ptq->tid);
 217                 else
 218                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 219                                                            ptq->tid);
 220         }
 221 }
 222
 223 /* This function assumes data is processed sequentially only */
 224 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 225 {
 226         struct intel_pt_queue *ptq = data;
 227         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 228         struct auxtrace_queue *queue;
 229
 230         if (ptq->stop) {
 231                 b->len = 0;
 232                 return 0;
 233         }
 234
 235         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 236
 237         buffer = auxtrace_buffer__next(queue, buffer);
 238         if (!buffer) {
 239                 if (old_buffer)
 240                         auxtrace_buffer__drop_data(old_buffer);
 241                 b->len = 0;
 242                 return 0;
 243         }
 244
 245         ptq->buffer = buffer;
 246
 247         if (!buffer->data) {
 248                 int fd = perf_data_file__fd(ptq->pt->session->file);
 249
 250                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 251                 if (!buffer->data)
 252                         return -ENOMEM;
 253         }
 254
 255         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 256             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 257                 return -ENOMEM;
 258
 259         if (old_buffer)
 260                 auxtrace_buffer__drop_data(old_buffer);
 261
 262         if (buffer->use_data) {
 263                 b->len = buffer->use_size;
 264                 b->buf = buffer->use_data;
 265         } else {
 266                 b->len = buffer->size;
 267                 b->buf = buffer->data;
 268         }
 269         b->ref_timestamp = buffer->reference;
 270
 271         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 272                                                       !buffer->consecutive)) {
 273                 b->consecutive = false;
 274                 b->trace_nr = buffer->buffer_nr + 1;
 275         } else {
 276                 b->consecutive = true;
 277         }
 278
 279         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 280                                         ptq->tid != buffer->tid))
 281                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 282
 283         if (ptq->step_through_buffers)
 284                 ptq->stop = true;
 285
 286         if (!b->len)
 287                 return intel_pt_get_trace(b, data);
 288
 289         return 0;
 290 }
 291
 292 struct intel_pt_cache_entry {
 293         struct auxtrace_cache_entry     entry;
 294         u64                             insn_cnt;
 295         u64                             byte_cnt;
 296         enum intel_pt_insn_op           op;
 297         enum intel_pt_insn_branch       branch;
 298         int                             length;
 299         int32_t                         rel;
 300 };
 301
 302 static int intel_pt_config_div(const char *var, const char *value, void *data)
 303 {
 304         int *d = data;
 305         long val;
 306
 307         if (!strcmp(var, "intel-pt.cache-divisor")) {
 308                 val = strtol(value, NULL, 0);
 309                 if (val > 0 && val <= INT_MAX)
 310                         *d = val;
 311         }
 312
 313         return 0;
 314 }
 315
 316 static int intel_pt_cache_divisor(void)
 317 {
 318         static int d;
 319
 320         if (d)
 321                 return d;
 322
 323         perf_config(intel_pt_config_div, &d);
 324
 325         if (!d)
 326                 d = 64;
 327
 328         return d;
 329 }
 330
 331 static unsigned int intel_pt_cache_size(struct dso *dso,
 332                                         struct machine *machine)
 333 {
 334         off_t size;
 335
 336         size = dso__data_size(dso, machine);
 337         size /= intel_pt_cache_divisor();
 338         if (size < 1000)
 339                 return 10;
 340         if (size > (1 << 21))
 341                 return 21;
 342         return 32 - __builtin_clz(size);
 343 }
 344
 345 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 346                                              struct machine *machine)
 347 {
 348         struct auxtrace_cache *c;
 349         unsigned int bits;
 350
 351         if (dso->auxtrace_cache)
 352                 return dso->auxtrace_cache;
 353
 354         bits = intel_pt_cache_size(dso, machine);
 355
 356         /* Ignoring cache creation failure */
 357         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 358
 359         dso->auxtrace_cache = c;
 360
 361         return c;
 362 }
 363
 364 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 365                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 366                               struct intel_pt_insn *intel_pt_insn)
 367 {
 368         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 369         struct intel_pt_cache_entry *e;
 370         int err;
 371
 372         if (!c)
 373                 return -ENOMEM;
 374
 375         e = auxtrace_cache__alloc_entry(c);
 376         if (!e)
 377                 return -ENOMEM;
 378
 379         e->insn_cnt = insn_cnt;
 380         e->byte_cnt = byte_cnt;
 381         e->op = intel_pt_insn->op;
 382         e->branch = intel_pt_insn->branch;
 383         e->length = intel_pt_insn->length;
 384         e->rel = intel_pt_insn->rel;
 385
 386         err = auxtrace_cache__add(c, offset, &e->entry);
 387         if (err)
 388                 auxtrace_cache__free_entry(c, e);
 389
 390         return err;
 391 }
 392
 393 static struct intel_pt_cache_entry *
 394 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 395 {
 396         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 397
 398         if (!c)
 399                 return NULL;
 400
 401         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 402 }
 403
 404 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 405                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 406                                    uint64_t to_ip, uint64_t max_insn_cnt,
 407                                    void *data)
 408 {
 409         struct intel_pt_queue *ptq = data;
 410         struct machine *machine = ptq->pt->machine;
 411         struct thread *thread;
 412         struct addr_location al;
 413         unsigned char buf[1024];
 414         size_t bufsz;
 415         ssize_t len;
 416         int x86_64;
 417         u8 cpumode;
 418         u64 offset, start_offset, start_ip;
 419         u64 insn_cnt = 0;
 420         bool one_map = true;
 421
 422         if (to_ip && *ip == to_ip)
 423                 goto out_no_cache;
 424
 425         bufsz = intel_pt_insn_max_size();
 426
 427         if (*ip >= ptq->pt->kernel_start)
 428                 cpumode = PERF_RECORD_MISC_KERNEL;
 429         else
 430                 cpumode = PERF_RECORD_MISC_USER;
 431
 432         thread = ptq->thread;
 433         if (!thread) {
 434                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 435                         return -EINVAL;
 436                 thread = ptq->pt->unknown_thread;
 437         }
 438
 439         while (1) {
 440                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 441                 if (!al.map || !al.map->dso)
 442                         return -EINVAL;
 443
 444                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 445                     dso__data_status_seen(al.map->dso,
 446                                           DSO_DATA_STATUS_SEEN_ITRACE))
 447                         return -ENOENT;
 448
 449                 offset = al.map->map_ip(al.map, *ip);
 450
 451                 if (!to_ip && one_map) {
 452                         struct intel_pt_cache_entry *e;
 453
 454                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 455                         if (e &&
 456                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 457                                 *insn_cnt_ptr = e->insn_cnt;
 458                                 *ip += e->byte_cnt;
 459                                 intel_pt_insn->op = e->op;
 460                                 intel_pt_insn->branch = e->branch;
 461                                 intel_pt_insn->length = e->length;
 462                                 intel_pt_insn->rel = e->rel;
 463                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 464                                 return 0;
 465                         }
 466                 }
 467
 468                 start_offset = offset;
 469                 start_ip = *ip;
 470
 471                 /* Load maps to ensure dso->is_64_bit has been updated */
 472                 map__load(al.map, machine->symbol_filter);
 473
 474                 x86_64 = al.map->dso->is_64_bit;
 475
 476                 while (1) {
 477                         len = dso__data_read_offset(al.map->dso, machine,
 478                                                     offset, buf, bufsz);
 479                         if (len <= 0)
 480                                 return -EINVAL;
 481
 482                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 483                                 return -EINVAL;
 484
 485                         intel_pt_log_insn(intel_pt_insn, *ip);
 486
 487                         insn_cnt += 1;
 488
 489                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 490                                 goto out;
 491
 492                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 493                                 goto out_no_cache;
 494
 495                         *ip += intel_pt_insn->length;
 496
 497                         if (to_ip && *ip == to_ip)
 498                                 goto out_no_cache;
 499
 500                         if (*ip >= al.map->end)
 501                                 break;
 502
 503                         offset += intel_pt_insn->length;
 504                 }
 505                 one_map = false;
 506         }
 507 out:
 508         *insn_cnt_ptr = insn_cnt;
 509
 510         if (!one_map)
 511                 goto out_no_cache;
 512
 513         /*
 514          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 515          * entries.
 516          */
 517         if (to_ip) {
 518                 struct intel_pt_cache_entry *e;
 519
 520                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 521                 if (e)
 522                         return 0;
 523         }
 524
 525         /* Ignore cache errors */
 526         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 527                            *ip - start_ip, intel_pt_insn);
 528
 529         return 0;
 530
 531 out_no_cache:
 532         *insn_cnt_ptr = insn_cnt;
 533         return 0;
 534 }
 535
 536 static bool intel_pt_get_config(struct intel_pt *pt,
 537                                 struct perf_event_attr *attr, u64 *config)
 538 {
 539         if (attr->type == pt->pmu_type) {
 540                 if (config)
 541                         *config = attr->config;
 542                 return true;
 543         }
 544
 545         return false;
 546 }
 547
 548 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 549 {
 550         struct perf_evsel *evsel;
 551
 552         evlist__for_each(pt->session->evlist, evsel) {
 553                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 554                     !evsel->attr.exclude_kernel)
 555                         return false;
 556         }
 557         return true;
 558 }
 559
 560 static bool intel_pt_return_compression(struct intel_pt *pt)
 561 {
 562         struct perf_evsel *evsel;
 563         u64 config;
 564
 565         if (!pt->noretcomp_bit)
 566                 return true;
 567
 568         evlist__for_each(pt->session->evlist, evsel) {
 569                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 570                     (config & pt->noretcomp_bit))
 571                         return false;
 572         }
 573         return true;
 574 }
 575
 576 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 577 {
 578         struct perf_evsel *evsel;
 579         unsigned int shift;
 580         u64 config;
 581
 582         if (!pt->mtc_freq_bits)
 583                 return 0;
 584
 585         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 586                 config >>= 1;
 587
 588         evlist__for_each(pt->session->evlist, evsel) {
 589                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 590                         return (config & pt->mtc_freq_bits) >> shift;
 591         }
 592         return 0;
 593 }
 594
 595 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 596 {
 597         struct perf_evsel *evsel;
 598         bool timeless_decoding = true;
 599         u64 config;
 600
 601         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 602                 return true;
 603
 604         evlist__for_each(pt->session->evlist, evsel) {
 605                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 606                         return true;
 607                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 608                         if (config & pt->tsc_bit)
 609                                 timeless_decoding = false;
 610                         else
 611                                 return true;
 612                 }
 613         }
 614         return timeless_decoding;
 615 }
 616
 617 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 618 {
 619         struct perf_evsel *evsel;
 620
 621         evlist__for_each(pt->session->evlist, evsel) {
 622                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 623                     !evsel->attr.exclude_kernel)
 624                         return true;
 625         }
 626         return false;
 627 }
 628
 629 static bool intel_pt_have_tsc(struct intel_pt *pt)
 630 {
 631         struct perf_evsel *evsel;
 632         bool have_tsc = false;
 633         u64 config;
 634
 635         if (!pt->tsc_bit)
 636                 return false;
 637
 638         evlist__for_each(pt->session->evlist, evsel) {
 639                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 640                         if (config & pt->tsc_bit)
 641                                 have_tsc = true;
 642                         else
 643                                 return false;
 644                 }
 645         }
 646         return have_tsc;
 647 }
 648
 649 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 650 {
 651         u64 quot, rem;
 652
 653         quot = ns / pt->tc.time_mult;
 654         rem  = ns % pt->tc.time_mult;
 655         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 656                 pt->tc.time_mult;
 657 }
 658
 659 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 660                                                    unsigned int queue_nr)
 661 {
 662         struct intel_pt_params params = { .get_trace = 0, };
 663         struct intel_pt_queue *ptq;
 664
 665         ptq = zalloc(sizeof(struct intel_pt_queue));
 666         if (!ptq)
 667                 return NULL;
 668
 669         if (pt->synth_opts.callchain) {
 670                 size_t sz = sizeof(struct ip_callchain);
 671
 672                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 673                 ptq->chain = zalloc(sz);
 674                 if (!ptq->chain)
 675                         goto out_free;
 676         }
 677
 678         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 679         if (!ptq->event_buf)
 680                 goto out_free;
 681
 682         ptq->pt = pt;
 683         ptq->queue_nr = queue_nr;
 684         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 685         ptq->pid = -1;
 686         ptq->tid = -1;
 687         ptq->cpu = -1;
 688         ptq->next_tid = -1;
 689
 690         params.get_trace = intel_pt_get_trace;
 691         params.walk_insn = intel_pt_walk_next_insn;
 692         params.data = ptq;
 693         params.return_compression = intel_pt_return_compression(pt);
 694         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 695         params.mtc_period = intel_pt_mtc_period(pt);
 696         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 697         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 698
 699         if (pt->synth_opts.instructions) {
 700                 if (pt->synth_opts.period) {
 701                         switch (pt->synth_opts.period_type) {
 702                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 703                                 params.period_type =
 704                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 705                                 params.period = pt->synth_opts.period;
 706                                 break;
 707                         case PERF_ITRACE_PERIOD_TICKS:
 708                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 709                                 params.period = pt->synth_opts.period;
 710                                 break;
 711                         case PERF_ITRACE_PERIOD_NANOSECS:
 712                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 713                                 params.period = intel_pt_ns_to_ticks(pt,
 714                                                         pt->synth_opts.period);
 715                                 break;
 716                         default:
 717                                 break;
 718                         }
 719                 }
 720
 721                 if (!params.period) {
 722                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 723                         params.period = 1000;
 724                 }
 725         }
 726
 727         ptq->decoder = intel_pt_decoder_new(&params);
 728         if (!ptq->decoder)
 729                 goto out_free;
 730
 731         return ptq;
 732
 733 out_free:
 734         zfree(&ptq->event_buf);
 735         zfree(&ptq->chain);
 736         free(ptq);
 737         return NULL;
 738 }
 739
 740 static void intel_pt_free_queue(void *priv)
 741 {
 742         struct intel_pt_queue *ptq = priv;
 743
 744         if (!ptq)
 745                 return;
 746         thread__zput(ptq->thread);
 747         intel_pt_decoder_free(ptq->decoder);
 748         zfree(&ptq->event_buf);
 749         zfree(&ptq->chain);
 750         free(ptq);
 751 }
 752
 753 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 754                                      struct auxtrace_queue *queue)
 755 {
 756         struct intel_pt_queue *ptq = queue->priv;
 757
 758         if (queue->tid == -1 || pt->have_sched_switch) {
 759                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 760                 thread__zput(ptq->thread);
 761         }
 762
 763         if (!ptq->thread && ptq->tid != -1)
 764                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 765
 766         if (ptq->thread) {
 767                 ptq->pid = ptq->thread->pid_;
 768                 if (queue->cpu == -1)
 769                         ptq->cpu = ptq->thread->cpu;
 770         }
 771 }
 772
 773 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 774 {
 775         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 776                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 777         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 778                 if (ptq->state->to_ip)
 779                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 780                                      PERF_IP_FLAG_ASYNC |
 781                                      PERF_IP_FLAG_INTERRUPT;
 782                 else
 783                         ptq->flags = PERF_IP_FLAG_BRANCH |
 784                                      PERF_IP_FLAG_TRACE_END;
 785                 ptq->insn_len = 0;
 786         } else {
 787                 if (ptq->state->from_ip)
 788                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 789                 else
 790                         ptq->flags = PERF_IP_FLAG_BRANCH |
 791                                      PERF_IP_FLAG_TRACE_BEGIN;
 792                 if (ptq->state->flags & INTEL_PT_IN_TX)
 793                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 794                 ptq->insn_len = ptq->state->insn_len;
 795         }
 796 }
 797
 798 static int intel_pt_setup_queue(struct intel_pt *pt,
 799                                 struct auxtrace_queue *queue,
 800                                 unsigned int queue_nr)
 801 {
 802         struct intel_pt_queue *ptq = queue->priv;
 803
 804         if (list_empty(&queue->head))
 805                 return 0;
 806
 807         if (!ptq) {
 808                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 809                 if (!ptq)
 810                         return -ENOMEM;
 811                 queue->priv = ptq;
 812
 813                 if (queue->cpu != -1)
 814                         ptq->cpu = queue->cpu;
 815                 ptq->tid = queue->tid;
 816
 817                 if (pt->sampling_mode) {
 818                         if (pt->timeless_decoding)
 819                                 ptq->step_through_buffers = true;
 820                         if (pt->timeless_decoding || !pt->have_sched_switch)
 821                                 ptq->use_buffer_pid_tid = true;
 822                 }
 823         }
 824
 825         if (!ptq->on_heap &&
 826             (!pt->sync_switch ||
 827              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 828                 const struct intel_pt_state *state;
 829                 int ret;
 830
 831                 if (pt->timeless_decoding)
 832                         return 0;
 833
 834                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 835                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 836                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 837                 while (1) {
 838                         state = intel_pt_decode(ptq->decoder);
 839                         if (state->err) {
 840                                 if (state->err == INTEL_PT_ERR_NODATA) {
 841                                         intel_pt_log("queue %u has no timestamp\n",
 842                                                      queue_nr);
 843                                         return 0;
 844                                 }
 845                                 continue;
 846                         }
 847                         if (state->timestamp)
 848                                 break;
 849                 }
 850
 851                 ptq->timestamp = state->timestamp;
 852                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 853                              queue_nr, ptq->timestamp);
 854                 ptq->state = state;
 855                 ptq->have_sample = true;
 856                 intel_pt_sample_flags(ptq);
 857                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 858                 if (ret)
 859                         return ret;
 860                 ptq->on_heap = true;
 861         }
 862
 863         return 0;
 864 }
 865
 866 static int intel_pt_setup_queues(struct intel_pt *pt)
 867 {
 868         unsigned int i;
 869         int ret;
 870
 871         for (i = 0; i < pt->queues.nr_queues; i++) {
 872                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 873                 if (ret)
 874                         return ret;
 875         }
 876         return 0;
 877 }
 878
 879 static int intel_pt_inject_event(union perf_event *event,
 880                                  struct perf_sample *sample, u64 type,
 881                                  bool swapped)
 882 {
 883         event->header.size = perf_event__sample_event_size(sample, type, 0);
 884         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
 885 }
 886
 887 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 888 {
 889         int ret;
 890         struct intel_pt *pt = ptq->pt;
 891         union perf_event *event = ptq->event_buf;
 892         struct perf_sample sample = { .ip = 0, };
 893
 894         event->sample.header.type = PERF_RECORD_SAMPLE;
 895         event->sample.header.misc = PERF_RECORD_MISC_USER;
 896         event->sample.header.size = sizeof(struct perf_event_header);
 897
 898         if (!pt->timeless_decoding)
 899                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 900
 901         sample.ip = ptq->state->from_ip;
 902         sample.pid = ptq->pid;
 903         sample.tid = ptq->tid;
 904         sample.addr = ptq->state->to_ip;
 905         sample.id = ptq->pt->branches_id;
 906         sample.stream_id = ptq->pt->branches_id;
 907         sample.period = 1;
 908         sample.cpu = ptq->cpu;
 909         sample.flags = ptq->flags;
 910         sample.insn_len = ptq->insn_len;
 911
 912         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
 913                 return 0;
 914
 915         if (pt->synth_opts.inject) {
 916                 ret = intel_pt_inject_event(event, &sample,
 917                                             pt->branches_sample_type,
 918                                             pt->synth_needs_swap);
 919                 if (ret)
 920                         return ret;
 921         }
 922
 923         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
 924         if (ret)
 925                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
 926                        ret);
 927
 928         return ret;
 929 }
 930
 931 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
 932 {
 933         int ret;
 934         struct intel_pt *pt = ptq->pt;
 935         union perf_event *event = ptq->event_buf;
 936         struct perf_sample sample = { .ip = 0, };
 937
 938         event->sample.header.type = PERF_RECORD_SAMPLE;
 939         event->sample.header.misc = PERF_RECORD_MISC_USER;
 940         event->sample.header.size = sizeof(struct perf_event_header);
 941
 942         if (!pt->timeless_decoding)
 943                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 944
 945         sample.ip = ptq->state->from_ip;
 946         sample.pid = ptq->pid;
 947         sample.tid = ptq->tid;
 948         sample.addr = ptq->state->to_ip;
 949         sample.id = ptq->pt->instructions_id;
 950         sample.stream_id = ptq->pt->instructions_id;
 951         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
 952         sample.cpu = ptq->cpu;
 953         sample.flags = ptq->flags;
 954         sample.insn_len = ptq->insn_len;
 955
 956         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
 957
 958         if (pt->synth_opts.callchain) {
 959                 thread_stack__sample(ptq->thread, ptq->chain,
 960                                      pt->synth_opts.callchain_sz, sample.ip);
 961                 sample.callchain = ptq->chain;
 962         }
 963
 964         if (pt->synth_opts.inject) {
 965                 ret = intel_pt_inject_event(event, &sample,
 966                                             pt->instructions_sample_type,
 967                                             pt->synth_needs_swap);
 968                 if (ret)
 969                         return ret;
 970         }
 971
 972         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
 973         if (ret)
 974                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
 975                        ret);
 976
 977         return ret;
 978 }
 979
 980 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
 981 {
 982         int ret;
 983         struct intel_pt *pt = ptq->pt;
 984         union perf_event *event = ptq->event_buf;
 985         struct perf_sample sample = { .ip = 0, };
 986
 987         event->sample.header.type = PERF_RECORD_SAMPLE;
 988         event->sample.header.misc = PERF_RECORD_MISC_USER;
 989         event->sample.header.size = sizeof(struct perf_event_header);
 990
 991         if (!pt->timeless_decoding)
 992                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 993
 994         sample.ip = ptq->state->from_ip;
 995         sample.pid = ptq->pid;
 996         sample.tid = ptq->tid;
 997         sample.addr = ptq->state->to_ip;
 998         sample.id = ptq->pt->transactions_id;
 999         sample.stream_id = ptq->pt->transactions_id;
1000         sample.period = 1;
1001         sample.cpu = ptq->cpu;
1002         sample.flags = ptq->flags;
1003         sample.insn_len = ptq->insn_len;
1004
1005         if (pt->synth_opts.callchain) {
1006                 thread_stack__sample(ptq->thread, ptq->chain,
1007                                      pt->synth_opts.callchain_sz, sample.ip);
1008                 sample.callchain = ptq->chain;
1009         }
1010
1011         if (pt->synth_opts.inject) {
1012                 ret = intel_pt_inject_event(event, &sample,
1013                                             pt->transactions_sample_type,
1014                                             pt->synth_needs_swap);
1015                 if (ret)
1016                         return ret;
1017         }
1018
1019         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1020         if (ret)
1021                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1022                        ret);
1023
1024         return ret;
1025 }
1026
1027 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1028                                 pid_t pid, pid_t tid, u64 ip)
1029 {
1030         union perf_event event;
1031         char msg[MAX_AUXTRACE_ERROR_MSG];
1032         int err;
1033
1034         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1035
1036         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1037                              code, cpu, pid, tid, ip, msg);
1038
1039         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1040         if (err)
1041                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1042                        err);
1043
1044         return err;
1045 }
1046
1047 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1048 {
1049         struct auxtrace_queue *queue;
1050         pid_t tid = ptq->next_tid;
1051         int err;
1052
1053         if (tid == -1)
1054                 return 0;
1055
1056         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1057
1058         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1059
1060         queue = &pt->queues.queue_array[ptq->queue_nr];
1061         intel_pt_set_pid_tid_cpu(pt, queue);
1062
1063         ptq->next_tid = -1;
1064
1065         return err;
1066 }
1067
1068 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1069 {
1070         struct intel_pt *pt = ptq->pt;
1071
1072         return ip == pt->switch_ip &&
1073                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1074                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1075                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1076 }
1077
1078 static int intel_pt_sample(struct intel_pt_queue *ptq)
1079 {
1080         const struct intel_pt_state *state = ptq->state;
1081         struct intel_pt *pt = ptq->pt;
1082         int err;
1083
1084         if (!ptq->have_sample)
1085                 return 0;
1086
1087         ptq->have_sample = false;
1088
1089         if (pt->sample_instructions &&
1090             (state->type & INTEL_PT_INSTRUCTION)) {
1091                 err = intel_pt_synth_instruction_sample(ptq);
1092                 if (err)
1093                         return err;
1094         }
1095
1096         if (pt->sample_transactions &&
1097             (state->type & INTEL_PT_TRANSACTION)) {
1098                 err = intel_pt_synth_transaction_sample(ptq);
1099                 if (err)
1100                         return err;
1101         }
1102
1103         if (!(state->type & INTEL_PT_BRANCH))
1104                 return 0;
1105
1106         if (pt->synth_opts.callchain)
1107                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1108                                     state->to_ip, ptq->insn_len,
1109                                     state->trace_nr);
1110         else
1111                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1112
1113         if (pt->sample_branches) {
1114                 err = intel_pt_synth_branch_sample(ptq);
1115                 if (err)
1116                         return err;
1117         }
1118
1119         if (!pt->sync_switch)
1120                 return 0;
1121
1122         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1123                 switch (ptq->switch_state) {
1124                 case INTEL_PT_SS_UNKNOWN:
1125                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1126                         err = intel_pt_next_tid(pt, ptq);
1127                         if (err)
1128                                 return err;
1129                         ptq->switch_state = INTEL_PT_SS_TRACING;
1130                         break;
1131                 default:
1132                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1133                         return 1;
1134                 }
1135         } else if (!state->to_ip) {
1136                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1137         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1138                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1139         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1140                    state->to_ip == pt->ptss_ip &&
1141                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1142                 ptq->switch_state = INTEL_PT_SS_TRACING;
1143         }
1144
1145         return 0;
1146 }
1147
1148 static u64 intel_pt_switch_ip(struct machine *machine, u64 *ptss_ip)
1149 {
1150         struct map *map;
1151         struct symbol *sym, *start;
1152         u64 ip, switch_ip = 0;
1153
1154         if (ptss_ip)
1155                 *ptss_ip = 0;
1156
1157         map = machine__kernel_map(machine, MAP__FUNCTION);
1158         if (!map)
1159                 return 0;
1160
1161         if (map__load(map, machine->symbol_filter))
1162                 return 0;
1163
1164         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1165
1166         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1167                 if (sym->binding == STB_GLOBAL &&
1168                     !strcmp(sym->name, "__switch_to")) {
1169                         ip = map->unmap_ip(map, sym->start);
1170                         if (ip >= map->start && ip < map->end) {
1171                                 switch_ip = ip;
1172                                 break;
1173                         }
1174                 }
1175         }
1176
1177         if (!switch_ip || !ptss_ip)
1178                 return 0;
1179
1180         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1181                 if (!strcmp(sym->name, "perf_trace_sched_switch")) {
1182                         ip = map->unmap_ip(map, sym->start);
1183                         if (ip >= map->start && ip < map->end) {
1184                                 *ptss_ip = ip;
1185                                 break;
1186                         }
1187                 }
1188         }
1189
1190         return switch_ip;
1191 }
1192
1193 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1194 {
1195         const struct intel_pt_state *state = ptq->state;
1196         struct intel_pt *pt = ptq->pt;
1197         int err;
1198
1199         if (!pt->kernel_start) {
1200                 pt->kernel_start = machine__kernel_start(pt->machine);
1201                 if (pt->per_cpu_mmaps && pt->have_sched_switch &&
1202                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1203                     !pt->sampling_mode) {
1204                         pt->switch_ip = intel_pt_switch_ip(pt->machine,
1205                                                            &pt->ptss_ip);
1206                         if (pt->switch_ip) {
1207                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1208                                              pt->switch_ip, pt->ptss_ip);
1209                                 pt->sync_switch = true;
1210                         }
1211                 }
1212         }
1213
1214         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1215                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1216         while (1) {
1217                 err = intel_pt_sample(ptq);
1218                 if (err)
1219                         return err;
1220
1221                 state = intel_pt_decode(ptq->decoder);
1222                 if (state->err) {
1223                         if (state->err == INTEL_PT_ERR_NODATA)
1224                                 return 1;
1225                         if (pt->sync_switch &&
1226                             state->from_ip >= pt->kernel_start) {
1227                                 pt->sync_switch = false;
1228                                 intel_pt_next_tid(pt, ptq);
1229                         }
1230                         if (pt->synth_opts.errors) {
1231                                 err = intel_pt_synth_error(pt, state->err,
1232                                                            ptq->cpu, ptq->pid,
1233                                                            ptq->tid,
1234                                                            state->from_ip);
1235                                 if (err)
1236                                         return err;
1237                         }
1238                         continue;
1239                 }
1240
1241                 ptq->state = state;
1242                 ptq->have_sample = true;
1243                 intel_pt_sample_flags(ptq);
1244
1245                 /* Use estimated TSC upon return to user space */
1246                 if (pt->est_tsc &&
1247                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1248                     state->to_ip && state->to_ip < pt->kernel_start) {
1249                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1250                                      state->timestamp, state->est_timestamp);
1251                         ptq->timestamp = state->est_timestamp;
1252                 /* Use estimated TSC in unknown switch state */
1253                 } else if (pt->sync_switch &&
1254                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1255                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1256                            ptq->next_tid == -1) {
1257                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1258                                      state->timestamp, state->est_timestamp);
1259                         ptq->timestamp = state->est_timestamp;
1260                 } else if (state->timestamp > ptq->timestamp) {
1261                         ptq->timestamp = state->timestamp;
1262                 }
1263
1264                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1265                         *timestamp = ptq->timestamp;
1266                         return 0;
1267                 }
1268         }
1269         return 0;
1270 }
1271
1272 static inline int intel_pt_update_queues(struct intel_pt *pt)
1273 {
1274         if (pt->queues.new_data) {
1275                 pt->queues.new_data = false;
1276                 return intel_pt_setup_queues(pt);
1277         }
1278         return 0;
1279 }
1280
1281 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1282 {
1283         unsigned int queue_nr;
1284         u64 ts;
1285         int ret;
1286
1287         while (1) {
1288                 struct auxtrace_queue *queue;
1289                 struct intel_pt_queue *ptq;
1290
1291                 if (!pt->heap.heap_cnt)
1292                         return 0;
1293
1294                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1295                         return 0;
1296
1297                 queue_nr = pt->heap.heap_array[0].queue_nr;
1298                 queue = &pt->queues.queue_array[queue_nr];
1299                 ptq = queue->priv;
1300
1301                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1302                              queue_nr, pt->heap.heap_array[0].ordinal,
1303                              timestamp);
1304
1305                 auxtrace_heap__pop(&pt->heap);
1306
1307                 if (pt->heap.heap_cnt) {
1308                         ts = pt->heap.heap_array[0].ordinal + 1;
1309                         if (ts > timestamp)
1310                                 ts = timestamp;
1311                 } else {
1312                         ts = timestamp;
1313                 }
1314
1315                 intel_pt_set_pid_tid_cpu(pt, queue);
1316
1317                 ret = intel_pt_run_decoder(ptq, &ts);
1318
1319                 if (ret < 0) {
1320                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1321                         return ret;
1322                 }
1323
1324                 if (!ret) {
1325                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1326                         if (ret < 0)
1327                                 return ret;
1328                 } else {
1329                         ptq->on_heap = false;
1330                 }
1331         }
1332
1333         return 0;
1334 }
1335
1336 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1337                                             u64 time_)
1338 {
1339         struct auxtrace_queues *queues = &pt->queues;
1340         unsigned int i;
1341         u64 ts = 0;
1342
1343         for (i = 0; i < queues->nr_queues; i++) {
1344                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1345                 struct intel_pt_queue *ptq = queue->priv;
1346
1347                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1348                         ptq->time = time_;
1349                         intel_pt_set_pid_tid_cpu(pt, queue);
1350                         intel_pt_run_decoder(ptq, &ts);
1351                 }
1352         }
1353         return 0;
1354 }
1355
1356 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1357 {
1358         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1359                                     sample->pid, sample->tid, 0);
1360 }
1361
1362 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1363 {
1364         unsigned i, j;
1365
1366         if (cpu < 0 || !pt->queues.nr_queues)
1367                 return NULL;
1368
1369         if ((unsigned)cpu >= pt->queues.nr_queues)
1370                 i = pt->queues.nr_queues - 1;
1371         else
1372                 i = cpu;
1373
1374         if (pt->queues.queue_array[i].cpu == cpu)
1375                 return pt->queues.queue_array[i].priv;
1376
1377         for (j = 0; i > 0; j++) {
1378                 if (pt->queues.queue_array[--i].cpu == cpu)
1379                         return pt->queues.queue_array[i].priv;
1380         }
1381
1382         for (; j < pt->queues.nr_queues; j++) {
1383                 if (pt->queues.queue_array[j].cpu == cpu)
1384                         return pt->queues.queue_array[j].priv;
1385         }
1386
1387         return NULL;
1388 }
1389
1390 static int intel_pt_process_switch(struct intel_pt *pt,
1391                                    struct perf_sample *sample)
1392 {
1393         struct intel_pt_queue *ptq;
1394         struct perf_evsel *evsel;
1395         pid_t tid;
1396         int cpu, err;
1397
1398         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1399         if (evsel != pt->switch_evsel)
1400                 return 0;
1401
1402         tid = perf_evsel__intval(evsel, sample, "next_pid");
1403         cpu = sample->cpu;
1404
1405         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1406                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1407                      &pt->tc));
1408
1409         if (!pt->sync_switch)
1410                 goto out;
1411
1412         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1413         if (!ptq)
1414                 goto out;
1415
1416         switch (ptq->switch_state) {
1417         case INTEL_PT_SS_NOT_TRACING:
1418                 ptq->next_tid = -1;
1419                 break;
1420         case INTEL_PT_SS_UNKNOWN:
1421         case INTEL_PT_SS_TRACING:
1422                 ptq->next_tid = tid;
1423                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1424                 return 0;
1425         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1426                 if (!ptq->on_heap) {
1427                         ptq->timestamp = perf_time_to_tsc(sample->time,
1428                                                           &pt->tc);
1429                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1430                                                  ptq->timestamp);
1431                         if (err)
1432                                 return err;
1433                         ptq->on_heap = true;
1434                 }
1435                 ptq->switch_state = INTEL_PT_SS_TRACING;
1436                 break;
1437         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1438                 ptq->next_tid = tid;
1439                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1440                 break;
1441         default:
1442                 break;
1443         }
1444 out:
1445         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1446 }
1447
1448 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1449                                          union perf_event *event,
1450                                          struct perf_sample *sample)
1451 {
1452         if (!pt->per_cpu_mmaps)
1453                 return 0;
1454
1455         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1456                      sample->cpu, event->itrace_start.pid,
1457                      event->itrace_start.tid, sample->time,
1458                      perf_time_to_tsc(sample->time, &pt->tc));
1459
1460         return machine__set_current_tid(pt->machine, sample->cpu,
1461                                         event->itrace_start.pid,
1462                                         event->itrace_start.tid);
1463 }
1464
1465 static int intel_pt_process_event(struct perf_session *session,
1466                                   union perf_event *event,
1467                                   struct perf_sample *sample,
1468                                   struct perf_tool *tool)
1469 {
1470         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1471                                            auxtrace);
1472         u64 timestamp;
1473         int err = 0;
1474
1475         if (dump_trace)
1476                 return 0;
1477
1478         if (!tool->ordered_events) {
1479                 pr_err("Intel Processor Trace requires ordered events\n");
1480                 return -EINVAL;
1481         }
1482
1483         if (sample->time && sample->time != (u64)-1)
1484                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1485         else
1486                 timestamp = 0;
1487
1488         if (timestamp || pt->timeless_decoding) {
1489                 err = intel_pt_update_queues(pt);
1490                 if (err)
1491                         return err;
1492         }
1493
1494         if (pt->timeless_decoding) {
1495                 if (event->header.type == PERF_RECORD_EXIT) {
1496                         err = intel_pt_process_timeless_queues(pt,
1497                                                                event->fork.tid,
1498                                                                sample->time);
1499                 }
1500         } else if (timestamp) {
1501                 err = intel_pt_process_queues(pt, timestamp);
1502         }
1503         if (err)
1504                 return err;
1505
1506         if (event->header.type == PERF_RECORD_AUX &&
1507             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1508             pt->synth_opts.errors) {
1509                 err = intel_pt_lost(pt, sample);
1510                 if (err)
1511                         return err;
1512         }
1513
1514         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1515                 err = intel_pt_process_switch(pt, sample);
1516         else if (event->header.type == PERF_RECORD_ITRACE_START)
1517                 err = intel_pt_process_itrace_start(pt, event, sample);
1518
1519         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1520                      perf_event__name(event->header.type), event->header.type,
1521                      sample->cpu, sample->time, timestamp);
1522
1523         return err;
1524 }
1525
1526 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1527 {
1528         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1529                                            auxtrace);
1530         int ret;
1531
1532         if (dump_trace)
1533                 return 0;
1534
1535         if (!tool->ordered_events)
1536                 return -EINVAL;
1537
1538         ret = intel_pt_update_queues(pt);
1539         if (ret < 0)
1540                 return ret;
1541
1542         if (pt->timeless_decoding)
1543                 return intel_pt_process_timeless_queues(pt, -1,
1544                                                         MAX_TIMESTAMP - 1);
1545
1546         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1547 }
1548
1549 static void intel_pt_free_events(struct perf_session *session)
1550 {
1551         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1552                                            auxtrace);
1553         struct auxtrace_queues *queues = &pt->queues;
1554         unsigned int i;
1555
1556         for (i = 0; i < queues->nr_queues; i++) {
1557                 intel_pt_free_queue(queues->queue_array[i].priv);
1558                 queues->queue_array[i].priv = NULL;
1559         }
1560         intel_pt_log_disable();
1561         auxtrace_queues__free(queues);
1562 }
1563
1564 static void intel_pt_free(struct perf_session *session)
1565 {
1566         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1567                                            auxtrace);
1568
1569         auxtrace_heap__free(&pt->heap);
1570         intel_pt_free_events(session);
1571         session->auxtrace = NULL;
1572         thread__delete(pt->unknown_thread);
1573         free(pt);
1574 }
1575
1576 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1577                                            union perf_event *event,
1578                                            struct perf_tool *tool __maybe_unused)
1579 {
1580         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1581                                            auxtrace);
1582
1583         if (pt->sampling_mode)
1584                 return 0;
1585
1586         if (!pt->data_queued) {
1587                 struct auxtrace_buffer *buffer;
1588                 off_t data_offset;
1589                 int fd = perf_data_file__fd(session->file);
1590                 int err;
1591
1592                 if (perf_data_file__is_pipe(session->file)) {
1593                         data_offset = 0;
1594                 } else {
1595                         data_offset = lseek(fd, 0, SEEK_CUR);
1596                         if (data_offset == -1)
1597                                 return -errno;
1598                 }
1599
1600                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1601                                                  data_offset, &buffer);
1602                 if (err)
1603                         return err;
1604
1605                 /* Dump here now we have copied a piped trace out of the pipe */
1606                 if (dump_trace) {
1607                         if (auxtrace_buffer__get_data(buffer, fd)) {
1608                                 intel_pt_dump_event(pt, buffer->data,
1609                                                     buffer->size);
1610                                 auxtrace_buffer__put_data(buffer);
1611                         }
1612                 }
1613         }
1614
1615         return 0;
1616 }
1617
1618 struct intel_pt_synth {
1619         struct perf_tool dummy_tool;
1620         struct perf_session *session;
1621 };
1622
1623 static int intel_pt_event_synth(struct perf_tool *tool,
1624                                 union perf_event *event,
1625                                 struct perf_sample *sample __maybe_unused,
1626                                 struct machine *machine __maybe_unused)
1627 {
1628         struct intel_pt_synth *intel_pt_synth =
1629                         container_of(tool, struct intel_pt_synth, dummy_tool);
1630
1631         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1632                                                  NULL);
1633 }
1634
1635 static int intel_pt_synth_event(struct perf_session *session,
1636                                 struct perf_event_attr *attr, u64 id)
1637 {
1638         struct intel_pt_synth intel_pt_synth;
1639
1640         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1641         intel_pt_synth.session = session;
1642
1643         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1644                                            &id, intel_pt_event_synth);
1645 }
1646
1647 static int intel_pt_synth_events(struct intel_pt *pt,
1648                                  struct perf_session *session)
1649 {
1650         struct perf_evlist *evlist = session->evlist;
1651         struct perf_evsel *evsel;
1652         struct perf_event_attr attr;
1653         bool found = false;
1654         u64 id;
1655         int err;
1656
1657         evlist__for_each(evlist, evsel) {
1658                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1659                         found = true;
1660                         break;
1661                 }
1662         }
1663
1664         if (!found) {
1665                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1666                 return 0;
1667         }
1668
1669         memset(&attr, 0, sizeof(struct perf_event_attr));
1670         attr.size = sizeof(struct perf_event_attr);
1671         attr.type = PERF_TYPE_HARDWARE;
1672         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1673         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1674                             PERF_SAMPLE_PERIOD;
1675         if (pt->timeless_decoding)
1676                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1677         else
1678                 attr.sample_type |= PERF_SAMPLE_TIME;
1679         if (!pt->per_cpu_mmaps)
1680                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1681         attr.exclude_user = evsel->attr.exclude_user;
1682         attr.exclude_kernel = evsel->attr.exclude_kernel;
1683         attr.exclude_hv = evsel->attr.exclude_hv;
1684         attr.exclude_host = evsel->attr.exclude_host;
1685         attr.exclude_guest = evsel->attr.exclude_guest;
1686         attr.sample_id_all = evsel->attr.sample_id_all;
1687         attr.read_format = evsel->attr.read_format;
1688
1689         id = evsel->id[0] + 1000000000;
1690         if (!id)
1691                 id = 1;
1692
1693         if (pt->synth_opts.instructions) {
1694                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1695                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1696                         attr.sample_period =
1697                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1698                 else
1699                         attr.sample_period = pt->synth_opts.period;
1700                 pt->instructions_sample_period = attr.sample_period;
1701                 if (pt->synth_opts.callchain)
1702                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1703                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1704                          id, (u64)attr.sample_type);
1705                 err = intel_pt_synth_event(session, &attr, id);
1706                 if (err) {
1707                         pr_err("%s: failed to synthesize 'instructions' event type\n",
1708                                __func__);
1709                         return err;
1710                 }
1711                 pt->sample_instructions = true;
1712                 pt->instructions_sample_type = attr.sample_type;
1713                 pt->instructions_id = id;
1714                 id += 1;
1715         }
1716
1717         if (pt->synth_opts.transactions) {
1718                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1719                 attr.sample_period = 1;
1720                 if (pt->synth_opts.callchain)
1721                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1722                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1723                          id, (u64)attr.sample_type);
1724                 err = intel_pt_synth_event(session, &attr, id);
1725                 if (err) {
1726                         pr_err("%s: failed to synthesize 'transactions' event type\n",
1727                                __func__);
1728                         return err;
1729                 }
1730                 pt->sample_transactions = true;
1731                 pt->transactions_id = id;
1732                 id += 1;
1733                 evlist__for_each(evlist, evsel) {
1734                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
1735                                 if (evsel->name)
1736                                         zfree(&evsel->name);
1737                                 evsel->name = strdup("transactions");
1738                                 break;
1739                         }
1740                 }
1741         }
1742
1743         if (pt->synth_opts.branches) {
1744                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1745                 attr.sample_period = 1;
1746                 attr.sample_type |= PERF_SAMPLE_ADDR;
1747                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1748                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1749                          id, (u64)attr.sample_type);
1750                 err = intel_pt_synth_event(session, &attr, id);
1751                 if (err) {
1752                         pr_err("%s: failed to synthesize 'branches' event type\n",
1753                                __func__);
1754                         return err;
1755                 }
1756                 pt->sample_branches = true;
1757                 pt->branches_sample_type = attr.sample_type;
1758                 pt->branches_id = id;
1759         }
1760
1761         pt->synth_needs_swap = evsel->needs_swap;
1762
1763         return 0;
1764 }
1765
1766 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1767 {
1768         struct perf_evsel *evsel;
1769
1770         evlist__for_each_reverse(evlist, evsel) {
1771                 const char *name = perf_evsel__name(evsel);
1772
1773                 if (!strcmp(name, "sched:sched_switch"))
1774                         return evsel;
1775         }
1776
1777         return NULL;
1778 }
1779
1780 static const char * const intel_pt_info_fmts[] = {
1781         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
1782         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
1783         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
1784         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
1785         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
1786         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
1787         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
1788         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
1789         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
1790         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
1791         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
1792         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
1793         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
1794         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
1795 };
1796
1797 static void intel_pt_print_info(u64 *arr, int start, int finish)
1798 {
1799         int i;
1800
1801         if (!dump_trace)
1802                 return;
1803
1804         for (i = start; i <= finish; i++)
1805                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
1806 }
1807
1808 int intel_pt_process_auxtrace_info(union perf_event *event,
1809                                    struct perf_session *session)
1810 {
1811         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
1812         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
1813         struct intel_pt *pt;
1814         int err;
1815
1816         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
1817                                         min_sz)
1818                 return -EINVAL;
1819
1820         pt = zalloc(sizeof(struct intel_pt));
1821         if (!pt)
1822                 return -ENOMEM;
1823
1824         err = auxtrace_queues__init(&pt->queues);
1825         if (err)
1826                 goto err_free;
1827
1828         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
1829
1830         pt->session = session;
1831         pt->machine = &session->machines.host; /* No kvm support */
1832         pt->auxtrace_type = auxtrace_info->type;
1833         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
1834         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
1835         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
1836         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
1837         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
1838         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
1839         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
1840         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
1841         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
1842         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
1843         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
1844                             INTEL_PT_PER_CPU_MMAPS);
1845
1846         if (auxtrace_info->header.size >= sizeof(struct auxtrace_info_event) +
1847                                         (sizeof(u64) * INTEL_PT_CYC_BIT)) {
1848                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
1849                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
1850                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
1851                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
1852                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
1853                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
1854                                     INTEL_PT_CYC_BIT);
1855         }
1856
1857         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
1858         pt->have_tsc = intel_pt_have_tsc(pt);
1859         pt->sampling_mode = false;
1860         pt->est_tsc = !pt->timeless_decoding;
1861
1862         pt->unknown_thread = thread__new(999999999, 999999999);
1863         if (!pt->unknown_thread) {
1864                 err = -ENOMEM;
1865                 goto err_free_queues;
1866         }
1867         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
1868         if (err)
1869                 goto err_delete_thread;
1870         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
1871                 err = -ENOMEM;
1872                 goto err_delete_thread;
1873         }
1874
1875         pt->auxtrace.process_event = intel_pt_process_event;
1876         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
1877         pt->auxtrace.flush_events = intel_pt_flush;
1878         pt->auxtrace.free_events = intel_pt_free_events;
1879         pt->auxtrace.free = intel_pt_free;
1880         session->auxtrace = &pt->auxtrace;
1881
1882         if (dump_trace)
1883                 return 0;
1884
1885         if (pt->have_sched_switch == 1) {
1886                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
1887                 if (!pt->switch_evsel) {
1888                         pr_err("%s: missing sched_switch event\n", __func__);
1889                         goto err_delete_thread;
1890                 }
1891         }
1892
1893         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
1894                 pt->synth_opts = *session->itrace_synth_opts;
1895         } else {
1896                 itrace_synth_opts__set_default(&pt->synth_opts);
1897                 if (use_browser != -1) {
1898                         pt->synth_opts.branches = false;
1899                         pt->synth_opts.callchain = true;
1900                 }
1901         }
1902
1903         if (pt->synth_opts.log)
1904                 intel_pt_log_enable();
1905
1906         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
1907         if (pt->tc.time_mult) {
1908                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
1909
1910                 pt->max_non_turbo_ratio = (tsc_freq + 50000000) / 100000000;
1911                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
1912                 intel_pt_log("Maximum non-turbo ratio %u\n",
1913                              pt->max_non_turbo_ratio);
1914         }
1915
1916         if (pt->synth_opts.calls)
1917                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
1918                                        PERF_IP_FLAG_TRACE_END;
1919         if (pt->synth_opts.returns)
1920                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
1921                                        PERF_IP_FLAG_TRACE_BEGIN;
1922
1923         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
1924                 symbol_conf.use_callchain = true;
1925                 if (callchain_register_param(&callchain_param) < 0) {
1926                         symbol_conf.use_callchain = false;
1927                         pt->synth_opts.callchain = false;
1928                 }
1929         }
1930
1931         err = intel_pt_synth_events(pt, session);
1932         if (err)
1933                 goto err_delete_thread;
1934
1935         err = auxtrace_queues__process_index(&pt->queues, session);
1936         if (err)
1937                 goto err_delete_thread;
1938
1939         if (pt->queues.populated)
1940                 pt->data_queued = true;
1941
1942         if (pt->timeless_decoding)
1943                 pr_debug2("Intel PT decoding without timestamps\n");
1944
1945         return 0;
1946
1947 err_delete_thread:
1948         thread__delete(pt->unknown_thread);
1949 err_free_queues:
1950         intel_pt_log_disable();
1951         auxtrace_queues__free(&pt->queues);
1952         session->auxtrace = NULL;
1953 err_free:
1954         free(pt);
1955         return err;
1956 }