tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <[email protected]>
   6  * Author: Mathieu Poirier <[email protected]>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/coresight-pmu.h>
  11 #include <linux/err.h>
  12 #include <linux/kernel.h>
  13 #include <linux/log2.h>
  14 #include <linux/types.h>
  15 #include <linux/zalloc.h>
  16
  17 #include <stdlib.h>
  18
  19 #include "auxtrace.h"
  20 #include "color.h"
  21 #include "cs-etm.h"
  22 #include "cs-etm-decoder/cs-etm-decoder.h"
  23 #include "debug.h"
  24 #include "dso.h"
  25 #include "evlist.h"
  26 #include "intlist.h"
  27 #include "machine.h"
  28 #include "map.h"
  29 #include "perf.h"
  30 #include "session.h"
  31 #include "map_symbol.h"
  32 #include "branch.h"
  33 #include "symbol.h"
  34 #include "tool.h"
  35 #include "thread.h"
  36 #include "thread-stack.h"
  37 #include "tsc.h"
  38 #include <tools/libc_compat.h>
  39 #include "util/synthetic-events.h"
  40 #include "util/util.h"
  41
  42 struct cs_etm_auxtrace {
  43         struct auxtrace auxtrace;
  44         struct auxtrace_queues queues;
  45         struct auxtrace_heap heap;
  46         struct itrace_synth_opts synth_opts;
  47         struct perf_session *session;
  48         struct perf_tsc_conversion tc;
  49
  50         /*
  51          * Timeless has no timestamps in the trace so overlapping mmap lookups
  52          * are less accurate but produces smaller trace data. We use context IDs
  53          * in the trace instead of matching timestamps with fork records so
  54          * they're not really needed in the general case. Overlapping mmaps
  55          * happen in cases like between a fork and an exec.
  56          */
  57         bool timeless_decoding;
  58
  59         /*
  60          * Per-thread ignores the trace channel ID and instead assumes that
  61          * everything in a buffer comes from the same process regardless of
  62          * which CPU it ran on. It also implies no context IDs so the TID is
  63          * taken from the auxtrace buffer.
  64          */
  65         bool per_thread_decoding;
  66         bool snapshot_mode;
  67         bool data_queued;
  68         bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
  69
  70         int num_cpu;
  71         u64 latest_kernel_timestamp;
  72         u32 auxtrace_type;
  73         u64 branches_sample_type;
  74         u64 branches_id;
  75         u64 instructions_sample_type;
  76         u64 instructions_sample_period;
  77         u64 instructions_id;
  78         u64 **metadata;
  79         unsigned int pmu_type;
  80         enum cs_etm_pid_fmt pid_fmt;
  81 };
  82
  83 struct cs_etm_traceid_queue {
  84         u8 trace_chan_id;
  85         u64 period_instructions;
  86         size_t last_branch_pos;
  87         union perf_event *event_buf;
  88         struct thread *thread;
  89         struct thread *prev_packet_thread;
  90         ocsd_ex_level prev_packet_el;
  91         ocsd_ex_level el;
  92         struct branch_stack *last_branch;
  93         struct branch_stack *last_branch_rb;
  94         struct cs_etm_packet *prev_packet;
  95         struct cs_etm_packet *packet;
  96         struct cs_etm_packet_queue packet_queue;
  97 };
  98
  99 struct cs_etm_queue {
 100         struct cs_etm_auxtrace *etm;
 101         struct cs_etm_decoder *decoder;
 102         struct auxtrace_buffer *buffer;
 103         unsigned int queue_nr;
 104         u8 pending_timestamp_chan_id;
 105         u64 offset;
 106         const unsigned char *buf;
 107         size_t buf_len, buf_used;
 108         /* Conversion between traceID and index in traceid_queues array */
 109         struct intlist *traceid_queues_list;
 110         struct cs_etm_traceid_queue **traceid_queues;
 111 };
 112
 113 /* RB tree for quick conversion between traceID and metadata pointers */
 114 static struct intlist *traceid_list;
 115
 116 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
 117 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
 118                                            pid_t tid);
 119 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
 120 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
 121
 122 /* PTMs ETMIDR [11:8] set to b0011 */
 123 #define ETMIDR_PTM_VERSION 0x00000300
 124
 125 /*
 126  * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
 127  * work with.  One option is to modify to auxtrace_heap_XYZ() API or simply
 128  * encode the etm queue number as the upper 16 bit and the channel as
 129  * the lower 16 bit.
 130  */
 131 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
 132                       (queue_nr << 16 | trace_chan_id)
 133 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
 134 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
 135
 136 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
 137 {
 138         etmidr &= ETMIDR_PTM_VERSION;
 139
 140         if (etmidr == ETMIDR_PTM_VERSION)
 141                 return CS_ETM_PROTO_PTM;
 142
 143         return CS_ETM_PROTO_ETMV3;
 144 }
 145
 146 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 147 {
 148         struct int_node *inode;
 149         u64 *metadata;
 150
 151         inode = intlist__find(traceid_list, trace_chan_id);
 152         if (!inode)
 153                 return -EINVAL;
 154
 155         metadata = inode->priv;
 156         *magic = metadata[CS_ETM_MAGIC];
 157         return 0;
 158 }
 159
 160 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 161 {
 162         struct int_node *inode;
 163         u64 *metadata;
 164
 165         inode = intlist__find(traceid_list, trace_chan_id);
 166         if (!inode)
 167                 return -EINVAL;
 168
 169         metadata = inode->priv;
 170         *cpu = (int)metadata[CS_ETM_CPU];
 171         return 0;
 172 }
 173
 174 /*
 175  * The returned PID format is presented as an enum:
 176  *
 177  *   CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced.
 178  *   CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced.
 179  *   CS_ETM_PIDFMT_NONE: No context IDs
 180  *
 181  * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
 182  * are enabled at the same time when the session runs on an EL2 kernel.
 183  * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
 184  * recorded in the trace data, the tool will selectively use
 185  * CONTEXTIDR_EL2 as PID.
 186  *
 187  * The result is cached in etm->pid_fmt so this function only needs to be called
 188  * when processing the aux info.
 189  */
 190 static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata)
 191 {
 192         u64 val;
 193
 194         if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
 195                 val = metadata[CS_ETM_ETMCR];
 196                 /* CONTEXTIDR is traced */
 197                 if (val & BIT(ETM_OPT_CTXTID))
 198                         return CS_ETM_PIDFMT_CTXTID;
 199         } else {
 200                 val = metadata[CS_ETMV4_TRCCONFIGR];
 201                 /* CONTEXTIDR_EL2 is traced */
 202                 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
 203                         return CS_ETM_PIDFMT_CTXTID2;
 204                 /* CONTEXTIDR_EL1 is traced */
 205                 else if (val & BIT(ETM4_CFG_BIT_CTXTID))
 206                         return CS_ETM_PIDFMT_CTXTID;
 207         }
 208
 209         return CS_ETM_PIDFMT_NONE;
 210 }
 211
 212 enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq)
 213 {
 214         return etmq->etm->pid_fmt;
 215 }
 216
 217 static int cs_etm__map_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
 218 {
 219         struct int_node *inode;
 220
 221         /* Get an RB node for this CPU */
 222         inode = intlist__findnew(traceid_list, trace_chan_id);
 223
 224         /* Something went wrong, no need to continue */
 225         if (!inode)
 226                 return -ENOMEM;
 227
 228         /*
 229          * The node for that CPU should not be taken.
 230          * Back out if that's the case.
 231          */
 232         if (inode->priv)
 233                 return -EINVAL;
 234
 235         /* All good, associate the traceID with the metadata pointer */
 236         inode->priv = cpu_metadata;
 237
 238         return 0;
 239 }
 240
 241 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
 242 {
 243         u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
 244
 245         switch (cs_etm_magic) {
 246         case __perf_cs_etmv3_magic:
 247                 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
 248                                       CORESIGHT_TRACE_ID_VAL_MASK);
 249                 break;
 250         case __perf_cs_etmv4_magic:
 251         case __perf_cs_ete_magic:
 252                 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
 253                                       CORESIGHT_TRACE_ID_VAL_MASK);
 254                 break;
 255         default:
 256                 return -EINVAL;
 257         }
 258         return 0;
 259 }
 260
 261 /*
 262  * update metadata trace ID from the value found in the AUX_HW_INFO packet.
 263  * This will also clear the CORESIGHT_TRACE_ID_UNUSED_FLAG flag if present.
 264  */
 265 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
 266 {
 267         u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
 268
 269         switch (cs_etm_magic) {
 270         case __perf_cs_etmv3_magic:
 271                  cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
 272                 break;
 273         case __perf_cs_etmv4_magic:
 274         case __perf_cs_ete_magic:
 275                 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
 276                 break;
 277
 278         default:
 279                 return -EINVAL;
 280         }
 281         return 0;
 282 }
 283
 284 /*
 285  * FIELD_GET (linux/bitfield.h) not available outside kernel code,
 286  * and the header contains too many dependencies to just copy over,
 287  * so roll our own based on the original
 288  */
 289 #define __bf_shf(x) (__builtin_ffsll(x) - 1)
 290 #define FIELD_GET(_mask, _reg)                                          \
 291         ({                                                              \
 292                 (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
 293         })
 294
 295 /*
 296  * Get a metadata for a specific cpu from an array.
 297  *
 298  */
 299 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu)
 300 {
 301         int i;
 302         u64 *metadata = NULL;
 303
 304         for (i = 0; i < etm->num_cpu; i++) {
 305                 if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) {
 306                         metadata = etm->metadata[i];
 307                         break;
 308                 }
 309         }
 310
 311         return metadata;
 312 }
 313
 314 /*
 315  * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
 316  *
 317  * The payload associates the Trace ID and the CPU.
 318  * The routine is tolerant of seeing multiple packets with the same association,
 319  * but a CPU / Trace ID association changing during a session is an error.
 320  */
 321 static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
 322                                             union perf_event *event)
 323 {
 324         struct cs_etm_auxtrace *etm;
 325         struct perf_sample sample;
 326         struct int_node *inode;
 327         struct evsel *evsel;
 328         u64 *cpu_data;
 329         u64 hw_id;
 330         int cpu, version, err;
 331         u8 trace_chan_id, curr_chan_id;
 332
 333         /* extract and parse the HW ID */
 334         hw_id = event->aux_output_hw_id.hw_id;
 335         version = FIELD_GET(CS_AUX_HW_ID_VERSION_MASK, hw_id);
 336         trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
 337
 338         /* check that we can handle this version */
 339         if (version > CS_AUX_HW_ID_CURR_VERSION)
 340                 return -EINVAL;
 341
 342         /* get access to the etm metadata */
 343         etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
 344         if (!etm || !etm->metadata)
 345                 return -EINVAL;
 346
 347         /* parse the sample to get the CPU */
 348         evsel = evlist__event2evsel(session->evlist, event);
 349         if (!evsel)
 350                 return -EINVAL;
 351         err = evsel__parse_sample(evsel, event, &sample);
 352         if (err)
 353                 return err;
 354         cpu = sample.cpu;
 355         if (cpu == -1) {
 356                 /* no CPU in the sample - possibly recorded with an old version of perf */
 357                 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
 358                 return -EINVAL;
 359         }
 360
 361         /* See if the ID is mapped to a CPU, and it matches the current CPU */
 362         inode = intlist__find(traceid_list, trace_chan_id);
 363         if (inode) {
 364                 cpu_data = inode->priv;
 365                 if ((int)cpu_data[CS_ETM_CPU] != cpu) {
 366                         pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
 367                         return -EINVAL;
 368                 }
 369
 370                 /* check that the mapped ID matches */
 371                 err = cs_etm__metadata_get_trace_id(&curr_chan_id, cpu_data);
 372                 if (err)
 373                         return err;
 374                 if (curr_chan_id != trace_chan_id) {
 375                         pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
 376                         return -EINVAL;
 377                 }
 378
 379                 /* mapped and matched - return OK */
 380                 return 0;
 381         }
 382
 383         cpu_data = get_cpu_data(etm, cpu);
 384         if (cpu_data == NULL)
 385                 return err;
 386
 387         /* not one we've seen before - lets map it */
 388         err = cs_etm__map_trace_id(trace_chan_id, cpu_data);
 389         if (err)
 390                 return err;
 391
 392         /*
 393          * if we are picking up the association from the packet, need to plug
 394          * the correct trace ID into the metadata for setting up decoders later.
 395          */
 396         err = cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
 397         return err;
 398 }
 399
 400 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
 401                                               u8 trace_chan_id)
 402 {
 403         /*
 404          * When a timestamp packet is encountered the backend code
 405          * is stopped so that the front end has time to process packets
 406          * that were accumulated in the traceID queue.  Since there can
 407          * be more than one channel per cs_etm_queue, we need to specify
 408          * what traceID queue needs servicing.
 409          */
 410         etmq->pending_timestamp_chan_id = trace_chan_id;
 411 }
 412
 413 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
 414                                       u8 *trace_chan_id)
 415 {
 416         struct cs_etm_packet_queue *packet_queue;
 417
 418         if (!etmq->pending_timestamp_chan_id)
 419                 return 0;
 420
 421         if (trace_chan_id)
 422                 *trace_chan_id = etmq->pending_timestamp_chan_id;
 423
 424         packet_queue = cs_etm__etmq_get_packet_queue(etmq,
 425                                                      etmq->pending_timestamp_chan_id);
 426         if (!packet_queue)
 427                 return 0;
 428
 429         /* Acknowledge pending status */
 430         etmq->pending_timestamp_chan_id = 0;
 431
 432         /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
 433         return packet_queue->cs_timestamp;
 434 }
 435
 436 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 437 {
 438         int i;
 439
 440         queue->head = 0;
 441         queue->tail = 0;
 442         queue->packet_count = 0;
 443         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 444                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 445                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 446                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 447                 queue->packet_buffer[i].instr_count = 0;
 448                 queue->packet_buffer[i].last_instr_taken_branch = false;
 449                 queue->packet_buffer[i].last_instr_size = 0;
 450                 queue->packet_buffer[i].last_instr_type = 0;
 451                 queue->packet_buffer[i].last_instr_subtype = 0;
 452                 queue->packet_buffer[i].last_instr_cond = 0;
 453                 queue->packet_buffer[i].flags = 0;
 454                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 455                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 456                 queue->packet_buffer[i].cpu = INT_MIN;
 457         }
 458 }
 459
 460 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
 461 {
 462         int idx;
 463         struct int_node *inode;
 464         struct cs_etm_traceid_queue *tidq;
 465         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 466
 467         intlist__for_each_entry(inode, traceid_queues_list) {
 468                 idx = (int)(intptr_t)inode->priv;
 469                 tidq = etmq->traceid_queues[idx];
 470                 cs_etm__clear_packet_queue(&tidq->packet_queue);
 471         }
 472 }
 473
 474 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 475                                       struct cs_etm_traceid_queue *tidq,
 476                                       u8 trace_chan_id)
 477 {
 478         int rc = -ENOMEM;
 479         struct auxtrace_queue *queue;
 480         struct cs_etm_auxtrace *etm = etmq->etm;
 481
 482         cs_etm__clear_packet_queue(&tidq->packet_queue);
 483
 484         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 485         tidq->trace_chan_id = trace_chan_id;
 486         tidq->el = tidq->prev_packet_el = ocsd_EL_unknown;
 487         tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,
 488                                                queue->tid);
 489         tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);
 490
 491         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 492         if (!tidq->packet)
 493                 goto out;
 494
 495         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 496         if (!tidq->prev_packet)
 497                 goto out_free;
 498
 499         if (etm->synth_opts.last_branch) {
 500                 size_t sz = sizeof(struct branch_stack);
 501
 502                 sz += etm->synth_opts.last_branch_sz *
 503                       sizeof(struct branch_entry);
 504                 tidq->last_branch = zalloc(sz);
 505                 if (!tidq->last_branch)
 506                         goto out_free;
 507                 tidq->last_branch_rb = zalloc(sz);
 508                 if (!tidq->last_branch_rb)
 509                         goto out_free;
 510         }
 511
 512         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 513         if (!tidq->event_buf)
 514                 goto out_free;
 515
 516         return 0;
 517
 518 out_free:
 519         zfree(&tidq->last_branch_rb);
 520         zfree(&tidq->last_branch);
 521         zfree(&tidq->prev_packet);
 522         zfree(&tidq->packet);
 523 out:
 524         return rc;
 525 }
 526
 527 static struct cs_etm_traceid_queue
 528 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 529 {
 530         int idx;
 531         struct int_node *inode;
 532         struct intlist *traceid_queues_list;
 533         struct cs_etm_traceid_queue *tidq, **traceid_queues;
 534         struct cs_etm_auxtrace *etm = etmq->etm;
 535
 536         if (etm->per_thread_decoding)
 537                 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
 538
 539         traceid_queues_list = etmq->traceid_queues_list;
 540
 541         /*
 542          * Check if the traceid_queue exist for this traceID by looking
 543          * in the queue list.
 544          */
 545         inode = intlist__find(traceid_queues_list, trace_chan_id);
 546         if (inode) {
 547                 idx = (int)(intptr_t)inode->priv;
 548                 return etmq->traceid_queues[idx];
 549         }
 550
 551         /* We couldn't find a traceid_queue for this traceID, allocate one */
 552         tidq = malloc(sizeof(*tidq));
 553         if (!tidq)
 554                 return NULL;
 555
 556         memset(tidq, 0, sizeof(*tidq));
 557
 558         /* Get a valid index for the new traceid_queue */
 559         idx = intlist__nr_entries(traceid_queues_list);
 560         /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
 561         inode = intlist__findnew(traceid_queues_list, trace_chan_id);
 562         if (!inode)
 563                 goto out_free;
 564
 565         /* Associate this traceID with this index */
 566         inode->priv = (void *)(intptr_t)idx;
 567
 568         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 569                 goto out_free;
 570
 571         /* Grow the traceid_queues array by one unit */
 572         traceid_queues = etmq->traceid_queues;
 573         traceid_queues = reallocarray(traceid_queues,
 574                                       idx + 1,
 575                                       sizeof(*traceid_queues));
 576
 577         /*
 578          * On failure reallocarray() returns NULL and the original block of
 579          * memory is left untouched.
 580          */
 581         if (!traceid_queues)
 582                 goto out_free;
 583
 584         traceid_queues[idx] = tidq;
 585         etmq->traceid_queues = traceid_queues;
 586
 587         return etmq->traceid_queues[idx];
 588
 589 out_free:
 590         /*
 591          * Function intlist__remove() removes the inode from the list
 592          * and delete the memory associated to it.
 593          */
 594         intlist__remove(traceid_queues_list, inode);
 595         free(tidq);
 596
 597         return NULL;
 598 }
 599
 600 struct cs_etm_packet_queue
 601 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 602 {
 603         struct cs_etm_traceid_queue *tidq;
 604
 605         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 606         if (tidq)
 607                 return &tidq->packet_queue;
 608
 609         return NULL;
 610 }
 611
 612 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
 613                                 struct cs_etm_traceid_queue *tidq)
 614 {
 615         struct cs_etm_packet *tmp;
 616
 617         if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
 618             etm->synth_opts.instructions) {
 619                 /*
 620                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
 621                  * the next incoming packet.
 622                  *
 623                  * Threads and exception levels are also tracked for both the
 624                  * previous and current packets. This is because the previous
 625                  * packet is used for the 'from' IP for branch samples, so the
 626                  * thread at that time must also be assigned to that sample.
 627                  * Across discontinuity packets the thread can change, so by
 628                  * tracking the thread for the previous packet the branch sample
 629                  * will have the correct info.
 630                  */
 631                 tmp = tidq->packet;
 632                 tidq->packet = tidq->prev_packet;
 633                 tidq->prev_packet = tmp;
 634                 tidq->prev_packet_el = tidq->el;
 635                 thread__put(tidq->prev_packet_thread);
 636                 tidq->prev_packet_thread = thread__get(tidq->thread);
 637         }
 638 }
 639
 640 static void cs_etm__packet_dump(const char *pkt_string)
 641 {
 642         const char *color = PERF_COLOR_BLUE;
 643         int len = strlen(pkt_string);
 644
 645         if (len && (pkt_string[len-1] == '\n'))
 646                 color_fprintf(stdout, color, "  %s", pkt_string);
 647         else
 648                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 649
 650         fflush(stdout);
 651 }
 652
 653 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 654                                           struct cs_etm_auxtrace *etm, int idx,
 655                                           u32 etmidr)
 656 {
 657         u64 **metadata = etm->metadata;
 658
 659         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 660         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 661         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 662 }
 663
 664 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 665                                           struct cs_etm_auxtrace *etm, int idx)
 666 {
 667         u64 **metadata = etm->metadata;
 668
 669         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 670         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 671         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 672         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 673         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 674         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 675         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 676 }
 677
 678 static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
 679                                           struct cs_etm_auxtrace *etm, int idx)
 680 {
 681         u64 **metadata = etm->metadata;
 682
 683         t_params[idx].protocol = CS_ETM_PROTO_ETE;
 684         t_params[idx].ete.reg_idr0 = metadata[idx][CS_ETE_TRCIDR0];
 685         t_params[idx].ete.reg_idr1 = metadata[idx][CS_ETE_TRCIDR1];
 686         t_params[idx].ete.reg_idr2 = metadata[idx][CS_ETE_TRCIDR2];
 687         t_params[idx].ete.reg_idr8 = metadata[idx][CS_ETE_TRCIDR8];
 688         t_params[idx].ete.reg_configr = metadata[idx][CS_ETE_TRCCONFIGR];
 689         t_params[idx].ete.reg_traceidr = metadata[idx][CS_ETE_TRCTRACEIDR];
 690         t_params[idx].ete.reg_devarch = metadata[idx][CS_ETE_TRCDEVARCH];
 691 }
 692
 693 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 694                                      struct cs_etm_auxtrace *etm,
 695                                      int decoders)
 696 {
 697         int i;
 698         u32 etmidr;
 699         u64 architecture;
 700
 701         for (i = 0; i < decoders; i++) {
 702                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 703
 704                 switch (architecture) {
 705                 case __perf_cs_etmv3_magic:
 706                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 707                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 708                         break;
 709                 case __perf_cs_etmv4_magic:
 710                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 711                         break;
 712                 case __perf_cs_ete_magic:
 713                         cs_etm__set_trace_param_ete(t_params, etm, i);
 714                         break;
 715                 default:
 716                         return -EINVAL;
 717                 }
 718         }
 719
 720         return 0;
 721 }
 722
 723 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 724                                        struct cs_etm_queue *etmq,
 725                                        enum cs_etm_decoder_operation mode,
 726                                        bool formatted)
 727 {
 728         int ret = -EINVAL;
 729
 730         if (!(mode < CS_ETM_OPERATION_MAX))
 731                 goto out;
 732
 733         d_params->packet_printer = cs_etm__packet_dump;
 734         d_params->operation = mode;
 735         d_params->data = etmq;
 736         d_params->formatted = formatted;
 737         d_params->fsyncs = false;
 738         d_params->hsyncs = false;
 739         d_params->frame_aligned = true;
 740
 741         ret = 0;
 742 out:
 743         return ret;
 744 }
 745
 746 static void cs_etm__dump_event(struct cs_etm_queue *etmq,
 747                                struct auxtrace_buffer *buffer)
 748 {
 749         int ret;
 750         const char *color = PERF_COLOR_BLUE;
 751         size_t buffer_used = 0;
 752
 753         fprintf(stdout, "\n");
 754         color_fprintf(stdout, color,
 755                      ". ... CoreSight %s Trace data: size %#zx bytes\n",
 756                      cs_etm_decoder__get_name(etmq->decoder), buffer->size);
 757
 758         do {
 759                 size_t consumed;
 760
 761                 ret = cs_etm_decoder__process_data_block(
 762                                 etmq->decoder, buffer->offset,
 763                                 &((u8 *)buffer->data)[buffer_used],
 764                                 buffer->size - buffer_used, &consumed);
 765                 if (ret)
 766                         break;
 767
 768                 buffer_used += consumed;
 769         } while (buffer_used < buffer->size);
 770
 771         cs_etm_decoder__reset(etmq->decoder);
 772 }
 773
 774 static int cs_etm__flush_events(struct perf_session *session,
 775                                 struct perf_tool *tool)
 776 {
 777         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 778                                                    struct cs_etm_auxtrace,
 779                                                    auxtrace);
 780         if (dump_trace)
 781                 return 0;
 782
 783         if (!tool->ordered_events)
 784                 return -EINVAL;
 785
 786         if (etm->timeless_decoding) {
 787                 /*
 788                  * Pass tid = -1 to process all queues. But likely they will have
 789                  * already been processed on PERF_RECORD_EXIT anyway.
 790                  */
 791                 return cs_etm__process_timeless_queues(etm, -1);
 792         }
 793
 794         return cs_etm__process_timestamped_queues(etm);
 795 }
 796
 797 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
 798 {
 799         int idx;
 800         uintptr_t priv;
 801         struct int_node *inode, *tmp;
 802         struct cs_etm_traceid_queue *tidq;
 803         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 804
 805         intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
 806                 priv = (uintptr_t)inode->priv;
 807                 idx = priv;
 808
 809                 /* Free this traceid_queue from the array */
 810                 tidq = etmq->traceid_queues[idx];
 811                 thread__zput(tidq->thread);
 812                 thread__zput(tidq->prev_packet_thread);
 813                 zfree(&tidq->event_buf);
 814                 zfree(&tidq->last_branch);
 815                 zfree(&tidq->last_branch_rb);
 816                 zfree(&tidq->prev_packet);
 817                 zfree(&tidq->packet);
 818                 zfree(&tidq);
 819
 820                 /*
 821                  * Function intlist__remove() removes the inode from the list
 822                  * and delete the memory associated to it.
 823                  */
 824                 intlist__remove(traceid_queues_list, inode);
 825         }
 826
 827         /* Then the RB tree itself */
 828         intlist__delete(traceid_queues_list);
 829         etmq->traceid_queues_list = NULL;
 830
 831         /* finally free the traceid_queues array */
 832         zfree(&etmq->traceid_queues);
 833 }
 834
 835 static void cs_etm__free_queue(void *priv)
 836 {
 837         struct cs_etm_queue *etmq = priv;
 838
 839         if (!etmq)
 840                 return;
 841
 842         cs_etm_decoder__free(etmq->decoder);
 843         cs_etm__free_traceid_queues(etmq);
 844         free(etmq);
 845 }
 846
 847 static void cs_etm__free_events(struct perf_session *session)
 848 {
 849         unsigned int i;
 850         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 851                                                    struct cs_etm_auxtrace,
 852                                                    auxtrace);
 853         struct auxtrace_queues *queues = &aux->queues;
 854
 855         for (i = 0; i < queues->nr_queues; i++) {
 856                 cs_etm__free_queue(queues->queue_array[i].priv);
 857                 queues->queue_array[i].priv = NULL;
 858         }
 859
 860         auxtrace_queues__free(queues);
 861 }
 862
 863 static void cs_etm__free(struct perf_session *session)
 864 {
 865         int i;
 866         struct int_node *inode, *tmp;
 867         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 868                                                    struct cs_etm_auxtrace,
 869                                                    auxtrace);
 870         cs_etm__free_events(session);
 871         session->auxtrace = NULL;
 872
 873         /* First remove all traceID/metadata nodes for the RB tree */
 874         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 875                 intlist__remove(traceid_list, inode);
 876         /* Then the RB tree itself */
 877         intlist__delete(traceid_list);
 878
 879         for (i = 0; i < aux->num_cpu; i++)
 880                 zfree(&aux->metadata[i]);
 881
 882         zfree(&aux->metadata);
 883         zfree(&aux);
 884 }
 885
 886 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
 887                                       struct evsel *evsel)
 888 {
 889         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 890                                                    struct cs_etm_auxtrace,
 891                                                    auxtrace);
 892
 893         return evsel->core.attr.type == aux->pmu_type;
 894 }
 895
 896 static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq,
 897                                            ocsd_ex_level el)
 898 {
 899         enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq);
 900
 901         /*
 902          * For any virtualisation based on nVHE (e.g. pKVM), or host kernels
 903          * running at EL1 assume everything is the host.
 904          */
 905         if (pid_fmt == CS_ETM_PIDFMT_CTXTID)
 906                 return &etmq->etm->session->machines.host;
 907
 908         /*
 909          * Not perfect, but otherwise assume anything in EL1 is the default
 910          * guest, and everything else is the host. Distinguishing between guest
 911          * and host userspaces isn't currently supported either. Neither is
 912          * multiple guest support. All this does is reduce the likeliness of
 913          * decode errors where we look into the host kernel maps when it should
 914          * have been the guest maps.
 915          */
 916         switch (el) {
 917         case ocsd_EL1:
 918                 return machines__find_guest(&etmq->etm->session->machines,
 919                                             DEFAULT_GUEST_KERNEL_ID);
 920         case ocsd_EL3:
 921         case ocsd_EL2:
 922         case ocsd_EL0:
 923         case ocsd_EL_unknown:
 924         default:
 925                 return &etmq->etm->session->machines.host;
 926         }
 927 }
 928
 929 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address,
 930                            ocsd_ex_level el)
 931 {
 932         struct machine *machine = cs_etm__get_machine(etmq, el);
 933
 934         if (address >= machine__kernel_start(machine)) {
 935                 if (machine__is_host(machine))
 936                         return PERF_RECORD_MISC_KERNEL;
 937                 else
 938                         return PERF_RECORD_MISC_GUEST_KERNEL;
 939         } else {
 940                 if (machine__is_host(machine))
 941                         return PERF_RECORD_MISC_USER;
 942                 else {
 943                         /*
 944                          * Can't really happen at the moment because
 945                          * cs_etm__get_machine() will always return
 946                          * machines.host for any non EL1 trace.
 947                          */
 948                         return PERF_RECORD_MISC_GUEST_USER;
 949                 }
 950         }
 951 }
 952
 953 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
 954                               u64 address, size_t size, u8 *buffer,
 955                               const ocsd_mem_space_acc_t mem_space)
 956 {
 957         u8  cpumode;
 958         u64 offset;
 959         int len;
 960         struct addr_location al;
 961         struct dso *dso;
 962         struct cs_etm_traceid_queue *tidq;
 963         int ret = 0;
 964
 965         if (!etmq)
 966                 return 0;
 967
 968         addr_location__init(&al);
 969         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 970         if (!tidq)
 971                 goto out;
 972
 973         /*
 974          * We've already tracked EL along side the PID in cs_etm__set_thread()
 975          * so double check that it matches what OpenCSD thinks as well. It
 976          * doesn't distinguish between EL0 and EL1 for this mem access callback
 977          * so we had to do the extra tracking. Skip validation if it's any of
 978          * the 'any' values.
 979          */
 980         if (!(mem_space == OCSD_MEM_SPACE_ANY ||
 981               mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) {
 982                 if (mem_space & OCSD_MEM_SPACE_EL1N) {
 983                         /* Includes both non secure EL1 and EL0 */
 984                         assert(tidq->el == ocsd_EL1 || tidq->el == ocsd_EL0);
 985                 } else if (mem_space & OCSD_MEM_SPACE_EL2)
 986                         assert(tidq->el == ocsd_EL2);
 987                 else if (mem_space & OCSD_MEM_SPACE_EL3)
 988                         assert(tidq->el == ocsd_EL3);
 989         }
 990
 991         cpumode = cs_etm__cpu_mode(etmq, address, tidq->el);
 992
 993         if (!thread__find_map(tidq->thread, cpumode, address, &al))
 994                 goto out;
 995
 996         dso = map__dso(al.map);
 997         if (!dso)
 998                 goto out;
 999
1000         if (dso->data.status == DSO_DATA_STATUS_ERROR &&
1001             dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
1002                 goto out;
1003
1004         offset = map__map_ip(al.map, address);
1005
1006         map__load(al.map);
1007
1008         len = dso__data_read_offset(dso, maps__machine(thread__maps(tidq->thread)),
1009                                     offset, buffer, size);
1010
1011         if (len <= 0) {
1012                 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
1013                                  "              Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
1014                 if (!dso->auxtrace_warned) {
1015                         pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
1016                                     address,
1017                                     dso->long_name ? dso->long_name : "Unknown");
1018                         dso->auxtrace_warned = true;
1019                 }
1020                 goto out;
1021         }
1022         ret = len;
1023 out:
1024         addr_location__exit(&al);
1025         return ret;
1026 }
1027
1028 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
1029                                                 bool formatted)
1030 {
1031         struct cs_etm_decoder_params d_params;
1032         struct cs_etm_trace_params  *t_params = NULL;
1033         struct cs_etm_queue *etmq;
1034         /*
1035          * Each queue can only contain data from one CPU when unformatted, so only one decoder is
1036          * needed.
1037          */
1038         int decoders = formatted ? etm->num_cpu : 1;
1039
1040         etmq = zalloc(sizeof(*etmq));
1041         if (!etmq)
1042                 return NULL;
1043
1044         etmq->traceid_queues_list = intlist__new(NULL);
1045         if (!etmq->traceid_queues_list)
1046                 goto out_free;
1047
1048         /* Use metadata to fill in trace parameters for trace decoder */
1049         t_params = zalloc(sizeof(*t_params) * decoders);
1050
1051         if (!t_params)
1052                 goto out_free;
1053
1054         if (cs_etm__init_trace_params(t_params, etm, decoders))
1055                 goto out_free;
1056
1057         /* Set decoder parameters to decode trace packets */
1058         if (cs_etm__init_decoder_params(&d_params, etmq,
1059                                         dump_trace ? CS_ETM_OPERATION_PRINT :
1060                                                      CS_ETM_OPERATION_DECODE,
1061                                         formatted))
1062                 goto out_free;
1063
1064         etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
1065                                             t_params);
1066
1067         if (!etmq->decoder)
1068                 goto out_free;
1069
1070         /*
1071          * Register a function to handle all memory accesses required by
1072          * the trace decoder library.
1073          */
1074         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
1075                                               0x0L, ((u64) -1L),
1076                                               cs_etm__mem_access))
1077                 goto out_free_decoder;
1078
1079         zfree(&t_params);
1080         return etmq;
1081
1082 out_free_decoder:
1083         cs_etm_decoder__free(etmq->decoder);
1084 out_free:
1085         intlist__delete(etmq->traceid_queues_list);
1086         free(etmq);
1087
1088         return NULL;
1089 }
1090
1091 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
1092                                struct auxtrace_queue *queue,
1093                                unsigned int queue_nr,
1094                                bool formatted)
1095 {
1096         struct cs_etm_queue *etmq = queue->priv;
1097
1098         if (list_empty(&queue->head) || etmq)
1099                 return 0;
1100
1101         etmq = cs_etm__alloc_queue(etm, formatted);
1102
1103         if (!etmq)
1104                 return -ENOMEM;
1105
1106         queue->priv = etmq;
1107         etmq->etm = etm;
1108         etmq->queue_nr = queue_nr;
1109         etmq->offset = 0;
1110
1111         return 0;
1112 }
1113
1114 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1115                                             struct cs_etm_queue *etmq,
1116                                             unsigned int queue_nr)
1117 {
1118         int ret = 0;
1119         unsigned int cs_queue_nr;
1120         u8 trace_chan_id;
1121         u64 cs_timestamp;
1122
1123         /*
1124          * We are under a CPU-wide trace scenario.  As such we need to know
1125          * when the code that generated the traces started to execute so that
1126          * it can be correlated with execution on other CPUs.  So we get a
1127          * handle on the beginning of traces and decode until we find a
1128          * timestamp.  The timestamp is then added to the auxtrace min heap
1129          * in order to know what nibble (of all the etmqs) to decode first.
1130          */
1131         while (1) {
1132                 /*
1133                  * Fetch an aux_buffer from this etmq.  Bail if no more
1134                  * blocks or an error has been encountered.
1135                  */
1136                 ret = cs_etm__get_data_block(etmq);
1137                 if (ret <= 0)
1138                         goto out;
1139
1140                 /*
1141                  * Run decoder on the trace block.  The decoder will stop when
1142                  * encountering a CS timestamp, a full packet queue or the end of
1143                  * trace for that block.
1144                  */
1145                 ret = cs_etm__decode_data_block(etmq);
1146                 if (ret)
1147                         goto out;
1148
1149                 /*
1150                  * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1151                  * the timestamp calculation for us.
1152                  */
1153                 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1154
1155                 /* We found a timestamp, no need to continue. */
1156                 if (cs_timestamp)
1157                         break;
1158
1159                 /*
1160                  * We didn't find a timestamp so empty all the traceid packet
1161                  * queues before looking for another timestamp packet, either
1162                  * in the current data block or a new one.  Packets that were
1163                  * just decoded are useless since no timestamp has been
1164                  * associated with them.  As such simply discard them.
1165                  */
1166                 cs_etm__clear_all_packet_queues(etmq);
1167         }
1168
1169         /*
1170          * We have a timestamp.  Add it to the min heap to reflect when
1171          * instructions conveyed by the range packets of this traceID queue
1172          * started to execute.  Once the same has been done for all the traceID
1173          * queues of each etmq, redenring and decoding can start in
1174          * chronological order.
1175          *
1176          * Note that packets decoded above are still in the traceID's packet
1177          * queue and will be processed in cs_etm__process_timestamped_queues().
1178          */
1179         cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1180         ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1181 out:
1182         return ret;
1183 }
1184
1185 static inline
1186 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1187                                  struct cs_etm_traceid_queue *tidq)
1188 {
1189         struct branch_stack *bs_src = tidq->last_branch_rb;
1190         struct branch_stack *bs_dst = tidq->last_branch;
1191         size_t nr = 0;
1192
1193         /*
1194          * Set the number of records before early exit: ->nr is used to
1195          * determine how many branches to copy from ->entries.
1196          */
1197         bs_dst->nr = bs_src->nr;
1198
1199         /*
1200          * Early exit when there is nothing to copy.
1201          */
1202         if (!bs_src->nr)
1203                 return;
1204
1205         /*
1206          * As bs_src->entries is a circular buffer, we need to copy from it in
1207          * two steps.  First, copy the branches from the most recently inserted
1208          * branch ->last_branch_pos until the end of bs_src->entries buffer.
1209          */
1210         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1211         memcpy(&bs_dst->entries[0],
1212                &bs_src->entries[tidq->last_branch_pos],
1213                sizeof(struct branch_entry) * nr);
1214
1215         /*
1216          * If we wrapped around at least once, the branches from the beginning
1217          * of the bs_src->entries buffer and until the ->last_branch_pos element
1218          * are older valid branches: copy them over.  The total number of
1219          * branches copied over will be equal to the number of branches asked by
1220          * the user in last_branch_sz.
1221          */
1222         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1223                 memcpy(&bs_dst->entries[nr],
1224                        &bs_src->entries[0],
1225                        sizeof(struct branch_entry) * tidq->last_branch_pos);
1226         }
1227 }
1228
1229 static inline
1230 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1231 {
1232         tidq->last_branch_pos = 0;
1233         tidq->last_branch_rb->nr = 0;
1234 }
1235
1236 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1237                                          u8 trace_chan_id, u64 addr)
1238 {
1239         u8 instrBytes[2];
1240
1241         cs_etm__mem_access(etmq, trace_chan_id, addr, ARRAY_SIZE(instrBytes),
1242                            instrBytes, 0);
1243         /*
1244          * T32 instruction size is indicated by bits[15:11] of the first
1245          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1246          * denote a 32-bit instruction.
1247          */
1248         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1249 }
1250
1251 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1252 {
1253         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1254         if (packet->sample_type == CS_ETM_DISCONTINUITY)
1255                 return 0;
1256
1257         return packet->start_addr;
1258 }
1259
1260 static inline
1261 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1262 {
1263         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1264         if (packet->sample_type == CS_ETM_DISCONTINUITY)
1265                 return 0;
1266
1267         return packet->end_addr - packet->last_instr_size;
1268 }
1269
1270 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1271                                      u64 trace_chan_id,
1272                                      const struct cs_etm_packet *packet,
1273                                      u64 offset)
1274 {
1275         if (packet->isa == CS_ETM_ISA_T32) {
1276                 u64 addr = packet->start_addr;
1277
1278                 while (offset) {
1279                         addr += cs_etm__t32_instr_size(etmq,
1280                                                        trace_chan_id, addr);
1281                         offset--;
1282                 }
1283                 return addr;
1284         }
1285
1286         /* Assume a 4 byte instruction size (A32/A64) */
1287         return packet->start_addr + offset * 4;
1288 }
1289
1290 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1291                                           struct cs_etm_traceid_queue *tidq)
1292 {
1293         struct branch_stack *bs = tidq->last_branch_rb;
1294         struct branch_entry *be;
1295
1296         /*
1297          * The branches are recorded in a circular buffer in reverse
1298          * chronological order: we start recording from the last element of the
1299          * buffer down.  After writing the first element of the stack, move the
1300          * insert position back to the end of the buffer.
1301          */
1302         if (!tidq->last_branch_pos)
1303                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1304
1305         tidq->last_branch_pos -= 1;
1306
1307         be       = &bs->entries[tidq->last_branch_pos];
1308         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1309         be->to   = cs_etm__first_executed_instr(tidq->packet);
1310         /* No support for mispredict */
1311         be->flags.mispred = 0;
1312         be->flags.predicted = 1;
1313
1314         /*
1315          * Increment bs->nr until reaching the number of last branches asked by
1316          * the user on the command line.
1317          */
1318         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1319                 bs->nr += 1;
1320 }
1321
1322 static int cs_etm__inject_event(union perf_event *event,
1323                                struct perf_sample *sample, u64 type)
1324 {
1325         event->header.size = perf_event__sample_event_size(sample, type, 0);
1326         return perf_event__synthesize_sample(event, type, 0, sample);
1327 }
1328
1329
1330 static int
1331 cs_etm__get_trace(struct cs_etm_queue *etmq)
1332 {
1333         struct auxtrace_buffer *aux_buffer = etmq->buffer;
1334         struct auxtrace_buffer *old_buffer = aux_buffer;
1335         struct auxtrace_queue *queue;
1336
1337         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1338
1339         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1340
1341         /* If no more data, drop the previous auxtrace_buffer and return */
1342         if (!aux_buffer) {
1343                 if (old_buffer)
1344                         auxtrace_buffer__drop_data(old_buffer);
1345                 etmq->buf_len = 0;
1346                 return 0;
1347         }
1348
1349         etmq->buffer = aux_buffer;
1350
1351         /* If the aux_buffer doesn't have data associated, try to load it */
1352         if (!aux_buffer->data) {
1353                 /* get the file desc associated with the perf data file */
1354                 int fd = perf_data__fd(etmq->etm->session->data);
1355
1356                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1357                 if (!aux_buffer->data)
1358                         return -ENOMEM;
1359         }
1360
1361         /* If valid, drop the previous buffer */
1362         if (old_buffer)
1363                 auxtrace_buffer__drop_data(old_buffer);
1364
1365         etmq->buf_used = 0;
1366         etmq->buf_len = aux_buffer->size;
1367         etmq->buf = aux_buffer->data;
1368
1369         return etmq->buf_len;
1370 }
1371
1372 static void cs_etm__set_thread(struct cs_etm_queue *etmq,
1373                                struct cs_etm_traceid_queue *tidq, pid_t tid,
1374                                ocsd_ex_level el)
1375 {
1376         struct machine *machine = cs_etm__get_machine(etmq, el);
1377
1378         if (tid != -1) {
1379                 thread__zput(tidq->thread);
1380                 tidq->thread = machine__find_thread(machine, -1, tid);
1381         }
1382
1383         /* Couldn't find a known thread */
1384         if (!tidq->thread)
1385                 tidq->thread = machine__idle_thread(machine);
1386
1387         tidq->el = el;
1388 }
1389
1390 int cs_etm__etmq_set_tid_el(struct cs_etm_queue *etmq, pid_t tid,
1391                             u8 trace_chan_id, ocsd_ex_level el)
1392 {
1393         struct cs_etm_traceid_queue *tidq;
1394
1395         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1396         if (!tidq)
1397                 return -EINVAL;
1398
1399         cs_etm__set_thread(etmq, tidq, tid, el);
1400         return 0;
1401 }
1402
1403 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1404 {
1405         return !!etmq->etm->timeless_decoding;
1406 }
1407
1408 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1409                               u64 trace_chan_id,
1410                               const struct cs_etm_packet *packet,
1411                               struct perf_sample *sample)
1412 {
1413         /*
1414          * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1415          * packet, so directly bail out with 'insn_len' = 0.
1416          */
1417         if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1418                 sample->insn_len = 0;
1419                 return;
1420         }
1421
1422         /*
1423          * T32 instruction size might be 32-bit or 16-bit, decide by calling
1424          * cs_etm__t32_instr_size().
1425          */
1426         if (packet->isa == CS_ETM_ISA_T32)
1427                 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1428                                                           sample->ip);
1429         /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1430         else
1431                 sample->insn_len = 4;
1432
1433         cs_etm__mem_access(etmq, trace_chan_id, sample->ip, sample->insn_len,
1434                            (void *)sample->insn, 0);
1435 }
1436
1437 u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1438 {
1439         struct cs_etm_auxtrace *etm = etmq->etm;
1440
1441         if (etm->has_virtual_ts)
1442                 return tsc_to_perf_time(cs_timestamp, &etm->tc);
1443         else
1444                 return cs_timestamp;
1445 }
1446
1447 static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1448                                                struct cs_etm_traceid_queue *tidq)
1449 {
1450         struct cs_etm_auxtrace *etm = etmq->etm;
1451         struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1452
1453         if (!etm->timeless_decoding && etm->has_virtual_ts)
1454                 return packet_queue->cs_timestamp;
1455         else
1456                 return etm->latest_kernel_timestamp;
1457 }
1458
1459 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1460                                             struct cs_etm_traceid_queue *tidq,
1461                                             u64 addr, u64 period)
1462 {
1463         int ret = 0;
1464         struct cs_etm_auxtrace *etm = etmq->etm;
1465         union perf_event *event = tidq->event_buf;
1466         struct perf_sample sample = {.ip = 0,};
1467
1468         event->sample.header.type = PERF_RECORD_SAMPLE;
1469         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, tidq->el);
1470         event->sample.header.size = sizeof(struct perf_event_header);
1471
1472         /* Set time field based on etm auxtrace config. */
1473         sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1474
1475         sample.ip = addr;
1476         sample.pid = thread__pid(tidq->thread);
1477         sample.tid = thread__tid(tidq->thread);
1478         sample.id = etmq->etm->instructions_id;
1479         sample.stream_id = etmq->etm->instructions_id;
1480         sample.period = period;
1481         sample.cpu = tidq->packet->cpu;
1482         sample.flags = tidq->prev_packet->flags;
1483         sample.cpumode = event->sample.header.misc;
1484
1485         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1486
1487         if (etm->synth_opts.last_branch)
1488                 sample.branch_stack = tidq->last_branch;
1489
1490         if (etm->synth_opts.inject) {
1491                 ret = cs_etm__inject_event(event, &sample,
1492                                            etm->instructions_sample_type);
1493                 if (ret)
1494                         return ret;
1495         }
1496
1497         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1498
1499         if (ret)
1500                 pr_err(
1501                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
1502                         ret);
1503
1504         return ret;
1505 }
1506
1507 /*
1508  * The cs etm packet encodes an instruction range between a branch target
1509  * and the next taken branch. Generate sample accordingly.
1510  */
1511 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1512                                        struct cs_etm_traceid_queue *tidq)
1513 {
1514         int ret = 0;
1515         struct cs_etm_auxtrace *etm = etmq->etm;
1516         struct perf_sample sample = {.ip = 0,};
1517         union perf_event *event = tidq->event_buf;
1518         struct dummy_branch_stack {
1519                 u64                     nr;
1520                 u64                     hw_idx;
1521                 struct branch_entry     entries;
1522         } dummy_bs;
1523         u64 ip;
1524
1525         ip = cs_etm__last_executed_instr(tidq->prev_packet);
1526
1527         event->sample.header.type = PERF_RECORD_SAMPLE;
1528         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip,
1529                                                      tidq->prev_packet_el);
1530         event->sample.header.size = sizeof(struct perf_event_header);
1531
1532         /* Set time field based on etm auxtrace config. */
1533         sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1534
1535         sample.ip = ip;
1536         sample.pid = thread__pid(tidq->prev_packet_thread);
1537         sample.tid = thread__tid(tidq->prev_packet_thread);
1538         sample.addr = cs_etm__first_executed_instr(tidq->packet);
1539         sample.id = etmq->etm->branches_id;
1540         sample.stream_id = etmq->etm->branches_id;
1541         sample.period = 1;
1542         sample.cpu = tidq->packet->cpu;
1543         sample.flags = tidq->prev_packet->flags;
1544         sample.cpumode = event->sample.header.misc;
1545
1546         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1547                           &sample);
1548
1549         /*
1550          * perf report cannot handle events without a branch stack
1551          */
1552         if (etm->synth_opts.last_branch) {
1553                 dummy_bs = (struct dummy_branch_stack){
1554                         .nr = 1,
1555                         .hw_idx = -1ULL,
1556                         .entries = {
1557                                 .from = sample.ip,
1558                                 .to = sample.addr,
1559                         },
1560                 };
1561                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1562         }
1563
1564         if (etm->synth_opts.inject) {
1565                 ret = cs_etm__inject_event(event, &sample,
1566                                            etm->branches_sample_type);
1567                 if (ret)
1568                         return ret;
1569         }
1570
1571         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1572
1573         if (ret)
1574                 pr_err(
1575                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1576                 ret);
1577
1578         return ret;
1579 }
1580
1581 struct cs_etm_synth {
1582         struct perf_tool dummy_tool;
1583         struct perf_session *session;
1584 };
1585
1586 static int cs_etm__event_synth(struct perf_tool *tool,
1587                                union perf_event *event,
1588                                struct perf_sample *sample __maybe_unused,
1589                                struct machine *machine __maybe_unused)
1590 {
1591         struct cs_etm_synth *cs_etm_synth =
1592                       container_of(tool, struct cs_etm_synth, dummy_tool);
1593
1594         return perf_session__deliver_synth_event(cs_etm_synth->session,
1595                                                  event, NULL);
1596 }
1597
1598 static int cs_etm__synth_event(struct perf_session *session,
1599                                struct perf_event_attr *attr, u64 id)
1600 {
1601         struct cs_etm_synth cs_etm_synth;
1602
1603         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1604         cs_etm_synth.session = session;
1605
1606         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1607                                            &id, cs_etm__event_synth);
1608 }
1609
1610 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1611                                 struct perf_session *session)
1612 {
1613         struct evlist *evlist = session->evlist;
1614         struct evsel *evsel;
1615         struct perf_event_attr attr;
1616         bool found = false;
1617         u64 id;
1618         int err;
1619
1620         evlist__for_each_entry(evlist, evsel) {
1621                 if (evsel->core.attr.type == etm->pmu_type) {
1622                         found = true;
1623                         break;
1624                 }
1625         }
1626
1627         if (!found) {
1628                 pr_debug("No selected events with CoreSight Trace data\n");
1629                 return 0;
1630         }
1631
1632         memset(&attr, 0, sizeof(struct perf_event_attr));
1633         attr.size = sizeof(struct perf_event_attr);
1634         attr.type = PERF_TYPE_HARDWARE;
1635         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1636         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1637                             PERF_SAMPLE_PERIOD;
1638         if (etm->timeless_decoding)
1639                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1640         else
1641                 attr.sample_type |= PERF_SAMPLE_TIME;
1642
1643         attr.exclude_user = evsel->core.attr.exclude_user;
1644         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1645         attr.exclude_hv = evsel->core.attr.exclude_hv;
1646         attr.exclude_host = evsel->core.attr.exclude_host;
1647         attr.exclude_guest = evsel->core.attr.exclude_guest;
1648         attr.sample_id_all = evsel->core.attr.sample_id_all;
1649         attr.read_format = evsel->core.attr.read_format;
1650
1651         /* create new id val to be a fixed offset from evsel id */
1652         id = evsel->core.id[0] + 1000000000;
1653
1654         if (!id)
1655                 id = 1;
1656
1657         if (etm->synth_opts.branches) {
1658                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1659                 attr.sample_period = 1;
1660                 attr.sample_type |= PERF_SAMPLE_ADDR;
1661                 err = cs_etm__synth_event(session, &attr, id);
1662                 if (err)
1663                         return err;
1664                 etm->branches_sample_type = attr.sample_type;
1665                 etm->branches_id = id;
1666                 id += 1;
1667                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1668         }
1669
1670         if (etm->synth_opts.last_branch) {
1671                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1672                 /*
1673                  * We don't use the hardware index, but the sample generation
1674                  * code uses the new format branch_stack with this field,
1675                  * so the event attributes must indicate that it's present.
1676                  */
1677                 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1678         }
1679
1680         if (etm->synth_opts.instructions) {
1681                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1682                 attr.sample_period = etm->synth_opts.period;
1683                 etm->instructions_sample_period = attr.sample_period;
1684                 err = cs_etm__synth_event(session, &attr, id);
1685                 if (err)
1686                         return err;
1687                 etm->instructions_sample_type = attr.sample_type;
1688                 etm->instructions_id = id;
1689                 id += 1;
1690         }
1691
1692         return 0;
1693 }
1694
1695 static int cs_etm__sample(struct cs_etm_queue *etmq,
1696                           struct cs_etm_traceid_queue *tidq)
1697 {
1698         struct cs_etm_auxtrace *etm = etmq->etm;
1699         int ret;
1700         u8 trace_chan_id = tidq->trace_chan_id;
1701         u64 instrs_prev;
1702
1703         /* Get instructions remainder from previous packet */
1704         instrs_prev = tidq->period_instructions;
1705
1706         tidq->period_instructions += tidq->packet->instr_count;
1707
1708         /*
1709          * Record a branch when the last instruction in
1710          * PREV_PACKET is a branch.
1711          */
1712         if (etm->synth_opts.last_branch &&
1713             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1714             tidq->prev_packet->last_instr_taken_branch)
1715                 cs_etm__update_last_branch_rb(etmq, tidq);
1716
1717         if (etm->synth_opts.instructions &&
1718             tidq->period_instructions >= etm->instructions_sample_period) {
1719                 /*
1720                  * Emit instruction sample periodically
1721                  * TODO: allow period to be defined in cycles and clock time
1722                  */
1723
1724                 /*
1725                  * Below diagram demonstrates the instruction samples
1726                  * generation flows:
1727                  *
1728                  *    Instrs     Instrs       Instrs       Instrs
1729                  *   Sample(n)  Sample(n+1)  Sample(n+2)  Sample(n+3)
1730                  *    |            |            |            |
1731                  *    V            V            V            V
1732                  *   --------------------------------------------------
1733                  *            ^                                  ^
1734                  *            |                                  |
1735                  *         Period                             Period
1736                  *    instructions(Pi)                   instructions(Pi')
1737                  *
1738                  *            |                                  |
1739                  *            \---------------- -----------------/
1740                  *                             V
1741                  *                 tidq->packet->instr_count
1742                  *
1743                  * Instrs Sample(n...) are the synthesised samples occurring
1744                  * every etm->instructions_sample_period instructions - as
1745                  * defined on the perf command line.  Sample(n) is being the
1746                  * last sample before the current etm packet, n+1 to n+3
1747                  * samples are generated from the current etm packet.
1748                  *
1749                  * tidq->packet->instr_count represents the number of
1750                  * instructions in the current etm packet.
1751                  *
1752                  * Period instructions (Pi) contains the number of
1753                  * instructions executed after the sample point(n) from the
1754                  * previous etm packet.  This will always be less than
1755                  * etm->instructions_sample_period.
1756                  *
1757                  * When generate new samples, it combines with two parts
1758                  * instructions, one is the tail of the old packet and another
1759                  * is the head of the new coming packet, to generate
1760                  * sample(n+1); sample(n+2) and sample(n+3) consume the
1761                  * instructions with sample period.  After sample(n+3), the rest
1762                  * instructions will be used by later packet and it is assigned
1763                  * to tidq->period_instructions for next round calculation.
1764                  */
1765
1766                 /*
1767                  * Get the initial offset into the current packet instructions;
1768                  * entry conditions ensure that instrs_prev is less than
1769                  * etm->instructions_sample_period.
1770                  */
1771                 u64 offset = etm->instructions_sample_period - instrs_prev;
1772                 u64 addr;
1773
1774                 /* Prepare last branches for instruction sample */
1775                 if (etm->synth_opts.last_branch)
1776                         cs_etm__copy_last_branch_rb(etmq, tidq);
1777
1778                 while (tidq->period_instructions >=
1779                                 etm->instructions_sample_period) {
1780                         /*
1781                          * Calculate the address of the sampled instruction (-1
1782                          * as sample is reported as though instruction has just
1783                          * been executed, but PC has not advanced to next
1784                          * instruction)
1785                          */
1786                         addr = cs_etm__instr_addr(etmq, trace_chan_id,
1787                                                   tidq->packet, offset - 1);
1788                         ret = cs_etm__synth_instruction_sample(
1789                                 etmq, tidq, addr,
1790                                 etm->instructions_sample_period);
1791                         if (ret)
1792                                 return ret;
1793
1794                         offset += etm->instructions_sample_period;
1795                         tidq->period_instructions -=
1796                                 etm->instructions_sample_period;
1797                 }
1798         }
1799
1800         if (etm->synth_opts.branches) {
1801                 bool generate_sample = false;
1802
1803                 /* Generate sample for tracing on packet */
1804                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1805                         generate_sample = true;
1806
1807                 /* Generate sample for branch taken packet */
1808                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1809                     tidq->prev_packet->last_instr_taken_branch)
1810                         generate_sample = true;
1811
1812                 if (generate_sample) {
1813                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1814                         if (ret)
1815                                 return ret;
1816                 }
1817         }
1818
1819         cs_etm__packet_swap(etm, tidq);
1820
1821         return 0;
1822 }
1823
1824 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1825 {
1826         /*
1827          * When the exception packet is inserted, whether the last instruction
1828          * in previous range packet is taken branch or not, we need to force
1829          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1830          * to generate branch sample for the instruction range before the
1831          * exception is trapped to kernel or before the exception returning.
1832          *
1833          * The exception packet includes the dummy address values, so don't
1834          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1835          * for generating instruction and branch samples.
1836          */
1837         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1838                 tidq->prev_packet->last_instr_taken_branch = true;
1839
1840         return 0;
1841 }
1842
1843 static int cs_etm__flush(struct cs_etm_queue *etmq,
1844                          struct cs_etm_traceid_queue *tidq)
1845 {
1846         int err = 0;
1847         struct cs_etm_auxtrace *etm = etmq->etm;
1848
1849         /* Handle start tracing packet */
1850         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1851                 goto swap_packet;
1852
1853         if (etmq->etm->synth_opts.last_branch &&
1854             etmq->etm->synth_opts.instructions &&
1855             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1856                 u64 addr;
1857
1858                 /* Prepare last branches for instruction sample */
1859                 cs_etm__copy_last_branch_rb(etmq, tidq);
1860
1861                 /*
1862                  * Generate a last branch event for the branches left in the
1863                  * circular buffer at the end of the trace.
1864                  *
1865                  * Use the address of the end of the last reported execution
1866                  * range
1867                  */
1868                 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1869
1870                 err = cs_etm__synth_instruction_sample(
1871                         etmq, tidq, addr,
1872                         tidq->period_instructions);
1873                 if (err)
1874                         return err;
1875
1876                 tidq->period_instructions = 0;
1877
1878         }
1879
1880         if (etm->synth_opts.branches &&
1881             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1882                 err = cs_etm__synth_branch_sample(etmq, tidq);
1883                 if (err)
1884                         return err;
1885         }
1886
1887 swap_packet:
1888         cs_etm__packet_swap(etm, tidq);
1889
1890         /* Reset last branches after flush the trace */
1891         if (etm->synth_opts.last_branch)
1892                 cs_etm__reset_last_branch_rb(tidq);
1893
1894         return err;
1895 }
1896
1897 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1898                              struct cs_etm_traceid_queue *tidq)
1899 {
1900         int err;
1901
1902         /*
1903          * It has no new packet coming and 'etmq->packet' contains the stale
1904          * packet which was set at the previous time with packets swapping;
1905          * so skip to generate branch sample to avoid stale packet.
1906          *
1907          * For this case only flush branch stack and generate a last branch
1908          * event for the branches left in the circular buffer at the end of
1909          * the trace.
1910          */
1911         if (etmq->etm->synth_opts.last_branch &&
1912             etmq->etm->synth_opts.instructions &&
1913             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1914                 u64 addr;
1915
1916                 /* Prepare last branches for instruction sample */
1917                 cs_etm__copy_last_branch_rb(etmq, tidq);
1918
1919                 /*
1920                  * Use the address of the end of the last reported execution
1921                  * range.
1922                  */
1923                 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1924
1925                 err = cs_etm__synth_instruction_sample(
1926                         etmq, tidq, addr,
1927                         tidq->period_instructions);
1928                 if (err)
1929                         return err;
1930
1931                 tidq->period_instructions = 0;
1932         }
1933
1934         return 0;
1935 }
1936 /*
1937  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1938  *                         if need be.
1939  * Returns:     < 0     if error
1940  *              = 0     if no more auxtrace_buffer to read
1941  *              > 0     if the current buffer isn't empty yet
1942  */
1943 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1944 {
1945         int ret;
1946
1947         if (!etmq->buf_len) {
1948                 ret = cs_etm__get_trace(etmq);
1949                 if (ret <= 0)
1950                         return ret;
1951                 /*
1952                  * We cannot assume consecutive blocks in the data file
1953                  * are contiguous, reset the decoder to force re-sync.
1954                  */
1955                 ret = cs_etm_decoder__reset(etmq->decoder);
1956                 if (ret)
1957                         return ret;
1958         }
1959
1960         return etmq->buf_len;
1961 }
1962
1963 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1964                                  struct cs_etm_packet *packet,
1965                                  u64 end_addr)
1966 {
1967         /* Initialise to keep compiler happy */
1968         u16 instr16 = 0;
1969         u32 instr32 = 0;
1970         u64 addr;
1971
1972         switch (packet->isa) {
1973         case CS_ETM_ISA_T32:
1974                 /*
1975                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1976                  *
1977                  *  b'15         b'8
1978                  * +-----------------+--------+
1979                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1980                  * +-----------------+--------+
1981                  *
1982                  * According to the specification, it only defines SVC for T32
1983                  * with 16 bits instruction and has no definition for 32bits;
1984                  * so below only read 2 bytes as instruction size for T32.
1985                  */
1986                 addr = end_addr - 2;
1987                 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr16),
1988                                    (u8 *)&instr16, 0);
1989                 if ((instr16 & 0xFF00) == 0xDF00)
1990                         return true;
1991
1992                 break;
1993         case CS_ETM_ISA_A32:
1994                 /*
1995                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1996                  *
1997                  *  b'31 b'28 b'27 b'24
1998                  * +---------+---------+-------------------------+
1999                  * |  !1111  | 1 1 1 1 |        imm24            |
2000                  * +---------+---------+-------------------------+
2001                  */
2002                 addr = end_addr - 4;
2003                 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2004                                    (u8 *)&instr32, 0);
2005                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
2006                     (instr32 & 0xF0000000) != 0xF0000000)
2007                         return true;
2008
2009                 break;
2010         case CS_ETM_ISA_A64:
2011                 /*
2012                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
2013                  *
2014                  *  b'31               b'21           b'4     b'0
2015                  * +-----------------------+---------+-----------+
2016                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
2017                  * +-----------------------+---------+-----------+
2018                  */
2019                 addr = end_addr - 4;
2020                 cs_etm__mem_access(etmq, trace_chan_id, addr, sizeof(instr32),
2021                                    (u8 *)&instr32, 0);
2022                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
2023                         return true;
2024
2025                 break;
2026         case CS_ETM_ISA_UNKNOWN:
2027         default:
2028                 break;
2029         }
2030
2031         return false;
2032 }
2033
2034 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
2035                                struct cs_etm_traceid_queue *tidq, u64 magic)
2036 {
2037         u8 trace_chan_id = tidq->trace_chan_id;
2038         struct cs_etm_packet *packet = tidq->packet;
2039         struct cs_etm_packet *prev_packet = tidq->prev_packet;
2040
2041         if (magic == __perf_cs_etmv3_magic)
2042                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
2043                         return true;
2044
2045         /*
2046          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
2047          * HVC cases; need to check if it's SVC instruction based on
2048          * packet address.
2049          */
2050         if (magic == __perf_cs_etmv4_magic) {
2051                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2052                     cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2053                                          prev_packet->end_addr))
2054                         return true;
2055         }
2056
2057         return false;
2058 }
2059
2060 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
2061                                        u64 magic)
2062 {
2063         struct cs_etm_packet *packet = tidq->packet;
2064
2065         if (magic == __perf_cs_etmv3_magic)
2066                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
2067                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
2068                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
2069                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
2070                     packet->exception_number == CS_ETMV3_EXC_FIQ)
2071                         return true;
2072
2073         if (magic == __perf_cs_etmv4_magic)
2074                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
2075                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
2076                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
2077                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
2078                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
2079                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
2080                     packet->exception_number == CS_ETMV4_EXC_FIQ)
2081                         return true;
2082
2083         return false;
2084 }
2085
2086 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
2087                                       struct cs_etm_traceid_queue *tidq,
2088                                       u64 magic)
2089 {
2090         u8 trace_chan_id = tidq->trace_chan_id;
2091         struct cs_etm_packet *packet = tidq->packet;
2092         struct cs_etm_packet *prev_packet = tidq->prev_packet;
2093
2094         if (magic == __perf_cs_etmv3_magic)
2095                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
2096                     packet->exception_number == CS_ETMV3_EXC_HYP ||
2097                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
2098                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
2099                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
2100                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
2101                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
2102                         return true;
2103
2104         if (magic == __perf_cs_etmv4_magic) {
2105                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2106                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2107                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2108                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2109                         return true;
2110
2111                 /*
2112                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
2113                  * (SMC, HVC) are taken as sync exceptions.
2114                  */
2115                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2116                     !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2117                                           prev_packet->end_addr))
2118                         return true;
2119
2120                 /*
2121                  * ETMv4 has 5 bits for exception number; if the numbers
2122                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2123                  * they are implementation defined exceptions.
2124                  *
2125                  * For this case, simply take it as sync exception.
2126                  */
2127                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2128                     packet->exception_number <= CS_ETMV4_EXC_END)
2129                         return true;
2130         }
2131
2132         return false;
2133 }
2134
2135 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2136                                     struct cs_etm_traceid_queue *tidq)
2137 {
2138         struct cs_etm_packet *packet = tidq->packet;
2139         struct cs_etm_packet *prev_packet = tidq->prev_packet;
2140         u8 trace_chan_id = tidq->trace_chan_id;
2141         u64 magic;
2142         int ret;
2143
2144         switch (packet->sample_type) {
2145         case CS_ETM_RANGE:
2146                 /*
2147                  * Immediate branch instruction without neither link nor
2148                  * return flag, it's normal branch instruction within
2149                  * the function.
2150                  */
2151                 if (packet->last_instr_type == OCSD_INSTR_BR &&
2152                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2153                         packet->flags = PERF_IP_FLAG_BRANCH;
2154
2155                         if (packet->last_instr_cond)
2156                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2157                 }
2158
2159                 /*
2160                  * Immediate branch instruction with link (e.g. BL), this is
2161                  * branch instruction for function call.
2162                  */
2163                 if (packet->last_instr_type == OCSD_INSTR_BR &&
2164                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2165                         packet->flags = PERF_IP_FLAG_BRANCH |
2166                                         PERF_IP_FLAG_CALL;
2167
2168                 /*
2169                  * Indirect branch instruction with link (e.g. BLR), this is
2170                  * branch instruction for function call.
2171                  */
2172                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2173                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2174                         packet->flags = PERF_IP_FLAG_BRANCH |
2175                                         PERF_IP_FLAG_CALL;
2176
2177                 /*
2178                  * Indirect branch instruction with subtype of
2179                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2180                  * function return for A32/T32.
2181                  */
2182                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2183                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2184                         packet->flags = PERF_IP_FLAG_BRANCH |
2185                                         PERF_IP_FLAG_RETURN;
2186
2187                 /*
2188                  * Indirect branch instruction without link (e.g. BR), usually
2189                  * this is used for function return, especially for functions
2190                  * within dynamic link lib.
2191                  */
2192                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2193                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2194                         packet->flags = PERF_IP_FLAG_BRANCH |
2195                                         PERF_IP_FLAG_RETURN;
2196
2197                 /* Return instruction for function return. */
2198                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2199                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2200                         packet->flags = PERF_IP_FLAG_BRANCH |
2201                                         PERF_IP_FLAG_RETURN;
2202
2203                 /*
2204                  * Decoder might insert a discontinuity in the middle of
2205                  * instruction packets, fixup prev_packet with flag
2206                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2207                  */
2208                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2209                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2210                                               PERF_IP_FLAG_TRACE_BEGIN;
2211
2212                 /*
2213                  * If the previous packet is an exception return packet
2214                  * and the return address just follows SVC instruction,
2215                  * it needs to calibrate the previous packet sample flags
2216                  * as PERF_IP_FLAG_SYSCALLRET.
2217                  */
2218                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2219                                            PERF_IP_FLAG_RETURN |
2220                                            PERF_IP_FLAG_INTERRUPT) &&
2221                     cs_etm__is_svc_instr(etmq, trace_chan_id,
2222                                          packet, packet->start_addr))
2223                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
2224                                              PERF_IP_FLAG_RETURN |
2225                                              PERF_IP_FLAG_SYSCALLRET;
2226                 break;
2227         case CS_ETM_DISCONTINUITY:
2228                 /*
2229                  * The trace is discontinuous, if the previous packet is
2230                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2231                  * for previous packet.
2232                  */
2233                 if (prev_packet->sample_type == CS_ETM_RANGE)
2234                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2235                                               PERF_IP_FLAG_TRACE_END;
2236                 break;
2237         case CS_ETM_EXCEPTION:
2238                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
2239                 if (ret)
2240                         return ret;
2241
2242                 /* The exception is for system call. */
2243                 if (cs_etm__is_syscall(etmq, tidq, magic))
2244                         packet->flags = PERF_IP_FLAG_BRANCH |
2245                                         PERF_IP_FLAG_CALL |
2246                                         PERF_IP_FLAG_SYSCALLRET;
2247                 /*
2248                  * The exceptions are triggered by external signals from bus,
2249                  * interrupt controller, debug module, PE reset or halt.
2250                  */
2251                 else if (cs_etm__is_async_exception(tidq, magic))
2252                         packet->flags = PERF_IP_FLAG_BRANCH |
2253                                         PERF_IP_FLAG_CALL |
2254                                         PERF_IP_FLAG_ASYNC |
2255                                         PERF_IP_FLAG_INTERRUPT;
2256                 /*
2257                  * Otherwise, exception is caused by trap, instruction &
2258                  * data fault, or alignment errors.
2259                  */
2260                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2261                         packet->flags = PERF_IP_FLAG_BRANCH |
2262                                         PERF_IP_FLAG_CALL |
2263                                         PERF_IP_FLAG_INTERRUPT;
2264
2265                 /*
2266                  * When the exception packet is inserted, since exception
2267                  * packet is not used standalone for generating samples
2268                  * and it's affiliation to the previous instruction range
2269                  * packet; so set previous range packet flags to tell perf
2270                  * it is an exception taken branch.
2271                  */
2272                 if (prev_packet->sample_type == CS_ETM_RANGE)
2273                         prev_packet->flags = packet->flags;
2274                 break;
2275         case CS_ETM_EXCEPTION_RET:
2276                 /*
2277                  * When the exception return packet is inserted, since
2278                  * exception return packet is not used standalone for
2279                  * generating samples and it's affiliation to the previous
2280                  * instruction range packet; so set previous range packet
2281                  * flags to tell perf it is an exception return branch.
2282                  *
2283                  * The exception return can be for either system call or
2284                  * other exception types; unfortunately the packet doesn't
2285                  * contain exception type related info so we cannot decide
2286                  * the exception type purely based on exception return packet.
2287                  * If we record the exception number from exception packet and
2288                  * reuse it for exception return packet, this is not reliable
2289                  * due the trace can be discontinuity or the interrupt can
2290                  * be nested, thus the recorded exception number cannot be
2291                  * used for exception return packet for these two cases.
2292                  *
2293                  * For exception return packet, we only need to distinguish the
2294                  * packet is for system call or for other types.  Thus the
2295                  * decision can be deferred when receive the next packet which
2296                  * contains the return address, based on the return address we
2297                  * can read out the previous instruction and check if it's a
2298                  * system call instruction and then calibrate the sample flag
2299                  * as needed.
2300                  */
2301                 if (prev_packet->sample_type == CS_ETM_RANGE)
2302                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
2303                                              PERF_IP_FLAG_RETURN |
2304                                              PERF_IP_FLAG_INTERRUPT;
2305                 break;
2306         case CS_ETM_EMPTY:
2307         default:
2308                 break;
2309         }
2310
2311         return 0;
2312 }
2313
2314 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2315 {
2316         int ret = 0;
2317         size_t processed = 0;
2318
2319         /*
2320          * Packets are decoded and added to the decoder's packet queue
2321          * until the decoder packet processing callback has requested that
2322          * processing stops or there is nothing left in the buffer.  Normal
2323          * operations that stop processing are a timestamp packet or a full
2324          * decoder buffer queue.
2325          */
2326         ret = cs_etm_decoder__process_data_block(etmq->decoder,
2327                                                  etmq->offset,
2328                                                  &etmq->buf[etmq->buf_used],
2329                                                  etmq->buf_len,
2330                                                  &processed);
2331         if (ret)
2332                 goto out;
2333
2334         etmq->offset += processed;
2335         etmq->buf_used += processed;
2336         etmq->buf_len -= processed;
2337
2338 out:
2339         return ret;
2340 }
2341
2342 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2343                                          struct cs_etm_traceid_queue *tidq)
2344 {
2345         int ret;
2346         struct cs_etm_packet_queue *packet_queue;
2347
2348         packet_queue = &tidq->packet_queue;
2349
2350         /* Process each packet in this chunk */
2351         while (1) {
2352                 ret = cs_etm_decoder__get_packet(packet_queue,
2353                                                  tidq->packet);
2354                 if (ret <= 0)
2355                         /*
2356                          * Stop processing this chunk on
2357                          * end of data or error
2358                          */
2359                         break;
2360
2361                 /*
2362                  * Since packet addresses are swapped in packet
2363                  * handling within below switch() statements,
2364                  * thus setting sample flags must be called
2365                  * prior to switch() statement to use address
2366                  * information before packets swapping.
2367                  */
2368                 ret = cs_etm__set_sample_flags(etmq, tidq);
2369                 if (ret < 0)
2370                         break;
2371
2372                 switch (tidq->packet->sample_type) {
2373                 case CS_ETM_RANGE:
2374                         /*
2375                          * If the packet contains an instruction
2376                          * range, generate instruction sequence
2377                          * events.
2378                          */
2379                         cs_etm__sample(etmq, tidq);
2380                         break;
2381                 case CS_ETM_EXCEPTION:
2382                 case CS_ETM_EXCEPTION_RET:
2383                         /*
2384                          * If the exception packet is coming,
2385                          * make sure the previous instruction
2386                          * range packet to be handled properly.
2387                          */
2388                         cs_etm__exception(tidq);
2389                         break;
2390                 case CS_ETM_DISCONTINUITY:
2391                         /*
2392                          * Discontinuity in trace, flush
2393                          * previous branch stack
2394                          */
2395                         cs_etm__flush(etmq, tidq);
2396                         break;
2397                 case CS_ETM_EMPTY:
2398                         /*
2399                          * Should not receive empty packet,
2400                          * report error.
2401                          */
2402                         pr_err("CS ETM Trace: empty packet\n");
2403                         return -EINVAL;
2404                 default:
2405                         break;
2406                 }
2407         }
2408
2409         return ret;
2410 }
2411
2412 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2413 {
2414         int idx;
2415         struct int_node *inode;
2416         struct cs_etm_traceid_queue *tidq;
2417         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2418
2419         intlist__for_each_entry(inode, traceid_queues_list) {
2420                 idx = (int)(intptr_t)inode->priv;
2421                 tidq = etmq->traceid_queues[idx];
2422
2423                 /* Ignore return value */
2424                 cs_etm__process_traceid_queue(etmq, tidq);
2425
2426                 /*
2427                  * Generate an instruction sample with the remaining
2428                  * branchstack entries.
2429                  */
2430                 cs_etm__flush(etmq, tidq);
2431         }
2432 }
2433
2434 static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq)
2435 {
2436         int err = 0;
2437         struct cs_etm_traceid_queue *tidq;
2438
2439         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2440         if (!tidq)
2441                 return -EINVAL;
2442
2443         /* Go through each buffer in the queue and decode them one by one */
2444         while (1) {
2445                 err = cs_etm__get_data_block(etmq);
2446                 if (err <= 0)
2447                         return err;
2448
2449                 /* Run trace decoder until buffer consumed or end of trace */
2450                 do {
2451                         err = cs_etm__decode_data_block(etmq);
2452                         if (err)
2453                                 return err;
2454
2455                         /*
2456                          * Process each packet in this chunk, nothing to do if
2457                          * an error occurs other than hoping the next one will
2458                          * be better.
2459                          */
2460                         err = cs_etm__process_traceid_queue(etmq, tidq);
2461
2462                 } while (etmq->buf_len);
2463
2464                 if (err == 0)
2465                         /* Flush any remaining branch stack entries */
2466                         err = cs_etm__end_block(etmq, tidq);
2467         }
2468
2469         return err;
2470 }
2471
2472 static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq)
2473 {
2474         int idx, err = 0;
2475         struct cs_etm_traceid_queue *tidq;
2476         struct int_node *inode;
2477
2478         /* Go through each buffer in the queue and decode them one by one */
2479         while (1) {
2480                 err = cs_etm__get_data_block(etmq);
2481                 if (err <= 0)
2482                         return err;
2483
2484                 /* Run trace decoder until buffer consumed or end of trace */
2485                 do {
2486                         err = cs_etm__decode_data_block(etmq);
2487                         if (err)
2488                                 return err;
2489
2490                         /*
2491                          * cs_etm__run_per_thread_timeless_decoder() runs on a
2492                          * single traceID queue because each TID has a separate
2493                          * buffer. But here in per-cpu mode we need to iterate
2494                          * over each channel instead.
2495                          */
2496                         intlist__for_each_entry(inode,
2497                                                 etmq->traceid_queues_list) {
2498                                 idx = (int)(intptr_t)inode->priv;
2499                                 tidq = etmq->traceid_queues[idx];
2500                                 cs_etm__process_traceid_queue(etmq, tidq);
2501                         }
2502                 } while (etmq->buf_len);
2503
2504                 intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2505                         idx = (int)(intptr_t)inode->priv;
2506                         tidq = etmq->traceid_queues[idx];
2507                         /* Flush any remaining branch stack entries */
2508                         err = cs_etm__end_block(etmq, tidq);
2509                         if (err)
2510                                 return err;
2511                 }
2512         }
2513
2514         return err;
2515 }
2516
2517 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2518                                            pid_t tid)
2519 {
2520         unsigned int i;
2521         struct auxtrace_queues *queues = &etm->queues;
2522
2523         for (i = 0; i < queues->nr_queues; i++) {
2524                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2525                 struct cs_etm_queue *etmq = queue->priv;
2526                 struct cs_etm_traceid_queue *tidq;
2527
2528                 if (!etmq)
2529                         continue;
2530
2531                 if (etm->per_thread_decoding) {
2532                         tidq = cs_etm__etmq_get_traceid_queue(
2533                                 etmq, CS_ETM_PER_THREAD_TRACEID);
2534
2535                         if (!tidq)
2536                                 continue;
2537
2538                         if (tid == -1 || thread__tid(tidq->thread) == tid)
2539                                 cs_etm__run_per_thread_timeless_decoder(etmq);
2540                 } else
2541                         cs_etm__run_per_cpu_timeless_decoder(etmq);
2542         }
2543
2544         return 0;
2545 }
2546
2547 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)
2548 {
2549         int ret = 0;
2550         unsigned int cs_queue_nr, queue_nr, i;
2551         u8 trace_chan_id;
2552         u64 cs_timestamp;
2553         struct auxtrace_queue *queue;
2554         struct cs_etm_queue *etmq;
2555         struct cs_etm_traceid_queue *tidq;
2556
2557         /*
2558          * Pre-populate the heap with one entry from each queue so that we can
2559          * start processing in time order across all queues.
2560          */
2561         for (i = 0; i < etm->queues.nr_queues; i++) {
2562                 etmq = etm->queues.queue_array[i].priv;
2563                 if (!etmq)
2564                         continue;
2565
2566                 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2567                 if (ret)
2568                         return ret;
2569         }
2570
2571         while (1) {
2572                 if (!etm->heap.heap_cnt)
2573                         goto out;
2574
2575                 /* Take the entry at the top of the min heap */
2576                 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2577                 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2578                 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2579                 queue = &etm->queues.queue_array[queue_nr];
2580                 etmq = queue->priv;
2581
2582                 /*
2583                  * Remove the top entry from the heap since we are about
2584                  * to process it.
2585                  */
2586                 auxtrace_heap__pop(&etm->heap);
2587
2588                 tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2589                 if (!tidq) {
2590                         /*
2591                          * No traceID queue has been allocated for this traceID,
2592                          * which means something somewhere went very wrong.  No
2593                          * other choice than simply exit.
2594                          */
2595                         ret = -EINVAL;
2596                         goto out;
2597                 }
2598
2599                 /*
2600                  * Packets associated with this timestamp are already in
2601                  * the etmq's traceID queue, so process them.
2602                  */
2603                 ret = cs_etm__process_traceid_queue(etmq, tidq);
2604                 if (ret < 0)
2605                         goto out;
2606
2607                 /*
2608                  * Packets for this timestamp have been processed, time to
2609                  * move on to the next timestamp, fetching a new auxtrace_buffer
2610                  * if need be.
2611                  */
2612 refetch:
2613                 ret = cs_etm__get_data_block(etmq);
2614                 if (ret < 0)
2615                         goto out;
2616
2617                 /*
2618                  * No more auxtrace_buffers to process in this etmq, simply
2619                  * move on to another entry in the auxtrace_heap.
2620                  */
2621                 if (!ret)
2622                         continue;
2623
2624                 ret = cs_etm__decode_data_block(etmq);
2625                 if (ret)
2626                         goto out;
2627
2628                 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2629
2630                 if (!cs_timestamp) {
2631                         /*
2632                          * Function cs_etm__decode_data_block() returns when
2633                          * there is no more traces to decode in the current
2634                          * auxtrace_buffer OR when a timestamp has been
2635                          * encountered on any of the traceID queues.  Since we
2636                          * did not get a timestamp, there is no more traces to
2637                          * process in this auxtrace_buffer.  As such empty and
2638                          * flush all traceID queues.
2639                          */
2640                         cs_etm__clear_all_traceid_queues(etmq);
2641
2642                         /* Fetch another auxtrace_buffer for this etmq */
2643                         goto refetch;
2644                 }
2645
2646                 /*
2647                  * Add to the min heap the timestamp for packets that have
2648                  * just been decoded.  They will be processed and synthesized
2649                  * during the next call to cs_etm__process_traceid_queue() for
2650                  * this queue/traceID.
2651                  */
2652                 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2653                 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2654         }
2655
2656 out:
2657         return ret;
2658 }
2659
2660 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2661                                         union perf_event *event)
2662 {
2663         struct thread *th;
2664
2665         if (etm->timeless_decoding)
2666                 return 0;
2667
2668         /*
2669          * Add the tid/pid to the log so that we can get a match when we get a
2670          * contextID from the decoder. Only track for the host: only kernel
2671          * trace is supported for guests which wouldn't need pids so this should
2672          * be fine.
2673          */
2674         th = machine__findnew_thread(&etm->session->machines.host,
2675                                      event->itrace_start.pid,
2676                                      event->itrace_start.tid);
2677         if (!th)
2678                 return -ENOMEM;
2679
2680         thread__put(th);
2681
2682         return 0;
2683 }
2684
2685 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2686                                            union perf_event *event)
2687 {
2688         struct thread *th;
2689         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2690
2691         /*
2692          * Context switch in per-thread mode are irrelevant since perf
2693          * will start/stop tracing as the process is scheduled.
2694          */
2695         if (etm->timeless_decoding)
2696                 return 0;
2697
2698         /*
2699          * SWITCH_IN events carry the next process to be switched out while
2700          * SWITCH_OUT events carry the process to be switched in.  As such
2701          * we don't care about IN events.
2702          */
2703         if (!out)
2704                 return 0;
2705
2706         /*
2707          * Add the tid/pid to the log so that we can get a match when we get a
2708          * contextID from the decoder. Only track for the host: only kernel
2709          * trace is supported for guests which wouldn't need pids so this should
2710          * be fine.
2711          */
2712         th = machine__findnew_thread(&etm->session->machines.host,
2713                                      event->context_switch.next_prev_pid,
2714                                      event->context_switch.next_prev_tid);
2715         if (!th)
2716                 return -ENOMEM;
2717
2718         thread__put(th);
2719
2720         return 0;
2721 }
2722
2723 static int cs_etm__process_event(struct perf_session *session,
2724                                  union perf_event *event,
2725                                  struct perf_sample *sample,
2726                                  struct perf_tool *tool)
2727 {
2728         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2729                                                    struct cs_etm_auxtrace,
2730                                                    auxtrace);
2731
2732         if (dump_trace)
2733                 return 0;
2734
2735         if (!tool->ordered_events) {
2736                 pr_err("CoreSight ETM Trace requires ordered events\n");
2737                 return -EINVAL;
2738         }
2739
2740         switch (event->header.type) {
2741         case PERF_RECORD_EXIT:
2742                 /*
2743                  * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2744                  * start the decode because we know there will be no more trace from
2745                  * this thread. All this does is emit samples earlier than waiting for
2746                  * the flush in other modes, but with timestamps it makes sense to wait
2747                  * for flush so that events from different threads are interleaved
2748                  * properly.
2749                  */
2750                 if (etm->per_thread_decoding && etm->timeless_decoding)
2751                         return cs_etm__process_timeless_queues(etm,
2752                                                                event->fork.tid);
2753                 break;
2754
2755         case PERF_RECORD_ITRACE_START:
2756                 return cs_etm__process_itrace_start(etm, event);
2757
2758         case PERF_RECORD_SWITCH_CPU_WIDE:
2759                 return cs_etm__process_switch_cpu_wide(etm, event);
2760
2761         case PERF_RECORD_AUX:
2762                 /*
2763                  * Record the latest kernel timestamp available in the header
2764                  * for samples so that synthesised samples occur from this point
2765                  * onwards.
2766                  */
2767                 if (sample->time && (sample->time != (u64)-1))
2768                         etm->latest_kernel_timestamp = sample->time;
2769                 break;
2770
2771         default:
2772                 break;
2773         }
2774
2775         return 0;
2776 }
2777
2778 static void dump_queued_data(struct cs_etm_auxtrace *etm,
2779                              struct perf_record_auxtrace *event)
2780 {
2781         struct auxtrace_buffer *buf;
2782         unsigned int i;
2783         /*
2784          * Find all buffers with same reference in the queues and dump them.
2785          * This is because the queues can contain multiple entries of the same
2786          * buffer that were split on aux records.
2787          */
2788         for (i = 0; i < etm->queues.nr_queues; ++i)
2789                 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2790                         if (buf->reference == event->reference)
2791                                 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2792 }
2793
2794 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2795                                           union perf_event *event,
2796                                           struct perf_tool *tool __maybe_unused)
2797 {
2798         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2799                                                    struct cs_etm_auxtrace,
2800                                                    auxtrace);
2801         if (!etm->data_queued) {
2802                 struct auxtrace_buffer *buffer;
2803                 off_t  data_offset;
2804                 int fd = perf_data__fd(session->data);
2805                 bool is_pipe = perf_data__is_pipe(session->data);
2806                 int err;
2807                 int idx = event->auxtrace.idx;
2808
2809                 if (is_pipe)
2810                         data_offset = 0;
2811                 else {
2812                         data_offset = lseek(fd, 0, SEEK_CUR);
2813                         if (data_offset == -1)
2814                                 return -errno;
2815                 }
2816
2817                 err = auxtrace_queues__add_event(&etm->queues, session,
2818                                                  event, data_offset, &buffer);
2819                 if (err)
2820                         return err;
2821
2822                 /*
2823                  * Knowing if the trace is formatted or not requires a lookup of
2824                  * the aux record so only works in non-piped mode where data is
2825                  * queued in cs_etm__queue_aux_records(). Always assume
2826                  * formatted in piped mode (true).
2827                  */
2828                 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2829                                           idx, true);
2830                 if (err)
2831                         return err;
2832
2833                 if (dump_trace)
2834                         if (auxtrace_buffer__get_data(buffer, fd)) {
2835                                 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2836                                 auxtrace_buffer__put_data(buffer);
2837                         }
2838         } else if (dump_trace)
2839                 dump_queued_data(etm, &event->auxtrace);
2840
2841         return 0;
2842 }
2843
2844 static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2845 {
2846         struct evsel *evsel;
2847         struct evlist *evlist = etm->session->evlist;
2848
2849         /* Override timeless mode with user input from --itrace=Z */
2850         if (etm->synth_opts.timeless_decoding) {
2851                 etm->timeless_decoding = true;
2852                 return 0;
2853         }
2854
2855         /*
2856          * Find the cs_etm evsel and look at what its timestamp setting was
2857          */
2858         evlist__for_each_entry(evlist, evsel)
2859                 if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2860                         etm->timeless_decoding =
2861                                 !(evsel->core.attr.config & BIT(ETM_OPT_TS));
2862                         return 0;
2863                 }
2864
2865         pr_err("CS ETM: Couldn't find ETM evsel\n");
2866         return -EINVAL;
2867 }
2868
2869 /*
2870  * Read a single cpu parameter block from the auxtrace_info priv block.
2871  *
2872  * For version 1 there is a per cpu nr_params entry. If we are handling
2873  * version 1 file, then there may be less, the same, or more params
2874  * indicated by this value than the compile time number we understand.
2875  *
2876  * For a version 0 info block, there are a fixed number, and we need to
2877  * fill out the nr_param value in the metadata we create.
2878  */
2879 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2880                                     int out_blk_size, int nr_params_v0)
2881 {
2882         u64 *metadata = NULL;
2883         int hdr_version;
2884         int nr_in_params, nr_out_params, nr_cmn_params;
2885         int i, k;
2886
2887         metadata = zalloc(sizeof(*metadata) * out_blk_size);
2888         if (!metadata)
2889                 return NULL;
2890
2891         /* read block current index & version */
2892         i = *buff_in_offset;
2893         hdr_version = buff_in[CS_HEADER_VERSION];
2894
2895         if (!hdr_version) {
2896         /* read version 0 info block into a version 1 metadata block  */
2897                 nr_in_params = nr_params_v0;
2898                 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2899                 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2900                 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2901                 /* remaining block params at offset +1 from source */
2902                 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2903                         metadata[k + 1] = buff_in[i + k];
2904                 /* version 0 has 2 common params */
2905                 nr_cmn_params = 2;
2906         } else {
2907         /* read version 1 info block - input and output nr_params may differ */
2908                 /* version 1 has 3 common params */
2909                 nr_cmn_params = 3;
2910                 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2911
2912                 /* if input has more params than output - skip excess */
2913                 nr_out_params = nr_in_params + nr_cmn_params;
2914                 if (nr_out_params > out_blk_size)
2915                         nr_out_params = out_blk_size;
2916
2917                 for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2918                         metadata[k] = buff_in[i + k];
2919
2920                 /* record the actual nr params we copied */
2921                 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2922         }
2923
2924         /* adjust in offset by number of in params used */
2925         i += nr_in_params + nr_cmn_params;
2926         *buff_in_offset = i;
2927         return metadata;
2928 }
2929
2930 /**
2931  * Puts a fragment of an auxtrace buffer into the auxtrace queues based
2932  * on the bounds of aux_event, if it matches with the buffer that's at
2933  * file_offset.
2934  *
2935  * Normally, whole auxtrace buffers would be added to the queue. But we
2936  * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
2937  * is reset across each buffer, so splitting the buffers up in advance has
2938  * the same effect.
2939  */
2940 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
2941                                       struct perf_record_aux *aux_event, struct perf_sample *sample)
2942 {
2943         int err;
2944         char buf[PERF_SAMPLE_MAX_SIZE];
2945         union perf_event *auxtrace_event_union;
2946         struct perf_record_auxtrace *auxtrace_event;
2947         union perf_event auxtrace_fragment;
2948         __u64 aux_offset, aux_size;
2949         __u32 idx;
2950         bool formatted;
2951
2952         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2953                                                    struct cs_etm_auxtrace,
2954                                                    auxtrace);
2955
2956         /*
2957          * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
2958          * from looping through the auxtrace index.
2959          */
2960         err = perf_session__peek_event(session, file_offset, buf,
2961                                        PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
2962         if (err)
2963                 return err;
2964         auxtrace_event = &auxtrace_event_union->auxtrace;
2965         if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
2966                 return -EINVAL;
2967
2968         if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
2969                 auxtrace_event->header.size != sz) {
2970                 return -EINVAL;
2971         }
2972
2973         /*
2974          * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
2975          * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
2976          * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
2977          * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
2978          * Return 'not found' if mismatch.
2979          */
2980         if (auxtrace_event->cpu == (__u32) -1) {
2981                 etm->per_thread_decoding = true;
2982                 if (auxtrace_event->tid != sample->tid)
2983                         return 1;
2984         } else if (auxtrace_event->cpu != sample->cpu) {
2985                 if (etm->per_thread_decoding) {
2986                         /*
2987                          * Found a per-cpu buffer after a per-thread one was
2988                          * already found
2989                          */
2990                         pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n");
2991                         return -EINVAL;
2992                 }
2993                 return 1;
2994         }
2995
2996         if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
2997                 /*
2998                  * Clamp size in snapshot mode. The buffer size is clamped in
2999                  * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
3000                  * the buffer size.
3001                  */
3002                 aux_size = min(aux_event->aux_size, auxtrace_event->size);
3003
3004                 /*
3005                  * In this mode, the head also points to the end of the buffer so aux_offset
3006                  * needs to have the size subtracted so it points to the beginning as in normal mode
3007                  */
3008                 aux_offset = aux_event->aux_offset - aux_size;
3009         } else {
3010                 aux_size = aux_event->aux_size;
3011                 aux_offset = aux_event->aux_offset;
3012         }
3013
3014         if (aux_offset >= auxtrace_event->offset &&
3015             aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
3016                 /*
3017                  * If this AUX event was inside this buffer somewhere, create a new auxtrace event
3018                  * based on the sizes of the aux event, and queue that fragment.
3019                  */
3020                 auxtrace_fragment.auxtrace = *auxtrace_event;
3021                 auxtrace_fragment.auxtrace.size = aux_size;
3022                 auxtrace_fragment.auxtrace.offset = aux_offset;
3023                 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
3024
3025                 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
3026                           " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
3027                 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
3028                                                  file_offset, NULL);
3029                 if (err)
3030                         return err;
3031
3032                 idx = auxtrace_event->idx;
3033                 formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
3034                 return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
3035                                            idx, formatted);
3036         }
3037
3038         /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
3039         return 1;
3040 }
3041
3042 static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
3043                                         u64 offset __maybe_unused, void *data __maybe_unused)
3044 {
3045         /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
3046         if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
3047                 (*(int *)data)++; /* increment found count */
3048                 return cs_etm__process_aux_output_hw_id(session, event);
3049         }
3050         return 0;
3051 }
3052
3053 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
3054                                         u64 offset __maybe_unused, void *data __maybe_unused)
3055 {
3056         struct perf_sample sample;
3057         int ret;
3058         struct auxtrace_index_entry *ent;
3059         struct auxtrace_index *auxtrace_index;
3060         struct evsel *evsel;
3061         size_t i;
3062
3063         /* Don't care about any other events, we're only queuing buffers for AUX events */
3064         if (event->header.type != PERF_RECORD_AUX)
3065                 return 0;
3066
3067         if (event->header.size < sizeof(struct perf_record_aux))
3068                 return -EINVAL;
3069
3070         /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
3071         if (!event->aux.aux_size)
3072                 return 0;
3073
3074         /*
3075          * Parse the sample, we need the sample_id_all data that comes after the event so that the
3076          * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
3077          */
3078         evsel = evlist__event2evsel(session->evlist, event);
3079         if (!evsel)
3080                 return -EINVAL;
3081         ret = evsel__parse_sample(evsel, event, &sample);
3082         if (ret)
3083                 return ret;
3084
3085         /*
3086          * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3087          */
3088         list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
3089                 for (i = 0; i < auxtrace_index->nr; i++) {
3090                         ent = &auxtrace_index->entries[i];
3091                         ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
3092                                                          ent->sz, &event->aux, &sample);
3093                         /*
3094                          * Stop search on error or successful values. Continue search on
3095                          * 1 ('not found')
3096                          */
3097                         if (ret != 1)
3098                                 return ret;
3099                 }
3100         }
3101
3102         /*
3103          * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3104          * don't exit with an error because it will still be possible to decode other aux records.
3105          */
3106         pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
3107                " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
3108         return 0;
3109 }
3110
3111 static int cs_etm__queue_aux_records(struct perf_session *session)
3112 {
3113         struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
3114                                                                 struct auxtrace_index, list);
3115         if (index && index->nr > 0)
3116                 return perf_session__peek_events(session, session->header.data_offset,
3117                                                  session->header.data_size,
3118                                                  cs_etm__queue_aux_records_cb, NULL);
3119
3120         /*
3121          * We would get here if there are no entries in the index (either no auxtrace
3122          * buffers or no index at all). Fail silently as there is the possibility of
3123          * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3124          * false.
3125          *
3126          * In that scenario, buffers will not be split by AUX records.
3127          */
3128         return 0;
3129 }
3130
3131 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3132                                   (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3133
3134 /*
3135  * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3136  * timestamps).
3137  */
3138 static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
3139 {
3140         int j;
3141
3142         for (j = 0; j < num_cpu; j++) {
3143                 switch (metadata[j][CS_ETM_MAGIC]) {
3144                 case __perf_cs_etmv4_magic:
3145                         if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
3146                                 return false;
3147                         break;
3148                 case __perf_cs_ete_magic:
3149                         if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
3150                                 return false;
3151                         break;
3152                 default:
3153                         /* Unknown / unsupported magic number. */
3154                         return false;
3155                 }
3156         }
3157         return true;
3158 }
3159
3160 /* map trace ids to correct metadata block, from information in metadata */
3161 static int cs_etm__map_trace_ids_metadata(int num_cpu, u64 **metadata)
3162 {
3163         u64 cs_etm_magic;
3164         u8 trace_chan_id;
3165         int i, err;
3166
3167         for (i = 0; i < num_cpu; i++) {
3168                 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3169                 switch (cs_etm_magic) {
3170                 case __perf_cs_etmv3_magic:
3171                         metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3172                         trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3173                         break;
3174                 case __perf_cs_etmv4_magic:
3175                 case __perf_cs_ete_magic:
3176                         metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3177                         trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3178                         break;
3179                 default:
3180                         /* unknown magic number */
3181                         return -EINVAL;
3182                 }
3183                 err = cs_etm__map_trace_id(trace_chan_id, metadata[i]);
3184                 if (err)
3185                         return err;
3186         }
3187         return 0;
3188 }
3189
3190 /*
3191  * If we found AUX_HW_ID packets, then set any metadata marked as unused to the
3192  * unused value to reduce the number of unneeded decoders created.
3193  */
3194 static int cs_etm__clear_unused_trace_ids_metadata(int num_cpu, u64 **metadata)
3195 {
3196         u64 cs_etm_magic;
3197         int i;
3198
3199         for (i = 0; i < num_cpu; i++) {
3200                 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3201                 switch (cs_etm_magic) {
3202                 case __perf_cs_etmv3_magic:
3203                         if (metadata[i][CS_ETM_ETMTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3204                                 metadata[i][CS_ETM_ETMTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3205                         break;
3206                 case __perf_cs_etmv4_magic:
3207                 case __perf_cs_ete_magic:
3208                         if (metadata[i][CS_ETMV4_TRCTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3209                                 metadata[i][CS_ETMV4_TRCTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3210                         break;
3211                 default:
3212                         /* unknown magic number */
3213                         return -EINVAL;
3214                 }
3215         }
3216         return 0;
3217 }
3218
3219 int cs_etm__process_auxtrace_info_full(union perf_event *event,
3220                                        struct perf_session *session)
3221 {
3222         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3223         struct cs_etm_auxtrace *etm = NULL;
3224         struct perf_record_time_conv *tc = &session->time_conv;
3225         int event_header_size = sizeof(struct perf_event_header);
3226         int total_size = auxtrace_info->header.size;
3227         int priv_size = 0;
3228         int num_cpu;
3229         int err = 0;
3230         int aux_hw_id_found;
3231         int i, j;
3232         u64 *ptr = NULL;
3233         u64 **metadata = NULL;
3234
3235         /*
3236          * Create an RB tree for traceID-metadata tuple.  Since the conversion
3237          * has to be made for each packet that gets decoded, optimizing access
3238          * in anything other than a sequential array is worth doing.
3239          */
3240         traceid_list = intlist__new(NULL);
3241         if (!traceid_list)
3242                 return -ENOMEM;
3243
3244         /* First the global part */
3245         ptr = (u64 *) auxtrace_info->priv;
3246         num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3247         metadata = zalloc(sizeof(*metadata) * num_cpu);
3248         if (!metadata) {
3249                 err = -ENOMEM;
3250                 goto err_free_traceid_list;
3251         }
3252
3253         /* Start parsing after the common part of the header */
3254         i = CS_HEADER_VERSION_MAX;
3255
3256         /*
3257          * The metadata is stored in the auxtrace_info section and encodes
3258          * the configuration of the ARM embedded trace macrocell which is
3259          * required by the trace decoder to properly decode the trace due
3260          * to its highly compressed nature.
3261          */
3262         for (j = 0; j < num_cpu; j++) {
3263                 if (ptr[i] == __perf_cs_etmv3_magic) {
3264                         metadata[j] =
3265                                 cs_etm__create_meta_blk(ptr, &i,
3266                                                         CS_ETM_PRIV_MAX,
3267                                                         CS_ETM_NR_TRC_PARAMS_V0);
3268                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
3269                         metadata[j] =
3270                                 cs_etm__create_meta_blk(ptr, &i,
3271                                                         CS_ETMV4_PRIV_MAX,
3272                                                         CS_ETMV4_NR_TRC_PARAMS_V0);
3273                 } else if (ptr[i] == __perf_cs_ete_magic) {
3274                         metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3275                 } else {
3276                         ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3277                                   ptr[i]);
3278                         err = -EINVAL;
3279                         goto err_free_metadata;
3280                 }
3281
3282                 if (!metadata[j]) {
3283                         err = -ENOMEM;
3284                         goto err_free_metadata;
3285                 }
3286         }
3287
3288         /*
3289          * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3290          * CS_ETMV4_PRIV_MAX mark how many double words are in the
3291          * global metadata, and each cpu's metadata respectively.
3292          * The following tests if the correct number of double words was
3293          * present in the auxtrace info section.
3294          */
3295         priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3296         if (i * 8 != priv_size) {
3297                 err = -EINVAL;
3298                 goto err_free_metadata;
3299         }
3300
3301         etm = zalloc(sizeof(*etm));
3302
3303         if (!etm) {
3304                 err = -ENOMEM;
3305                 goto err_free_metadata;
3306         }
3307
3308         /*
3309          * As all the ETMs run at the same exception level, the system should
3310          * have the same PID format crossing CPUs.  So cache the PID format
3311          * and reuse it for sequential decoding.
3312          */
3313         etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]);
3314
3315         err = auxtrace_queues__init(&etm->queues);
3316         if (err)
3317                 goto err_free_etm;
3318
3319         if (session->itrace_synth_opts->set) {
3320                 etm->synth_opts = *session->itrace_synth_opts;
3321         } else {
3322                 itrace_synth_opts__set_default(&etm->synth_opts,
3323                                 session->itrace_synth_opts->default_no_sample);
3324                 etm->synth_opts.callchain = false;
3325         }
3326
3327         etm->session = session;
3328
3329         etm->num_cpu = num_cpu;
3330         etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3331         etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3332         etm->metadata = metadata;
3333         etm->auxtrace_type = auxtrace_info->type;
3334
3335         /* Use virtual timestamps if all ETMs report ts_source = 1 */
3336         etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3337
3338         if (!etm->has_virtual_ts)
3339                 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3340                             "The time field of the samples will not be set accurately.\n\n");
3341
3342         etm->auxtrace.process_event = cs_etm__process_event;
3343         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3344         etm->auxtrace.flush_events = cs_etm__flush_events;
3345         etm->auxtrace.free_events = cs_etm__free_events;
3346         etm->auxtrace.free = cs_etm__free;
3347         etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3348         session->auxtrace = &etm->auxtrace;
3349
3350         err = cs_etm__setup_timeless_decoding(etm);
3351         if (err)
3352                 return err;
3353
3354         etm->tc.time_shift = tc->time_shift;
3355         etm->tc.time_mult = tc->time_mult;
3356         etm->tc.time_zero = tc->time_zero;
3357         if (event_contains(*tc, time_cycles)) {
3358                 etm->tc.time_cycles = tc->time_cycles;
3359                 etm->tc.time_mask = tc->time_mask;
3360                 etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3361                 etm->tc.cap_user_time_short = tc->cap_user_time_short;
3362         }
3363         err = cs_etm__synth_events(etm, session);
3364         if (err)
3365                 goto err_free_queues;
3366
3367         /*
3368          * Map Trace ID values to CPU metadata.
3369          *
3370          * Trace metadata will always contain Trace ID values from the legacy algorithm. If the
3371          * files has been recorded by a "new" perf updated to handle AUX_HW_ID then the metadata
3372          * ID value will also have the CORESIGHT_TRACE_ID_UNUSED_FLAG set.
3373          *
3374          * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3375          * the same IDs as the old algorithm as far as is possible, unless there are clashes
3376          * in which case a different value will be used. This means an older perf may still
3377          * be able to record and read files generate on a newer system.
3378          *
3379          * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3380          * those packets. If they are there then the values will be mapped and plugged into
3381          * the metadata. We then set any remaining metadata values with the used flag to a
3382          * value CORESIGHT_TRACE_ID_UNUSED_VAL - which indicates no decoder is required.
3383          *
3384          * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3385          * then we map Trace ID values to CPU directly from the metadata - clearing any unused
3386          * flags if present.
3387          */
3388
3389         /* first scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3390         aux_hw_id_found = 0;
3391         err = perf_session__peek_events(session, session->header.data_offset,
3392                                         session->header.data_size,
3393                                         cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3394         if (err)
3395                 goto err_free_queues;
3396
3397         /* if HW ID found then clear any unused metadata ID values */
3398         if (aux_hw_id_found)
3399                 err = cs_etm__clear_unused_trace_ids_metadata(num_cpu, metadata);
3400         /* otherwise, this is a file with metadata values only, map from metadata */
3401         else
3402                 err = cs_etm__map_trace_ids_metadata(num_cpu, metadata);
3403
3404         if (err)
3405                 goto err_free_queues;
3406
3407         err = cs_etm__queue_aux_records(session);
3408         if (err)
3409                 goto err_free_queues;
3410
3411         etm->data_queued = etm->queues.populated;
3412         return 0;
3413
3414 err_free_queues:
3415         auxtrace_queues__free(&etm->queues);
3416         session->auxtrace = NULL;
3417 err_free_etm:
3418         zfree(&etm);
3419 err_free_metadata:
3420         /* No need to check @metadata[j], free(NULL) is supported */
3421         for (j = 0; j < num_cpu; j++)
3422                 zfree(&metadata[j]);
3423         zfree(&metadata);
3424 err_free_traceid_list:
3425         intlist__delete(traceid_list);
3426         return err;
3427 }