tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <tor@ti.com>
   6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/coresight-pmu.h>
  11 #include <linux/err.h>
  12 #include <linux/kernel.h>
  13 #include <linux/log2.h>
  14 #include <linux/types.h>
  15 #include <linux/zalloc.h>
  16
  17 #include <opencsd/ocsd_if_types.h>
  18 #include <stdlib.h>
  19
  20 #include "auxtrace.h"
  21 #include "color.h"
  22 #include "cs-etm.h"
  23 #include "cs-etm-decoder/cs-etm-decoder.h"
  24 #include "debug.h"
  25 #include "dso.h"
  26 #include "evlist.h"
  27 #include "intlist.h"
  28 #include "machine.h"
  29 #include "map.h"
  30 #include "perf.h"
  31 #include "session.h"
  32 #include "map_symbol.h"
  33 #include "branch.h"
  34 #include "symbol.h"
  35 #include "tool.h"
  36 #include "thread.h"
  37 #include "thread-stack.h"
  38 #include "tsc.h"
  39 #include <tools/libc_compat.h>
  40 #include "util/synthetic-events.h"
  41 #include "util/util.h"
  42
  43 struct cs_etm_auxtrace {
  44         struct auxtrace auxtrace;
  45         struct auxtrace_queues queues;
  46         struct auxtrace_heap heap;
  47         struct itrace_synth_opts synth_opts;
  48         struct perf_session *session;
  49         struct machine *machine;
  50         struct thread *unknown_thread;
  51         struct perf_tsc_conversion tc;
  52
  53         /*
  54          * Timeless has no timestamps in the trace so overlapping mmap lookups
  55          * are less accurate but produces smaller trace data. We use context IDs
  56          * in the trace instead of matching timestamps with fork records so
  57          * they're not really needed in the general case. Overlapping mmaps
  58          * happen in cases like between a fork and an exec.
  59          */
  60         bool timeless_decoding;
  61
  62         /*
  63          * Per-thread ignores the trace channel ID and instead assumes that
  64          * everything in a buffer comes from the same process regardless of
  65          * which CPU it ran on. It also implies no context IDs so the TID is
  66          * taken from the auxtrace buffer.
  67          */
  68         bool per_thread_decoding;
  69         bool snapshot_mode;
  70         bool data_queued;
  71         bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
  72
  73         int num_cpu;
  74         u64 latest_kernel_timestamp;
  75         u32 auxtrace_type;
  76         u64 branches_sample_type;
  77         u64 branches_id;
  78         u64 instructions_sample_type;
  79         u64 instructions_sample_period;
  80         u64 instructions_id;
  81         u64 **metadata;
  82         unsigned int pmu_type;
  83 };
  84
  85 struct cs_etm_traceid_queue {
  86         u8 trace_chan_id;
  87         pid_t pid, tid;
  88         u64 period_instructions;
  89         size_t last_branch_pos;
  90         union perf_event *event_buf;
  91         struct thread *thread;
  92         struct branch_stack *last_branch;
  93         struct branch_stack *last_branch_rb;
  94         struct cs_etm_packet *prev_packet;
  95         struct cs_etm_packet *packet;
  96         struct cs_etm_packet_queue packet_queue;
  97 };
  98
  99 struct cs_etm_queue {
 100         struct cs_etm_auxtrace *etm;
 101         struct cs_etm_decoder *decoder;
 102         struct auxtrace_buffer *buffer;
 103         unsigned int queue_nr;
 104         u8 pending_timestamp_chan_id;
 105         u64 offset;
 106         const unsigned char *buf;
 107         size_t buf_len, buf_used;
 108         /* Conversion between traceID and index in traceid_queues array */
 109         struct intlist *traceid_queues_list;
 110         struct cs_etm_traceid_queue **traceid_queues;
 111 };
 112
 113 /* RB tree for quick conversion between traceID and metadata pointers */
 114 static struct intlist *traceid_list;
 115
 116 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
 117 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
 118                                            pid_t tid);
 119 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
 120 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
 121
 122 /* PTMs ETMIDR [11:8] set to b0011 */
 123 #define ETMIDR_PTM_VERSION 0x00000300
 124
 125 /*
 126  * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
 127  * work with.  One option is to modify to auxtrace_heap_XYZ() API or simply
 128  * encode the etm queue number as the upper 16 bit and the channel as
 129  * the lower 16 bit.
 130  */
 131 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
 132                       (queue_nr << 16 | trace_chan_id)
 133 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
 134 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
 135
 136 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
 137 {
 138         etmidr &= ETMIDR_PTM_VERSION;
 139
 140         if (etmidr == ETMIDR_PTM_VERSION)
 141                 return CS_ETM_PROTO_PTM;
 142
 143         return CS_ETM_PROTO_ETMV3;
 144 }
 145
 146 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 147 {
 148         struct int_node *inode;
 149         u64 *metadata;
 150
 151         inode = intlist__find(traceid_list, trace_chan_id);
 152         if (!inode)
 153                 return -EINVAL;
 154
 155         metadata = inode->priv;
 156         *magic = metadata[CS_ETM_MAGIC];
 157         return 0;
 158 }
 159
 160 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 161 {
 162         struct int_node *inode;
 163         u64 *metadata;
 164
 165         inode = intlist__find(traceid_list, trace_chan_id);
 166         if (!inode)
 167                 return -EINVAL;
 168
 169         metadata = inode->priv;
 170         *cpu = (int)metadata[CS_ETM_CPU];
 171         return 0;
 172 }
 173
 174 /*
 175  * The returned PID format is presented by two bits:
 176  *
 177  *   Bit ETM_OPT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced;
 178  *   Bit ETM_OPT_CTXTID2: CONTEXTIDR_EL2 is traced.
 179  *
 180  * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
 181  * are enabled at the same time when the session runs on an EL2 kernel.
 182  * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
 183  * recorded in the trace data, the tool will selectively use
 184  * CONTEXTIDR_EL2 as PID.
 185  */
 186 int cs_etm__get_pid_fmt(u8 trace_chan_id, u64 *pid_fmt)
 187 {
 188         struct int_node *inode;
 189         u64 *metadata, val;
 190
 191         inode = intlist__find(traceid_list, trace_chan_id);
 192         if (!inode)
 193                 return -EINVAL;
 194
 195         metadata = inode->priv;
 196
 197         if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
 198                 val = metadata[CS_ETM_ETMCR];
 199                 /* CONTEXTIDR is traced */
 200                 if (val & BIT(ETM_OPT_CTXTID))
 201                         *pid_fmt = BIT(ETM_OPT_CTXTID);
 202         } else {
 203                 val = metadata[CS_ETMV4_TRCCONFIGR];
 204                 /* CONTEXTIDR_EL2 is traced */
 205                 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
 206                         *pid_fmt = BIT(ETM_OPT_CTXTID2);
 207                 /* CONTEXTIDR_EL1 is traced */
 208                 else if (val & BIT(ETM4_CFG_BIT_CTXTID))
 209                         *pid_fmt = BIT(ETM_OPT_CTXTID);
 210         }
 211
 212         return 0;
 213 }
 214
 215 static int cs_etm__map_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
 216 {
 217         struct int_node *inode;
 218
 219         /* Get an RB node for this CPU */
 220         inode = intlist__findnew(traceid_list, trace_chan_id);
 221
 222         /* Something went wrong, no need to continue */
 223         if (!inode)
 224                 return -ENOMEM;
 225
 226         /*
 227          * The node for that CPU should not be taken.
 228          * Back out if that's the case.
 229          */
 230         if (inode->priv)
 231                 return -EINVAL;
 232
 233         /* All good, associate the traceID with the metadata pointer */
 234         inode->priv = cpu_metadata;
 235
 236         return 0;
 237 }
 238
 239 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
 240 {
 241         u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
 242
 243         switch (cs_etm_magic) {
 244         case __perf_cs_etmv3_magic:
 245                 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
 246                                       CORESIGHT_TRACE_ID_VAL_MASK);
 247                 break;
 248         case __perf_cs_etmv4_magic:
 249         case __perf_cs_ete_magic:
 250                 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
 251                                       CORESIGHT_TRACE_ID_VAL_MASK);
 252                 break;
 253         default:
 254                 return -EINVAL;
 255         }
 256         return 0;
 257 }
 258
 259 /*
 260  * update metadata trace ID from the value found in the AUX_HW_INFO packet.
 261  * This will also clear the CORESIGHT_TRACE_ID_UNUSED_FLAG flag if present.
 262  */
 263 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
 264 {
 265         u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
 266
 267         switch (cs_etm_magic) {
 268         case __perf_cs_etmv3_magic:
 269                  cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
 270                 break;
 271         case __perf_cs_etmv4_magic:
 272         case __perf_cs_ete_magic:
 273                 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
 274                 break;
 275
 276         default:
 277                 return -EINVAL;
 278         }
 279         return 0;
 280 }
 281
 282 /*
 283  * FIELD_GET (linux/bitfield.h) not available outside kernel code,
 284  * and the header contains too many dependencies to just copy over,
 285  * so roll our own based on the original
 286  */
 287 #define __bf_shf(x) (__builtin_ffsll(x) - 1)
 288 #define FIELD_GET(_mask, _reg)                                          \
 289         ({                                                              \
 290                 (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
 291         })
 292
 293 /*
 294  * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
 295  *
 296  * The payload associates the Trace ID and the CPU.
 297  * The routine is tolerant of seeing multiple packets with the same association,
 298  * but a CPU / Trace ID association changing during a session is an error.
 299  */
 300 static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
 301                                             union perf_event *event)
 302 {
 303         struct cs_etm_auxtrace *etm;
 304         struct perf_sample sample;
 305         struct int_node *inode;
 306         struct evsel *evsel;
 307         u64 *cpu_data;
 308         u64 hw_id;
 309         int cpu, version, err;
 310         u8 trace_chan_id, curr_chan_id;
 311
 312         /* extract and parse the HW ID */
 313         hw_id = event->aux_output_hw_id.hw_id;
 314         version = FIELD_GET(CS_AUX_HW_ID_VERSION_MASK, hw_id);
 315         trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
 316
 317         /* check that we can handle this version */
 318         if (version > CS_AUX_HW_ID_CURR_VERSION)
 319                 return -EINVAL;
 320
 321         /* get access to the etm metadata */
 322         etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
 323         if (!etm || !etm->metadata)
 324                 return -EINVAL;
 325
 326         /* parse the sample to get the CPU */
 327         evsel = evlist__event2evsel(session->evlist, event);
 328         if (!evsel)
 329                 return -EINVAL;
 330         err = evsel__parse_sample(evsel, event, &sample);
 331         if (err)
 332                 return err;
 333         cpu = sample.cpu;
 334         if (cpu == -1) {
 335                 /* no CPU in the sample - possibly recorded with an old version of perf */
 336                 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
 337                 return -EINVAL;
 338         }
 339
 340         /* See if the ID is mapped to a CPU, and it matches the current CPU */
 341         inode = intlist__find(traceid_list, trace_chan_id);
 342         if (inode) {
 343                 cpu_data = inode->priv;
 344                 if ((int)cpu_data[CS_ETM_CPU] != cpu) {
 345                         pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
 346                         return -EINVAL;
 347                 }
 348
 349                 /* check that the mapped ID matches */
 350                 err = cs_etm__metadata_get_trace_id(&curr_chan_id, cpu_data);
 351                 if (err)
 352                         return err;
 353                 if (curr_chan_id != trace_chan_id) {
 354                         pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
 355                         return -EINVAL;
 356                 }
 357
 358                 /* mapped and matched - return OK */
 359                 return 0;
 360         }
 361
 362         /* not one we've seen before - lets map it */
 363         cpu_data = etm->metadata[cpu];
 364         err = cs_etm__map_trace_id(trace_chan_id, cpu_data);
 365         if (err)
 366                 return err;
 367
 368         /*
 369          * if we are picking up the association from the packet, need to plug
 370          * the correct trace ID into the metadata for setting up decoders later.
 371          */
 372         err = cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
 373         return err;
 374 }
 375
 376 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
 377                                               u8 trace_chan_id)
 378 {
 379         /*
 380          * When a timestamp packet is encountered the backend code
 381          * is stopped so that the front end has time to process packets
 382          * that were accumulated in the traceID queue.  Since there can
 383          * be more than one channel per cs_etm_queue, we need to specify
 384          * what traceID queue needs servicing.
 385          */
 386         etmq->pending_timestamp_chan_id = trace_chan_id;
 387 }
 388
 389 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
 390                                       u8 *trace_chan_id)
 391 {
 392         struct cs_etm_packet_queue *packet_queue;
 393
 394         if (!etmq->pending_timestamp_chan_id)
 395                 return 0;
 396
 397         if (trace_chan_id)
 398                 *trace_chan_id = etmq->pending_timestamp_chan_id;
 399
 400         packet_queue = cs_etm__etmq_get_packet_queue(etmq,
 401                                                      etmq->pending_timestamp_chan_id);
 402         if (!packet_queue)
 403                 return 0;
 404
 405         /* Acknowledge pending status */
 406         etmq->pending_timestamp_chan_id = 0;
 407
 408         /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
 409         return packet_queue->cs_timestamp;
 410 }
 411
 412 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 413 {
 414         int i;
 415
 416         queue->head = 0;
 417         queue->tail = 0;
 418         queue->packet_count = 0;
 419         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 420                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 421                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 422                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 423                 queue->packet_buffer[i].instr_count = 0;
 424                 queue->packet_buffer[i].last_instr_taken_branch = false;
 425                 queue->packet_buffer[i].last_instr_size = 0;
 426                 queue->packet_buffer[i].last_instr_type = 0;
 427                 queue->packet_buffer[i].last_instr_subtype = 0;
 428                 queue->packet_buffer[i].last_instr_cond = 0;
 429                 queue->packet_buffer[i].flags = 0;
 430                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 431                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 432                 queue->packet_buffer[i].cpu = INT_MIN;
 433         }
 434 }
 435
 436 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
 437 {
 438         int idx;
 439         struct int_node *inode;
 440         struct cs_etm_traceid_queue *tidq;
 441         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 442
 443         intlist__for_each_entry(inode, traceid_queues_list) {
 444                 idx = (int)(intptr_t)inode->priv;
 445                 tidq = etmq->traceid_queues[idx];
 446                 cs_etm__clear_packet_queue(&tidq->packet_queue);
 447         }
 448 }
 449
 450 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 451                                       struct cs_etm_traceid_queue *tidq,
 452                                       u8 trace_chan_id)
 453 {
 454         int rc = -ENOMEM;
 455         struct auxtrace_queue *queue;
 456         struct cs_etm_auxtrace *etm = etmq->etm;
 457
 458         cs_etm__clear_packet_queue(&tidq->packet_queue);
 459
 460         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 461         tidq->tid = queue->tid;
 462         tidq->pid = -1;
 463         tidq->trace_chan_id = trace_chan_id;
 464
 465         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 466         if (!tidq->packet)
 467                 goto out;
 468
 469         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 470         if (!tidq->prev_packet)
 471                 goto out_free;
 472
 473         if (etm->synth_opts.last_branch) {
 474                 size_t sz = sizeof(struct branch_stack);
 475
 476                 sz += etm->synth_opts.last_branch_sz *
 477                       sizeof(struct branch_entry);
 478                 tidq->last_branch = zalloc(sz);
 479                 if (!tidq->last_branch)
 480                         goto out_free;
 481                 tidq->last_branch_rb = zalloc(sz);
 482                 if (!tidq->last_branch_rb)
 483                         goto out_free;
 484         }
 485
 486         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 487         if (!tidq->event_buf)
 488                 goto out_free;
 489
 490         return 0;
 491
 492 out_free:
 493         zfree(&tidq->last_branch_rb);
 494         zfree(&tidq->last_branch);
 495         zfree(&tidq->prev_packet);
 496         zfree(&tidq->packet);
 497 out:
 498         return rc;
 499 }
 500
 501 static struct cs_etm_traceid_queue
 502 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 503 {
 504         int idx;
 505         struct int_node *inode;
 506         struct intlist *traceid_queues_list;
 507         struct cs_etm_traceid_queue *tidq, **traceid_queues;
 508         struct cs_etm_auxtrace *etm = etmq->etm;
 509
 510         if (etm->per_thread_decoding)
 511                 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
 512
 513         traceid_queues_list = etmq->traceid_queues_list;
 514
 515         /*
 516          * Check if the traceid_queue exist for this traceID by looking
 517          * in the queue list.
 518          */
 519         inode = intlist__find(traceid_queues_list, trace_chan_id);
 520         if (inode) {
 521                 idx = (int)(intptr_t)inode->priv;
 522                 return etmq->traceid_queues[idx];
 523         }
 524
 525         /* We couldn't find a traceid_queue for this traceID, allocate one */
 526         tidq = malloc(sizeof(*tidq));
 527         if (!tidq)
 528                 return NULL;
 529
 530         memset(tidq, 0, sizeof(*tidq));
 531
 532         /* Get a valid index for the new traceid_queue */
 533         idx = intlist__nr_entries(traceid_queues_list);
 534         /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
 535         inode = intlist__findnew(traceid_queues_list, trace_chan_id);
 536         if (!inode)
 537                 goto out_free;
 538
 539         /* Associate this traceID with this index */
 540         inode->priv = (void *)(intptr_t)idx;
 541
 542         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 543                 goto out_free;
 544
 545         /* Grow the traceid_queues array by one unit */
 546         traceid_queues = etmq->traceid_queues;
 547         traceid_queues = reallocarray(traceid_queues,
 548                                       idx + 1,
 549                                       sizeof(*traceid_queues));
 550
 551         /*
 552          * On failure reallocarray() returns NULL and the original block of
 553          * memory is left untouched.
 554          */
 555         if (!traceid_queues)
 556                 goto out_free;
 557
 558         traceid_queues[idx] = tidq;
 559         etmq->traceid_queues = traceid_queues;
 560
 561         return etmq->traceid_queues[idx];
 562
 563 out_free:
 564         /*
 565          * Function intlist__remove() removes the inode from the list
 566          * and delete the memory associated to it.
 567          */
 568         intlist__remove(traceid_queues_list, inode);
 569         free(tidq);
 570
 571         return NULL;
 572 }
 573
 574 struct cs_etm_packet_queue
 575 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 576 {
 577         struct cs_etm_traceid_queue *tidq;
 578
 579         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 580         if (tidq)
 581                 return &tidq->packet_queue;
 582
 583         return NULL;
 584 }
 585
 586 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
 587                                 struct cs_etm_traceid_queue *tidq)
 588 {
 589         struct cs_etm_packet *tmp;
 590
 591         if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
 592             etm->synth_opts.instructions) {
 593                 /*
 594                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
 595                  * the next incoming packet.
 596                  */
 597                 tmp = tidq->packet;
 598                 tidq->packet = tidq->prev_packet;
 599                 tidq->prev_packet = tmp;
 600         }
 601 }
 602
 603 static void cs_etm__packet_dump(const char *pkt_string)
 604 {
 605         const char *color = PERF_COLOR_BLUE;
 606         int len = strlen(pkt_string);
 607
 608         if (len && (pkt_string[len-1] == '\n'))
 609                 color_fprintf(stdout, color, "  %s", pkt_string);
 610         else
 611                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 612
 613         fflush(stdout);
 614 }
 615
 616 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 617                                           struct cs_etm_auxtrace *etm, int idx,
 618                                           u32 etmidr)
 619 {
 620         u64 **metadata = etm->metadata;
 621
 622         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 623         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 624         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 625 }
 626
 627 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 628                                           struct cs_etm_auxtrace *etm, int idx)
 629 {
 630         u64 **metadata = etm->metadata;
 631
 632         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 633         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 634         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 635         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 636         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 637         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 638         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 639 }
 640
 641 static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
 642                                           struct cs_etm_auxtrace *etm, int idx)
 643 {
 644         u64 **metadata = etm->metadata;
 645
 646         t_params[idx].protocol = CS_ETM_PROTO_ETE;
 647         t_params[idx].ete.reg_idr0 = metadata[idx][CS_ETE_TRCIDR0];
 648         t_params[idx].ete.reg_idr1 = metadata[idx][CS_ETE_TRCIDR1];
 649         t_params[idx].ete.reg_idr2 = metadata[idx][CS_ETE_TRCIDR2];
 650         t_params[idx].ete.reg_idr8 = metadata[idx][CS_ETE_TRCIDR8];
 651         t_params[idx].ete.reg_configr = metadata[idx][CS_ETE_TRCCONFIGR];
 652         t_params[idx].ete.reg_traceidr = metadata[idx][CS_ETE_TRCTRACEIDR];
 653         t_params[idx].ete.reg_devarch = metadata[idx][CS_ETE_TRCDEVARCH];
 654 }
 655
 656 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 657                                      struct cs_etm_auxtrace *etm,
 658                                      int decoders)
 659 {
 660         int i;
 661         u32 etmidr;
 662         u64 architecture;
 663
 664         for (i = 0; i < decoders; i++) {
 665                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 666
 667                 switch (architecture) {
 668                 case __perf_cs_etmv3_magic:
 669                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 670                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 671                         break;
 672                 case __perf_cs_etmv4_magic:
 673                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 674                         break;
 675                 case __perf_cs_ete_magic:
 676                         cs_etm__set_trace_param_ete(t_params, etm, i);
 677                         break;
 678                 default:
 679                         return -EINVAL;
 680                 }
 681         }
 682
 683         return 0;
 684 }
 685
 686 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 687                                        struct cs_etm_queue *etmq,
 688                                        enum cs_etm_decoder_operation mode,
 689                                        bool formatted)
 690 {
 691         int ret = -EINVAL;
 692
 693         if (!(mode < CS_ETM_OPERATION_MAX))
 694                 goto out;
 695
 696         d_params->packet_printer = cs_etm__packet_dump;
 697         d_params->operation = mode;
 698         d_params->data = etmq;
 699         d_params->formatted = formatted;
 700         d_params->fsyncs = false;
 701         d_params->hsyncs = false;
 702         d_params->frame_aligned = true;
 703
 704         ret = 0;
 705 out:
 706         return ret;
 707 }
 708
 709 static void cs_etm__dump_event(struct cs_etm_queue *etmq,
 710                                struct auxtrace_buffer *buffer)
 711 {
 712         int ret;
 713         const char *color = PERF_COLOR_BLUE;
 714         size_t buffer_used = 0;
 715
 716         fprintf(stdout, "\n");
 717         color_fprintf(stdout, color,
 718                      ". ... CoreSight %s Trace data: size %#zx bytes\n",
 719                      cs_etm_decoder__get_name(etmq->decoder), buffer->size);
 720
 721         do {
 722                 size_t consumed;
 723
 724                 ret = cs_etm_decoder__process_data_block(
 725                                 etmq->decoder, buffer->offset,
 726                                 &((u8 *)buffer->data)[buffer_used],
 727                                 buffer->size - buffer_used, &consumed);
 728                 if (ret)
 729                         break;
 730
 731                 buffer_used += consumed;
 732         } while (buffer_used < buffer->size);
 733
 734         cs_etm_decoder__reset(etmq->decoder);
 735 }
 736
 737 static int cs_etm__flush_events(struct perf_session *session,
 738                                 struct perf_tool *tool)
 739 {
 740         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 741                                                    struct cs_etm_auxtrace,
 742                                                    auxtrace);
 743         if (dump_trace)
 744                 return 0;
 745
 746         if (!tool->ordered_events)
 747                 return -EINVAL;
 748
 749         if (etm->timeless_decoding) {
 750                 /*
 751                  * Pass tid = -1 to process all queues. But likely they will have
 752                  * already been processed on PERF_RECORD_EXIT anyway.
 753                  */
 754                 return cs_etm__process_timeless_queues(etm, -1);
 755         }
 756
 757         return cs_etm__process_timestamped_queues(etm);
 758 }
 759
 760 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
 761 {
 762         int idx;
 763         uintptr_t priv;
 764         struct int_node *inode, *tmp;
 765         struct cs_etm_traceid_queue *tidq;
 766         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
 767
 768         intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
 769                 priv = (uintptr_t)inode->priv;
 770                 idx = priv;
 771
 772                 /* Free this traceid_queue from the array */
 773                 tidq = etmq->traceid_queues[idx];
 774                 thread__zput(tidq->thread);
 775                 zfree(&tidq->event_buf);
 776                 zfree(&tidq->last_branch);
 777                 zfree(&tidq->last_branch_rb);
 778                 zfree(&tidq->prev_packet);
 779                 zfree(&tidq->packet);
 780                 zfree(&tidq);
 781
 782                 /*
 783                  * Function intlist__remove() removes the inode from the list
 784                  * and delete the memory associated to it.
 785                  */
 786                 intlist__remove(traceid_queues_list, inode);
 787         }
 788
 789         /* Then the RB tree itself */
 790         intlist__delete(traceid_queues_list);
 791         etmq->traceid_queues_list = NULL;
 792
 793         /* finally free the traceid_queues array */
 794         zfree(&etmq->traceid_queues);
 795 }
 796
 797 static void cs_etm__free_queue(void *priv)
 798 {
 799         struct cs_etm_queue *etmq = priv;
 800
 801         if (!etmq)
 802                 return;
 803
 804         cs_etm_decoder__free(etmq->decoder);
 805         cs_etm__free_traceid_queues(etmq);
 806         free(etmq);
 807 }
 808
 809 static void cs_etm__free_events(struct perf_session *session)
 810 {
 811         unsigned int i;
 812         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 813                                                    struct cs_etm_auxtrace,
 814                                                    auxtrace);
 815         struct auxtrace_queues *queues = &aux->queues;
 816
 817         for (i = 0; i < queues->nr_queues; i++) {
 818                 cs_etm__free_queue(queues->queue_array[i].priv);
 819                 queues->queue_array[i].priv = NULL;
 820         }
 821
 822         auxtrace_queues__free(queues);
 823 }
 824
 825 static void cs_etm__free(struct perf_session *session)
 826 {
 827         int i;
 828         struct int_node *inode, *tmp;
 829         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 830                                                    struct cs_etm_auxtrace,
 831                                                    auxtrace);
 832         cs_etm__free_events(session);
 833         session->auxtrace = NULL;
 834
 835         /* First remove all traceID/metadata nodes for the RB tree */
 836         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 837                 intlist__remove(traceid_list, inode);
 838         /* Then the RB tree itself */
 839         intlist__delete(traceid_list);
 840
 841         for (i = 0; i < aux->num_cpu; i++)
 842                 zfree(&aux->metadata[i]);
 843
 844         thread__zput(aux->unknown_thread);
 845         zfree(&aux->metadata);
 846         zfree(&aux);
 847 }
 848
 849 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
 850                                       struct evsel *evsel)
 851 {
 852         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 853                                                    struct cs_etm_auxtrace,
 854                                                    auxtrace);
 855
 856         return evsel->core.attr.type == aux->pmu_type;
 857 }
 858
 859 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
 860 {
 861         struct machine *machine;
 862
 863         machine = etmq->etm->machine;
 864
 865         if (address >= machine__kernel_start(machine)) {
 866                 if (machine__is_host(machine))
 867                         return PERF_RECORD_MISC_KERNEL;
 868                 else
 869                         return PERF_RECORD_MISC_GUEST_KERNEL;
 870         } else {
 871                 if (machine__is_host(machine))
 872                         return PERF_RECORD_MISC_USER;
 873                 else if (perf_guest)
 874                         return PERF_RECORD_MISC_GUEST_USER;
 875                 else
 876                         return PERF_RECORD_MISC_HYPERVISOR;
 877         }
 878 }
 879
 880 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
 881                               u64 address, size_t size, u8 *buffer)
 882 {
 883         u8  cpumode;
 884         u64 offset;
 885         int len;
 886         struct thread *thread;
 887         struct machine *machine;
 888         struct addr_location al;
 889         struct dso *dso;
 890         struct cs_etm_traceid_queue *tidq;
 891
 892         if (!etmq)
 893                 return 0;
 894
 895         machine = etmq->etm->machine;
 896         cpumode = cs_etm__cpu_mode(etmq, address);
 897         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 898         if (!tidq)
 899                 return 0;
 900
 901         thread = tidq->thread;
 902         if (!thread) {
 903                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 904                         return 0;
 905                 thread = etmq->etm->unknown_thread;
 906         }
 907
 908         if (!thread__find_map(thread, cpumode, address, &al))
 909                 return 0;
 910
 911         dso = map__dso(al.map);
 912         if (!dso)
 913                 return 0;
 914
 915         if (dso->data.status == DSO_DATA_STATUS_ERROR &&
 916             dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
 917                 return 0;
 918
 919         offset = map__map_ip(al.map, address);
 920
 921         map__load(al.map);
 922
 923         len = dso__data_read_offset(dso, machine, offset, buffer, size);
 924
 925         if (len <= 0) {
 926                 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
 927                                  "              Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
 928                 if (!dso->auxtrace_warned) {
 929                         pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
 930                                     address,
 931                                     dso->long_name ? dso->long_name : "Unknown");
 932                         dso->auxtrace_warned = true;
 933                 }
 934                 return 0;
 935         }
 936
 937         return len;
 938 }
 939
 940 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
 941                                                 bool formatted)
 942 {
 943         struct cs_etm_decoder_params d_params;
 944         struct cs_etm_trace_params  *t_params = NULL;
 945         struct cs_etm_queue *etmq;
 946         /*
 947          * Each queue can only contain data from one CPU when unformatted, so only one decoder is
 948          * needed.
 949          */
 950         int decoders = formatted ? etm->num_cpu : 1;
 951
 952         etmq = zalloc(sizeof(*etmq));
 953         if (!etmq)
 954                 return NULL;
 955
 956         etmq->traceid_queues_list = intlist__new(NULL);
 957         if (!etmq->traceid_queues_list)
 958                 goto out_free;
 959
 960         /* Use metadata to fill in trace parameters for trace decoder */
 961         t_params = zalloc(sizeof(*t_params) * decoders);
 962
 963         if (!t_params)
 964                 goto out_free;
 965
 966         if (cs_etm__init_trace_params(t_params, etm, decoders))
 967                 goto out_free;
 968
 969         /* Set decoder parameters to decode trace packets */
 970         if (cs_etm__init_decoder_params(&d_params, etmq,
 971                                         dump_trace ? CS_ETM_OPERATION_PRINT :
 972                                                      CS_ETM_OPERATION_DECODE,
 973                                         formatted))
 974                 goto out_free;
 975
 976         etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
 977                                             t_params);
 978
 979         if (!etmq->decoder)
 980                 goto out_free;
 981
 982         /*
 983          * Register a function to handle all memory accesses required by
 984          * the trace decoder library.
 985          */
 986         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
 987                                               0x0L, ((u64) -1L),
 988                                               cs_etm__mem_access))
 989                 goto out_free_decoder;
 990
 991         zfree(&t_params);
 992         return etmq;
 993
 994 out_free_decoder:
 995         cs_etm_decoder__free(etmq->decoder);
 996 out_free:
 997         intlist__delete(etmq->traceid_queues_list);
 998         free(etmq);
 999
1000         return NULL;
1001 }
1002
1003 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
1004                                struct auxtrace_queue *queue,
1005                                unsigned int queue_nr,
1006                                bool formatted)
1007 {
1008         struct cs_etm_queue *etmq = queue->priv;
1009
1010         if (list_empty(&queue->head) || etmq)
1011                 return 0;
1012
1013         etmq = cs_etm__alloc_queue(etm, formatted);
1014
1015         if (!etmq)
1016                 return -ENOMEM;
1017
1018         queue->priv = etmq;
1019         etmq->etm = etm;
1020         etmq->queue_nr = queue_nr;
1021         etmq->offset = 0;
1022
1023         return 0;
1024 }
1025
1026 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1027                                             struct cs_etm_queue *etmq,
1028                                             unsigned int queue_nr)
1029 {
1030         int ret = 0;
1031         unsigned int cs_queue_nr;
1032         u8 trace_chan_id;
1033         u64 cs_timestamp;
1034
1035         /*
1036          * We are under a CPU-wide trace scenario.  As such we need to know
1037          * when the code that generated the traces started to execute so that
1038          * it can be correlated with execution on other CPUs.  So we get a
1039          * handle on the beginning of traces and decode until we find a
1040          * timestamp.  The timestamp is then added to the auxtrace min heap
1041          * in order to know what nibble (of all the etmqs) to decode first.
1042          */
1043         while (1) {
1044                 /*
1045                  * Fetch an aux_buffer from this etmq.  Bail if no more
1046                  * blocks or an error has been encountered.
1047                  */
1048                 ret = cs_etm__get_data_block(etmq);
1049                 if (ret <= 0)
1050                         goto out;
1051
1052                 /*
1053                  * Run decoder on the trace block.  The decoder will stop when
1054                  * encountering a CS timestamp, a full packet queue or the end of
1055                  * trace for that block.
1056                  */
1057                 ret = cs_etm__decode_data_block(etmq);
1058                 if (ret)
1059                         goto out;
1060
1061                 /*
1062                  * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1063                  * the timestamp calculation for us.
1064                  */
1065                 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1066
1067                 /* We found a timestamp, no need to continue. */
1068                 if (cs_timestamp)
1069                         break;
1070
1071                 /*
1072                  * We didn't find a timestamp so empty all the traceid packet
1073                  * queues before looking for another timestamp packet, either
1074                  * in the current data block or a new one.  Packets that were
1075                  * just decoded are useless since no timestamp has been
1076                  * associated with them.  As such simply discard them.
1077                  */
1078                 cs_etm__clear_all_packet_queues(etmq);
1079         }
1080
1081         /*
1082          * We have a timestamp.  Add it to the min heap to reflect when
1083          * instructions conveyed by the range packets of this traceID queue
1084          * started to execute.  Once the same has been done for all the traceID
1085          * queues of each etmq, redenring and decoding can start in
1086          * chronological order.
1087          *
1088          * Note that packets decoded above are still in the traceID's packet
1089          * queue and will be processed in cs_etm__process_timestamped_queues().
1090          */
1091         cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1092         ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1093 out:
1094         return ret;
1095 }
1096
1097 static inline
1098 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1099                                  struct cs_etm_traceid_queue *tidq)
1100 {
1101         struct branch_stack *bs_src = tidq->last_branch_rb;
1102         struct branch_stack *bs_dst = tidq->last_branch;
1103         size_t nr = 0;
1104
1105         /*
1106          * Set the number of records before early exit: ->nr is used to
1107          * determine how many branches to copy from ->entries.
1108          */
1109         bs_dst->nr = bs_src->nr;
1110
1111         /*
1112          * Early exit when there is nothing to copy.
1113          */
1114         if (!bs_src->nr)
1115                 return;
1116
1117         /*
1118          * As bs_src->entries is a circular buffer, we need to copy from it in
1119          * two steps.  First, copy the branches from the most recently inserted
1120          * branch ->last_branch_pos until the end of bs_src->entries buffer.
1121          */
1122         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1123         memcpy(&bs_dst->entries[0],
1124                &bs_src->entries[tidq->last_branch_pos],
1125                sizeof(struct branch_entry) * nr);
1126
1127         /*
1128          * If we wrapped around at least once, the branches from the beginning
1129          * of the bs_src->entries buffer and until the ->last_branch_pos element
1130          * are older valid branches: copy them over.  The total number of
1131          * branches copied over will be equal to the number of branches asked by
1132          * the user in last_branch_sz.
1133          */
1134         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1135                 memcpy(&bs_dst->entries[nr],
1136                        &bs_src->entries[0],
1137                        sizeof(struct branch_entry) * tidq->last_branch_pos);
1138         }
1139 }
1140
1141 static inline
1142 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1143 {
1144         tidq->last_branch_pos = 0;
1145         tidq->last_branch_rb->nr = 0;
1146 }
1147
1148 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1149                                          u8 trace_chan_id, u64 addr)
1150 {
1151         u8 instrBytes[2];
1152
1153         cs_etm__mem_access(etmq, trace_chan_id, addr,
1154                            ARRAY_SIZE(instrBytes), instrBytes);
1155         /*
1156          * T32 instruction size is indicated by bits[15:11] of the first
1157          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1158          * denote a 32-bit instruction.
1159          */
1160         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1161 }
1162
1163 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1164 {
1165         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1166         if (packet->sample_type == CS_ETM_DISCONTINUITY)
1167                 return 0;
1168
1169         return packet->start_addr;
1170 }
1171
1172 static inline
1173 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1174 {
1175         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1176         if (packet->sample_type == CS_ETM_DISCONTINUITY)
1177                 return 0;
1178
1179         return packet->end_addr - packet->last_instr_size;
1180 }
1181
1182 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1183                                      u64 trace_chan_id,
1184                                      const struct cs_etm_packet *packet,
1185                                      u64 offset)
1186 {
1187         if (packet->isa == CS_ETM_ISA_T32) {
1188                 u64 addr = packet->start_addr;
1189
1190                 while (offset) {
1191                         addr += cs_etm__t32_instr_size(etmq,
1192                                                        trace_chan_id, addr);
1193                         offset--;
1194                 }
1195                 return addr;
1196         }
1197
1198         /* Assume a 4 byte instruction size (A32/A64) */
1199         return packet->start_addr + offset * 4;
1200 }
1201
1202 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1203                                           struct cs_etm_traceid_queue *tidq)
1204 {
1205         struct branch_stack *bs = tidq->last_branch_rb;
1206         struct branch_entry *be;
1207
1208         /*
1209          * The branches are recorded in a circular buffer in reverse
1210          * chronological order: we start recording from the last element of the
1211          * buffer down.  After writing the first element of the stack, move the
1212          * insert position back to the end of the buffer.
1213          */
1214         if (!tidq->last_branch_pos)
1215                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1216
1217         tidq->last_branch_pos -= 1;
1218
1219         be       = &bs->entries[tidq->last_branch_pos];
1220         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1221         be->to   = cs_etm__first_executed_instr(tidq->packet);
1222         /* No support for mispredict */
1223         be->flags.mispred = 0;
1224         be->flags.predicted = 1;
1225
1226         /*
1227          * Increment bs->nr until reaching the number of last branches asked by
1228          * the user on the command line.
1229          */
1230         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1231                 bs->nr += 1;
1232 }
1233
1234 static int cs_etm__inject_event(union perf_event *event,
1235                                struct perf_sample *sample, u64 type)
1236 {
1237         event->header.size = perf_event__sample_event_size(sample, type, 0);
1238         return perf_event__synthesize_sample(event, type, 0, sample);
1239 }
1240
1241
1242 static int
1243 cs_etm__get_trace(struct cs_etm_queue *etmq)
1244 {
1245         struct auxtrace_buffer *aux_buffer = etmq->buffer;
1246         struct auxtrace_buffer *old_buffer = aux_buffer;
1247         struct auxtrace_queue *queue;
1248
1249         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1250
1251         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1252
1253         /* If no more data, drop the previous auxtrace_buffer and return */
1254         if (!aux_buffer) {
1255                 if (old_buffer)
1256                         auxtrace_buffer__drop_data(old_buffer);
1257                 etmq->buf_len = 0;
1258                 return 0;
1259         }
1260
1261         etmq->buffer = aux_buffer;
1262
1263         /* If the aux_buffer doesn't have data associated, try to load it */
1264         if (!aux_buffer->data) {
1265                 /* get the file desc associated with the perf data file */
1266                 int fd = perf_data__fd(etmq->etm->session->data);
1267
1268                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1269                 if (!aux_buffer->data)
1270                         return -ENOMEM;
1271         }
1272
1273         /* If valid, drop the previous buffer */
1274         if (old_buffer)
1275                 auxtrace_buffer__drop_data(old_buffer);
1276
1277         etmq->buf_used = 0;
1278         etmq->buf_len = aux_buffer->size;
1279         etmq->buf = aux_buffer->data;
1280
1281         return etmq->buf_len;
1282 }
1283
1284 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1285                                     struct cs_etm_traceid_queue *tidq)
1286 {
1287         if ((!tidq->thread) && (tidq->tid != -1))
1288                 tidq->thread = machine__find_thread(etm->machine, -1,
1289                                                     tidq->tid);
1290
1291         if (tidq->thread)
1292                 tidq->pid = tidq->thread->pid_;
1293 }
1294
1295 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1296                          pid_t tid, u8 trace_chan_id)
1297 {
1298         int cpu, err = -EINVAL;
1299         struct cs_etm_auxtrace *etm = etmq->etm;
1300         struct cs_etm_traceid_queue *tidq;
1301
1302         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1303         if (!tidq)
1304                 return err;
1305
1306         if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1307                 return err;
1308
1309         err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1310         if (err)
1311                 return err;
1312
1313         tidq->tid = tid;
1314         thread__zput(tidq->thread);
1315
1316         cs_etm__set_pid_tid_cpu(etm, tidq);
1317         return 0;
1318 }
1319
1320 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1321 {
1322         return !!etmq->etm->timeless_decoding;
1323 }
1324
1325 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1326                               u64 trace_chan_id,
1327                               const struct cs_etm_packet *packet,
1328                               struct perf_sample *sample)
1329 {
1330         /*
1331          * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1332          * packet, so directly bail out with 'insn_len' = 0.
1333          */
1334         if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1335                 sample->insn_len = 0;
1336                 return;
1337         }
1338
1339         /*
1340          * T32 instruction size might be 32-bit or 16-bit, decide by calling
1341          * cs_etm__t32_instr_size().
1342          */
1343         if (packet->isa == CS_ETM_ISA_T32)
1344                 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1345                                                           sample->ip);
1346         /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1347         else
1348                 sample->insn_len = 4;
1349
1350         cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1351                            sample->insn_len, (void *)sample->insn);
1352 }
1353
1354 u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1355 {
1356         struct cs_etm_auxtrace *etm = etmq->etm;
1357
1358         if (etm->has_virtual_ts)
1359                 return tsc_to_perf_time(cs_timestamp, &etm->tc);
1360         else
1361                 return cs_timestamp;
1362 }
1363
1364 static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1365                                                struct cs_etm_traceid_queue *tidq)
1366 {
1367         struct cs_etm_auxtrace *etm = etmq->etm;
1368         struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1369
1370         if (!etm->timeless_decoding && etm->has_virtual_ts)
1371                 return packet_queue->cs_timestamp;
1372         else
1373                 return etm->latest_kernel_timestamp;
1374 }
1375
1376 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1377                                             struct cs_etm_traceid_queue *tidq,
1378                                             u64 addr, u64 period)
1379 {
1380         int ret = 0;
1381         struct cs_etm_auxtrace *etm = etmq->etm;
1382         union perf_event *event = tidq->event_buf;
1383         struct perf_sample sample = {.ip = 0,};
1384
1385         event->sample.header.type = PERF_RECORD_SAMPLE;
1386         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1387         event->sample.header.size = sizeof(struct perf_event_header);
1388
1389         /* Set time field based on etm auxtrace config. */
1390         sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1391
1392         sample.ip = addr;
1393         sample.pid = tidq->pid;
1394         sample.tid = tidq->tid;
1395         sample.id = etmq->etm->instructions_id;
1396         sample.stream_id = etmq->etm->instructions_id;
1397         sample.period = period;
1398         sample.cpu = tidq->packet->cpu;
1399         sample.flags = tidq->prev_packet->flags;
1400         sample.cpumode = event->sample.header.misc;
1401
1402         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1403
1404         if (etm->synth_opts.last_branch)
1405                 sample.branch_stack = tidq->last_branch;
1406
1407         if (etm->synth_opts.inject) {
1408                 ret = cs_etm__inject_event(event, &sample,
1409                                            etm->instructions_sample_type);
1410                 if (ret)
1411                         return ret;
1412         }
1413
1414         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1415
1416         if (ret)
1417                 pr_err(
1418                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
1419                         ret);
1420
1421         return ret;
1422 }
1423
1424 /*
1425  * The cs etm packet encodes an instruction range between a branch target
1426  * and the next taken branch. Generate sample accordingly.
1427  */
1428 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1429                                        struct cs_etm_traceid_queue *tidq)
1430 {
1431         int ret = 0;
1432         struct cs_etm_auxtrace *etm = etmq->etm;
1433         struct perf_sample sample = {.ip = 0,};
1434         union perf_event *event = tidq->event_buf;
1435         struct dummy_branch_stack {
1436                 u64                     nr;
1437                 u64                     hw_idx;
1438                 struct branch_entry     entries;
1439         } dummy_bs;
1440         u64 ip;
1441
1442         ip = cs_etm__last_executed_instr(tidq->prev_packet);
1443
1444         event->sample.header.type = PERF_RECORD_SAMPLE;
1445         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1446         event->sample.header.size = sizeof(struct perf_event_header);
1447
1448         /* Set time field based on etm auxtrace config. */
1449         sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1450
1451         sample.ip = ip;
1452         sample.pid = tidq->pid;
1453         sample.tid = tidq->tid;
1454         sample.addr = cs_etm__first_executed_instr(tidq->packet);
1455         sample.id = etmq->etm->branches_id;
1456         sample.stream_id = etmq->etm->branches_id;
1457         sample.period = 1;
1458         sample.cpu = tidq->packet->cpu;
1459         sample.flags = tidq->prev_packet->flags;
1460         sample.cpumode = event->sample.header.misc;
1461
1462         cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1463                           &sample);
1464
1465         /*
1466          * perf report cannot handle events without a branch stack
1467          */
1468         if (etm->synth_opts.last_branch) {
1469                 dummy_bs = (struct dummy_branch_stack){
1470                         .nr = 1,
1471                         .hw_idx = -1ULL,
1472                         .entries = {
1473                                 .from = sample.ip,
1474                                 .to = sample.addr,
1475                         },
1476                 };
1477                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1478         }
1479
1480         if (etm->synth_opts.inject) {
1481                 ret = cs_etm__inject_event(event, &sample,
1482                                            etm->branches_sample_type);
1483                 if (ret)
1484                         return ret;
1485         }
1486
1487         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1488
1489         if (ret)
1490                 pr_err(
1491                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1492                 ret);
1493
1494         return ret;
1495 }
1496
1497 struct cs_etm_synth {
1498         struct perf_tool dummy_tool;
1499         struct perf_session *session;
1500 };
1501
1502 static int cs_etm__event_synth(struct perf_tool *tool,
1503                                union perf_event *event,
1504                                struct perf_sample *sample __maybe_unused,
1505                                struct machine *machine __maybe_unused)
1506 {
1507         struct cs_etm_synth *cs_etm_synth =
1508                       container_of(tool, struct cs_etm_synth, dummy_tool);
1509
1510         return perf_session__deliver_synth_event(cs_etm_synth->session,
1511                                                  event, NULL);
1512 }
1513
1514 static int cs_etm__synth_event(struct perf_session *session,
1515                                struct perf_event_attr *attr, u64 id)
1516 {
1517         struct cs_etm_synth cs_etm_synth;
1518
1519         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1520         cs_etm_synth.session = session;
1521
1522         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1523                                            &id, cs_etm__event_synth);
1524 }
1525
1526 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1527                                 struct perf_session *session)
1528 {
1529         struct evlist *evlist = session->evlist;
1530         struct evsel *evsel;
1531         struct perf_event_attr attr;
1532         bool found = false;
1533         u64 id;
1534         int err;
1535
1536         evlist__for_each_entry(evlist, evsel) {
1537                 if (evsel->core.attr.type == etm->pmu_type) {
1538                         found = true;
1539                         break;
1540                 }
1541         }
1542
1543         if (!found) {
1544                 pr_debug("No selected events with CoreSight Trace data\n");
1545                 return 0;
1546         }
1547
1548         memset(&attr, 0, sizeof(struct perf_event_attr));
1549         attr.size = sizeof(struct perf_event_attr);
1550         attr.type = PERF_TYPE_HARDWARE;
1551         attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1552         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1553                             PERF_SAMPLE_PERIOD;
1554         if (etm->timeless_decoding)
1555                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1556         else
1557                 attr.sample_type |= PERF_SAMPLE_TIME;
1558
1559         attr.exclude_user = evsel->core.attr.exclude_user;
1560         attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1561         attr.exclude_hv = evsel->core.attr.exclude_hv;
1562         attr.exclude_host = evsel->core.attr.exclude_host;
1563         attr.exclude_guest = evsel->core.attr.exclude_guest;
1564         attr.sample_id_all = evsel->core.attr.sample_id_all;
1565         attr.read_format = evsel->core.attr.read_format;
1566
1567         /* create new id val to be a fixed offset from evsel id */
1568         id = evsel->core.id[0] + 1000000000;
1569
1570         if (!id)
1571                 id = 1;
1572
1573         if (etm->synth_opts.branches) {
1574                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1575                 attr.sample_period = 1;
1576                 attr.sample_type |= PERF_SAMPLE_ADDR;
1577                 err = cs_etm__synth_event(session, &attr, id);
1578                 if (err)
1579                         return err;
1580                 etm->branches_sample_type = attr.sample_type;
1581                 etm->branches_id = id;
1582                 id += 1;
1583                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1584         }
1585
1586         if (etm->synth_opts.last_branch) {
1587                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1588                 /*
1589                  * We don't use the hardware index, but the sample generation
1590                  * code uses the new format branch_stack with this field,
1591                  * so the event attributes must indicate that it's present.
1592                  */
1593                 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1594         }
1595
1596         if (etm->synth_opts.instructions) {
1597                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1598                 attr.sample_period = etm->synth_opts.period;
1599                 etm->instructions_sample_period = attr.sample_period;
1600                 err = cs_etm__synth_event(session, &attr, id);
1601                 if (err)
1602                         return err;
1603                 etm->instructions_sample_type = attr.sample_type;
1604                 etm->instructions_id = id;
1605                 id += 1;
1606         }
1607
1608         return 0;
1609 }
1610
1611 static int cs_etm__sample(struct cs_etm_queue *etmq,
1612                           struct cs_etm_traceid_queue *tidq)
1613 {
1614         struct cs_etm_auxtrace *etm = etmq->etm;
1615         int ret;
1616         u8 trace_chan_id = tidq->trace_chan_id;
1617         u64 instrs_prev;
1618
1619         /* Get instructions remainder from previous packet */
1620         instrs_prev = tidq->period_instructions;
1621
1622         tidq->period_instructions += tidq->packet->instr_count;
1623
1624         /*
1625          * Record a branch when the last instruction in
1626          * PREV_PACKET is a branch.
1627          */
1628         if (etm->synth_opts.last_branch &&
1629             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1630             tidq->prev_packet->last_instr_taken_branch)
1631                 cs_etm__update_last_branch_rb(etmq, tidq);
1632
1633         if (etm->synth_opts.instructions &&
1634             tidq->period_instructions >= etm->instructions_sample_period) {
1635                 /*
1636                  * Emit instruction sample periodically
1637                  * TODO: allow period to be defined in cycles and clock time
1638                  */
1639
1640                 /*
1641                  * Below diagram demonstrates the instruction samples
1642                  * generation flows:
1643                  *
1644                  *    Instrs     Instrs       Instrs       Instrs
1645                  *   Sample(n)  Sample(n+1)  Sample(n+2)  Sample(n+3)
1646                  *    |            |            |            |
1647                  *    V            V            V            V
1648                  *   --------------------------------------------------
1649                  *            ^                                  ^
1650                  *            |                                  |
1651                  *         Period                             Period
1652                  *    instructions(Pi)                   instructions(Pi')
1653                  *
1654                  *            |                                  |
1655                  *            \---------------- -----------------/
1656                  *                             V
1657                  *                 tidq->packet->instr_count
1658                  *
1659                  * Instrs Sample(n...) are the synthesised samples occurring
1660                  * every etm->instructions_sample_period instructions - as
1661                  * defined on the perf command line.  Sample(n) is being the
1662                  * last sample before the current etm packet, n+1 to n+3
1663                  * samples are generated from the current etm packet.
1664                  *
1665                  * tidq->packet->instr_count represents the number of
1666                  * instructions in the current etm packet.
1667                  *
1668                  * Period instructions (Pi) contains the number of
1669                  * instructions executed after the sample point(n) from the
1670                  * previous etm packet.  This will always be less than
1671                  * etm->instructions_sample_period.
1672                  *
1673                  * When generate new samples, it combines with two parts
1674                  * instructions, one is the tail of the old packet and another
1675                  * is the head of the new coming packet, to generate
1676                  * sample(n+1); sample(n+2) and sample(n+3) consume the
1677                  * instructions with sample period.  After sample(n+3), the rest
1678                  * instructions will be used by later packet and it is assigned
1679                  * to tidq->period_instructions for next round calculation.
1680                  */
1681
1682                 /*
1683                  * Get the initial offset into the current packet instructions;
1684                  * entry conditions ensure that instrs_prev is less than
1685                  * etm->instructions_sample_period.
1686                  */
1687                 u64 offset = etm->instructions_sample_period - instrs_prev;
1688                 u64 addr;
1689
1690                 /* Prepare last branches for instruction sample */
1691                 if (etm->synth_opts.last_branch)
1692                         cs_etm__copy_last_branch_rb(etmq, tidq);
1693
1694                 while (tidq->period_instructions >=
1695                                 etm->instructions_sample_period) {
1696                         /*
1697                          * Calculate the address of the sampled instruction (-1
1698                          * as sample is reported as though instruction has just
1699                          * been executed, but PC has not advanced to next
1700                          * instruction)
1701                          */
1702                         addr = cs_etm__instr_addr(etmq, trace_chan_id,
1703                                                   tidq->packet, offset - 1);
1704                         ret = cs_etm__synth_instruction_sample(
1705                                 etmq, tidq, addr,
1706                                 etm->instructions_sample_period);
1707                         if (ret)
1708                                 return ret;
1709
1710                         offset += etm->instructions_sample_period;
1711                         tidq->period_instructions -=
1712                                 etm->instructions_sample_period;
1713                 }
1714         }
1715
1716         if (etm->synth_opts.branches) {
1717                 bool generate_sample = false;
1718
1719                 /* Generate sample for tracing on packet */
1720                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1721                         generate_sample = true;
1722
1723                 /* Generate sample for branch taken packet */
1724                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1725                     tidq->prev_packet->last_instr_taken_branch)
1726                         generate_sample = true;
1727
1728                 if (generate_sample) {
1729                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1730                         if (ret)
1731                                 return ret;
1732                 }
1733         }
1734
1735         cs_etm__packet_swap(etm, tidq);
1736
1737         return 0;
1738 }
1739
1740 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1741 {
1742         /*
1743          * When the exception packet is inserted, whether the last instruction
1744          * in previous range packet is taken branch or not, we need to force
1745          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1746          * to generate branch sample for the instruction range before the
1747          * exception is trapped to kernel or before the exception returning.
1748          *
1749          * The exception packet includes the dummy address values, so don't
1750          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1751          * for generating instruction and branch samples.
1752          */
1753         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1754                 tidq->prev_packet->last_instr_taken_branch = true;
1755
1756         return 0;
1757 }
1758
1759 static int cs_etm__flush(struct cs_etm_queue *etmq,
1760                          struct cs_etm_traceid_queue *tidq)
1761 {
1762         int err = 0;
1763         struct cs_etm_auxtrace *etm = etmq->etm;
1764
1765         /* Handle start tracing packet */
1766         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1767                 goto swap_packet;
1768
1769         if (etmq->etm->synth_opts.last_branch &&
1770             etmq->etm->synth_opts.instructions &&
1771             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1772                 u64 addr;
1773
1774                 /* Prepare last branches for instruction sample */
1775                 cs_etm__copy_last_branch_rb(etmq, tidq);
1776
1777                 /*
1778                  * Generate a last branch event for the branches left in the
1779                  * circular buffer at the end of the trace.
1780                  *
1781                  * Use the address of the end of the last reported execution
1782                  * range
1783                  */
1784                 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1785
1786                 err = cs_etm__synth_instruction_sample(
1787                         etmq, tidq, addr,
1788                         tidq->period_instructions);
1789                 if (err)
1790                         return err;
1791
1792                 tidq->period_instructions = 0;
1793
1794         }
1795
1796         if (etm->synth_opts.branches &&
1797             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1798                 err = cs_etm__synth_branch_sample(etmq, tidq);
1799                 if (err)
1800                         return err;
1801         }
1802
1803 swap_packet:
1804         cs_etm__packet_swap(etm, tidq);
1805
1806         /* Reset last branches after flush the trace */
1807         if (etm->synth_opts.last_branch)
1808                 cs_etm__reset_last_branch_rb(tidq);
1809
1810         return err;
1811 }
1812
1813 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1814                              struct cs_etm_traceid_queue *tidq)
1815 {
1816         int err;
1817
1818         /*
1819          * It has no new packet coming and 'etmq->packet' contains the stale
1820          * packet which was set at the previous time with packets swapping;
1821          * so skip to generate branch sample to avoid stale packet.
1822          *
1823          * For this case only flush branch stack and generate a last branch
1824          * event for the branches left in the circular buffer at the end of
1825          * the trace.
1826          */
1827         if (etmq->etm->synth_opts.last_branch &&
1828             etmq->etm->synth_opts.instructions &&
1829             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1830                 u64 addr;
1831
1832                 /* Prepare last branches for instruction sample */
1833                 cs_etm__copy_last_branch_rb(etmq, tidq);
1834
1835                 /*
1836                  * Use the address of the end of the last reported execution
1837                  * range.
1838                  */
1839                 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1840
1841                 err = cs_etm__synth_instruction_sample(
1842                         etmq, tidq, addr,
1843                         tidq->period_instructions);
1844                 if (err)
1845                         return err;
1846
1847                 tidq->period_instructions = 0;
1848         }
1849
1850         return 0;
1851 }
1852 /*
1853  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1854  *                         if need be.
1855  * Returns:     < 0     if error
1856  *              = 0     if no more auxtrace_buffer to read
1857  *              > 0     if the current buffer isn't empty yet
1858  */
1859 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1860 {
1861         int ret;
1862
1863         if (!etmq->buf_len) {
1864                 ret = cs_etm__get_trace(etmq);
1865                 if (ret <= 0)
1866                         return ret;
1867                 /*
1868                  * We cannot assume consecutive blocks in the data file
1869                  * are contiguous, reset the decoder to force re-sync.
1870                  */
1871                 ret = cs_etm_decoder__reset(etmq->decoder);
1872                 if (ret)
1873                         return ret;
1874         }
1875
1876         return etmq->buf_len;
1877 }
1878
1879 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1880                                  struct cs_etm_packet *packet,
1881                                  u64 end_addr)
1882 {
1883         /* Initialise to keep compiler happy */
1884         u16 instr16 = 0;
1885         u32 instr32 = 0;
1886         u64 addr;
1887
1888         switch (packet->isa) {
1889         case CS_ETM_ISA_T32:
1890                 /*
1891                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1892                  *
1893                  *  b'15         b'8
1894                  * +-----------------+--------+
1895                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1896                  * +-----------------+--------+
1897                  *
1898                  * According to the specification, it only defines SVC for T32
1899                  * with 16 bits instruction and has no definition for 32bits;
1900                  * so below only read 2 bytes as instruction size for T32.
1901                  */
1902                 addr = end_addr - 2;
1903                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1904                                    sizeof(instr16), (u8 *)&instr16);
1905                 if ((instr16 & 0xFF00) == 0xDF00)
1906                         return true;
1907
1908                 break;
1909         case CS_ETM_ISA_A32:
1910                 /*
1911                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1912                  *
1913                  *  b'31 b'28 b'27 b'24
1914                  * +---------+---------+-------------------------+
1915                  * |  !1111  | 1 1 1 1 |        imm24            |
1916                  * +---------+---------+-------------------------+
1917                  */
1918                 addr = end_addr - 4;
1919                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1920                                    sizeof(instr32), (u8 *)&instr32);
1921                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1922                     (instr32 & 0xF0000000) != 0xF0000000)
1923                         return true;
1924
1925                 break;
1926         case CS_ETM_ISA_A64:
1927                 /*
1928                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1929                  *
1930                  *  b'31               b'21           b'4     b'0
1931                  * +-----------------------+---------+-----------+
1932                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
1933                  * +-----------------------+---------+-----------+
1934                  */
1935                 addr = end_addr - 4;
1936                 cs_etm__mem_access(etmq, trace_chan_id, addr,
1937                                    sizeof(instr32), (u8 *)&instr32);
1938                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1939                         return true;
1940
1941                 break;
1942         case CS_ETM_ISA_UNKNOWN:
1943         default:
1944                 break;
1945         }
1946
1947         return false;
1948 }
1949
1950 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1951                                struct cs_etm_traceid_queue *tidq, u64 magic)
1952 {
1953         u8 trace_chan_id = tidq->trace_chan_id;
1954         struct cs_etm_packet *packet = tidq->packet;
1955         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1956
1957         if (magic == __perf_cs_etmv3_magic)
1958                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1959                         return true;
1960
1961         /*
1962          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1963          * HVC cases; need to check if it's SVC instruction based on
1964          * packet address.
1965          */
1966         if (magic == __perf_cs_etmv4_magic) {
1967                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1968                     cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1969                                          prev_packet->end_addr))
1970                         return true;
1971         }
1972
1973         return false;
1974 }
1975
1976 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1977                                        u64 magic)
1978 {
1979         struct cs_etm_packet *packet = tidq->packet;
1980
1981         if (magic == __perf_cs_etmv3_magic)
1982                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1983                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1984                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1985                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
1986                     packet->exception_number == CS_ETMV3_EXC_FIQ)
1987                         return true;
1988
1989         if (magic == __perf_cs_etmv4_magic)
1990                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1991                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1992                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1993                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1994                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1995                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
1996                     packet->exception_number == CS_ETMV4_EXC_FIQ)
1997                         return true;
1998
1999         return false;
2000 }
2001
2002 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
2003                                       struct cs_etm_traceid_queue *tidq,
2004                                       u64 magic)
2005 {
2006         u8 trace_chan_id = tidq->trace_chan_id;
2007         struct cs_etm_packet *packet = tidq->packet;
2008         struct cs_etm_packet *prev_packet = tidq->prev_packet;
2009
2010         if (magic == __perf_cs_etmv3_magic)
2011                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
2012                     packet->exception_number == CS_ETMV3_EXC_HYP ||
2013                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
2014                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
2015                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
2016                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
2017                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
2018                         return true;
2019
2020         if (magic == __perf_cs_etmv4_magic) {
2021                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2022                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2023                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2024                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2025                         return true;
2026
2027                 /*
2028                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
2029                  * (SMC, HVC) are taken as sync exceptions.
2030                  */
2031                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2032                     !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2033                                           prev_packet->end_addr))
2034                         return true;
2035
2036                 /*
2037                  * ETMv4 has 5 bits for exception number; if the numbers
2038                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2039                  * they are implementation defined exceptions.
2040                  *
2041                  * For this case, simply take it as sync exception.
2042                  */
2043                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2044                     packet->exception_number <= CS_ETMV4_EXC_END)
2045                         return true;
2046         }
2047
2048         return false;
2049 }
2050
2051 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2052                                     struct cs_etm_traceid_queue *tidq)
2053 {
2054         struct cs_etm_packet *packet = tidq->packet;
2055         struct cs_etm_packet *prev_packet = tidq->prev_packet;
2056         u8 trace_chan_id = tidq->trace_chan_id;
2057         u64 magic;
2058         int ret;
2059
2060         switch (packet->sample_type) {
2061         case CS_ETM_RANGE:
2062                 /*
2063                  * Immediate branch instruction without neither link nor
2064                  * return flag, it's normal branch instruction within
2065                  * the function.
2066                  */
2067                 if (packet->last_instr_type == OCSD_INSTR_BR &&
2068                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2069                         packet->flags = PERF_IP_FLAG_BRANCH;
2070
2071                         if (packet->last_instr_cond)
2072                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2073                 }
2074
2075                 /*
2076                  * Immediate branch instruction with link (e.g. BL), this is
2077                  * branch instruction for function call.
2078                  */
2079                 if (packet->last_instr_type == OCSD_INSTR_BR &&
2080                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2081                         packet->flags = PERF_IP_FLAG_BRANCH |
2082                                         PERF_IP_FLAG_CALL;
2083
2084                 /*
2085                  * Indirect branch instruction with link (e.g. BLR), this is
2086                  * branch instruction for function call.
2087                  */
2088                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2089                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2090                         packet->flags = PERF_IP_FLAG_BRANCH |
2091                                         PERF_IP_FLAG_CALL;
2092
2093                 /*
2094                  * Indirect branch instruction with subtype of
2095                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2096                  * function return for A32/T32.
2097                  */
2098                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2099                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2100                         packet->flags = PERF_IP_FLAG_BRANCH |
2101                                         PERF_IP_FLAG_RETURN;
2102
2103                 /*
2104                  * Indirect branch instruction without link (e.g. BR), usually
2105                  * this is used for function return, especially for functions
2106                  * within dynamic link lib.
2107                  */
2108                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2109                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2110                         packet->flags = PERF_IP_FLAG_BRANCH |
2111                                         PERF_IP_FLAG_RETURN;
2112
2113                 /* Return instruction for function return. */
2114                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2115                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2116                         packet->flags = PERF_IP_FLAG_BRANCH |
2117                                         PERF_IP_FLAG_RETURN;
2118
2119                 /*
2120                  * Decoder might insert a discontinuity in the middle of
2121                  * instruction packets, fixup prev_packet with flag
2122                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2123                  */
2124                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2125                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2126                                               PERF_IP_FLAG_TRACE_BEGIN;
2127
2128                 /*
2129                  * If the previous packet is an exception return packet
2130                  * and the return address just follows SVC instruction,
2131                  * it needs to calibrate the previous packet sample flags
2132                  * as PERF_IP_FLAG_SYSCALLRET.
2133                  */
2134                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2135                                            PERF_IP_FLAG_RETURN |
2136                                            PERF_IP_FLAG_INTERRUPT) &&
2137                     cs_etm__is_svc_instr(etmq, trace_chan_id,
2138                                          packet, packet->start_addr))
2139                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
2140                                              PERF_IP_FLAG_RETURN |
2141                                              PERF_IP_FLAG_SYSCALLRET;
2142                 break;
2143         case CS_ETM_DISCONTINUITY:
2144                 /*
2145                  * The trace is discontinuous, if the previous packet is
2146                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2147                  * for previous packet.
2148                  */
2149                 if (prev_packet->sample_type == CS_ETM_RANGE)
2150                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2151                                               PERF_IP_FLAG_TRACE_END;
2152                 break;
2153         case CS_ETM_EXCEPTION:
2154                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
2155                 if (ret)
2156                         return ret;
2157
2158                 /* The exception is for system call. */
2159                 if (cs_etm__is_syscall(etmq, tidq, magic))
2160                         packet->flags = PERF_IP_FLAG_BRANCH |
2161                                         PERF_IP_FLAG_CALL |
2162                                         PERF_IP_FLAG_SYSCALLRET;
2163                 /*
2164                  * The exceptions are triggered by external signals from bus,
2165                  * interrupt controller, debug module, PE reset or halt.
2166                  */
2167                 else if (cs_etm__is_async_exception(tidq, magic))
2168                         packet->flags = PERF_IP_FLAG_BRANCH |
2169                                         PERF_IP_FLAG_CALL |
2170                                         PERF_IP_FLAG_ASYNC |
2171                                         PERF_IP_FLAG_INTERRUPT;
2172                 /*
2173                  * Otherwise, exception is caused by trap, instruction &
2174                  * data fault, or alignment errors.
2175                  */
2176                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2177                         packet->flags = PERF_IP_FLAG_BRANCH |
2178                                         PERF_IP_FLAG_CALL |
2179                                         PERF_IP_FLAG_INTERRUPT;
2180
2181                 /*
2182                  * When the exception packet is inserted, since exception
2183                  * packet is not used standalone for generating samples
2184                  * and it's affiliation to the previous instruction range
2185                  * packet; so set previous range packet flags to tell perf
2186                  * it is an exception taken branch.
2187                  */
2188                 if (prev_packet->sample_type == CS_ETM_RANGE)
2189                         prev_packet->flags = packet->flags;
2190                 break;
2191         case CS_ETM_EXCEPTION_RET:
2192                 /*
2193                  * When the exception return packet is inserted, since
2194                  * exception return packet is not used standalone for
2195                  * generating samples and it's affiliation to the previous
2196                  * instruction range packet; so set previous range packet
2197                  * flags to tell perf it is an exception return branch.
2198                  *
2199                  * The exception return can be for either system call or
2200                  * other exception types; unfortunately the packet doesn't
2201                  * contain exception type related info so we cannot decide
2202                  * the exception type purely based on exception return packet.
2203                  * If we record the exception number from exception packet and
2204                  * reuse it for exception return packet, this is not reliable
2205                  * due the trace can be discontinuity or the interrupt can
2206                  * be nested, thus the recorded exception number cannot be
2207                  * used for exception return packet for these two cases.
2208                  *
2209                  * For exception return packet, we only need to distinguish the
2210                  * packet is for system call or for other types.  Thus the
2211                  * decision can be deferred when receive the next packet which
2212                  * contains the return address, based on the return address we
2213                  * can read out the previous instruction and check if it's a
2214                  * system call instruction and then calibrate the sample flag
2215                  * as needed.
2216                  */
2217                 if (prev_packet->sample_type == CS_ETM_RANGE)
2218                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
2219                                              PERF_IP_FLAG_RETURN |
2220                                              PERF_IP_FLAG_INTERRUPT;
2221                 break;
2222         case CS_ETM_EMPTY:
2223         default:
2224                 break;
2225         }
2226
2227         return 0;
2228 }
2229
2230 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2231 {
2232         int ret = 0;
2233         size_t processed = 0;
2234
2235         /*
2236          * Packets are decoded and added to the decoder's packet queue
2237          * until the decoder packet processing callback has requested that
2238          * processing stops or there is nothing left in the buffer.  Normal
2239          * operations that stop processing are a timestamp packet or a full
2240          * decoder buffer queue.
2241          */
2242         ret = cs_etm_decoder__process_data_block(etmq->decoder,
2243                                                  etmq->offset,
2244                                                  &etmq->buf[etmq->buf_used],
2245                                                  etmq->buf_len,
2246                                                  &processed);
2247         if (ret)
2248                 goto out;
2249
2250         etmq->offset += processed;
2251         etmq->buf_used += processed;
2252         etmq->buf_len -= processed;
2253
2254 out:
2255         return ret;
2256 }
2257
2258 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2259                                          struct cs_etm_traceid_queue *tidq)
2260 {
2261         int ret;
2262         struct cs_etm_packet_queue *packet_queue;
2263
2264         packet_queue = &tidq->packet_queue;
2265
2266         /* Process each packet in this chunk */
2267         while (1) {
2268                 ret = cs_etm_decoder__get_packet(packet_queue,
2269                                                  tidq->packet);
2270                 if (ret <= 0)
2271                         /*
2272                          * Stop processing this chunk on
2273                          * end of data or error
2274                          */
2275                         break;
2276
2277                 /*
2278                  * Since packet addresses are swapped in packet
2279                  * handling within below switch() statements,
2280                  * thus setting sample flags must be called
2281                  * prior to switch() statement to use address
2282                  * information before packets swapping.
2283                  */
2284                 ret = cs_etm__set_sample_flags(etmq, tidq);
2285                 if (ret < 0)
2286                         break;
2287
2288                 switch (tidq->packet->sample_type) {
2289                 case CS_ETM_RANGE:
2290                         /*
2291                          * If the packet contains an instruction
2292                          * range, generate instruction sequence
2293                          * events.
2294                          */
2295                         cs_etm__sample(etmq, tidq);
2296                         break;
2297                 case CS_ETM_EXCEPTION:
2298                 case CS_ETM_EXCEPTION_RET:
2299                         /*
2300                          * If the exception packet is coming,
2301                          * make sure the previous instruction
2302                          * range packet to be handled properly.
2303                          */
2304                         cs_etm__exception(tidq);
2305                         break;
2306                 case CS_ETM_DISCONTINUITY:
2307                         /*
2308                          * Discontinuity in trace, flush
2309                          * previous branch stack
2310                          */
2311                         cs_etm__flush(etmq, tidq);
2312                         break;
2313                 case CS_ETM_EMPTY:
2314                         /*
2315                          * Should not receive empty packet,
2316                          * report error.
2317                          */
2318                         pr_err("CS ETM Trace: empty packet\n");
2319                         return -EINVAL;
2320                 default:
2321                         break;
2322                 }
2323         }
2324
2325         return ret;
2326 }
2327
2328 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2329 {
2330         int idx;
2331         struct int_node *inode;
2332         struct cs_etm_traceid_queue *tidq;
2333         struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2334
2335         intlist__for_each_entry(inode, traceid_queues_list) {
2336                 idx = (int)(intptr_t)inode->priv;
2337                 tidq = etmq->traceid_queues[idx];
2338
2339                 /* Ignore return value */
2340                 cs_etm__process_traceid_queue(etmq, tidq);
2341
2342                 /*
2343                  * Generate an instruction sample with the remaining
2344                  * branchstack entries.
2345                  */
2346                 cs_etm__flush(etmq, tidq);
2347         }
2348 }
2349
2350 static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq)
2351 {
2352         int err = 0;
2353         struct cs_etm_traceid_queue *tidq;
2354
2355         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2356         if (!tidq)
2357                 return -EINVAL;
2358
2359         /* Go through each buffer in the queue and decode them one by one */
2360         while (1) {
2361                 err = cs_etm__get_data_block(etmq);
2362                 if (err <= 0)
2363                         return err;
2364
2365                 /* Run trace decoder until buffer consumed or end of trace */
2366                 do {
2367                         err = cs_etm__decode_data_block(etmq);
2368                         if (err)
2369                                 return err;
2370
2371                         /*
2372                          * Process each packet in this chunk, nothing to do if
2373                          * an error occurs other than hoping the next one will
2374                          * be better.
2375                          */
2376                         err = cs_etm__process_traceid_queue(etmq, tidq);
2377
2378                 } while (etmq->buf_len);
2379
2380                 if (err == 0)
2381                         /* Flush any remaining branch stack entries */
2382                         err = cs_etm__end_block(etmq, tidq);
2383         }
2384
2385         return err;
2386 }
2387
2388 static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq)
2389 {
2390         int idx, err = 0;
2391         struct cs_etm_traceid_queue *tidq;
2392         struct int_node *inode;
2393
2394         /* Go through each buffer in the queue and decode them one by one */
2395         while (1) {
2396                 err = cs_etm__get_data_block(etmq);
2397                 if (err <= 0)
2398                         return err;
2399
2400                 /* Run trace decoder until buffer consumed or end of trace */
2401                 do {
2402                         err = cs_etm__decode_data_block(etmq);
2403                         if (err)
2404                                 return err;
2405
2406                         /*
2407                          * cs_etm__run_per_thread_timeless_decoder() runs on a
2408                          * single traceID queue because each TID has a separate
2409                          * buffer. But here in per-cpu mode we need to iterate
2410                          * over each channel instead.
2411                          */
2412                         intlist__for_each_entry(inode,
2413                                                 etmq->traceid_queues_list) {
2414                                 idx = (int)(intptr_t)inode->priv;
2415                                 tidq = etmq->traceid_queues[idx];
2416                                 cs_etm__process_traceid_queue(etmq, tidq);
2417                         }
2418                 } while (etmq->buf_len);
2419
2420                 intlist__for_each_entry(inode, etmq->traceid_queues_list) {
2421                         idx = (int)(intptr_t)inode->priv;
2422                         tidq = etmq->traceid_queues[idx];
2423                         /* Flush any remaining branch stack entries */
2424                         err = cs_etm__end_block(etmq, tidq);
2425                         if (err)
2426                                 return err;
2427                 }
2428         }
2429
2430         return err;
2431 }
2432
2433 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2434                                            pid_t tid)
2435 {
2436         unsigned int i;
2437         struct auxtrace_queues *queues = &etm->queues;
2438
2439         for (i = 0; i < queues->nr_queues; i++) {
2440                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2441                 struct cs_etm_queue *etmq = queue->priv;
2442                 struct cs_etm_traceid_queue *tidq;
2443
2444                 if (!etmq)
2445                         continue;
2446
2447                 /*
2448                  * Per-cpu mode has contextIDs in the trace and the decoder
2449                  * calls cs_etm__set_pid_tid_cpu() automatically so no need
2450                  * to do this here
2451                  */
2452                 if (etm->per_thread_decoding) {
2453                         tidq = cs_etm__etmq_get_traceid_queue(
2454                                 etmq, CS_ETM_PER_THREAD_TRACEID);
2455
2456                         if (!tidq)
2457                                 continue;
2458
2459                         if ((tid == -1) || (tidq->tid == tid)) {
2460                                 cs_etm__set_pid_tid_cpu(etm, tidq);
2461                                 cs_etm__run_per_thread_timeless_decoder(etmq);
2462                         }
2463                 } else
2464                         cs_etm__run_per_cpu_timeless_decoder(etmq);
2465         }
2466
2467         return 0;
2468 }
2469
2470 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)
2471 {
2472         int ret = 0;
2473         unsigned int cs_queue_nr, queue_nr, i;
2474         u8 trace_chan_id;
2475         u64 cs_timestamp;
2476         struct auxtrace_queue *queue;
2477         struct cs_etm_queue *etmq;
2478         struct cs_etm_traceid_queue *tidq;
2479
2480         /*
2481          * Pre-populate the heap with one entry from each queue so that we can
2482          * start processing in time order across all queues.
2483          */
2484         for (i = 0; i < etm->queues.nr_queues; i++) {
2485                 etmq = etm->queues.queue_array[i].priv;
2486                 if (!etmq)
2487                         continue;
2488
2489                 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2490                 if (ret)
2491                         return ret;
2492         }
2493
2494         while (1) {
2495                 if (!etm->heap.heap_cnt)
2496                         goto out;
2497
2498                 /* Take the entry at the top of the min heap */
2499                 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2500                 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2501                 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2502                 queue = &etm->queues.queue_array[queue_nr];
2503                 etmq = queue->priv;
2504
2505                 /*
2506                  * Remove the top entry from the heap since we are about
2507                  * to process it.
2508                  */
2509                 auxtrace_heap__pop(&etm->heap);
2510
2511                 tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2512                 if (!tidq) {
2513                         /*
2514                          * No traceID queue has been allocated for this traceID,
2515                          * which means something somewhere went very wrong.  No
2516                          * other choice than simply exit.
2517                          */
2518                         ret = -EINVAL;
2519                         goto out;
2520                 }
2521
2522                 /*
2523                  * Packets associated with this timestamp are already in
2524                  * the etmq's traceID queue, so process them.
2525                  */
2526                 ret = cs_etm__process_traceid_queue(etmq, tidq);
2527                 if (ret < 0)
2528                         goto out;
2529
2530                 /*
2531                  * Packets for this timestamp have been processed, time to
2532                  * move on to the next timestamp, fetching a new auxtrace_buffer
2533                  * if need be.
2534                  */
2535 refetch:
2536                 ret = cs_etm__get_data_block(etmq);
2537                 if (ret < 0)
2538                         goto out;
2539
2540                 /*
2541                  * No more auxtrace_buffers to process in this etmq, simply
2542                  * move on to another entry in the auxtrace_heap.
2543                  */
2544                 if (!ret)
2545                         continue;
2546
2547                 ret = cs_etm__decode_data_block(etmq);
2548                 if (ret)
2549                         goto out;
2550
2551                 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2552
2553                 if (!cs_timestamp) {
2554                         /*
2555                          * Function cs_etm__decode_data_block() returns when
2556                          * there is no more traces to decode in the current
2557                          * auxtrace_buffer OR when a timestamp has been
2558                          * encountered on any of the traceID queues.  Since we
2559                          * did not get a timestamp, there is no more traces to
2560                          * process in this auxtrace_buffer.  As such empty and
2561                          * flush all traceID queues.
2562                          */
2563                         cs_etm__clear_all_traceid_queues(etmq);
2564
2565                         /* Fetch another auxtrace_buffer for this etmq */
2566                         goto refetch;
2567                 }
2568
2569                 /*
2570                  * Add to the min heap the timestamp for packets that have
2571                  * just been decoded.  They will be processed and synthesized
2572                  * during the next call to cs_etm__process_traceid_queue() for
2573                  * this queue/traceID.
2574                  */
2575                 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2576                 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2577         }
2578
2579 out:
2580         return ret;
2581 }
2582
2583 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2584                                         union perf_event *event)
2585 {
2586         struct thread *th;
2587
2588         if (etm->timeless_decoding)
2589                 return 0;
2590
2591         /*
2592          * Add the tid/pid to the log so that we can get a match when
2593          * we get a contextID from the decoder.
2594          */
2595         th = machine__findnew_thread(etm->machine,
2596                                      event->itrace_start.pid,
2597                                      event->itrace_start.tid);
2598         if (!th)
2599                 return -ENOMEM;
2600
2601         thread__put(th);
2602
2603         return 0;
2604 }
2605
2606 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2607                                            union perf_event *event)
2608 {
2609         struct thread *th;
2610         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2611
2612         /*
2613          * Context switch in per-thread mode are irrelevant since perf
2614          * will start/stop tracing as the process is scheduled.
2615          */
2616         if (etm->timeless_decoding)
2617                 return 0;
2618
2619         /*
2620          * SWITCH_IN events carry the next process to be switched out while
2621          * SWITCH_OUT events carry the process to be switched in.  As such
2622          * we don't care about IN events.
2623          */
2624         if (!out)
2625                 return 0;
2626
2627         /*
2628          * Add the tid/pid to the log so that we can get a match when
2629          * we get a contextID from the decoder.
2630          */
2631         th = machine__findnew_thread(etm->machine,
2632                                      event->context_switch.next_prev_pid,
2633                                      event->context_switch.next_prev_tid);
2634         if (!th)
2635                 return -ENOMEM;
2636
2637         thread__put(th);
2638
2639         return 0;
2640 }
2641
2642 static int cs_etm__process_event(struct perf_session *session,
2643                                  union perf_event *event,
2644                                  struct perf_sample *sample,
2645                                  struct perf_tool *tool)
2646 {
2647         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2648                                                    struct cs_etm_auxtrace,
2649                                                    auxtrace);
2650
2651         if (dump_trace)
2652                 return 0;
2653
2654         if (!tool->ordered_events) {
2655                 pr_err("CoreSight ETM Trace requires ordered events\n");
2656                 return -EINVAL;
2657         }
2658
2659         switch (event->header.type) {
2660         case PERF_RECORD_EXIT:
2661                 /*
2662                  * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2663                  * start the decode because we know there will be no more trace from
2664                  * this thread. All this does is emit samples earlier than waiting for
2665                  * the flush in other modes, but with timestamps it makes sense to wait
2666                  * for flush so that events from different threads are interleaved
2667                  * properly.
2668                  */
2669                 if (etm->per_thread_decoding && etm->timeless_decoding)
2670                         return cs_etm__process_timeless_queues(etm,
2671                                                                event->fork.tid);
2672                 break;
2673
2674         case PERF_RECORD_ITRACE_START:
2675                 return cs_etm__process_itrace_start(etm, event);
2676
2677         case PERF_RECORD_SWITCH_CPU_WIDE:
2678                 return cs_etm__process_switch_cpu_wide(etm, event);
2679
2680         case PERF_RECORD_AUX:
2681                 /*
2682                  * Record the latest kernel timestamp available in the header
2683                  * for samples so that synthesised samples occur from this point
2684                  * onwards.
2685                  */
2686                 if (sample->time && (sample->time != (u64)-1))
2687                         etm->latest_kernel_timestamp = sample->time;
2688                 break;
2689
2690         default:
2691                 break;
2692         }
2693
2694         return 0;
2695 }
2696
2697 static void dump_queued_data(struct cs_etm_auxtrace *etm,
2698                              struct perf_record_auxtrace *event)
2699 {
2700         struct auxtrace_buffer *buf;
2701         unsigned int i;
2702         /*
2703          * Find all buffers with same reference in the queues and dump them.
2704          * This is because the queues can contain multiple entries of the same
2705          * buffer that were split on aux records.
2706          */
2707         for (i = 0; i < etm->queues.nr_queues; ++i)
2708                 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2709                         if (buf->reference == event->reference)
2710                                 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2711 }
2712
2713 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2714                                           union perf_event *event,
2715                                           struct perf_tool *tool __maybe_unused)
2716 {
2717         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2718                                                    struct cs_etm_auxtrace,
2719                                                    auxtrace);
2720         if (!etm->data_queued) {
2721                 struct auxtrace_buffer *buffer;
2722                 off_t  data_offset;
2723                 int fd = perf_data__fd(session->data);
2724                 bool is_pipe = perf_data__is_pipe(session->data);
2725                 int err;
2726                 int idx = event->auxtrace.idx;
2727
2728                 if (is_pipe)
2729                         data_offset = 0;
2730                 else {
2731                         data_offset = lseek(fd, 0, SEEK_CUR);
2732                         if (data_offset == -1)
2733                                 return -errno;
2734                 }
2735
2736                 err = auxtrace_queues__add_event(&etm->queues, session,
2737                                                  event, data_offset, &buffer);
2738                 if (err)
2739                         return err;
2740
2741                 /*
2742                  * Knowing if the trace is formatted or not requires a lookup of
2743                  * the aux record so only works in non-piped mode where data is
2744                  * queued in cs_etm__queue_aux_records(). Always assume
2745                  * formatted in piped mode (true).
2746                  */
2747                 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2748                                           idx, true);
2749                 if (err)
2750                         return err;
2751
2752                 if (dump_trace)
2753                         if (auxtrace_buffer__get_data(buffer, fd)) {
2754                                 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2755                                 auxtrace_buffer__put_data(buffer);
2756                         }
2757         } else if (dump_trace)
2758                 dump_queued_data(etm, &event->auxtrace);
2759
2760         return 0;
2761 }
2762
2763 static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2764 {
2765         struct evsel *evsel;
2766         struct evlist *evlist = etm->session->evlist;
2767
2768         /* Override timeless mode with user input from --itrace=Z */
2769         if (etm->synth_opts.timeless_decoding) {
2770                 etm->timeless_decoding = true;
2771                 return 0;
2772         }
2773
2774         /*
2775          * Find the cs_etm evsel and look at what its timestamp setting was
2776          */
2777         evlist__for_each_entry(evlist, evsel)
2778                 if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2779                         etm->timeless_decoding =
2780                                 !(evsel->core.attr.config & BIT(ETM_OPT_TS));
2781                         return 0;
2782                 }
2783
2784         pr_err("CS ETM: Couldn't find ETM evsel\n");
2785         return -EINVAL;
2786 }
2787
2788 /*
2789  * Read a single cpu parameter block from the auxtrace_info priv block.
2790  *
2791  * For version 1 there is a per cpu nr_params entry. If we are handling
2792  * version 1 file, then there may be less, the same, or more params
2793  * indicated by this value than the compile time number we understand.
2794  *
2795  * For a version 0 info block, there are a fixed number, and we need to
2796  * fill out the nr_param value in the metadata we create.
2797  */
2798 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2799                                     int out_blk_size, int nr_params_v0)
2800 {
2801         u64 *metadata = NULL;
2802         int hdr_version;
2803         int nr_in_params, nr_out_params, nr_cmn_params;
2804         int i, k;
2805
2806         metadata = zalloc(sizeof(*metadata) * out_blk_size);
2807         if (!metadata)
2808                 return NULL;
2809
2810         /* read block current index & version */
2811         i = *buff_in_offset;
2812         hdr_version = buff_in[CS_HEADER_VERSION];
2813
2814         if (!hdr_version) {
2815         /* read version 0 info block into a version 1 metadata block  */
2816                 nr_in_params = nr_params_v0;
2817                 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2818                 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2819                 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2820                 /* remaining block params at offset +1 from source */
2821                 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2822                         metadata[k + 1] = buff_in[i + k];
2823                 /* version 0 has 2 common params */
2824                 nr_cmn_params = 2;
2825         } else {
2826         /* read version 1 info block - input and output nr_params may differ */
2827                 /* version 1 has 3 common params */
2828                 nr_cmn_params = 3;
2829                 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2830
2831                 /* if input has more params than output - skip excess */
2832                 nr_out_params = nr_in_params + nr_cmn_params;
2833                 if (nr_out_params > out_blk_size)
2834                         nr_out_params = out_blk_size;
2835
2836                 for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2837                         metadata[k] = buff_in[i + k];
2838
2839                 /* record the actual nr params we copied */
2840                 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2841         }
2842
2843         /* adjust in offset by number of in params used */
2844         i += nr_in_params + nr_cmn_params;
2845         *buff_in_offset = i;
2846         return metadata;
2847 }
2848
2849 /**
2850  * Puts a fragment of an auxtrace buffer into the auxtrace queues based
2851  * on the bounds of aux_event, if it matches with the buffer that's at
2852  * file_offset.
2853  *
2854  * Normally, whole auxtrace buffers would be added to the queue. But we
2855  * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
2856  * is reset across each buffer, so splitting the buffers up in advance has
2857  * the same effect.
2858  */
2859 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
2860                                       struct perf_record_aux *aux_event, struct perf_sample *sample)
2861 {
2862         int err;
2863         char buf[PERF_SAMPLE_MAX_SIZE];
2864         union perf_event *auxtrace_event_union;
2865         struct perf_record_auxtrace *auxtrace_event;
2866         union perf_event auxtrace_fragment;
2867         __u64 aux_offset, aux_size;
2868         __u32 idx;
2869         bool formatted;
2870
2871         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2872                                                    struct cs_etm_auxtrace,
2873                                                    auxtrace);
2874
2875         /*
2876          * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
2877          * from looping through the auxtrace index.
2878          */
2879         err = perf_session__peek_event(session, file_offset, buf,
2880                                        PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
2881         if (err)
2882                 return err;
2883         auxtrace_event = &auxtrace_event_union->auxtrace;
2884         if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
2885                 return -EINVAL;
2886
2887         if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
2888                 auxtrace_event->header.size != sz) {
2889                 return -EINVAL;
2890         }
2891
2892         /*
2893          * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
2894          * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
2895          * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
2896          * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
2897          * Return 'not found' if mismatch.
2898          */
2899         if (auxtrace_event->cpu == (__u32) -1) {
2900                 etm->per_thread_decoding = true;
2901                 if (auxtrace_event->tid != sample->tid)
2902                         return 1;
2903         } else if (auxtrace_event->cpu != sample->cpu) {
2904                 if (etm->per_thread_decoding) {
2905                         /*
2906                          * Found a per-cpu buffer after a per-thread one was
2907                          * already found
2908                          */
2909                         pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n");
2910                         return -EINVAL;
2911                 }
2912                 return 1;
2913         }
2914
2915         if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
2916                 /*
2917                  * Clamp size in snapshot mode. The buffer size is clamped in
2918                  * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
2919                  * the buffer size.
2920                  */
2921                 aux_size = min(aux_event->aux_size, auxtrace_event->size);
2922
2923                 /*
2924                  * In this mode, the head also points to the end of the buffer so aux_offset
2925                  * needs to have the size subtracted so it points to the beginning as in normal mode
2926                  */
2927                 aux_offset = aux_event->aux_offset - aux_size;
2928         } else {
2929                 aux_size = aux_event->aux_size;
2930                 aux_offset = aux_event->aux_offset;
2931         }
2932
2933         if (aux_offset >= auxtrace_event->offset &&
2934             aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
2935                 /*
2936                  * If this AUX event was inside this buffer somewhere, create a new auxtrace event
2937                  * based on the sizes of the aux event, and queue that fragment.
2938                  */
2939                 auxtrace_fragment.auxtrace = *auxtrace_event;
2940                 auxtrace_fragment.auxtrace.size = aux_size;
2941                 auxtrace_fragment.auxtrace.offset = aux_offset;
2942                 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
2943
2944                 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
2945                           " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
2946                 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
2947                                                  file_offset, NULL);
2948                 if (err)
2949                         return err;
2950
2951                 idx = auxtrace_event->idx;
2952                 formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
2953                 return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2954                                            idx, formatted);
2955         }
2956
2957         /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
2958         return 1;
2959 }
2960
2961 static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
2962                                         u64 offset __maybe_unused, void *data __maybe_unused)
2963 {
2964         /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
2965         if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
2966                 (*(int *)data)++; /* increment found count */
2967                 return cs_etm__process_aux_output_hw_id(session, event);
2968         }
2969         return 0;
2970 }
2971
2972 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
2973                                         u64 offset __maybe_unused, void *data __maybe_unused)
2974 {
2975         struct perf_sample sample;
2976         int ret;
2977         struct auxtrace_index_entry *ent;
2978         struct auxtrace_index *auxtrace_index;
2979         struct evsel *evsel;
2980         size_t i;
2981
2982         /* Don't care about any other events, we're only queuing buffers for AUX events */
2983         if (event->header.type != PERF_RECORD_AUX)
2984                 return 0;
2985
2986         if (event->header.size < sizeof(struct perf_record_aux))
2987                 return -EINVAL;
2988
2989         /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
2990         if (!event->aux.aux_size)
2991                 return 0;
2992
2993         /*
2994          * Parse the sample, we need the sample_id_all data that comes after the event so that the
2995          * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
2996          */
2997         evsel = evlist__event2evsel(session->evlist, event);
2998         if (!evsel)
2999                 return -EINVAL;
3000         ret = evsel__parse_sample(evsel, event, &sample);
3001         if (ret)
3002                 return ret;
3003
3004         /*
3005          * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3006          */
3007         list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
3008                 for (i = 0; i < auxtrace_index->nr; i++) {
3009                         ent = &auxtrace_index->entries[i];
3010                         ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
3011                                                          ent->sz, &event->aux, &sample);
3012                         /*
3013                          * Stop search on error or successful values. Continue search on
3014                          * 1 ('not found')
3015                          */
3016                         if (ret != 1)
3017                                 return ret;
3018                 }
3019         }
3020
3021         /*
3022          * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3023          * don't exit with an error because it will still be possible to decode other aux records.
3024          */
3025         pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
3026                " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
3027         return 0;
3028 }
3029
3030 static int cs_etm__queue_aux_records(struct perf_session *session)
3031 {
3032         struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
3033                                                                 struct auxtrace_index, list);
3034         if (index && index->nr > 0)
3035                 return perf_session__peek_events(session, session->header.data_offset,
3036                                                  session->header.data_size,
3037                                                  cs_etm__queue_aux_records_cb, NULL);
3038
3039         /*
3040          * We would get here if there are no entries in the index (either no auxtrace
3041          * buffers or no index at all). Fail silently as there is the possibility of
3042          * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3043          * false.
3044          *
3045          * In that scenario, buffers will not be split by AUX records.
3046          */
3047         return 0;
3048 }
3049
3050 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3051                                   (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3052
3053 /*
3054  * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3055  * timestamps).
3056  */
3057 static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
3058 {
3059         int j;
3060
3061         for (j = 0; j < num_cpu; j++) {
3062                 switch (metadata[j][CS_ETM_MAGIC]) {
3063                 case __perf_cs_etmv4_magic:
3064                         if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
3065                                 return false;
3066                         break;
3067                 case __perf_cs_ete_magic:
3068                         if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
3069                                 return false;
3070                         break;
3071                 default:
3072                         /* Unknown / unsupported magic number. */
3073                         return false;
3074                 }
3075         }
3076         return true;
3077 }
3078
3079 /* map trace ids to correct metadata block, from information in metadata */
3080 static int cs_etm__map_trace_ids_metadata(int num_cpu, u64 **metadata)
3081 {
3082         u64 cs_etm_magic;
3083         u8 trace_chan_id;
3084         int i, err;
3085
3086         for (i = 0; i < num_cpu; i++) {
3087                 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3088                 switch (cs_etm_magic) {
3089                 case __perf_cs_etmv3_magic:
3090                         metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3091                         trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3092                         break;
3093                 case __perf_cs_etmv4_magic:
3094                 case __perf_cs_ete_magic:
3095                         metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3096                         trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3097                         break;
3098                 default:
3099                         /* unknown magic number */
3100                         return -EINVAL;
3101                 }
3102                 err = cs_etm__map_trace_id(trace_chan_id, metadata[i]);
3103                 if (err)
3104                         return err;
3105         }
3106         return 0;
3107 }
3108
3109 /*
3110  * If we found AUX_HW_ID packets, then set any metadata marked as unused to the
3111  * unused value to reduce the number of unneeded decoders created.
3112  */
3113 static int cs_etm__clear_unused_trace_ids_metadata(int num_cpu, u64 **metadata)
3114 {
3115         u64 cs_etm_magic;
3116         int i;
3117
3118         for (i = 0; i < num_cpu; i++) {
3119                 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3120                 switch (cs_etm_magic) {
3121                 case __perf_cs_etmv3_magic:
3122                         if (metadata[i][CS_ETM_ETMTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3123                                 metadata[i][CS_ETM_ETMTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3124                         break;
3125                 case __perf_cs_etmv4_magic:
3126                 case __perf_cs_ete_magic:
3127                         if (metadata[i][CS_ETMV4_TRCTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3128                                 metadata[i][CS_ETMV4_TRCTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3129                         break;
3130                 default:
3131                         /* unknown magic number */
3132                         return -EINVAL;
3133                 }
3134         }
3135         return 0;
3136 }
3137
3138 int cs_etm__process_auxtrace_info_full(union perf_event *event,
3139                                        struct perf_session *session)
3140 {
3141         struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3142         struct cs_etm_auxtrace *etm = NULL;
3143         struct perf_record_time_conv *tc = &session->time_conv;
3144         int event_header_size = sizeof(struct perf_event_header);
3145         int total_size = auxtrace_info->header.size;
3146         int priv_size = 0;
3147         int num_cpu;
3148         int err = 0;
3149         int aux_hw_id_found;
3150         int i, j;
3151         u64 *ptr = NULL;
3152         u64 **metadata = NULL;
3153
3154         /*
3155          * Create an RB tree for traceID-metadata tuple.  Since the conversion
3156          * has to be made for each packet that gets decoded, optimizing access
3157          * in anything other than a sequential array is worth doing.
3158          */
3159         traceid_list = intlist__new(NULL);
3160         if (!traceid_list)
3161                 return -ENOMEM;
3162
3163         /* First the global part */
3164         ptr = (u64 *) auxtrace_info->priv;
3165         num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3166         metadata = zalloc(sizeof(*metadata) * num_cpu);
3167         if (!metadata) {
3168                 err = -ENOMEM;
3169                 goto err_free_traceid_list;
3170         }
3171
3172         /* Start parsing after the common part of the header */
3173         i = CS_HEADER_VERSION_MAX;
3174
3175         /*
3176          * The metadata is stored in the auxtrace_info section and encodes
3177          * the configuration of the ARM embedded trace macrocell which is
3178          * required by the trace decoder to properly decode the trace due
3179          * to its highly compressed nature.
3180          */
3181         for (j = 0; j < num_cpu; j++) {
3182                 if (ptr[i] == __perf_cs_etmv3_magic) {
3183                         metadata[j] =
3184                                 cs_etm__create_meta_blk(ptr, &i,
3185                                                         CS_ETM_PRIV_MAX,
3186                                                         CS_ETM_NR_TRC_PARAMS_V0);
3187                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
3188                         metadata[j] =
3189                                 cs_etm__create_meta_blk(ptr, &i,
3190                                                         CS_ETMV4_PRIV_MAX,
3191                                                         CS_ETMV4_NR_TRC_PARAMS_V0);
3192                 } else if (ptr[i] == __perf_cs_ete_magic) {
3193                         metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3194                 } else {
3195                         ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3196                                   ptr[i]);
3197                         err = -EINVAL;
3198                         goto err_free_metadata;
3199                 }
3200
3201                 if (!metadata[j]) {
3202                         err = -ENOMEM;
3203                         goto err_free_metadata;
3204                 }
3205         }
3206
3207         /*
3208          * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3209          * CS_ETMV4_PRIV_MAX mark how many double words are in the
3210          * global metadata, and each cpu's metadata respectively.
3211          * The following tests if the correct number of double words was
3212          * present in the auxtrace info section.
3213          */
3214         priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3215         if (i * 8 != priv_size) {
3216                 err = -EINVAL;
3217                 goto err_free_metadata;
3218         }
3219
3220         etm = zalloc(sizeof(*etm));
3221
3222         if (!etm) {
3223                 err = -ENOMEM;
3224                 goto err_free_metadata;
3225         }
3226
3227         err = auxtrace_queues__init(&etm->queues);
3228         if (err)
3229                 goto err_free_etm;
3230
3231         if (session->itrace_synth_opts->set) {
3232                 etm->synth_opts = *session->itrace_synth_opts;
3233         } else {
3234                 itrace_synth_opts__set_default(&etm->synth_opts,
3235                                 session->itrace_synth_opts->default_no_sample);
3236                 etm->synth_opts.callchain = false;
3237         }
3238
3239         etm->session = session;
3240         etm->machine = &session->machines.host;
3241
3242         etm->num_cpu = num_cpu;
3243         etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3244         etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3245         etm->metadata = metadata;
3246         etm->auxtrace_type = auxtrace_info->type;
3247
3248         /* Use virtual timestamps if all ETMs report ts_source = 1 */
3249         etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3250
3251         if (!etm->has_virtual_ts)
3252                 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3253                             "The time field of the samples will not be set accurately.\n\n");
3254
3255         etm->auxtrace.process_event = cs_etm__process_event;
3256         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3257         etm->auxtrace.flush_events = cs_etm__flush_events;
3258         etm->auxtrace.free_events = cs_etm__free_events;
3259         etm->auxtrace.free = cs_etm__free;
3260         etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3261         session->auxtrace = &etm->auxtrace;
3262
3263         err = cs_etm__setup_timeless_decoding(etm);
3264         if (err)
3265                 return err;
3266
3267         etm->unknown_thread = thread__new(999999999, 999999999);
3268         if (!etm->unknown_thread) {
3269                 err = -ENOMEM;
3270                 goto err_free_queues;
3271         }
3272
3273         /*
3274          * Initialize list node so that at thread__zput() we can avoid
3275          * segmentation fault at list_del_init().
3276          */
3277         INIT_LIST_HEAD(&etm->unknown_thread->node);
3278
3279         err = thread__set_comm(etm->unknown_thread, "unknown", 0);
3280         if (err)
3281                 goto err_delete_thread;
3282
3283         if (thread__init_maps(etm->unknown_thread, etm->machine)) {
3284                 err = -ENOMEM;
3285                 goto err_delete_thread;
3286         }
3287
3288         etm->tc.time_shift = tc->time_shift;
3289         etm->tc.time_mult = tc->time_mult;
3290         etm->tc.time_zero = tc->time_zero;
3291         if (event_contains(*tc, time_cycles)) {
3292                 etm->tc.time_cycles = tc->time_cycles;
3293                 etm->tc.time_mask = tc->time_mask;
3294                 etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3295                 etm->tc.cap_user_time_short = tc->cap_user_time_short;
3296         }
3297         err = cs_etm__synth_events(etm, session);
3298         if (err)
3299                 goto err_delete_thread;
3300
3301         /*
3302          * Map Trace ID values to CPU metadata.
3303          *
3304          * Trace metadata will always contain Trace ID values from the legacy algorithm. If the
3305          * files has been recorded by a "new" perf updated to handle AUX_HW_ID then the metadata
3306          * ID value will also have the CORESIGHT_TRACE_ID_UNUSED_FLAG set.
3307          *
3308          * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3309          * the same IDs as the old algorithm as far as is possible, unless there are clashes
3310          * in which case a different value will be used. This means an older perf may still
3311          * be able to record and read files generate on a newer system.
3312          *
3313          * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3314          * those packets. If they are there then the values will be mapped and plugged into
3315          * the metadata. We then set any remaining metadata values with the used flag to a
3316          * value CORESIGHT_TRACE_ID_UNUSED_VAL - which indicates no decoder is required.
3317          *
3318          * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3319          * then we map Trace ID values to CPU directly from the metadata - clearing any unused
3320          * flags if present.
3321          */
3322
3323         /* first scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3324         aux_hw_id_found = 0;
3325         err = perf_session__peek_events(session, session->header.data_offset,
3326                                         session->header.data_size,
3327                                         cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3328         if (err)
3329                 goto err_delete_thread;
3330
3331         /* if HW ID found then clear any unused metadata ID values */
3332         if (aux_hw_id_found)
3333                 err = cs_etm__clear_unused_trace_ids_metadata(num_cpu, metadata);
3334         /* otherwise, this is a file with metadata values only, map from metadata */
3335         else
3336                 err = cs_etm__map_trace_ids_metadata(num_cpu, metadata);
3337
3338         if (err)
3339                 goto err_delete_thread;
3340
3341         err = cs_etm__queue_aux_records(session);
3342         if (err)
3343                 goto err_delete_thread;
3344
3345         etm->data_queued = etm->queues.populated;
3346         return 0;
3347
3348 err_delete_thread:
3349         thread__zput(etm->unknown_thread);
3350 err_free_queues:
3351         auxtrace_queues__free(&etm->queues);
3352         session->auxtrace = NULL;
3353 err_free_etm:
3354         zfree(&etm);
3355 err_free_metadata:
3356         /* No need to check @metadata[j], free(NULL) is supported */
3357         for (j = 0; j < num_cpu; j++)
3358                 zfree(&metadata[j]);
3359         zfree(&metadata);
3360 err_free_traceid_list:
3361         intlist__delete(traceid_list);
3362         return err;
3363 }