1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2015-2018 Linaro Limited.
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
9 #include <linux/bitops.h>
10 #include <linux/coresight-pmu.h>
11 #include <linux/err.h>
12 #include <linux/kernel.h>
13 #include <linux/log2.h>
14 #include <linux/types.h>
15 #include <linux/zalloc.h>
17 #include <opencsd/ocsd_if_types.h>
23 #include "cs-etm-decoder/cs-etm-decoder.h"
32 #include "map_symbol.h"
37 #include "thread-stack.h"
39 #include <tools/libc_compat.h>
40 #include "util/synthetic-events.h"
41 #include "util/util.h"
43 struct cs_etm_auxtrace {
44 struct auxtrace auxtrace;
45 struct auxtrace_queues queues;
46 struct auxtrace_heap heap;
47 struct itrace_synth_opts synth_opts;
48 struct perf_session *session;
49 struct machine *machine;
50 struct thread *unknown_thread;
51 struct perf_tsc_conversion tc;
53 bool timeless_decoding;
56 bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */
59 u64 latest_kernel_timestamp;
61 u64 branches_sample_type;
63 u64 instructions_sample_type;
64 u64 instructions_sample_period;
67 unsigned int pmu_type;
70 struct cs_etm_traceid_queue {
73 u64 period_instructions;
74 size_t last_branch_pos;
75 union perf_event *event_buf;
76 struct thread *thread;
77 struct branch_stack *last_branch;
78 struct branch_stack *last_branch_rb;
79 struct cs_etm_packet *prev_packet;
80 struct cs_etm_packet *packet;
81 struct cs_etm_packet_queue packet_queue;
85 struct cs_etm_auxtrace *etm;
86 struct cs_etm_decoder *decoder;
87 struct auxtrace_buffer *buffer;
88 unsigned int queue_nr;
89 u8 pending_timestamp_chan_id;
91 const unsigned char *buf;
92 size_t buf_len, buf_used;
93 /* Conversion between traceID and index in traceid_queues array */
94 struct intlist *traceid_queues_list;
95 struct cs_etm_traceid_queue **traceid_queues;
98 /* RB tree for quick conversion between traceID and metadata pointers */
99 static struct intlist *traceid_list;
101 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
102 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
104 static int cs_etm__get_data_block(struct cs_etm_queue *etmq);
105 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq);
107 /* PTMs ETMIDR [11:8] set to b0011 */
108 #define ETMIDR_PTM_VERSION 0x00000300
111 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
112 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
113 * encode the etm queue number as the upper 16 bit and the channel as
116 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
117 (queue_nr << 16 | trace_chan_id)
118 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
119 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
121 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
123 etmidr &= ETMIDR_PTM_VERSION;
125 if (etmidr == ETMIDR_PTM_VERSION)
126 return CS_ETM_PROTO_PTM;
128 return CS_ETM_PROTO_ETMV3;
131 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
133 struct int_node *inode;
136 inode = intlist__find(traceid_list, trace_chan_id);
140 metadata = inode->priv;
141 *magic = metadata[CS_ETM_MAGIC];
145 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
147 struct int_node *inode;
150 inode = intlist__find(traceid_list, trace_chan_id);
154 metadata = inode->priv;
155 *cpu = (int)metadata[CS_ETM_CPU];
160 * The returned PID format is presented by two bits:
162 * Bit ETM_OPT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced;
163 * Bit ETM_OPT_CTXTID2: CONTEXTIDR_EL2 is traced.
165 * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
166 * are enabled at the same time when the session runs on an EL2 kernel.
167 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
168 * recorded in the trace data, the tool will selectively use
169 * CONTEXTIDR_EL2 as PID.
171 int cs_etm__get_pid_fmt(u8 trace_chan_id, u64 *pid_fmt)
173 struct int_node *inode;
176 inode = intlist__find(traceid_list, trace_chan_id);
180 metadata = inode->priv;
182 if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) {
183 val = metadata[CS_ETM_ETMCR];
184 /* CONTEXTIDR is traced */
185 if (val & BIT(ETM_OPT_CTXTID))
186 *pid_fmt = BIT(ETM_OPT_CTXTID);
188 val = metadata[CS_ETMV4_TRCCONFIGR];
189 /* CONTEXTIDR_EL2 is traced */
190 if (val & (BIT(ETM4_CFG_BIT_VMID) | BIT(ETM4_CFG_BIT_VMID_OPT)))
191 *pid_fmt = BIT(ETM_OPT_CTXTID2);
192 /* CONTEXTIDR_EL1 is traced */
193 else if (val & BIT(ETM4_CFG_BIT_CTXTID))
194 *pid_fmt = BIT(ETM_OPT_CTXTID);
200 static int cs_etm__map_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
202 struct int_node *inode;
204 /* Get an RB node for this CPU */
205 inode = intlist__findnew(traceid_list, trace_chan_id);
207 /* Something went wrong, no need to continue */
212 * The node for that CPU should not be taken.
213 * Back out if that's the case.
218 /* All good, associate the traceID with the metadata pointer */
219 inode->priv = cpu_metadata;
224 static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata)
226 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
228 switch (cs_etm_magic) {
229 case __perf_cs_etmv3_magic:
230 *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] &
231 CORESIGHT_TRACE_ID_VAL_MASK);
233 case __perf_cs_etmv4_magic:
234 case __perf_cs_ete_magic:
235 *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] &
236 CORESIGHT_TRACE_ID_VAL_MASK);
245 * update metadata trace ID from the value found in the AUX_HW_INFO packet.
246 * This will also clear the CORESIGHT_TRACE_ID_UNUSED_FLAG flag if present.
248 static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata)
250 u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC];
252 switch (cs_etm_magic) {
253 case __perf_cs_etmv3_magic:
254 cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id;
256 case __perf_cs_etmv4_magic:
257 case __perf_cs_ete_magic:
258 cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id;
268 * FIELD_GET (linux/bitfield.h) not available outside kernel code,
269 * and the header contains too many dependencies to just copy over,
270 * so roll our own based on the original
272 #define __bf_shf(x) (__builtin_ffsll(x) - 1)
273 #define FIELD_GET(_mask, _reg) \
275 (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \
279 * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
281 * The payload associates the Trace ID and the CPU.
282 * The routine is tolerant of seeing multiple packets with the same association,
283 * but a CPU / Trace ID association changing during a session is an error.
285 static int cs_etm__process_aux_output_hw_id(struct perf_session *session,
286 union perf_event *event)
288 struct cs_etm_auxtrace *etm;
289 struct perf_sample sample;
290 struct int_node *inode;
294 int cpu, version, err;
295 u8 trace_chan_id, curr_chan_id;
297 /* extract and parse the HW ID */
298 hw_id = event->aux_output_hw_id.hw_id;
299 version = FIELD_GET(CS_AUX_HW_ID_VERSION_MASK, hw_id);
300 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id);
302 /* check that we can handle this version */
303 if (version > CS_AUX_HW_ID_CURR_VERSION)
306 /* get access to the etm metadata */
307 etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);
308 if (!etm || !etm->metadata)
311 /* parse the sample to get the CPU */
312 evsel = evlist__event2evsel(session->evlist, event);
315 err = evsel__parse_sample(evsel, event, &sample);
320 /* no CPU in the sample - possibly recorded with an old version of perf */
321 pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record.");
325 /* See if the ID is mapped to a CPU, and it matches the current CPU */
326 inode = intlist__find(traceid_list, trace_chan_id);
328 cpu_data = inode->priv;
329 if ((int)cpu_data[CS_ETM_CPU] != cpu) {
330 pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n");
334 /* check that the mapped ID matches */
335 err = cs_etm__metadata_get_trace_id(&curr_chan_id, cpu_data);
338 if (curr_chan_id != trace_chan_id) {
339 pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n");
343 /* mapped and matched - return OK */
347 /* not one we've seen before - lets map it */
348 cpu_data = etm->metadata[cpu];
349 err = cs_etm__map_trace_id(trace_chan_id, cpu_data);
354 * if we are picking up the association from the packet, need to plug
355 * the correct trace ID into the metadata for setting up decoders later.
357 err = cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data);
361 void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq,
365 * When a timestamp packet is encountered the backend code
366 * is stopped so that the front end has time to process packets
367 * that were accumulated in the traceID queue. Since there can
368 * be more than one channel per cs_etm_queue, we need to specify
369 * what traceID queue needs servicing.
371 etmq->pending_timestamp_chan_id = trace_chan_id;
374 static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq,
377 struct cs_etm_packet_queue *packet_queue;
379 if (!etmq->pending_timestamp_chan_id)
383 *trace_chan_id = etmq->pending_timestamp_chan_id;
385 packet_queue = cs_etm__etmq_get_packet_queue(etmq,
386 etmq->pending_timestamp_chan_id);
390 /* Acknowledge pending status */
391 etmq->pending_timestamp_chan_id = 0;
393 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
394 return packet_queue->cs_timestamp;
397 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
403 queue->packet_count = 0;
404 for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
405 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
406 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
407 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
408 queue->packet_buffer[i].instr_count = 0;
409 queue->packet_buffer[i].last_instr_taken_branch = false;
410 queue->packet_buffer[i].last_instr_size = 0;
411 queue->packet_buffer[i].last_instr_type = 0;
412 queue->packet_buffer[i].last_instr_subtype = 0;
413 queue->packet_buffer[i].last_instr_cond = 0;
414 queue->packet_buffer[i].flags = 0;
415 queue->packet_buffer[i].exception_number = UINT32_MAX;
416 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
417 queue->packet_buffer[i].cpu = INT_MIN;
421 static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq)
424 struct int_node *inode;
425 struct cs_etm_traceid_queue *tidq;
426 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
428 intlist__for_each_entry(inode, traceid_queues_list) {
429 idx = (int)(intptr_t)inode->priv;
430 tidq = etmq->traceid_queues[idx];
431 cs_etm__clear_packet_queue(&tidq->packet_queue);
435 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
436 struct cs_etm_traceid_queue *tidq,
440 struct auxtrace_queue *queue;
441 struct cs_etm_auxtrace *etm = etmq->etm;
443 cs_etm__clear_packet_queue(&tidq->packet_queue);
445 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
446 tidq->tid = queue->tid;
448 tidq->trace_chan_id = trace_chan_id;
450 tidq->packet = zalloc(sizeof(struct cs_etm_packet));
454 tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
455 if (!tidq->prev_packet)
458 if (etm->synth_opts.last_branch) {
459 size_t sz = sizeof(struct branch_stack);
461 sz += etm->synth_opts.last_branch_sz *
462 sizeof(struct branch_entry);
463 tidq->last_branch = zalloc(sz);
464 if (!tidq->last_branch)
466 tidq->last_branch_rb = zalloc(sz);
467 if (!tidq->last_branch_rb)
471 tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
472 if (!tidq->event_buf)
478 zfree(&tidq->last_branch_rb);
479 zfree(&tidq->last_branch);
480 zfree(&tidq->prev_packet);
481 zfree(&tidq->packet);
486 static struct cs_etm_traceid_queue
487 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
490 struct int_node *inode;
491 struct intlist *traceid_queues_list;
492 struct cs_etm_traceid_queue *tidq, **traceid_queues;
493 struct cs_etm_auxtrace *etm = etmq->etm;
495 if (etm->timeless_decoding)
496 trace_chan_id = CS_ETM_PER_THREAD_TRACEID;
498 traceid_queues_list = etmq->traceid_queues_list;
501 * Check if the traceid_queue exist for this traceID by looking
504 inode = intlist__find(traceid_queues_list, trace_chan_id);
506 idx = (int)(intptr_t)inode->priv;
507 return etmq->traceid_queues[idx];
510 /* We couldn't find a traceid_queue for this traceID, allocate one */
511 tidq = malloc(sizeof(*tidq));
515 memset(tidq, 0, sizeof(*tidq));
517 /* Get a valid index for the new traceid_queue */
518 idx = intlist__nr_entries(traceid_queues_list);
519 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
520 inode = intlist__findnew(traceid_queues_list, trace_chan_id);
524 /* Associate this traceID with this index */
525 inode->priv = (void *)(intptr_t)idx;
527 if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
530 /* Grow the traceid_queues array by one unit */
531 traceid_queues = etmq->traceid_queues;
532 traceid_queues = reallocarray(traceid_queues,
534 sizeof(*traceid_queues));
537 * On failure reallocarray() returns NULL and the original block of
538 * memory is left untouched.
543 traceid_queues[idx] = tidq;
544 etmq->traceid_queues = traceid_queues;
546 return etmq->traceid_queues[idx];
550 * Function intlist__remove() removes the inode from the list
551 * and delete the memory associated to it.
553 intlist__remove(traceid_queues_list, inode);
559 struct cs_etm_packet_queue
560 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
562 struct cs_etm_traceid_queue *tidq;
564 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
566 return &tidq->packet_queue;
571 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,
572 struct cs_etm_traceid_queue *tidq)
574 struct cs_etm_packet *tmp;
576 if (etm->synth_opts.branches || etm->synth_opts.last_branch ||
577 etm->synth_opts.instructions) {
579 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
580 * the next incoming packet.
583 tidq->packet = tidq->prev_packet;
584 tidq->prev_packet = tmp;
588 static void cs_etm__packet_dump(const char *pkt_string)
590 const char *color = PERF_COLOR_BLUE;
591 int len = strlen(pkt_string);
593 if (len && (pkt_string[len-1] == '\n'))
594 color_fprintf(stdout, color, " %s", pkt_string);
596 color_fprintf(stdout, color, " %s\n", pkt_string);
601 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
602 struct cs_etm_auxtrace *etm, int idx,
605 u64 **metadata = etm->metadata;
607 t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
608 t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
609 t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
612 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
613 struct cs_etm_auxtrace *etm, int idx)
615 u64 **metadata = etm->metadata;
617 t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
618 t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
619 t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
620 t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
621 t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
622 t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
623 t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
626 static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params,
627 struct cs_etm_auxtrace *etm, int idx)
629 u64 **metadata = etm->metadata;
631 t_params[idx].protocol = CS_ETM_PROTO_ETE;
632 t_params[idx].ete.reg_idr0 = metadata[idx][CS_ETE_TRCIDR0];
633 t_params[idx].ete.reg_idr1 = metadata[idx][CS_ETE_TRCIDR1];
634 t_params[idx].ete.reg_idr2 = metadata[idx][CS_ETE_TRCIDR2];
635 t_params[idx].ete.reg_idr8 = metadata[idx][CS_ETE_TRCIDR8];
636 t_params[idx].ete.reg_configr = metadata[idx][CS_ETE_TRCCONFIGR];
637 t_params[idx].ete.reg_traceidr = metadata[idx][CS_ETE_TRCTRACEIDR];
638 t_params[idx].ete.reg_devarch = metadata[idx][CS_ETE_TRCDEVARCH];
641 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
642 struct cs_etm_auxtrace *etm,
649 for (i = 0; i < decoders; i++) {
650 architecture = etm->metadata[i][CS_ETM_MAGIC];
652 switch (architecture) {
653 case __perf_cs_etmv3_magic:
654 etmidr = etm->metadata[i][CS_ETM_ETMIDR];
655 cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
657 case __perf_cs_etmv4_magic:
658 cs_etm__set_trace_param_etmv4(t_params, etm, i);
660 case __perf_cs_ete_magic:
661 cs_etm__set_trace_param_ete(t_params, etm, i);
671 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
672 struct cs_etm_queue *etmq,
673 enum cs_etm_decoder_operation mode,
678 if (!(mode < CS_ETM_OPERATION_MAX))
681 d_params->packet_printer = cs_etm__packet_dump;
682 d_params->operation = mode;
683 d_params->data = etmq;
684 d_params->formatted = formatted;
685 d_params->fsyncs = false;
686 d_params->hsyncs = false;
687 d_params->frame_aligned = true;
694 static void cs_etm__dump_event(struct cs_etm_queue *etmq,
695 struct auxtrace_buffer *buffer)
698 const char *color = PERF_COLOR_BLUE;
699 size_t buffer_used = 0;
701 fprintf(stdout, "\n");
702 color_fprintf(stdout, color,
703 ". ... CoreSight %s Trace data: size %#zx bytes\n",
704 cs_etm_decoder__get_name(etmq->decoder), buffer->size);
709 ret = cs_etm_decoder__process_data_block(
710 etmq->decoder, buffer->offset,
711 &((u8 *)buffer->data)[buffer_used],
712 buffer->size - buffer_used, &consumed);
716 buffer_used += consumed;
717 } while (buffer_used < buffer->size);
719 cs_etm_decoder__reset(etmq->decoder);
722 static int cs_etm__flush_events(struct perf_session *session,
723 struct perf_tool *tool)
725 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
726 struct cs_etm_auxtrace,
731 if (!tool->ordered_events)
734 if (etm->timeless_decoding)
735 return cs_etm__process_timeless_queues(etm, -1);
737 return cs_etm__process_queues(etm);
740 static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq)
744 struct int_node *inode, *tmp;
745 struct cs_etm_traceid_queue *tidq;
746 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
748 intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) {
749 priv = (uintptr_t)inode->priv;
752 /* Free this traceid_queue from the array */
753 tidq = etmq->traceid_queues[idx];
754 thread__zput(tidq->thread);
755 zfree(&tidq->event_buf);
756 zfree(&tidq->last_branch);
757 zfree(&tidq->last_branch_rb);
758 zfree(&tidq->prev_packet);
759 zfree(&tidq->packet);
763 * Function intlist__remove() removes the inode from the list
764 * and delete the memory associated to it.
766 intlist__remove(traceid_queues_list, inode);
769 /* Then the RB tree itself */
770 intlist__delete(traceid_queues_list);
771 etmq->traceid_queues_list = NULL;
773 /* finally free the traceid_queues array */
774 zfree(&etmq->traceid_queues);
777 static void cs_etm__free_queue(void *priv)
779 struct cs_etm_queue *etmq = priv;
784 cs_etm_decoder__free(etmq->decoder);
785 cs_etm__free_traceid_queues(etmq);
789 static void cs_etm__free_events(struct perf_session *session)
792 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
793 struct cs_etm_auxtrace,
795 struct auxtrace_queues *queues = &aux->queues;
797 for (i = 0; i < queues->nr_queues; i++) {
798 cs_etm__free_queue(queues->queue_array[i].priv);
799 queues->queue_array[i].priv = NULL;
802 auxtrace_queues__free(queues);
805 static void cs_etm__free(struct perf_session *session)
808 struct int_node *inode, *tmp;
809 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
810 struct cs_etm_auxtrace,
812 cs_etm__free_events(session);
813 session->auxtrace = NULL;
815 /* First remove all traceID/metadata nodes for the RB tree */
816 intlist__for_each_entry_safe(inode, tmp, traceid_list)
817 intlist__remove(traceid_list, inode);
818 /* Then the RB tree itself */
819 intlist__delete(traceid_list);
821 for (i = 0; i < aux->num_cpu; i++)
822 zfree(&aux->metadata[i]);
824 thread__zput(aux->unknown_thread);
825 zfree(&aux->metadata);
829 static bool cs_etm__evsel_is_auxtrace(struct perf_session *session,
832 struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
833 struct cs_etm_auxtrace,
836 return evsel->core.attr.type == aux->pmu_type;
839 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
841 struct machine *machine;
843 machine = etmq->etm->machine;
845 if (address >= machine__kernel_start(machine)) {
846 if (machine__is_host(machine))
847 return PERF_RECORD_MISC_KERNEL;
849 return PERF_RECORD_MISC_GUEST_KERNEL;
851 if (machine__is_host(machine))
852 return PERF_RECORD_MISC_USER;
854 return PERF_RECORD_MISC_GUEST_USER;
856 return PERF_RECORD_MISC_HYPERVISOR;
860 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id,
861 u64 address, size_t size, u8 *buffer)
866 struct thread *thread;
867 struct machine *machine;
868 struct addr_location al;
870 struct cs_etm_traceid_queue *tidq;
875 machine = etmq->etm->machine;
876 cpumode = cs_etm__cpu_mode(etmq, address);
877 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
881 thread = tidq->thread;
883 if (cpumode != PERF_RECORD_MISC_KERNEL)
885 thread = etmq->etm->unknown_thread;
888 if (!thread__find_map(thread, cpumode, address, &al))
891 dso = map__dso(al.map);
895 if (dso->data.status == DSO_DATA_STATUS_ERROR &&
896 dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE))
899 offset = map__map_ip(al.map, address);
903 len = dso__data_read_offset(dso, machine, offset, buffer, size);
906 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
907 " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n");
908 if (!dso->auxtrace_warned) {
909 pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n",
911 dso->long_name ? dso->long_name : "Unknown");
912 dso->auxtrace_warned = true;
920 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm,
923 struct cs_etm_decoder_params d_params;
924 struct cs_etm_trace_params *t_params = NULL;
925 struct cs_etm_queue *etmq;
927 * Each queue can only contain data from one CPU when unformatted, so only one decoder is
930 int decoders = formatted ? etm->num_cpu : 1;
932 etmq = zalloc(sizeof(*etmq));
936 etmq->traceid_queues_list = intlist__new(NULL);
937 if (!etmq->traceid_queues_list)
940 /* Use metadata to fill in trace parameters for trace decoder */
941 t_params = zalloc(sizeof(*t_params) * decoders);
946 if (cs_etm__init_trace_params(t_params, etm, decoders))
949 /* Set decoder parameters to decode trace packets */
950 if (cs_etm__init_decoder_params(&d_params, etmq,
951 dump_trace ? CS_ETM_OPERATION_PRINT :
952 CS_ETM_OPERATION_DECODE,
956 etmq->decoder = cs_etm_decoder__new(decoders, &d_params,
963 * Register a function to handle all memory accesses required by
964 * the trace decoder library.
966 if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
969 goto out_free_decoder;
975 cs_etm_decoder__free(etmq->decoder);
977 intlist__delete(etmq->traceid_queues_list);
983 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
984 struct auxtrace_queue *queue,
985 unsigned int queue_nr,
988 struct cs_etm_queue *etmq = queue->priv;
990 if (list_empty(&queue->head) || etmq)
993 etmq = cs_etm__alloc_queue(etm, formatted);
1000 etmq->queue_nr = queue_nr;
1006 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,
1007 struct cs_etm_queue *etmq,
1008 unsigned int queue_nr)
1011 unsigned int cs_queue_nr;
1016 * We are under a CPU-wide trace scenario. As such we need to know
1017 * when the code that generated the traces started to execute so that
1018 * it can be correlated with execution on other CPUs. So we get a
1019 * handle on the beginning of traces and decode until we find a
1020 * timestamp. The timestamp is then added to the auxtrace min heap
1021 * in order to know what nibble (of all the etmqs) to decode first.
1025 * Fetch an aux_buffer from this etmq. Bail if no more
1026 * blocks or an error has been encountered.
1028 ret = cs_etm__get_data_block(etmq);
1033 * Run decoder on the trace block. The decoder will stop when
1034 * encountering a CS timestamp, a full packet queue or the end of
1035 * trace for that block.
1037 ret = cs_etm__decode_data_block(etmq);
1042 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1043 * the timestamp calculation for us.
1045 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
1047 /* We found a timestamp, no need to continue. */
1052 * We didn't find a timestamp so empty all the traceid packet
1053 * queues before looking for another timestamp packet, either
1054 * in the current data block or a new one. Packets that were
1055 * just decoded are useless since no timestamp has been
1056 * associated with them. As such simply discard them.
1058 cs_etm__clear_all_packet_queues(etmq);
1062 * We have a timestamp. Add it to the min heap to reflect when
1063 * instructions conveyed by the range packets of this traceID queue
1064 * started to execute. Once the same has been done for all the traceID
1065 * queues of each etmq, redenring and decoding can start in
1066 * chronological order.
1068 * Note that packets decoded above are still in the traceID's packet
1069 * queue and will be processed in cs_etm__process_queues().
1071 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
1072 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
1078 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
1079 struct cs_etm_traceid_queue *tidq)
1081 struct branch_stack *bs_src = tidq->last_branch_rb;
1082 struct branch_stack *bs_dst = tidq->last_branch;
1086 * Set the number of records before early exit: ->nr is used to
1087 * determine how many branches to copy from ->entries.
1089 bs_dst->nr = bs_src->nr;
1092 * Early exit when there is nothing to copy.
1098 * As bs_src->entries is a circular buffer, we need to copy from it in
1099 * two steps. First, copy the branches from the most recently inserted
1100 * branch ->last_branch_pos until the end of bs_src->entries buffer.
1102 nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
1103 memcpy(&bs_dst->entries[0],
1104 &bs_src->entries[tidq->last_branch_pos],
1105 sizeof(struct branch_entry) * nr);
1108 * If we wrapped around at least once, the branches from the beginning
1109 * of the bs_src->entries buffer and until the ->last_branch_pos element
1110 * are older valid branches: copy them over. The total number of
1111 * branches copied over will be equal to the number of branches asked by
1112 * the user in last_branch_sz.
1114 if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
1115 memcpy(&bs_dst->entries[nr],
1116 &bs_src->entries[0],
1117 sizeof(struct branch_entry) * tidq->last_branch_pos);
1122 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
1124 tidq->last_branch_pos = 0;
1125 tidq->last_branch_rb->nr = 0;
1128 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
1129 u8 trace_chan_id, u64 addr)
1133 cs_etm__mem_access(etmq, trace_chan_id, addr,
1134 ARRAY_SIZE(instrBytes), instrBytes);
1136 * T32 instruction size is indicated by bits[15:11] of the first
1137 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1138 * denote a 32-bit instruction.
1140 return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
1143 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
1145 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1146 if (packet->sample_type == CS_ETM_DISCONTINUITY)
1149 return packet->start_addr;
1153 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
1155 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1156 if (packet->sample_type == CS_ETM_DISCONTINUITY)
1159 return packet->end_addr - packet->last_instr_size;
1162 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
1164 const struct cs_etm_packet *packet,
1167 if (packet->isa == CS_ETM_ISA_T32) {
1168 u64 addr = packet->start_addr;
1171 addr += cs_etm__t32_instr_size(etmq,
1172 trace_chan_id, addr);
1178 /* Assume a 4 byte instruction size (A32/A64) */
1179 return packet->start_addr + offset * 4;
1182 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
1183 struct cs_etm_traceid_queue *tidq)
1185 struct branch_stack *bs = tidq->last_branch_rb;
1186 struct branch_entry *be;
1189 * The branches are recorded in a circular buffer in reverse
1190 * chronological order: we start recording from the last element of the
1191 * buffer down. After writing the first element of the stack, move the
1192 * insert position back to the end of the buffer.
1194 if (!tidq->last_branch_pos)
1195 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
1197 tidq->last_branch_pos -= 1;
1199 be = &bs->entries[tidq->last_branch_pos];
1200 be->from = cs_etm__last_executed_instr(tidq->prev_packet);
1201 be->to = cs_etm__first_executed_instr(tidq->packet);
1202 /* No support for mispredict */
1203 be->flags.mispred = 0;
1204 be->flags.predicted = 1;
1207 * Increment bs->nr until reaching the number of last branches asked by
1208 * the user on the command line.
1210 if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
1214 static int cs_etm__inject_event(union perf_event *event,
1215 struct perf_sample *sample, u64 type)
1217 event->header.size = perf_event__sample_event_size(sample, type, 0);
1218 return perf_event__synthesize_sample(event, type, 0, sample);
1223 cs_etm__get_trace(struct cs_etm_queue *etmq)
1225 struct auxtrace_buffer *aux_buffer = etmq->buffer;
1226 struct auxtrace_buffer *old_buffer = aux_buffer;
1227 struct auxtrace_queue *queue;
1229 queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
1231 aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
1233 /* If no more data, drop the previous auxtrace_buffer and return */
1236 auxtrace_buffer__drop_data(old_buffer);
1241 etmq->buffer = aux_buffer;
1243 /* If the aux_buffer doesn't have data associated, try to load it */
1244 if (!aux_buffer->data) {
1245 /* get the file desc associated with the perf data file */
1246 int fd = perf_data__fd(etmq->etm->session->data);
1248 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
1249 if (!aux_buffer->data)
1253 /* If valid, drop the previous buffer */
1255 auxtrace_buffer__drop_data(old_buffer);
1258 etmq->buf_len = aux_buffer->size;
1259 etmq->buf = aux_buffer->data;
1261 return etmq->buf_len;
1264 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
1265 struct cs_etm_traceid_queue *tidq)
1267 if ((!tidq->thread) && (tidq->tid != -1))
1268 tidq->thread = machine__find_thread(etm->machine, -1,
1272 tidq->pid = tidq->thread->pid_;
1275 int cs_etm__etmq_set_tid(struct cs_etm_queue *etmq,
1276 pid_t tid, u8 trace_chan_id)
1278 int cpu, err = -EINVAL;
1279 struct cs_etm_auxtrace *etm = etmq->etm;
1280 struct cs_etm_traceid_queue *tidq;
1282 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
1286 if (cs_etm__get_cpu(trace_chan_id, &cpu) < 0)
1289 err = machine__set_current_tid(etm->machine, cpu, tid, tid);
1294 thread__zput(tidq->thread);
1296 cs_etm__set_pid_tid_cpu(etm, tidq);
1300 bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq)
1302 return !!etmq->etm->timeless_decoding;
1305 static void cs_etm__copy_insn(struct cs_etm_queue *etmq,
1307 const struct cs_etm_packet *packet,
1308 struct perf_sample *sample)
1311 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1312 * packet, so directly bail out with 'insn_len' = 0.
1314 if (packet->sample_type == CS_ETM_DISCONTINUITY) {
1315 sample->insn_len = 0;
1320 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1321 * cs_etm__t32_instr_size().
1323 if (packet->isa == CS_ETM_ISA_T32)
1324 sample->insn_len = cs_etm__t32_instr_size(etmq, trace_chan_id,
1326 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1328 sample->insn_len = 4;
1330 cs_etm__mem_access(etmq, trace_chan_id, sample->ip,
1331 sample->insn_len, (void *)sample->insn);
1334 u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp)
1336 struct cs_etm_auxtrace *etm = etmq->etm;
1338 if (etm->has_virtual_ts)
1339 return tsc_to_perf_time(cs_timestamp, &etm->tc);
1341 return cs_timestamp;
1344 static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq,
1345 struct cs_etm_traceid_queue *tidq)
1347 struct cs_etm_auxtrace *etm = etmq->etm;
1348 struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue;
1350 if (etm->timeless_decoding)
1352 else if (etm->has_virtual_ts)
1353 return packet_queue->cs_timestamp;
1355 return etm->latest_kernel_timestamp;
1358 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
1359 struct cs_etm_traceid_queue *tidq,
1360 u64 addr, u64 period)
1363 struct cs_etm_auxtrace *etm = etmq->etm;
1364 union perf_event *event = tidq->event_buf;
1365 struct perf_sample sample = {.ip = 0,};
1367 event->sample.header.type = PERF_RECORD_SAMPLE;
1368 event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
1369 event->sample.header.size = sizeof(struct perf_event_header);
1371 /* Set time field based on etm auxtrace config. */
1372 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1375 sample.pid = tidq->pid;
1376 sample.tid = tidq->tid;
1377 sample.id = etmq->etm->instructions_id;
1378 sample.stream_id = etmq->etm->instructions_id;
1379 sample.period = period;
1380 sample.cpu = tidq->packet->cpu;
1381 sample.flags = tidq->prev_packet->flags;
1382 sample.cpumode = event->sample.header.misc;
1384 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);
1386 if (etm->synth_opts.last_branch)
1387 sample.branch_stack = tidq->last_branch;
1389 if (etm->synth_opts.inject) {
1390 ret = cs_etm__inject_event(event, &sample,
1391 etm->instructions_sample_type);
1396 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1400 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1407 * The cs etm packet encodes an instruction range between a branch target
1408 * and the next taken branch. Generate sample accordingly.
1410 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
1411 struct cs_etm_traceid_queue *tidq)
1414 struct cs_etm_auxtrace *etm = etmq->etm;
1415 struct perf_sample sample = {.ip = 0,};
1416 union perf_event *event = tidq->event_buf;
1417 struct dummy_branch_stack {
1420 struct branch_entry entries;
1424 ip = cs_etm__last_executed_instr(tidq->prev_packet);
1426 event->sample.header.type = PERF_RECORD_SAMPLE;
1427 event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
1428 event->sample.header.size = sizeof(struct perf_event_header);
1430 /* Set time field based on etm auxtrace config. */
1431 sample.time = cs_etm__resolve_sample_time(etmq, tidq);
1434 sample.pid = tidq->pid;
1435 sample.tid = tidq->tid;
1436 sample.addr = cs_etm__first_executed_instr(tidq->packet);
1437 sample.id = etmq->etm->branches_id;
1438 sample.stream_id = etmq->etm->branches_id;
1440 sample.cpu = tidq->packet->cpu;
1441 sample.flags = tidq->prev_packet->flags;
1442 sample.cpumode = event->sample.header.misc;
1444 cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,
1448 * perf report cannot handle events without a branch stack
1450 if (etm->synth_opts.last_branch) {
1451 dummy_bs = (struct dummy_branch_stack){
1459 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1462 if (etm->synth_opts.inject) {
1463 ret = cs_etm__inject_event(event, &sample,
1464 etm->branches_sample_type);
1469 ret = perf_session__deliver_synth_event(etm->session, event, &sample);
1473 "CS ETM Trace: failed to deliver instruction event, error %d\n",
1479 struct cs_etm_synth {
1480 struct perf_tool dummy_tool;
1481 struct perf_session *session;
1484 static int cs_etm__event_synth(struct perf_tool *tool,
1485 union perf_event *event,
1486 struct perf_sample *sample __maybe_unused,
1487 struct machine *machine __maybe_unused)
1489 struct cs_etm_synth *cs_etm_synth =
1490 container_of(tool, struct cs_etm_synth, dummy_tool);
1492 return perf_session__deliver_synth_event(cs_etm_synth->session,
1496 static int cs_etm__synth_event(struct perf_session *session,
1497 struct perf_event_attr *attr, u64 id)
1499 struct cs_etm_synth cs_etm_synth;
1501 memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
1502 cs_etm_synth.session = session;
1504 return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
1505 &id, cs_etm__event_synth);
1508 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
1509 struct perf_session *session)
1511 struct evlist *evlist = session->evlist;
1512 struct evsel *evsel;
1513 struct perf_event_attr attr;
1518 evlist__for_each_entry(evlist, evsel) {
1519 if (evsel->core.attr.type == etm->pmu_type) {
1526 pr_debug("No selected events with CoreSight Trace data\n");
1530 memset(&attr, 0, sizeof(struct perf_event_attr));
1531 attr.size = sizeof(struct perf_event_attr);
1532 attr.type = PERF_TYPE_HARDWARE;
1533 attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK;
1534 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1536 if (etm->timeless_decoding)
1537 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1539 attr.sample_type |= PERF_SAMPLE_TIME;
1541 attr.exclude_user = evsel->core.attr.exclude_user;
1542 attr.exclude_kernel = evsel->core.attr.exclude_kernel;
1543 attr.exclude_hv = evsel->core.attr.exclude_hv;
1544 attr.exclude_host = evsel->core.attr.exclude_host;
1545 attr.exclude_guest = evsel->core.attr.exclude_guest;
1546 attr.sample_id_all = evsel->core.attr.sample_id_all;
1547 attr.read_format = evsel->core.attr.read_format;
1549 /* create new id val to be a fixed offset from evsel id */
1550 id = evsel->core.id[0] + 1000000000;
1555 if (etm->synth_opts.branches) {
1556 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1557 attr.sample_period = 1;
1558 attr.sample_type |= PERF_SAMPLE_ADDR;
1559 err = cs_etm__synth_event(session, &attr, id);
1562 etm->branches_sample_type = attr.sample_type;
1563 etm->branches_id = id;
1565 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1568 if (etm->synth_opts.last_branch) {
1569 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1571 * We don't use the hardware index, but the sample generation
1572 * code uses the new format branch_stack with this field,
1573 * so the event attributes must indicate that it's present.
1575 attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX;
1578 if (etm->synth_opts.instructions) {
1579 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1580 attr.sample_period = etm->synth_opts.period;
1581 etm->instructions_sample_period = attr.sample_period;
1582 err = cs_etm__synth_event(session, &attr, id);
1585 etm->instructions_sample_type = attr.sample_type;
1586 etm->instructions_id = id;
1593 static int cs_etm__sample(struct cs_etm_queue *etmq,
1594 struct cs_etm_traceid_queue *tidq)
1596 struct cs_etm_auxtrace *etm = etmq->etm;
1598 u8 trace_chan_id = tidq->trace_chan_id;
1601 /* Get instructions remainder from previous packet */
1602 instrs_prev = tidq->period_instructions;
1604 tidq->period_instructions += tidq->packet->instr_count;
1607 * Record a branch when the last instruction in
1608 * PREV_PACKET is a branch.
1610 if (etm->synth_opts.last_branch &&
1611 tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1612 tidq->prev_packet->last_instr_taken_branch)
1613 cs_etm__update_last_branch_rb(etmq, tidq);
1615 if (etm->synth_opts.instructions &&
1616 tidq->period_instructions >= etm->instructions_sample_period) {
1618 * Emit instruction sample periodically
1619 * TODO: allow period to be defined in cycles and clock time
1623 * Below diagram demonstrates the instruction samples
1626 * Instrs Instrs Instrs Instrs
1627 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3)
1630 * --------------------------------------------------
1634 * instructions(Pi) instructions(Pi')
1637 * \---------------- -----------------/
1639 * tidq->packet->instr_count
1641 * Instrs Sample(n...) are the synthesised samples occurring
1642 * every etm->instructions_sample_period instructions - as
1643 * defined on the perf command line. Sample(n) is being the
1644 * last sample before the current etm packet, n+1 to n+3
1645 * samples are generated from the current etm packet.
1647 * tidq->packet->instr_count represents the number of
1648 * instructions in the current etm packet.
1650 * Period instructions (Pi) contains the number of
1651 * instructions executed after the sample point(n) from the
1652 * previous etm packet. This will always be less than
1653 * etm->instructions_sample_period.
1655 * When generate new samples, it combines with two parts
1656 * instructions, one is the tail of the old packet and another
1657 * is the head of the new coming packet, to generate
1658 * sample(n+1); sample(n+2) and sample(n+3) consume the
1659 * instructions with sample period. After sample(n+3), the rest
1660 * instructions will be used by later packet and it is assigned
1661 * to tidq->period_instructions for next round calculation.
1665 * Get the initial offset into the current packet instructions;
1666 * entry conditions ensure that instrs_prev is less than
1667 * etm->instructions_sample_period.
1669 u64 offset = etm->instructions_sample_period - instrs_prev;
1672 /* Prepare last branches for instruction sample */
1673 if (etm->synth_opts.last_branch)
1674 cs_etm__copy_last_branch_rb(etmq, tidq);
1676 while (tidq->period_instructions >=
1677 etm->instructions_sample_period) {
1679 * Calculate the address of the sampled instruction (-1
1680 * as sample is reported as though instruction has just
1681 * been executed, but PC has not advanced to next
1684 addr = cs_etm__instr_addr(etmq, trace_chan_id,
1685 tidq->packet, offset - 1);
1686 ret = cs_etm__synth_instruction_sample(
1688 etm->instructions_sample_period);
1692 offset += etm->instructions_sample_period;
1693 tidq->period_instructions -=
1694 etm->instructions_sample_period;
1698 if (etm->synth_opts.branches) {
1699 bool generate_sample = false;
1701 /* Generate sample for tracing on packet */
1702 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1703 generate_sample = true;
1705 /* Generate sample for branch taken packet */
1706 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1707 tidq->prev_packet->last_instr_taken_branch)
1708 generate_sample = true;
1710 if (generate_sample) {
1711 ret = cs_etm__synth_branch_sample(etmq, tidq);
1717 cs_etm__packet_swap(etm, tidq);
1722 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1725 * When the exception packet is inserted, whether the last instruction
1726 * in previous range packet is taken branch or not, we need to force
1727 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1728 * to generate branch sample for the instruction range before the
1729 * exception is trapped to kernel or before the exception returning.
1731 * The exception packet includes the dummy address values, so don't
1732 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1733 * for generating instruction and branch samples.
1735 if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1736 tidq->prev_packet->last_instr_taken_branch = true;
1741 static int cs_etm__flush(struct cs_etm_queue *etmq,
1742 struct cs_etm_traceid_queue *tidq)
1745 struct cs_etm_auxtrace *etm = etmq->etm;
1747 /* Handle start tracing packet */
1748 if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1751 if (etmq->etm->synth_opts.last_branch &&
1752 etmq->etm->synth_opts.instructions &&
1753 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1756 /* Prepare last branches for instruction sample */
1757 cs_etm__copy_last_branch_rb(etmq, tidq);
1760 * Generate a last branch event for the branches left in the
1761 * circular buffer at the end of the trace.
1763 * Use the address of the end of the last reported execution
1766 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1768 err = cs_etm__synth_instruction_sample(
1770 tidq->period_instructions);
1774 tidq->period_instructions = 0;
1778 if (etm->synth_opts.branches &&
1779 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1780 err = cs_etm__synth_branch_sample(etmq, tidq);
1786 cs_etm__packet_swap(etm, tidq);
1788 /* Reset last branches after flush the trace */
1789 if (etm->synth_opts.last_branch)
1790 cs_etm__reset_last_branch_rb(tidq);
1795 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1796 struct cs_etm_traceid_queue *tidq)
1801 * It has no new packet coming and 'etmq->packet' contains the stale
1802 * packet which was set at the previous time with packets swapping;
1803 * so skip to generate branch sample to avoid stale packet.
1805 * For this case only flush branch stack and generate a last branch
1806 * event for the branches left in the circular buffer at the end of
1809 if (etmq->etm->synth_opts.last_branch &&
1810 etmq->etm->synth_opts.instructions &&
1811 tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1814 /* Prepare last branches for instruction sample */
1815 cs_etm__copy_last_branch_rb(etmq, tidq);
1818 * Use the address of the end of the last reported execution
1821 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1823 err = cs_etm__synth_instruction_sample(
1825 tidq->period_instructions);
1829 tidq->period_instructions = 0;
1835 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1837 * Returns: < 0 if error
1838 * = 0 if no more auxtrace_buffer to read
1839 * > 0 if the current buffer isn't empty yet
1841 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1845 if (!etmq->buf_len) {
1846 ret = cs_etm__get_trace(etmq);
1850 * We cannot assume consecutive blocks in the data file
1851 * are contiguous, reset the decoder to force re-sync.
1853 ret = cs_etm_decoder__reset(etmq->decoder);
1858 return etmq->buf_len;
1861 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, u8 trace_chan_id,
1862 struct cs_etm_packet *packet,
1865 /* Initialise to keep compiler happy */
1870 switch (packet->isa) {
1871 case CS_ETM_ISA_T32:
1873 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1876 * +-----------------+--------+
1877 * | 1 1 0 1 1 1 1 1 | imm8 |
1878 * +-----------------+--------+
1880 * According to the specification, it only defines SVC for T32
1881 * with 16 bits instruction and has no definition for 32bits;
1882 * so below only read 2 bytes as instruction size for T32.
1884 addr = end_addr - 2;
1885 cs_etm__mem_access(etmq, trace_chan_id, addr,
1886 sizeof(instr16), (u8 *)&instr16);
1887 if ((instr16 & 0xFF00) == 0xDF00)
1891 case CS_ETM_ISA_A32:
1893 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1895 * b'31 b'28 b'27 b'24
1896 * +---------+---------+-------------------------+
1897 * | !1111 | 1 1 1 1 | imm24 |
1898 * +---------+---------+-------------------------+
1900 addr = end_addr - 4;
1901 cs_etm__mem_access(etmq, trace_chan_id, addr,
1902 sizeof(instr32), (u8 *)&instr32);
1903 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1904 (instr32 & 0xF0000000) != 0xF0000000)
1908 case CS_ETM_ISA_A64:
1910 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1913 * +-----------------------+---------+-----------+
1914 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
1915 * +-----------------------+---------+-----------+
1917 addr = end_addr - 4;
1918 cs_etm__mem_access(etmq, trace_chan_id, addr,
1919 sizeof(instr32), (u8 *)&instr32);
1920 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1924 case CS_ETM_ISA_UNKNOWN:
1932 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1933 struct cs_etm_traceid_queue *tidq, u64 magic)
1935 u8 trace_chan_id = tidq->trace_chan_id;
1936 struct cs_etm_packet *packet = tidq->packet;
1937 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1939 if (magic == __perf_cs_etmv3_magic)
1940 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1944 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1945 * HVC cases; need to check if it's SVC instruction based on
1948 if (magic == __perf_cs_etmv4_magic) {
1949 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1950 cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
1951 prev_packet->end_addr))
1958 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1961 struct cs_etm_packet *packet = tidq->packet;
1963 if (magic == __perf_cs_etmv3_magic)
1964 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1965 packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1966 packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1967 packet->exception_number == CS_ETMV3_EXC_IRQ ||
1968 packet->exception_number == CS_ETMV3_EXC_FIQ)
1971 if (magic == __perf_cs_etmv4_magic)
1972 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1973 packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1974 packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1975 packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1976 packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1977 packet->exception_number == CS_ETMV4_EXC_IRQ ||
1978 packet->exception_number == CS_ETMV4_EXC_FIQ)
1984 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1985 struct cs_etm_traceid_queue *tidq,
1988 u8 trace_chan_id = tidq->trace_chan_id;
1989 struct cs_etm_packet *packet = tidq->packet;
1990 struct cs_etm_packet *prev_packet = tidq->prev_packet;
1992 if (magic == __perf_cs_etmv3_magic)
1993 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1994 packet->exception_number == CS_ETMV3_EXC_HYP ||
1995 packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1996 packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1997 packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1998 packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1999 packet->exception_number == CS_ETMV3_EXC_GENERIC)
2002 if (magic == __perf_cs_etmv4_magic) {
2003 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
2004 packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
2005 packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
2006 packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
2010 * For CS_ETMV4_EXC_CALL, except SVC other instructions
2011 * (SMC, HVC) are taken as sync exceptions.
2013 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
2014 !cs_etm__is_svc_instr(etmq, trace_chan_id, prev_packet,
2015 prev_packet->end_addr))
2019 * ETMv4 has 5 bits for exception number; if the numbers
2020 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2021 * they are implementation defined exceptions.
2023 * For this case, simply take it as sync exception.
2025 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
2026 packet->exception_number <= CS_ETMV4_EXC_END)
2033 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
2034 struct cs_etm_traceid_queue *tidq)
2036 struct cs_etm_packet *packet = tidq->packet;
2037 struct cs_etm_packet *prev_packet = tidq->prev_packet;
2038 u8 trace_chan_id = tidq->trace_chan_id;
2042 switch (packet->sample_type) {
2045 * Immediate branch instruction without neither link nor
2046 * return flag, it's normal branch instruction within
2049 if (packet->last_instr_type == OCSD_INSTR_BR &&
2050 packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
2051 packet->flags = PERF_IP_FLAG_BRANCH;
2053 if (packet->last_instr_cond)
2054 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
2058 * Immediate branch instruction with link (e.g. BL), this is
2059 * branch instruction for function call.
2061 if (packet->last_instr_type == OCSD_INSTR_BR &&
2062 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2063 packet->flags = PERF_IP_FLAG_BRANCH |
2067 * Indirect branch instruction with link (e.g. BLR), this is
2068 * branch instruction for function call.
2070 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2071 packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
2072 packet->flags = PERF_IP_FLAG_BRANCH |
2076 * Indirect branch instruction with subtype of
2077 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2078 * function return for A32/T32.
2080 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2081 packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
2082 packet->flags = PERF_IP_FLAG_BRANCH |
2083 PERF_IP_FLAG_RETURN;
2086 * Indirect branch instruction without link (e.g. BR), usually
2087 * this is used for function return, especially for functions
2088 * within dynamic link lib.
2090 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2091 packet->last_instr_subtype == OCSD_S_INSTR_NONE)
2092 packet->flags = PERF_IP_FLAG_BRANCH |
2093 PERF_IP_FLAG_RETURN;
2095 /* Return instruction for function return. */
2096 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
2097 packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
2098 packet->flags = PERF_IP_FLAG_BRANCH |
2099 PERF_IP_FLAG_RETURN;
2102 * Decoder might insert a discontinuity in the middle of
2103 * instruction packets, fixup prev_packet with flag
2104 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2106 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
2107 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2108 PERF_IP_FLAG_TRACE_BEGIN;
2111 * If the previous packet is an exception return packet
2112 * and the return address just follows SVC instruction,
2113 * it needs to calibrate the previous packet sample flags
2114 * as PERF_IP_FLAG_SYSCALLRET.
2116 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
2117 PERF_IP_FLAG_RETURN |
2118 PERF_IP_FLAG_INTERRUPT) &&
2119 cs_etm__is_svc_instr(etmq, trace_chan_id,
2120 packet, packet->start_addr))
2121 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2122 PERF_IP_FLAG_RETURN |
2123 PERF_IP_FLAG_SYSCALLRET;
2125 case CS_ETM_DISCONTINUITY:
2127 * The trace is discontinuous, if the previous packet is
2128 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2129 * for previous packet.
2131 if (prev_packet->sample_type == CS_ETM_RANGE)
2132 prev_packet->flags |= PERF_IP_FLAG_BRANCH |
2133 PERF_IP_FLAG_TRACE_END;
2135 case CS_ETM_EXCEPTION:
2136 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
2140 /* The exception is for system call. */
2141 if (cs_etm__is_syscall(etmq, tidq, magic))
2142 packet->flags = PERF_IP_FLAG_BRANCH |
2144 PERF_IP_FLAG_SYSCALLRET;
2146 * The exceptions are triggered by external signals from bus,
2147 * interrupt controller, debug module, PE reset or halt.
2149 else if (cs_etm__is_async_exception(tidq, magic))
2150 packet->flags = PERF_IP_FLAG_BRANCH |
2152 PERF_IP_FLAG_ASYNC |
2153 PERF_IP_FLAG_INTERRUPT;
2155 * Otherwise, exception is caused by trap, instruction &
2156 * data fault, or alignment errors.
2158 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
2159 packet->flags = PERF_IP_FLAG_BRANCH |
2161 PERF_IP_FLAG_INTERRUPT;
2164 * When the exception packet is inserted, since exception
2165 * packet is not used standalone for generating samples
2166 * and it's affiliation to the previous instruction range
2167 * packet; so set previous range packet flags to tell perf
2168 * it is an exception taken branch.
2170 if (prev_packet->sample_type == CS_ETM_RANGE)
2171 prev_packet->flags = packet->flags;
2173 case CS_ETM_EXCEPTION_RET:
2175 * When the exception return packet is inserted, since
2176 * exception return packet is not used standalone for
2177 * generating samples and it's affiliation to the previous
2178 * instruction range packet; so set previous range packet
2179 * flags to tell perf it is an exception return branch.
2181 * The exception return can be for either system call or
2182 * other exception types; unfortunately the packet doesn't
2183 * contain exception type related info so we cannot decide
2184 * the exception type purely based on exception return packet.
2185 * If we record the exception number from exception packet and
2186 * reuse it for exception return packet, this is not reliable
2187 * due the trace can be discontinuity or the interrupt can
2188 * be nested, thus the recorded exception number cannot be
2189 * used for exception return packet for these two cases.
2191 * For exception return packet, we only need to distinguish the
2192 * packet is for system call or for other types. Thus the
2193 * decision can be deferred when receive the next packet which
2194 * contains the return address, based on the return address we
2195 * can read out the previous instruction and check if it's a
2196 * system call instruction and then calibrate the sample flag
2199 if (prev_packet->sample_type == CS_ETM_RANGE)
2200 prev_packet->flags = PERF_IP_FLAG_BRANCH |
2201 PERF_IP_FLAG_RETURN |
2202 PERF_IP_FLAG_INTERRUPT;
2212 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
2215 size_t processed = 0;
2218 * Packets are decoded and added to the decoder's packet queue
2219 * until the decoder packet processing callback has requested that
2220 * processing stops or there is nothing left in the buffer. Normal
2221 * operations that stop processing are a timestamp packet or a full
2222 * decoder buffer queue.
2224 ret = cs_etm_decoder__process_data_block(etmq->decoder,
2226 &etmq->buf[etmq->buf_used],
2232 etmq->offset += processed;
2233 etmq->buf_used += processed;
2234 etmq->buf_len -= processed;
2240 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
2241 struct cs_etm_traceid_queue *tidq)
2244 struct cs_etm_packet_queue *packet_queue;
2246 packet_queue = &tidq->packet_queue;
2248 /* Process each packet in this chunk */
2250 ret = cs_etm_decoder__get_packet(packet_queue,
2254 * Stop processing this chunk on
2255 * end of data or error
2260 * Since packet addresses are swapped in packet
2261 * handling within below switch() statements,
2262 * thus setting sample flags must be called
2263 * prior to switch() statement to use address
2264 * information before packets swapping.
2266 ret = cs_etm__set_sample_flags(etmq, tidq);
2270 switch (tidq->packet->sample_type) {
2273 * If the packet contains an instruction
2274 * range, generate instruction sequence
2277 cs_etm__sample(etmq, tidq);
2279 case CS_ETM_EXCEPTION:
2280 case CS_ETM_EXCEPTION_RET:
2282 * If the exception packet is coming,
2283 * make sure the previous instruction
2284 * range packet to be handled properly.
2286 cs_etm__exception(tidq);
2288 case CS_ETM_DISCONTINUITY:
2290 * Discontinuity in trace, flush
2291 * previous branch stack
2293 cs_etm__flush(etmq, tidq);
2297 * Should not receive empty packet,
2300 pr_err("CS ETM Trace: empty packet\n");
2310 static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq)
2313 struct int_node *inode;
2314 struct cs_etm_traceid_queue *tidq;
2315 struct intlist *traceid_queues_list = etmq->traceid_queues_list;
2317 intlist__for_each_entry(inode, traceid_queues_list) {
2318 idx = (int)(intptr_t)inode->priv;
2319 tidq = etmq->traceid_queues[idx];
2321 /* Ignore return value */
2322 cs_etm__process_traceid_queue(etmq, tidq);
2325 * Generate an instruction sample with the remaining
2326 * branchstack entries.
2328 cs_etm__flush(etmq, tidq);
2332 static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
2335 struct cs_etm_traceid_queue *tidq;
2337 tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
2341 /* Go through each buffer in the queue and decode them one by one */
2343 err = cs_etm__get_data_block(etmq);
2347 /* Run trace decoder until buffer consumed or end of trace */
2349 err = cs_etm__decode_data_block(etmq);
2354 * Process each packet in this chunk, nothing to do if
2355 * an error occurs other than hoping the next one will
2358 err = cs_etm__process_traceid_queue(etmq, tidq);
2360 } while (etmq->buf_len);
2363 /* Flush any remaining branch stack entries */
2364 err = cs_etm__end_block(etmq, tidq);
2370 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
2374 struct auxtrace_queues *queues = &etm->queues;
2376 for (i = 0; i < queues->nr_queues; i++) {
2377 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
2378 struct cs_etm_queue *etmq = queue->priv;
2379 struct cs_etm_traceid_queue *tidq;
2384 tidq = cs_etm__etmq_get_traceid_queue(etmq,
2385 CS_ETM_PER_THREAD_TRACEID);
2390 if ((tid == -1) || (tidq->tid == tid)) {
2391 cs_etm__set_pid_tid_cpu(etm, tidq);
2392 cs_etm__run_decoder(etmq);
2399 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)
2402 unsigned int cs_queue_nr, queue_nr, i;
2405 struct auxtrace_queue *queue;
2406 struct cs_etm_queue *etmq;
2407 struct cs_etm_traceid_queue *tidq;
2410 * Pre-populate the heap with one entry from each queue so that we can
2411 * start processing in time order across all queues.
2413 for (i = 0; i < etm->queues.nr_queues; i++) {
2414 etmq = etm->queues.queue_array[i].priv;
2418 ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);
2424 if (!etm->heap.heap_cnt)
2427 /* Take the entry at the top of the min heap */
2428 cs_queue_nr = etm->heap.heap_array[0].queue_nr;
2429 queue_nr = TO_QUEUE_NR(cs_queue_nr);
2430 trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr);
2431 queue = &etm->queues.queue_array[queue_nr];
2435 * Remove the top entry from the heap since we are about
2438 auxtrace_heap__pop(&etm->heap);
2440 tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
2443 * No traceID queue has been allocated for this traceID,
2444 * which means something somewhere went very wrong. No
2445 * other choice than simply exit.
2452 * Packets associated with this timestamp are already in
2453 * the etmq's traceID queue, so process them.
2455 ret = cs_etm__process_traceid_queue(etmq, tidq);
2460 * Packets for this timestamp have been processed, time to
2461 * move on to the next timestamp, fetching a new auxtrace_buffer
2465 ret = cs_etm__get_data_block(etmq);
2470 * No more auxtrace_buffers to process in this etmq, simply
2471 * move on to another entry in the auxtrace_heap.
2476 ret = cs_etm__decode_data_block(etmq);
2480 cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id);
2482 if (!cs_timestamp) {
2484 * Function cs_etm__decode_data_block() returns when
2485 * there is no more traces to decode in the current
2486 * auxtrace_buffer OR when a timestamp has been
2487 * encountered on any of the traceID queues. Since we
2488 * did not get a timestamp, there is no more traces to
2489 * process in this auxtrace_buffer. As such empty and
2490 * flush all traceID queues.
2492 cs_etm__clear_all_traceid_queues(etmq);
2494 /* Fetch another auxtrace_buffer for this etmq */
2499 * Add to the min heap the timestamp for packets that have
2500 * just been decoded. They will be processed and synthesized
2501 * during the next call to cs_etm__process_traceid_queue() for
2502 * this queue/traceID.
2504 cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id);
2505 ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);
2512 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
2513 union perf_event *event)
2517 if (etm->timeless_decoding)
2521 * Add the tid/pid to the log so that we can get a match when
2522 * we get a contextID from the decoder.
2524 th = machine__findnew_thread(etm->machine,
2525 event->itrace_start.pid,
2526 event->itrace_start.tid);
2535 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
2536 union perf_event *event)
2539 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
2542 * Context switch in per-thread mode are irrelevant since perf
2543 * will start/stop tracing as the process is scheduled.
2545 if (etm->timeless_decoding)
2549 * SWITCH_IN events carry the next process to be switched out while
2550 * SWITCH_OUT events carry the process to be switched in. As such
2551 * we don't care about IN events.
2557 * Add the tid/pid to the log so that we can get a match when
2558 * we get a contextID from the decoder.
2560 th = machine__findnew_thread(etm->machine,
2561 event->context_switch.next_prev_pid,
2562 event->context_switch.next_prev_tid);
2571 static int cs_etm__process_event(struct perf_session *session,
2572 union perf_event *event,
2573 struct perf_sample *sample,
2574 struct perf_tool *tool)
2576 u64 sample_kernel_timestamp;
2577 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2578 struct cs_etm_auxtrace,
2584 if (!tool->ordered_events) {
2585 pr_err("CoreSight ETM Trace requires ordered events\n");
2589 if (sample->time && (sample->time != (u64) -1))
2590 sample_kernel_timestamp = sample->time;
2592 sample_kernel_timestamp = 0;
2595 * Don't wait for cs_etm__flush_events() in per-thread/timeless mode to start the decode. We
2596 * need the tid of the PERF_RECORD_EXIT event to assign to the synthesised samples because
2597 * ETM_OPT_CTXTID is not enabled.
2599 if (etm->timeless_decoding &&
2600 event->header.type == PERF_RECORD_EXIT)
2601 return cs_etm__process_timeless_queues(etm,
2604 if (event->header.type == PERF_RECORD_ITRACE_START)
2605 return cs_etm__process_itrace_start(etm, event);
2606 else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2607 return cs_etm__process_switch_cpu_wide(etm, event);
2609 if (!etm->timeless_decoding && event->header.type == PERF_RECORD_AUX) {
2611 * Record the latest kernel timestamp available in the header
2612 * for samples so that synthesised samples occur from this point
2615 etm->latest_kernel_timestamp = sample_kernel_timestamp;
2621 static void dump_queued_data(struct cs_etm_auxtrace *etm,
2622 struct perf_record_auxtrace *event)
2624 struct auxtrace_buffer *buf;
2627 * Find all buffers with same reference in the queues and dump them.
2628 * This is because the queues can contain multiple entries of the same
2629 * buffer that were split on aux records.
2631 for (i = 0; i < etm->queues.nr_queues; ++i)
2632 list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)
2633 if (buf->reference == event->reference)
2634 cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);
2637 static int cs_etm__process_auxtrace_event(struct perf_session *session,
2638 union perf_event *event,
2639 struct perf_tool *tool __maybe_unused)
2641 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2642 struct cs_etm_auxtrace,
2644 if (!etm->data_queued) {
2645 struct auxtrace_buffer *buffer;
2647 int fd = perf_data__fd(session->data);
2648 bool is_pipe = perf_data__is_pipe(session->data);
2650 int idx = event->auxtrace.idx;
2655 data_offset = lseek(fd, 0, SEEK_CUR);
2656 if (data_offset == -1)
2660 err = auxtrace_queues__add_event(&etm->queues, session,
2661 event, data_offset, &buffer);
2666 * Knowing if the trace is formatted or not requires a lookup of
2667 * the aux record so only works in non-piped mode where data is
2668 * queued in cs_etm__queue_aux_records(). Always assume
2669 * formatted in piped mode (true).
2671 err = cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2677 if (auxtrace_buffer__get_data(buffer, fd)) {
2678 cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);
2679 auxtrace_buffer__put_data(buffer);
2681 } else if (dump_trace)
2682 dump_queued_data(etm, &event->auxtrace);
2687 static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)
2689 struct evsel *evsel;
2690 struct evlist *evlist = etm->session->evlist;
2692 /* Override timeless mode with user input from --itrace=Z */
2693 if (etm->synth_opts.timeless_decoding) {
2694 etm->timeless_decoding = true;
2699 * Find the cs_etm evsel and look at what its timestamp setting was
2701 evlist__for_each_entry(evlist, evsel)
2702 if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {
2703 etm->timeless_decoding =
2704 !(evsel->core.attr.config & BIT(ETM_OPT_TS));
2708 pr_err("CS ETM: Couldn't find ETM evsel\n");
2713 * Read a single cpu parameter block from the auxtrace_info priv block.
2715 * For version 1 there is a per cpu nr_params entry. If we are handling
2716 * version 1 file, then there may be less, the same, or more params
2717 * indicated by this value than the compile time number we understand.
2719 * For a version 0 info block, there are a fixed number, and we need to
2720 * fill out the nr_param value in the metadata we create.
2722 static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset,
2723 int out_blk_size, int nr_params_v0)
2725 u64 *metadata = NULL;
2727 int nr_in_params, nr_out_params, nr_cmn_params;
2730 metadata = zalloc(sizeof(*metadata) * out_blk_size);
2734 /* read block current index & version */
2735 i = *buff_in_offset;
2736 hdr_version = buff_in[CS_HEADER_VERSION];
2739 /* read version 0 info block into a version 1 metadata block */
2740 nr_in_params = nr_params_v0;
2741 metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC];
2742 metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU];
2743 metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params;
2744 /* remaining block params at offset +1 from source */
2745 for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++)
2746 metadata[k + 1] = buff_in[i + k];
2747 /* version 0 has 2 common params */
2750 /* read version 1 info block - input and output nr_params may differ */
2751 /* version 1 has 3 common params */
2753 nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS];
2755 /* if input has more params than output - skip excess */
2756 nr_out_params = nr_in_params + nr_cmn_params;
2757 if (nr_out_params > out_blk_size)
2758 nr_out_params = out_blk_size;
2760 for (k = CS_ETM_MAGIC; k < nr_out_params; k++)
2761 metadata[k] = buff_in[i + k];
2763 /* record the actual nr params we copied */
2764 metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params;
2767 /* adjust in offset by number of in params used */
2768 i += nr_in_params + nr_cmn_params;
2769 *buff_in_offset = i;
2774 * Puts a fragment of an auxtrace buffer into the auxtrace queues based
2775 * on the bounds of aux_event, if it matches with the buffer that's at
2778 * Normally, whole auxtrace buffers would be added to the queue. But we
2779 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
2780 * is reset across each buffer, so splitting the buffers up in advance has
2783 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
2784 struct perf_record_aux *aux_event, struct perf_sample *sample)
2787 char buf[PERF_SAMPLE_MAX_SIZE];
2788 union perf_event *auxtrace_event_union;
2789 struct perf_record_auxtrace *auxtrace_event;
2790 union perf_event auxtrace_fragment;
2791 __u64 aux_offset, aux_size;
2795 struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
2796 struct cs_etm_auxtrace,
2800 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
2801 * from looping through the auxtrace index.
2803 err = perf_session__peek_event(session, file_offset, buf,
2804 PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL);
2807 auxtrace_event = &auxtrace_event_union->auxtrace;
2808 if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE)
2811 if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) ||
2812 auxtrace_event->header.size != sz) {
2817 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
2818 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
2819 * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
2820 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
2821 * Return 'not found' if mismatch.
2823 if (auxtrace_event->cpu == (__u32) -1) {
2824 if (auxtrace_event->tid != sample->tid)
2826 } else if (auxtrace_event->cpu != sample->cpu)
2829 if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) {
2831 * Clamp size in snapshot mode. The buffer size is clamped in
2832 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
2835 aux_size = min(aux_event->aux_size, auxtrace_event->size);
2838 * In this mode, the head also points to the end of the buffer so aux_offset
2839 * needs to have the size subtracted so it points to the beginning as in normal mode
2841 aux_offset = aux_event->aux_offset - aux_size;
2843 aux_size = aux_event->aux_size;
2844 aux_offset = aux_event->aux_offset;
2847 if (aux_offset >= auxtrace_event->offset &&
2848 aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) {
2850 * If this AUX event was inside this buffer somewhere, create a new auxtrace event
2851 * based on the sizes of the aux event, and queue that fragment.
2853 auxtrace_fragment.auxtrace = *auxtrace_event;
2854 auxtrace_fragment.auxtrace.size = aux_size;
2855 auxtrace_fragment.auxtrace.offset = aux_offset;
2856 file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size;
2858 pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64
2859 " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu);
2860 err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,
2865 idx = auxtrace_event->idx;
2866 formatted = !(aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
2867 return cs_etm__setup_queue(etm, &etm->queues.queue_array[idx],
2871 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
2875 static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event,
2876 u64 offset __maybe_unused, void *data __maybe_unused)
2878 /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
2879 if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) {
2880 (*(int *)data)++; /* increment found count */
2881 return cs_etm__process_aux_output_hw_id(session, event);
2886 static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event,
2887 u64 offset __maybe_unused, void *data __maybe_unused)
2889 struct perf_sample sample;
2891 struct auxtrace_index_entry *ent;
2892 struct auxtrace_index *auxtrace_index;
2893 struct evsel *evsel;
2896 /* Don't care about any other events, we're only queuing buffers for AUX events */
2897 if (event->header.type != PERF_RECORD_AUX)
2900 if (event->header.size < sizeof(struct perf_record_aux))
2903 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
2904 if (!event->aux.aux_size)
2908 * Parse the sample, we need the sample_id_all data that comes after the event so that the
2909 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
2911 evsel = evlist__event2evsel(session->evlist, event);
2914 ret = evsel__parse_sample(evsel, event, &sample);
2919 * Loop through the auxtrace index to find the buffer that matches up with this aux event.
2921 list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) {
2922 for (i = 0; i < auxtrace_index->nr; i++) {
2923 ent = &auxtrace_index->entries[i];
2924 ret = cs_etm__queue_aux_fragment(session, ent->file_offset,
2925 ent->sz, &event->aux, &sample);
2927 * Stop search on error or successful values. Continue search on
2936 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
2937 * don't exit with an error because it will still be possible to decode other aux records.
2939 pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64
2940 " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu);
2944 static int cs_etm__queue_aux_records(struct perf_session *session)
2946 struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index,
2947 struct auxtrace_index, list);
2948 if (index && index->nr > 0)
2949 return perf_session__peek_events(session, session->header.data_offset,
2950 session->header.data_size,
2951 cs_etm__queue_aux_records_cb, NULL);
2954 * We would get here if there are no entries in the index (either no auxtrace
2955 * buffers or no index at all). Fail silently as there is the possibility of
2956 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
2959 * In that scenario, buffers will not be split by AUX records.
2964 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
2965 (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
2968 * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
2971 static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu)
2975 for (j = 0; j < num_cpu; j++) {
2976 switch (metadata[j][CS_ETM_MAGIC]) {
2977 case __perf_cs_etmv4_magic:
2978 if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1)
2981 case __perf_cs_ete_magic:
2982 if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1)
2986 /* Unknown / unsupported magic number. */
2993 /* map trace ids to correct metadata block, from information in metadata */
2994 static int cs_etm__map_trace_ids_metadata(int num_cpu, u64 **metadata)
3000 for (i = 0; i < num_cpu; i++) {
3001 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3002 switch (cs_etm_magic) {
3003 case __perf_cs_etmv3_magic:
3004 metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3005 trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]);
3007 case __perf_cs_etmv4_magic:
3008 case __perf_cs_ete_magic:
3009 metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK;
3010 trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]);
3013 /* unknown magic number */
3016 err = cs_etm__map_trace_id(trace_chan_id, metadata[i]);
3024 * If we found AUX_HW_ID packets, then set any metadata marked as unused to the
3025 * unused value to reduce the number of unneeded decoders created.
3027 static int cs_etm__clear_unused_trace_ids_metadata(int num_cpu, u64 **metadata)
3032 for (i = 0; i < num_cpu; i++) {
3033 cs_etm_magic = metadata[i][CS_ETM_MAGIC];
3034 switch (cs_etm_magic) {
3035 case __perf_cs_etmv3_magic:
3036 if (metadata[i][CS_ETM_ETMTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3037 metadata[i][CS_ETM_ETMTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3039 case __perf_cs_etmv4_magic:
3040 case __perf_cs_ete_magic:
3041 if (metadata[i][CS_ETMV4_TRCTRACEIDR] & CORESIGHT_TRACE_ID_UNUSED_FLAG)
3042 metadata[i][CS_ETMV4_TRCTRACEIDR] = CORESIGHT_TRACE_ID_UNUSED_VAL;
3045 /* unknown magic number */
3052 int cs_etm__process_auxtrace_info_full(union perf_event *event,
3053 struct perf_session *session)
3055 struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info;
3056 struct cs_etm_auxtrace *etm = NULL;
3057 struct perf_record_time_conv *tc = &session->time_conv;
3058 int event_header_size = sizeof(struct perf_event_header);
3059 int total_size = auxtrace_info->header.size;
3063 int aux_hw_id_found;
3066 u64 **metadata = NULL;
3069 * Create an RB tree for traceID-metadata tuple. Since the conversion
3070 * has to be made for each packet that gets decoded, optimizing access
3071 * in anything other than a sequential array is worth doing.
3073 traceid_list = intlist__new(NULL);
3077 /* First the global part */
3078 ptr = (u64 *) auxtrace_info->priv;
3079 num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff;
3080 metadata = zalloc(sizeof(*metadata) * num_cpu);
3083 goto err_free_traceid_list;
3086 /* Start parsing after the common part of the header */
3087 i = CS_HEADER_VERSION_MAX;
3090 * The metadata is stored in the auxtrace_info section and encodes
3091 * the configuration of the ARM embedded trace macrocell which is
3092 * required by the trace decoder to properly decode the trace due
3093 * to its highly compressed nature.
3095 for (j = 0; j < num_cpu; j++) {
3096 if (ptr[i] == __perf_cs_etmv3_magic) {
3098 cs_etm__create_meta_blk(ptr, &i,
3100 CS_ETM_NR_TRC_PARAMS_V0);
3101 } else if (ptr[i] == __perf_cs_etmv4_magic) {
3103 cs_etm__create_meta_blk(ptr, &i,
3105 CS_ETMV4_NR_TRC_PARAMS_V0);
3106 } else if (ptr[i] == __perf_cs_ete_magic) {
3107 metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1);
3109 ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3112 goto err_free_metadata;
3117 goto err_free_metadata;
3122 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3123 * CS_ETMV4_PRIV_MAX mark how many double words are in the
3124 * global metadata, and each cpu's metadata respectively.
3125 * The following tests if the correct number of double words was
3126 * present in the auxtrace info section.
3128 priv_size = total_size - event_header_size - INFO_HEADER_SIZE;
3129 if (i * 8 != priv_size) {
3131 goto err_free_metadata;
3134 etm = zalloc(sizeof(*etm));
3138 goto err_free_metadata;
3141 err = auxtrace_queues__init(&etm->queues);
3145 if (session->itrace_synth_opts->set) {
3146 etm->synth_opts = *session->itrace_synth_opts;
3148 itrace_synth_opts__set_default(&etm->synth_opts,
3149 session->itrace_synth_opts->default_no_sample);
3150 etm->synth_opts.callchain = false;
3153 etm->session = session;
3154 etm->machine = &session->machines.host;
3156 etm->num_cpu = num_cpu;
3157 etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);
3158 etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);
3159 etm->metadata = metadata;
3160 etm->auxtrace_type = auxtrace_info->type;
3162 /* Use virtual timestamps if all ETMs report ts_source = 1 */
3163 etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);
3165 if (!etm->has_virtual_ts)
3166 ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n"
3167 "The time field of the samples will not be set accurately.\n\n");
3169 etm->auxtrace.process_event = cs_etm__process_event;
3170 etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
3171 etm->auxtrace.flush_events = cs_etm__flush_events;
3172 etm->auxtrace.free_events = cs_etm__free_events;
3173 etm->auxtrace.free = cs_etm__free;
3174 etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;
3175 session->auxtrace = &etm->auxtrace;
3177 err = cs_etm__setup_timeless_decoding(etm);
3181 etm->unknown_thread = thread__new(999999999, 999999999);
3182 if (!etm->unknown_thread) {
3184 goto err_free_queues;
3188 * Initialize list node so that at thread__zput() we can avoid
3189 * segmentation fault at list_del_init().
3191 INIT_LIST_HEAD(&etm->unknown_thread->node);
3193 err = thread__set_comm(etm->unknown_thread, "unknown", 0);
3195 goto err_delete_thread;
3197 if (thread__init_maps(etm->unknown_thread, etm->machine)) {
3199 goto err_delete_thread;
3202 etm->tc.time_shift = tc->time_shift;
3203 etm->tc.time_mult = tc->time_mult;
3204 etm->tc.time_zero = tc->time_zero;
3205 if (event_contains(*tc, time_cycles)) {
3206 etm->tc.time_cycles = tc->time_cycles;
3207 etm->tc.time_mask = tc->time_mask;
3208 etm->tc.cap_user_time_zero = tc->cap_user_time_zero;
3209 etm->tc.cap_user_time_short = tc->cap_user_time_short;
3211 err = cs_etm__synth_events(etm, session);
3213 goto err_delete_thread;
3216 * Map Trace ID values to CPU metadata.
3218 * Trace metadata will always contain Trace ID values from the legacy algorithm. If the
3219 * files has been recorded by a "new" perf updated to handle AUX_HW_ID then the metadata
3220 * ID value will also have the CORESIGHT_TRACE_ID_UNUSED_FLAG set.
3222 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3223 * the same IDs as the old algorithm as far as is possible, unless there are clashes
3224 * in which case a different value will be used. This means an older perf may still
3225 * be able to record and read files generate on a newer system.
3227 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3228 * those packets. If they are there then the values will be mapped and plugged into
3229 * the metadata. We then set any remaining metadata values with the used flag to a
3230 * value CORESIGHT_TRACE_ID_UNUSED_VAL - which indicates no decoder is required.
3232 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3233 * then we map Trace ID values to CPU directly from the metadata - clearing any unused
3237 /* first scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3238 aux_hw_id_found = 0;
3239 err = perf_session__peek_events(session, session->header.data_offset,
3240 session->header.data_size,
3241 cs_etm__process_aux_hw_id_cb, &aux_hw_id_found);
3243 goto err_delete_thread;
3245 /* if HW ID found then clear any unused metadata ID values */
3246 if (aux_hw_id_found)
3247 err = cs_etm__clear_unused_trace_ids_metadata(num_cpu, metadata);
3248 /* otherwise, this is a file with metadata values only, map from metadata */
3250 err = cs_etm__map_trace_ids_metadata(num_cpu, metadata);
3253 goto err_delete_thread;
3255 err = cs_etm__queue_aux_records(session);
3257 goto err_delete_thread;
3259 etm->data_queued = etm->queues.populated;
3263 thread__zput(etm->unknown_thread);
3265 auxtrace_queues__free(&etm->queues);
3266 session->auxtrace = NULL;
3270 /* No need to check @metadata[j], free(NULL) is supported */
3271 for (j = 0; j < num_cpu; j++)
3272 zfree(&metadata[j]);
3274 err_free_traceid_list:
3275 intlist__delete(traceid_list);