1 // SPDX-License-Identifier: GPL-2.0
3 * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight
4 * sink device could then pair with an appropriate per-cpu coresight source
5 * device (ETE) thus generating required trace data. Trace can be enabled
6 * via the perf framework.
8 * The AUX buffer handling is inspired from Arm SPE PMU driver.
10 * Copyright (C) 2020 ARM Ltd.
12 * Author: Anshuman Khandual <anshuman.khandual@arm.com>
14 #define DRVNAME "arm_trbe"
16 #define pr_fmt(fmt) DRVNAME ": " fmt
18 #include <asm/barrier.h>
19 #include <asm/cpufeature.h>
21 #include "coresight-self-hosted-trace.h"
22 #include "coresight-trbe.h"
24 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT))
27 * A padding packet that will help the user space tools
28 * in skipping relevant sections in the captured trace
29 * data which could not be decoded. TRBE doesn't support
30 * formatting the trace data, unlike the legacy CoreSight
31 * sinks and thus we use ETE trace packets to pad the
32 * sections of the buffer.
34 #define ETE_IGNORE_PACKET 0x70
37 * Minimum amount of meaningful trace will contain:
38 * A-Sync, Trace Info, Trace On, Address, Atom.
39 * This is about 44bytes of ETE trace. To be on
40 * the safer side, we assume 64bytes is the minimum
41 * space required for a meaningful session, before
42 * we hit a "WRAP" event.
44 #define TRBE_TRACE_MIN_BUF_SIZE 64
46 enum trbe_fault_action {
48 TRBE_FAULT_ACT_SPURIOUS,
54 * Even though trbe_base represents vmap()
55 * mapped allocated buffer's start address,
56 * it's being as unsigned long for various
57 * arithmetic and comparision operations &
58 * also to be consistent with trbe_write &
59 * trbe_limit sibling pointers.
61 unsigned long trbe_base;
62 /* The base programmed into the TRBE */
63 unsigned long trbe_hw_base;
64 unsigned long trbe_limit;
65 unsigned long trbe_write;
69 struct trbe_cpudata *cpudata;
75 * The errata are defined in arm64 generic cpu_errata framework.
76 * Since the errata work arounds could be applied individually
77 * to the affected CPUs inside the TRBE driver, we need to know if
78 * a given CPU is affected by the erratum. Unlike the other erratum
79 * work arounds, TRBE driver needs to check multiple times during
80 * a trace session. Thus we need a quicker access to per-CPU
81 * errata and not issue costly this_cpu_has_cap() everytime.
82 * We keep a set of the affected errata in trbe_cpudata, per TRBE.
84 * We rely on the corresponding cpucaps to be defined for a given
85 * TRBE erratum. We map the given cpucap into a TRBE internal number
86 * to make the tracking of the errata lean.
89 * - Not duplicating the detection logic
90 * - Streamlined detection of erratum across the system
92 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE 0
93 #define TRBE_WORKAROUND_WRITE_OUT_OF_RANGE 1
94 #define TRBE_NEEDS_DRAIN_AFTER_DISABLE 2
95 #define TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE 3
96 #define TRBE_IS_BROKEN 4
98 static int trbe_errata_cpucaps[] = {
99 [TRBE_WORKAROUND_OVERWRITE_FILL_MODE] = ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE,
100 [TRBE_WORKAROUND_WRITE_OUT_OF_RANGE] = ARM64_WORKAROUND_TRBE_WRITE_OUT_OF_RANGE,
101 [TRBE_NEEDS_DRAIN_AFTER_DISABLE] = ARM64_WORKAROUND_2064142,
102 [TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE] = ARM64_WORKAROUND_2038923,
103 [TRBE_IS_BROKEN] = ARM64_WORKAROUND_1902691,
104 -1, /* Sentinel, must be the last entry */
107 /* The total number of listed errata in trbe_errata_cpucaps */
108 #define TRBE_ERRATA_MAX (ARRAY_SIZE(trbe_errata_cpucaps) - 1)
111 * Safe limit for the number of bytes that may be overwritten
112 * when ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE is triggered.
114 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES 256
117 * struct trbe_cpudata: TRBE instance specific data
118 * @trbe_flag - TRBE dirty/access flag support
119 * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1.
120 * @trbe_align - Software alignment used for the TRBPTR_EL1.
121 * @cpu - CPU this TRBE belongs to.
122 * @mode - Mode of current operation. (perf/disabled)
123 * @drvdata - TRBE specific drvdata
124 * @errata - Bit map for the errata on this TRBE.
126 struct trbe_cpudata {
132 struct trbe_buf *buf;
133 struct trbe_drvdata *drvdata;
134 DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
137 struct trbe_drvdata {
138 struct trbe_cpudata __percpu *cpudata;
139 struct perf_output_handle * __percpu *handle;
140 struct hlist_node hotplug_node;
142 cpumask_t supported_cpus;
143 enum cpuhp_state trbe_online;
144 struct platform_device *pdev;
147 static void trbe_check_errata(struct trbe_cpudata *cpudata)
151 for (i = 0; i < TRBE_ERRATA_MAX; i++) {
152 int cap = trbe_errata_cpucaps[i];
154 if (WARN_ON_ONCE(cap < 0))
156 if (this_cpu_has_cap(cap))
157 set_bit(i, cpudata->errata);
161 static inline bool trbe_has_erratum(struct trbe_cpudata *cpudata, int i)
163 return (i < TRBE_ERRATA_MAX) && test_bit(i, cpudata->errata);
166 static inline bool trbe_may_overwrite_in_fill_mode(struct trbe_cpudata *cpudata)
168 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE);
171 static inline bool trbe_may_write_out_of_range(struct trbe_cpudata *cpudata)
173 return trbe_has_erratum(cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE);
176 static inline bool trbe_needs_drain_after_disable(struct trbe_cpudata *cpudata)
179 * Errata affected TRBE implementation will need TSB CSYNC and
180 * DSB in order to prevent subsequent writes into certain TRBE
181 * system registers from being ignored and not effected.
183 return trbe_has_erratum(cpudata, TRBE_NEEDS_DRAIN_AFTER_DISABLE);
186 static inline bool trbe_needs_ctxt_sync_after_enable(struct trbe_cpudata *cpudata)
189 * Errata affected TRBE implementation will need an additional
190 * context synchronization in order to prevent an inconsistent
191 * TRBE prohibited region view on the CPU which could possibly
192 * corrupt the TRBE buffer or the TRBE state.
194 return trbe_has_erratum(cpudata, TRBE_NEEDS_CTXT_SYNC_AFTER_ENABLE);
197 static inline bool trbe_is_broken(struct trbe_cpudata *cpudata)
199 return trbe_has_erratum(cpudata, TRBE_IS_BROKEN);
202 static int trbe_alloc_node(struct perf_event *event)
204 if (event->cpu == -1)
206 return cpu_to_node(event->cpu);
209 static inline void trbe_drain_buffer(void)
215 static inline void set_trbe_enabled(struct trbe_cpudata *cpudata, u64 trblimitr)
218 * Enable the TRBE without clearing LIMITPTR which
219 * might be required for fetching the buffer limits.
221 trblimitr |= TRBLIMITR_EL1_E;
222 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
224 /* Synchronize the TRBE enable event */
227 if (trbe_needs_ctxt_sync_after_enable(cpudata))
231 static inline void set_trbe_disabled(struct trbe_cpudata *cpudata)
233 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
236 * Disable the TRBE without clearing LIMITPTR which
237 * might be required for fetching the buffer limits.
239 trblimitr &= ~TRBLIMITR_EL1_E;
240 write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1);
242 if (trbe_needs_drain_after_disable(cpudata))
247 static void trbe_drain_and_disable_local(struct trbe_cpudata *cpudata)
250 set_trbe_disabled(cpudata);
253 static void trbe_reset_local(struct trbe_cpudata *cpudata)
255 trbe_drain_and_disable_local(cpudata);
256 write_sysreg_s(0, SYS_TRBLIMITR_EL1);
257 write_sysreg_s(0, SYS_TRBPTR_EL1);
258 write_sysreg_s(0, SYS_TRBBASER_EL1);
259 write_sysreg_s(0, SYS_TRBSR_EL1);
262 static void trbe_report_wrap_event(struct perf_output_handle *handle)
265 * Mark the buffer to indicate that there was a WRAP event by
266 * setting the COLLISION flag. This indicates to the user that
267 * the TRBE trace collection was stopped without stopping the
268 * ETE and thus there might be some amount of trace that was
269 * lost between the time the WRAP was detected and the IRQ
270 * was consumed by the CPU.
272 * Setting the TRUNCATED flag would move the event to STOPPED
273 * state unnecessarily, even when there is space left in the
274 * ring buffer. Using the COLLISION flag doesn't have this side
275 * effect. We only set TRUNCATED flag when there is no space
276 * left in the ring buffer.
278 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
281 static void trbe_stop_and_truncate_event(struct perf_output_handle *handle)
283 struct trbe_buf *buf = etm_perf_sink_config(handle);
286 * We cannot proceed with the buffer collection and we
287 * do not have any data for the current session. The
288 * etm_perf driver expects to close out the aux_buffer
289 * at event_stop(). So disable the TRBE here and leave
290 * the update_buffer() to return a 0 size.
292 trbe_drain_and_disable_local(buf->cpudata);
293 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
294 perf_aux_output_end(handle, 0);
295 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
299 * TRBE Buffer Management
301 * The TRBE buffer spans from the base pointer till the limit pointer. When enabled,
302 * it starts writing trace data from the write pointer onward till the limit pointer.
303 * When the write pointer reaches the address just before the limit pointer, it gets
304 * wrapped around again to the base pointer. This is called a TRBE wrap event, which
305 * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver
306 * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ
307 * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and
310 * Wrap around with an IRQ
311 * ------ < ------ < ------- < ----- < -----
313 * ------ > ------ > ------- > ----- > -----
315 * +---------------+-----------------------+
317 * +---------------+-----------------------+
318 * Base Pointer Write Pointer Limit Pointer
320 * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write
321 * pointer can be aligned to the implementation defined TRBE trace buffer alignment
322 * as captured in trbe_cpudata->trbe_align.
326 * +---------------------------------------+----- ~ ~ ------
327 * |$$$$$$$|################|$$$$$$$$$$$$$$| |
328 * +---------------------------------------+----- ~ ~ ------
329 * Base Pointer Write Pointer Limit Pointer
331 * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing
332 * values which tracks all the driver writes and user reads from the perf auxiliary
333 * buffer. Generally [head..tail] is the area where the driver can write into unless
334 * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and
335 * configured depending on the perf_output_handle indices, so that the driver does
336 * not override into areas in the perf auxiliary buffer which is being or yet to be
337 * consumed from the user space. The enabled TRBE buffer area is a moving subset of
338 * the allocated perf auxiliary buffer.
341 static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len)
343 memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len);
346 static void trbe_pad_buf(struct perf_output_handle *handle, int len)
348 struct trbe_buf *buf = etm_perf_sink_config(handle);
349 u64 head = PERF_IDX2OFF(handle->head, buf);
351 __trbe_pad_buf(buf, head, len);
353 perf_aux_output_skip(handle, len);
356 static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle)
358 struct trbe_buf *buf = etm_perf_sink_config(handle);
361 * The ETE trace has alignment synchronization packets allowing
362 * the decoder to reset in case of an overflow or corruption.
363 * So we can use the entire buffer for the snapshot mode.
365 return buf->nr_pages * PAGE_SIZE;
368 static u64 trbe_min_trace_buf_size(struct perf_output_handle *handle)
370 u64 size = TRBE_TRACE_MIN_BUF_SIZE;
371 struct trbe_buf *buf = etm_perf_sink_config(handle);
372 struct trbe_cpudata *cpudata = buf->cpudata;
375 * When the TRBE is affected by an erratum that could make it
376 * write to the next "virtually addressed" page beyond the LIMIT.
377 * We need to make sure there is always a PAGE after the LIMIT,
378 * within the buffer. Thus we ensure there is at least an extra
379 * page than normal. With this we could then adjust the LIMIT
380 * pointer down by a PAGE later.
382 if (trbe_may_write_out_of_range(cpudata))
388 * TRBE Limit Calculation
390 * The following markers are used to illustrate various TRBE buffer situations.
392 * $$$$ - Data area, unconsumed captured trace data, not to be overridden
393 * #### - Free area, enabled, trace will be written
394 * %%%% - Free area, disabled, trace will not be written
395 * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped
397 static unsigned long __trbe_normal_offset(struct perf_output_handle *handle)
399 struct trbe_buf *buf = etm_perf_sink_config(handle);
400 struct trbe_cpudata *cpudata = buf->cpudata;
401 const u64 bufsize = buf->nr_pages * PAGE_SIZE;
403 u64 head, tail, wakeup;
405 head = PERF_IDX2OFF(handle->head, buf);
411 * head TRBE align tail
412 * +----|-------|---------------|-------+
413 * |$$$$|=======|###############|$$$$$$$|
414 * +----|-------|---------------|-------+
415 * trbe_base trbe_base + nr_pages
417 * Perf aux buffer output head position can be misaligned depending on
418 * various factors including user space reads. In case misaligned, head
419 * needs to be aligned before TRBE can be configured. Pad the alignment
420 * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools
421 * and skip this section thus advancing the head.
423 if (!IS_ALIGNED(head, cpudata->trbe_align)) {
424 unsigned long delta = roundup(head, cpudata->trbe_align) - head;
426 delta = min(delta, handle->size);
427 trbe_pad_buf(handle, delta);
428 head = PERF_IDX2OFF(handle->head, buf);
432 * head = tail (size = 0)
433 * +----|-------------------------------+
434 * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ |
435 * +----|-------------------------------+
436 * trbe_base trbe_base + nr_pages
438 * Perf aux buffer does not have any space for the driver to write into.
443 /* Compute the tail and wakeup indices now that we've aligned head */
444 tail = PERF_IDX2OFF(handle->head + handle->size, buf);
445 wakeup = PERF_IDX2OFF(handle->wakeup, buf);
448 * Lets calculate the buffer area which TRBE could write into. There
449 * are three possible scenarios here. Limit needs to be aligned with
450 * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the
456 * +----|-----------------------|-------+
457 * |$$$$|#######################|$$$$$$$|
458 * +----|-----------------------|-------+
459 * trbe_base limit trbe_base + nr_pages
461 * TRBE could write into [head..tail] area. Unless the tail is right at
462 * the end of the buffer, neither an wrap around nor an IRQ is expected
463 * while being enabled.
467 * head = tail (size > 0)
468 * +----|-------------------------------+
469 * |%%%%|###############################|
470 * +----|-------------------------------+
471 * trbe_base limit = trbe_base + nr_pages
473 * TRBE should just write into [head..base + nr_pages] area even though
474 * the entire buffer is empty. Reason being, when the trace reaches the
475 * end of the buffer, it will just wrap around with an IRQ giving an
476 * opportunity to reconfigure the buffer.
481 * +----|-----------------------|-------+
482 * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######|
483 * +----|-----------------------|-------+
484 * trbe_base limit = trbe_base + nr_pages
486 * TRBE should just write into [head..base + nr_pages] area even though
487 * the [trbe_base..tail] is also empty. Reason being, when the trace
488 * reaches the end of the buffer, it will just wrap around with an IRQ
489 * giving an opportunity to reconfigure the buffer.
492 limit = round_down(tail, PAGE_SIZE);
495 * Wakeup may be arbitrarily far into the future. If it's not in the
496 * current generation, either we'll wrap before hitting it, or it's
497 * in the past and has been handled already.
499 * If there's a wakeup before we wrap, arrange to be woken up by the
500 * page boundary following it. Keep the tail boundary if that's lower.
503 * +----|---------------|-------|-------+
504 * |$$$$|###############|%%%%%%%|$$$$$$$|
505 * +----|---------------|-------|-------+
506 * trbe_base limit trbe_base + nr_pages
508 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
509 limit = min(limit, round_up(wakeup, PAGE_SIZE));
512 * There are two situation when this can happen i.e limit is before
513 * the head and hence TRBE cannot be configured.
515 * 1) head < tail (aligned down with PAGE_SIZE) and also they are both
516 * within the same PAGE size range.
519 * |----------------------|
522 * +------------|------|--------|-------+
523 * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$|
524 * +------------|------|--------|-------+
525 * trbe_base trbe_base + nr_pages
527 * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both
528 * head and wakeup are within same PAGE size range.
531 * |----------------------|
533 * limit head wakeup tail
534 * +----|------|-------|--------|-------+
535 * |$$$$$$$$$$$|=======|========|$$$$$$$|
536 * +----|------|-------|--------|-------+
537 * trbe_base trbe_base + nr_pages
542 trbe_pad_buf(handle, handle->size);
546 static unsigned long trbe_normal_offset(struct perf_output_handle *handle)
548 struct trbe_buf *buf = etm_perf_sink_config(handle);
549 u64 limit = __trbe_normal_offset(handle);
550 u64 head = PERF_IDX2OFF(handle->head, buf);
553 * If the head is too close to the limit and we don't
554 * have space for a meaningful run, we rather pad it
557 * We might have to do this more than once to make sure
558 * we have enough required space.
560 while (limit && ((limit - head) < trbe_min_trace_buf_size(handle))) {
561 trbe_pad_buf(handle, limit - head);
562 limit = __trbe_normal_offset(handle);
563 head = PERF_IDX2OFF(handle->head, buf);
568 static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle)
570 struct trbe_buf *buf = etm_perf_sink_config(handle);
571 unsigned long offset;
574 offset = trbe_snapshot_offset(handle);
576 offset = trbe_normal_offset(handle);
577 return buf->trbe_base + offset;
580 static void clr_trbe_status(void)
582 u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1);
584 WARN_ON(is_trbe_enabled());
585 trbsr &= ~TRBSR_EL1_IRQ;
586 trbsr &= ~TRBSR_EL1_TRG;
587 trbsr &= ~TRBSR_EL1_WRAP;
588 trbsr &= ~TRBSR_EL1_EC_MASK;
589 trbsr &= ~TRBSR_EL1_BSC_MASK;
590 trbsr &= ~TRBSR_EL1_S;
591 write_sysreg_s(trbsr, SYS_TRBSR_EL1);
594 static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf)
596 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
597 unsigned long addr = buf->trbe_limit;
599 WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT)));
600 WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE));
602 trblimitr &= ~TRBLIMITR_EL1_nVM;
603 trblimitr &= ~TRBLIMITR_EL1_FM_MASK;
604 trblimitr &= ~TRBLIMITR_EL1_TM_MASK;
605 trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;
608 * Fill trace buffer mode is used here while configuring the
609 * TRBE for trace capture. In this particular mode, the trace
610 * collection is stopped and a maintenance interrupt is raised
611 * when the current write pointer wraps. This pause in trace
612 * collection gives the software an opportunity to capture the
613 * trace data in the interrupt handler, before reconfiguring
616 trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) &
617 TRBLIMITR_EL1_FM_MASK;
620 * Trigger mode is not used here while configuring the TRBE for
621 * the trace capture. Hence just keep this in the ignore mode.
623 trblimitr |= (TRBLIMITR_EL1_TM_IGNR << TRBLIMITR_EL1_TM_SHIFT) &
624 TRBLIMITR_EL1_TM_MASK;
625 trblimitr |= (addr & PAGE_MASK);
626 set_trbe_enabled(buf->cpudata, trblimitr);
629 static void trbe_enable_hw(struct trbe_buf *buf)
631 WARN_ON(buf->trbe_hw_base < buf->trbe_base);
632 WARN_ON(buf->trbe_write < buf->trbe_hw_base);
633 WARN_ON(buf->trbe_write >= buf->trbe_limit);
634 set_trbe_disabled(buf->cpudata);
636 set_trbe_base_pointer(buf->trbe_hw_base);
637 set_trbe_write_pointer(buf->trbe_write);
640 * Synchronize all the register updates
641 * till now before enabling the TRBE.
644 set_trbe_limit_pointer_enabled(buf);
647 static enum trbe_fault_action trbe_get_fault_act(struct perf_output_handle *handle,
650 int ec = get_trbe_ec(trbsr);
651 int bsc = get_trbe_bsc(trbsr);
652 struct trbe_buf *buf = etm_perf_sink_config(handle);
653 struct trbe_cpudata *cpudata = buf->cpudata;
655 WARN_ON(is_trbe_running(trbsr));
656 if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr))
657 return TRBE_FAULT_ACT_FATAL;
659 if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT))
660 return TRBE_FAULT_ACT_FATAL;
663 * If the trbe is affected by TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
664 * it might write data after a WRAP event in the fill mode.
665 * Thus the check TRBPTR == TRBBASER will not be honored.
667 if ((is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) &&
668 (trbe_may_overwrite_in_fill_mode(cpudata) ||
669 get_trbe_write_pointer() == get_trbe_base_pointer()))
670 return TRBE_FAULT_ACT_WRAP;
672 return TRBE_FAULT_ACT_SPURIOUS;
675 static unsigned long trbe_get_trace_size(struct perf_output_handle *handle,
676 struct trbe_buf *buf, bool wrap)
679 u64 start_off, end_off;
681 u64 overwrite_skip = TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
684 * If the TRBE has wrapped around the write pointer has
685 * wrapped and should be treated as limit.
687 * When the TRBE is affected by TRBE_WORKAROUND_WRITE_OUT_OF_RANGE,
688 * it may write upto 64bytes beyond the "LIMIT". The driver already
689 * keeps a valid page next to the LIMIT and we could potentially
690 * consume the trace data that may have been collected there. But we
691 * cannot be really sure it is available, and the TRBPTR may not
692 * indicate the same. Also, affected cores are also affected by another
693 * erratum which forces the PAGE_SIZE alignment on the TRBPTR, and thus
694 * could potentially pad an entire PAGE_SIZE - 64bytes, to get those
695 * 64bytes. Thus we ignore the potential triggering of the erratum
696 * on WRAP and limit the data to LIMIT.
699 write = get_trbe_limit_pointer();
701 write = get_trbe_write_pointer();
704 * TRBE may use a different base address than the base
705 * of the ring buffer. Thus use the beginning of the ring
706 * buffer to compute the offsets.
708 end_off = write - buf->trbe_base;
709 start_off = PERF_IDX2OFF(handle->head, buf);
711 if (WARN_ON_ONCE(end_off < start_off))
714 size = end_off - start_off;
716 * If the TRBE is affected by the following erratum, we must fill
717 * the space we skipped with IGNORE packets. And we are always
718 * guaranteed to have at least a PAGE_SIZE space in the buffer.
720 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE) &&
721 !WARN_ON(size < overwrite_skip))
722 __trbe_pad_buf(buf, start_off, overwrite_skip);
727 static void *arm_trbe_alloc_buffer(struct coresight_device *csdev,
728 struct perf_event *event, void **pages,
729 int nr_pages, bool snapshot)
731 struct trbe_buf *buf;
732 struct page **pglist;
736 * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with
737 * just a single page, there would not be any room left while writing
738 * into a partially filled TRBE buffer after the page size alignment.
739 * Hence restrict the minimum buffer size as two pages.
744 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event));
746 return ERR_PTR(-ENOMEM);
748 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
751 return ERR_PTR(-ENOMEM);
754 for (i = 0; i < nr_pages; i++)
755 pglist[i] = virt_to_page(pages[i]);
757 buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
758 if (!buf->trbe_base) {
761 return ERR_PTR(-ENOMEM);
763 buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE;
764 buf->trbe_write = buf->trbe_base;
765 buf->snapshot = snapshot;
766 buf->nr_pages = nr_pages;
772 static void arm_trbe_free_buffer(void *config)
774 struct trbe_buf *buf = config;
776 vunmap((void *)buf->trbe_base);
780 static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev,
781 struct perf_output_handle *handle,
784 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
785 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
786 struct trbe_buf *buf = config;
787 enum trbe_fault_action act;
788 unsigned long size, status;
792 WARN_ON(buf->cpudata != cpudata);
793 WARN_ON(cpudata->cpu != smp_processor_id());
794 WARN_ON(cpudata->drvdata != drvdata);
795 if (cpudata->mode != CS_MODE_PERF)
799 * We are about to disable the TRBE. And this could in turn
800 * fill up the buffer triggering, an IRQ. This could be consumed
801 * by the PE asynchronously, causing a race here against
802 * the IRQ handler in closing out the handle. So, let us
803 * make sure the IRQ can't trigger while we are collecting
804 * the buffer. We also make sure that a WRAP event is handled
807 local_irq_save(flags);
810 * If the TRBE was disabled due to lack of space in the AUX buffer or a
811 * spurious fault, the driver leaves it disabled, truncating the buffer.
812 * Since the etm_perf driver expects to close out the AUX buffer, the
813 * driver skips it. Thus, just pass in 0 size here to indicate that the
814 * buffer was truncated.
816 if (!is_trbe_enabled()) {
821 * perf handle structure needs to be shared with the TRBE IRQ handler for
822 * capturing trace data and restarting the handle. There is a probability
823 * of an undefined reference based crash when etm event is being stopped
824 * while a TRBE IRQ also getting processed. This happens due the release
825 * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping
826 * the TRBE here will ensure that no IRQ could be generated when the perf
827 * handle gets freed in etm_event_stop().
829 trbe_drain_and_disable_local(cpudata);
831 /* Check if there is a pending interrupt and handle it here */
832 status = read_sysreg_s(SYS_TRBSR_EL1);
833 if (is_trbe_irq(status)) {
836 * Now that we are handling the IRQ here, clear the IRQ
837 * from the status, to let the irq handler know that it
843 act = trbe_get_fault_act(handle, status);
845 * If this was not due to a WRAP event, we have some
846 * errors and as such buffer is empty.
848 if (act != TRBE_FAULT_ACT_WRAP) {
853 trbe_report_wrap_event(handle);
857 size = trbe_get_trace_size(handle, buf, wrap);
860 local_irq_restore(flags);
863 handle->head += size;
868 static int trbe_apply_work_around_before_enable(struct trbe_buf *buf)
871 * TRBE_WORKAROUND_OVERWRITE_FILL_MODE causes the TRBE to overwrite a few cache
872 * line size from the "TRBBASER_EL1" in the event of a "FILL".
873 * Thus, we could loose some amount of the trace at the base.
877 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
879 * -------------------------------------------------------------
880 * | Pg0 | Pg1 | | | PgN |
881 * -------------------------------------------------------------
883 * In the normal course of action, we would set the TRBBASER to the
884 * beginning of the ring-buffer (normal-BASE). But with the erratum,
885 * the TRBE could overwrite the contents at the "normal-BASE", after
886 * hitting the "normal-LIMIT", since it doesn't stop as expected. And
887 * this is wrong. This could result in overwriting trace collected in
888 * one of the previous runs, being consumed by the user. So we must
889 * always make sure that the TRBBASER is within the region
890 * [head, head+size]. Note that TRBBASER must be PAGE aligned,
892 * After moving the BASE:
894 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
896 * -------------------------------------------------------------
897 * | | |xyzdef. |.. tuvw| |
898 * -------------------------------------------------------------
902 * Also, we would set the TRBPTR to head (after adjusting for
903 * alignment) at normal-PTR. This would mean that the last few bytes
904 * of the trace (say, "xyz") might overwrite the first few bytes of
905 * trace written ("abc"). More importantly they will appear in what
906 * userspace sees as the beginning of the trace, which is wrong. We may
907 * not always have space to move the latest trace "xyz" to the correct
908 * order as it must appear beyond the LIMIT. (i.e, [head..head+size]).
909 * Thus it is easier to ignore those bytes than to complicate the
910 * driver to move it, assuming that the erratum was triggered and
911 * doing additional checks to see if there is indeed allowed space at
914 * Thus the full workaround will move the BASE and the PTR and would
915 * look like (after padding at the skipped bytes at the end of
918 * normal-BASE head (normal-TRBPTR) tail (normal-LIMIT)
920 * -------------------------------------------------------------
921 * | | |///abc.. |.. rst| |
922 * -------------------------------------------------------------
924 * New-BASER New-TRBPTR
926 * To summarize, with the work around:
928 * - We always align the offset for the next session to PAGE_SIZE
929 * (This is to ensure we can program the TRBBASER to this offset
930 * within the region [head...head+size]).
933 * - Set the TRBBASER to the page aligned offset of the current
934 * proposed write offset. (which is guaranteed to be aligned
936 * - Move the TRBPTR to skip first 256bytes (that might be
937 * overwritten with the erratum). This ensures that the trace
938 * generated in the session is not re-written.
940 * - At trace collection:
941 * - Pad the 256bytes skipped above again with IGNORE packets.
943 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) {
944 if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE)))
946 buf->trbe_hw_base = buf->trbe_write;
947 buf->trbe_write += TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES;
951 * TRBE_WORKAROUND_WRITE_OUT_OF_RANGE could cause the TRBE to write to
952 * the next page after the TRBLIMITR.LIMIT. For perf, the "next page"
954 * - The page beyond the ring buffer. This could mean, TRBE could
955 * corrupt another entity (kernel / user)
956 * - A portion of the "ring buffer" consumed by the userspace.
957 * i.e, a page outisde [head, head + size].
959 * We work around this by:
960 * - Making sure that we have at least an extra space of PAGE left
961 * in the ring buffer [head, head + size], than we normally do
962 * without the erratum. See trbe_min_trace_buf_size().
964 * - Adjust the TRBLIMITR.LIMIT to leave the extra PAGE outside
965 * the TRBE's range (i.e [TRBBASER, TRBLIMITR.LIMI] ).
967 if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_WRITE_OUT_OF_RANGE)) {
968 s64 space = buf->trbe_limit - buf->trbe_write;
970 * We must have more than a PAGE_SIZE worth space in the proposed
971 * range for the TRBE.
973 if (WARN_ON(space <= PAGE_SIZE ||
974 !IS_ALIGNED(buf->trbe_limit, PAGE_SIZE)))
976 buf->trbe_limit -= PAGE_SIZE;
982 static int __arm_trbe_enable(struct trbe_buf *buf,
983 struct perf_output_handle *handle)
987 perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW);
988 buf->trbe_limit = compute_trbe_buffer_limit(handle);
989 buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf);
990 if (buf->trbe_limit == buf->trbe_base) {
994 /* Set the base of the TRBE to the buffer base */
995 buf->trbe_hw_base = buf->trbe_base;
997 ret = trbe_apply_work_around_before_enable(buf);
1001 *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle;
1002 trbe_enable_hw(buf);
1005 trbe_stop_and_truncate_event(handle);
1009 static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode,
1012 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1013 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1014 struct perf_output_handle *handle = data;
1015 struct trbe_buf *buf = etm_perf_sink_config(handle);
1017 WARN_ON(cpudata->cpu != smp_processor_id());
1018 WARN_ON(cpudata->drvdata != drvdata);
1019 if (mode != CS_MODE_PERF)
1023 cpudata->mode = mode;
1024 buf->cpudata = cpudata;
1026 return __arm_trbe_enable(buf, handle);
1029 static int arm_trbe_disable(struct coresight_device *csdev)
1031 struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
1032 struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
1033 struct trbe_buf *buf = cpudata->buf;
1035 WARN_ON(buf->cpudata != cpudata);
1036 WARN_ON(cpudata->cpu != smp_processor_id());
1037 WARN_ON(cpudata->drvdata != drvdata);
1038 if (cpudata->mode != CS_MODE_PERF)
1041 trbe_drain_and_disable_local(cpudata);
1042 buf->cpudata = NULL;
1043 cpudata->buf = NULL;
1044 cpudata->mode = CS_MODE_DISABLED;
1048 static void trbe_handle_spurious(struct perf_output_handle *handle)
1050 struct trbe_buf *buf = etm_perf_sink_config(handle);
1051 u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
1054 * If the IRQ was spurious, simply re-enable the TRBE
1055 * back without modifying the buffer parameters to
1056 * retain the trace collected so far.
1058 set_trbe_enabled(buf->cpudata, trblimitr);
1061 static int trbe_handle_overflow(struct perf_output_handle *handle)
1063 struct perf_event *event = handle->event;
1064 struct trbe_buf *buf = etm_perf_sink_config(handle);
1066 struct etm_event_data *event_data;
1068 size = trbe_get_trace_size(handle, buf, true);
1070 handle->head += size;
1072 trbe_report_wrap_event(handle);
1073 perf_aux_output_end(handle, size);
1074 event_data = perf_aux_output_begin(handle, event);
1077 * We are unable to restart the trace collection,
1078 * thus leave the TRBE disabled. The etm-perf driver
1079 * is able to detect this with a disconnected handle
1080 * (handle->event = NULL).
1082 trbe_drain_and_disable_local(buf->cpudata);
1083 *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL;
1087 return __arm_trbe_enable(buf, handle);
1090 static bool is_perf_trbe(struct perf_output_handle *handle)
1092 struct trbe_buf *buf = etm_perf_sink_config(handle);
1093 struct trbe_cpudata *cpudata = buf->cpudata;
1094 struct trbe_drvdata *drvdata = cpudata->drvdata;
1095 int cpu = smp_processor_id();
1097 WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer());
1098 WARN_ON(buf->trbe_limit != get_trbe_limit_pointer());
1100 if (cpudata->mode != CS_MODE_PERF)
1103 if (cpudata->cpu != cpu)
1106 if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1112 static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
1114 struct perf_output_handle **handle_ptr = dev;
1115 struct perf_output_handle *handle = *handle_ptr;
1116 struct trbe_buf *buf = etm_perf_sink_config(handle);
1117 enum trbe_fault_action act;
1119 bool truncated = false;
1122 /* Reads to TRBSR_EL1 is fine when TRBE is active */
1123 status = read_sysreg_s(SYS_TRBSR_EL1);
1125 * If the pending IRQ was handled by update_buffer callback
1126 * we have nothing to do here.
1128 if (!is_trbe_irq(status))
1131 /* Prohibit the CPU from tracing before we disable the TRBE */
1132 trfcr = cpu_prohibit_trace();
1134 * Ensure the trace is visible to the CPUs and
1135 * any external aborts have been resolved.
1137 trbe_drain_and_disable_local(buf->cpudata);
1141 if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle))
1144 if (!is_perf_trbe(handle))
1147 act = trbe_get_fault_act(handle, status);
1149 case TRBE_FAULT_ACT_WRAP:
1150 truncated = !!trbe_handle_overflow(handle);
1152 case TRBE_FAULT_ACT_SPURIOUS:
1153 trbe_handle_spurious(handle);
1155 case TRBE_FAULT_ACT_FATAL:
1156 trbe_stop_and_truncate_event(handle);
1162 * If the buffer was truncated, ensure perf callbacks
1163 * have completed, which will disable the event.
1165 * Otherwise, restore the trace filter controls to
1166 * allow the tracing.
1176 static const struct coresight_ops_sink arm_trbe_sink_ops = {
1177 .enable = arm_trbe_enable,
1178 .disable = arm_trbe_disable,
1179 .alloc_buffer = arm_trbe_alloc_buffer,
1180 .free_buffer = arm_trbe_free_buffer,
1181 .update_buffer = arm_trbe_update_buffer,
1184 static const struct coresight_ops arm_trbe_cs_ops = {
1185 .sink_ops = &arm_trbe_sink_ops,
1188 static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
1190 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1192 return sprintf(buf, "%llx\n", cpudata->trbe_hw_align);
1194 static DEVICE_ATTR_RO(align);
1196 static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf)
1198 struct trbe_cpudata *cpudata = dev_get_drvdata(dev);
1200 return sprintf(buf, "%d\n", cpudata->trbe_flag);
1202 static DEVICE_ATTR_RO(flag);
1204 static struct attribute *arm_trbe_attrs[] = {
1205 &dev_attr_align.attr,
1206 &dev_attr_flag.attr,
1210 static const struct attribute_group arm_trbe_group = {
1211 .attrs = arm_trbe_attrs,
1214 static const struct attribute_group *arm_trbe_groups[] = {
1219 static void arm_trbe_enable_cpu(void *info)
1221 struct trbe_drvdata *drvdata = info;
1222 struct trbe_cpudata *cpudata = this_cpu_ptr(drvdata->cpudata);
1224 trbe_reset_local(cpudata);
1225 enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE);
1228 static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu)
1230 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1231 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1232 struct coresight_desc desc = { 0 };
1235 if (WARN_ON(trbe_csdev))
1238 /* If the TRBE was not probed on the CPU, we shouldn't be here */
1239 if (WARN_ON(!cpudata->drvdata))
1242 dev = &cpudata->drvdata->pdev->dev;
1243 desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
1247 desc.type = CORESIGHT_DEV_TYPE_SINK;
1248 desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM;
1249 desc.ops = &arm_trbe_cs_ops;
1250 desc.pdata = dev_get_platdata(dev);
1251 desc.groups = arm_trbe_groups;
1253 trbe_csdev = coresight_register(&desc);
1254 if (IS_ERR(trbe_csdev))
1257 dev_set_drvdata(&trbe_csdev->dev, cpudata);
1258 coresight_set_percpu_sink(cpu, trbe_csdev);
1261 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1265 * Must be called with preemption disabled, for trbe_check_errata().
1267 static void arm_trbe_probe_cpu(void *info)
1269 struct trbe_drvdata *drvdata = info;
1270 int cpu = smp_processor_id();
1271 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1274 if (WARN_ON(!cpudata))
1277 if (!is_trbe_available()) {
1278 pr_err("TRBE is not implemented on cpu %d\n", cpu);
1282 trbidr = read_sysreg_s(SYS_TRBIDR_EL1);
1283 if (!is_trbe_programmable(trbidr)) {
1284 pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu);
1288 cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr);
1289 if (cpudata->trbe_hw_align > SZ_2K) {
1290 pr_err("Unsupported alignment on cpu %d\n", cpu);
1295 * Run the TRBE erratum checks, now that we know
1296 * this instance is about to be registered.
1298 trbe_check_errata(cpudata);
1300 if (trbe_is_broken(cpudata)) {
1301 pr_err("Disabling TRBE on cpu%d due to erratum\n", cpu);
1306 * If the TRBE is affected by erratum TRBE_WORKAROUND_OVERWRITE_FILL_MODE,
1307 * we must always program the TBRPTR_EL1, 256bytes from a page
1308 * boundary, with TRBBASER_EL1 set to the page, to prevent
1309 * TRBE over-writing 256bytes at TRBBASER_EL1 on FILL event.
1311 * Thus make sure we always align our write pointer to a PAGE_SIZE,
1312 * which also guarantees that we have at least a PAGE_SIZE space in
1313 * the buffer (TRBLIMITR is PAGE aligned) and thus we can skip
1314 * the required bytes at the base.
1316 if (trbe_may_overwrite_in_fill_mode(cpudata))
1317 cpudata->trbe_align = PAGE_SIZE;
1319 cpudata->trbe_align = cpudata->trbe_hw_align;
1321 cpudata->trbe_flag = get_trbe_flag_update(trbidr);
1323 cpudata->drvdata = drvdata;
1326 cpumask_clear_cpu(cpu, &drvdata->supported_cpus);
1329 static void arm_trbe_remove_coresight_cpu(void *info)
1331 int cpu = smp_processor_id();
1332 struct trbe_drvdata *drvdata = info;
1333 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1334 struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu);
1336 disable_percpu_irq(drvdata->irq);
1337 trbe_reset_local(cpudata);
1339 coresight_unregister(trbe_csdev);
1340 cpudata->drvdata = NULL;
1341 coresight_set_percpu_sink(cpu, NULL);
1345 static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata)
1349 drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata));
1350 if (!drvdata->cpudata)
1353 for_each_cpu(cpu, &drvdata->supported_cpus) {
1354 /* If we fail to probe the CPU, let us defer it to hotplug callbacks */
1355 if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1))
1357 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1358 arm_trbe_register_coresight_cpu(drvdata, cpu);
1359 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1360 smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1);
1365 static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata)
1369 for_each_cpu(cpu, &drvdata->supported_cpus)
1370 smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1);
1371 free_percpu(drvdata->cpudata);
1375 static void arm_trbe_probe_hotplugged_cpu(struct trbe_drvdata *drvdata)
1378 arm_trbe_probe_cpu(drvdata);
1382 static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node)
1384 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1386 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1389 * If this CPU was not probed for TRBE,
1390 * initialize it now.
1392 if (!coresight_get_percpu_sink(cpu)) {
1393 arm_trbe_probe_hotplugged_cpu(drvdata);
1394 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1395 arm_trbe_register_coresight_cpu(drvdata, cpu);
1396 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus))
1397 arm_trbe_enable_cpu(drvdata);
1399 arm_trbe_enable_cpu(drvdata);
1405 static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1407 struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node);
1409 if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) {
1410 struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu);
1412 disable_percpu_irq(drvdata->irq);
1413 trbe_reset_local(cpudata);
1418 static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata)
1420 enum cpuhp_state trbe_online;
1423 trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1424 arm_trbe_cpu_startup, arm_trbe_cpu_teardown);
1425 if (trbe_online < 0)
1428 ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node);
1430 cpuhp_remove_multi_state(trbe_online);
1433 drvdata->trbe_online = trbe_online;
1437 static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata)
1439 cpuhp_state_remove_instance(drvdata->trbe_online, &drvdata->hotplug_node);
1440 cpuhp_remove_multi_state(drvdata->trbe_online);
1443 static int arm_trbe_probe_irq(struct platform_device *pdev,
1444 struct trbe_drvdata *drvdata)
1448 drvdata->irq = platform_get_irq(pdev, 0);
1449 if (drvdata->irq < 0) {
1450 pr_err("IRQ not found for the platform device\n");
1451 return drvdata->irq;
1454 if (!irq_is_percpu(drvdata->irq)) {
1455 pr_err("IRQ is not a PPI\n");
1459 if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus))
1462 drvdata->handle = alloc_percpu(struct perf_output_handle *);
1463 if (!drvdata->handle)
1466 ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle);
1468 free_percpu(drvdata->handle);
1474 static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata)
1476 free_percpu_irq(drvdata->irq, drvdata->handle);
1477 free_percpu(drvdata->handle);
1480 static int arm_trbe_device_probe(struct platform_device *pdev)
1482 struct coresight_platform_data *pdata;
1483 struct trbe_drvdata *drvdata;
1484 struct device *dev = &pdev->dev;
1487 /* Trace capture is not possible with kernel page table isolation */
1488 if (arm64_kernel_unmapped_at_el0()) {
1489 pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n");
1493 drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1497 pdata = coresight_get_platform_data(dev);
1499 return PTR_ERR(pdata);
1501 dev_set_drvdata(dev, drvdata);
1502 dev->platform_data = pdata;
1503 drvdata->pdev = pdev;
1504 ret = arm_trbe_probe_irq(pdev, drvdata);
1508 ret = arm_trbe_probe_coresight(drvdata);
1512 ret = arm_trbe_probe_cpuhp(drvdata);
1518 arm_trbe_remove_coresight(drvdata);
1520 arm_trbe_remove_irq(drvdata);
1524 static int arm_trbe_device_remove(struct platform_device *pdev)
1526 struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
1528 arm_trbe_remove_cpuhp(drvdata);
1529 arm_trbe_remove_coresight(drvdata);
1530 arm_trbe_remove_irq(drvdata);
1534 static const struct of_device_id arm_trbe_of_match[] = {
1535 { .compatible = "arm,trace-buffer-extension"},
1538 MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
1540 static struct platform_driver arm_trbe_driver = {
1543 .of_match_table = of_match_ptr(arm_trbe_of_match),
1544 .suppress_bind_attrs = true,
1546 .probe = arm_trbe_device_probe,
1547 .remove = arm_trbe_device_remove,
1550 static int __init arm_trbe_init(void)
1554 ret = platform_driver_register(&arm_trbe_driver);
1558 pr_err("Error registering %s platform driver\n", DRVNAME);
1562 static void __exit arm_trbe_exit(void)
1564 platform_driver_unregister(&arm_trbe_driver);
1566 module_init(arm_trbe_init);
1567 module_exit(arm_trbe_exit);
1569 MODULE_AUTHOR("Anshuman Khandual <anshuman.khandual@arm.com>");
1570 MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver");
1571 MODULE_LICENSE("GPL v2");