From 4e32d573ace2611df25b59d0045833baac3e5708 Mon Sep 17 00:00:00 2001 From: Hanjie Lin Date: Fri, 6 Jul 2018 12:58:02 +0800 Subject: [PATCH] perf_event: aml pmu interrupt issue fixup PD#167574: perf_event: aml pmu interrupt issue fixup amlogic arm pmu have a issue that all core's interrupts routes to one gic SPI interrupt, when some core raise a pmu interrupt(arm pmu counter overflow), the global gic SPI interrupt will raise(default on cpu0), and we can obtain core info which caused interrupt from sys_cpu_status0 reg. In global pmu interrupt handler we distinguish interrupts from other cpu, then send a AML ipi interrupt and wait that cpu complete pmu interrupt. Change-Id: I28ada689e5b94671c8cfb6189e46134c3c6804cd Signed-off-by: Hanjie Lin --- arch/arm64/boot/dts/amlogic/mesonaxg.dtsi | 18 +- arch/arm64/boot/dts/amlogic/mesong12a.dtsi | 12 ++ arch/arm64/boot/dts/amlogic/mesongxl.dtsi | 17 +- arch/arm64/boot/dts/amlogic/mesontxlx.dtsi | 17 +- arch/arm64/include/asm/hardirq.h | 4 + arch/arm64/include/asm/perf_event.h | 50 +++++ arch/arm64/kernel/perf_event.c | 289 ++++++++++++++++++++++++++++- arch/arm64/kernel/smp.c | 26 +++ include/linux/smp.h | 6 + 9 files changed, 426 insertions(+), 13 deletions(-) diff --git a/arch/arm64/boot/dts/amlogic/mesonaxg.dtsi b/arch/arm64/boot/dts/amlogic/mesonaxg.dtsi index e3fb2d9..1330bf8 100644 --- a/arch/arm64/boot/dts/amlogic/mesonaxg.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesonaxg.dtsi @@ -121,12 +121,22 @@ }; arm_pmu { compatible = "arm,armv8-pmuv3"; - interrupts = <0 137 4>, - <0 138 4>, - <0 153 4>, - <0 154 4>; + interrupts = <0 137 4>; + reg = <0x0 0xff634400 0 0x1000>; + + /* addr = base + offset << 2 */ + sys_cpu_status0_offset = <0xa0>; + + sys_cpu_status0_pmuirq_mask = <0xf>; + + /* default 10ms */ + relax_timer_ns = <10000000>; + + /* default 10000us */ + max_wait_cnt = <10000>; }; + gic: interrupt-controller@2c001000 { compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic"; #interrupt-cells = <3>; diff --git a/arch/arm64/boot/dts/amlogic/mesong12a.dtsi b/arch/arm64/boot/dts/amlogic/mesong12a.dtsi index 0d752d9..1e5e1d9 100644 --- a/arch/arm64/boot/dts/amlogic/mesong12a.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesong12a.dtsi @@ -158,6 +158,18 @@ arm_pmu { compatible = "arm,armv8-pmuv3"; interrupts = <0 137 4>; + reg = <0x0 0xff634400 0 0x1000>; + + /* addr = base + offset << 2 */ + sys_cpu_status0_offset = <0xa0>; + + sys_cpu_status0_pmuirq_mask = <0xf>; + + /* default 10ms */ + relax_timer_ns = <10000000>; + + /* default 10000us */ + max_wait_cnt = <10000>; }; gic: interrupt-controller@2c001000 { diff --git a/arch/arm64/boot/dts/amlogic/mesongxl.dtsi b/arch/arm64/boot/dts/amlogic/mesongxl.dtsi index 05b0e8d..4729498 100644 --- a/arch/arm64/boot/dts/amlogic/mesongxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesongxl.dtsi @@ -135,10 +135,19 @@ }; arm_pmu { compatible = "arm,armv8-pmuv3"; - interrupts = <0 137 4>, - <0 138 4>, - <0 153 4>, - <0 154 4>; + interrupts = <0 137 4>; + reg = <0x0 0xc8834400 0 0x1000>; + + /* addr = base + offset << 2 */ + sys_cpu_status0_offset = <0xa0>; + + sys_cpu_status0_pmuirq_mask = <0xf>; + + /* default 10ms */ + relax_timer_ns = <10000000>; + + /* default 10000us */ + max_wait_cnt = <10000>; }; gic: interrupt-controller@2c001000 { diff --git a/arch/arm64/boot/dts/amlogic/mesontxlx.dtsi b/arch/arm64/boot/dts/amlogic/mesontxlx.dtsi index f56dc75..58da140 100644 --- a/arch/arm64/boot/dts/amlogic/mesontxlx.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesontxlx.dtsi @@ -134,10 +134,19 @@ arm_pmu { compatible = "arm,armv8-pmuv3"; - interrupts = <0 137 4>, - <0 138 4>, - <0 153 4>, - <0 154 4>; + interrupts = <0 137 4>; + reg = <0x0 0xff634400 0 0x1000>; + + /* addr = base + offset << 2 */ + sys_cpu_status0_offset = <0xa0>; + + sys_cpu_status0_pmuirq_mask = <0xf>; + + /* default 10ms */ + relax_timer_ns = <10000000>; + + /* default 10000us */ + max_wait_cnt = <10000>; }; gic: interrupt-controller@2c001000 { diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index 8740297..9745a1d 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -20,7 +20,11 @@ #include #include +#ifdef CONFIG_AMLOGIC_MODIFY +#define NR_IPI 7 +#else #define NR_IPI 6 +#endif typedef struct { unsigned int __softirq_pending; diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 8d5cbec..11185b4 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -18,6 +18,9 @@ #define __ASM_PERF_EVENT_H #include +#ifdef CONFIG_AMLOGIC_MODIFY +#include +#endif #define ARMV8_PMU_MAX_COUNTERS 32 #define ARMV8_PMU_COUNTER_MASK (ARMV8_PMU_MAX_COUNTERS - 1) @@ -88,4 +91,51 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); (regs)->pstate = PSR_MODE_EL1h; \ } + +#ifdef CONFIG_AMLOGIC_MODIFY + +extern void armv8pmu_handle_irq_ipi(void); + +struct amlpmu_fixup_cpuinfo { + int irq_num; + + int fix_done; + + unsigned long irq_cnt; + unsigned long empty_irq_cnt; + + unsigned long irq_time; + unsigned long empty_irq_time; + + unsigned long last_irq_cnt; + unsigned long last_empty_irq_cnt; + + unsigned long last_irq_time; + unsigned long last_empty_irq_time; +}; + +struct amlpmu_fixup_context { + struct amlpmu_fixup_cpuinfo __percpu *cpuinfo; + + /* struct arm_pmu */ + void *dev; + + /* sys_cpu_status0 reg */ + unsigned int *sys_cpu_status0; + + /* + * In main pmu irq route wait for other cpu fix done may cause lockup, + * when lockup we disable main irq for a while. + * relax_timer will enable main irq again. + */ + struct hrtimer relax_timer; + + /* dts prop */ + unsigned int sys_cpu_status0_offset; + unsigned int sys_cpu_status0_pmuirq_mask; + unsigned int relax_timer_ns; + unsigned int max_wait_cnt; +}; +#endif + #endif diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index 199a23f..77d8aa0 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -19,6 +19,10 @@ * along with this program. If not, see . */ +#ifdef CONFIG_AMLOGIC_MODIFY +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#endif + #include #include #include @@ -29,6 +33,19 @@ #include #include + +#ifdef CONFIG_AMLOGIC_MODIFY +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + /* * ARMv8 PMUv3 Performance Events handling code. * Common event types (some are defined in asm/perf_event.h). @@ -748,6 +765,165 @@ static void armv8pmu_disable_event(struct perf_event *event) raw_spin_unlock_irqrestore(&events->pmu_lock, flags); } +#ifdef CONFIG_AMLOGIC_MODIFY + +static struct amlpmu_fixup_context amlpmu_fixup_ctx; + +static enum hrtimer_restart amlpmu_relax_timer_func(struct hrtimer *timer) +{ + struct amlpmu_fixup_cpuinfo *ci; + + ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0); + + pr_alert("enable cpu0_irq %d again, irq cnt = %lu\n", + ci->irq_num, + ci->irq_cnt); + enable_irq(ci->irq_num); + + return HRTIMER_NORESTART; +} + + +static void amlpmu_relax_timer_start(int other_cpu) +{ + struct amlpmu_fixup_cpuinfo *ci; + int cpu; + + cpu = smp_processor_id(); + WARN_ON(cpu != 0); + + ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0); + + pr_alert("wait cpu %d fixup done timeout, main cpu irq cnt = %lu\n", + other_cpu, + ci->irq_cnt); + + if (hrtimer_active(&amlpmu_fixup_ctx.relax_timer)) { + pr_alert("relax_timer already active, return!\n"); + return; + } + + disable_irq_nosync(ci->irq_num); + + hrtimer_start(&amlpmu_fixup_ctx.relax_timer, + ns_to_ktime(amlpmu_fixup_ctx.relax_timer_ns), + HRTIMER_MODE_REL); +} + +static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev); + +void armv8pmu_handle_irq_ipi(void) +{ + int cpu = smp_processor_id(); + + WARN_ON(cpu == 0); + WARN_ON(!amlpmu_fixup_ctx.dev); + + armv8pmu_handle_irq(-1, amlpmu_fixup_ctx.dev); +} + +static int aml_pmu_fix(void) +{ + int i; + int cpu; + int pmuirq_val; + struct amlpmu_fixup_cpuinfo *ci; + + int max_wait_cnt = amlpmu_fixup_ctx.max_wait_cnt; + + pmuirq_val = readl(amlpmu_fixup_ctx.sys_cpu_status0); + pmuirq_val &= amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask; + + for (cpu = 0; cpu < num_possible_cpus(); cpu++) { + if (pmuirq_val & (1<fix_done = 0; + + /* aml pmu IPI will set fix_done to 1 */ + mb(); + + smp_send_aml_pmu(cpu); + + for (i = 0; i < max_wait_cnt; i++) { + if (READ_ONCE(ci->fix_done)) + break; + + udelay(1); + } + + if (i == amlpmu_fixup_ctx.max_wait_cnt) + amlpmu_relax_timer_start(cpu); + + return 0; + } + } + } + + return 1; +} + +static void aml_pmu_fix_stat_account(int is_empty_irq) +{ + int freq; + unsigned long time = jiffies; + struct amlpmu_fixup_cpuinfo *ci; + + ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo); + + ci->irq_cnt++; + ci->irq_time = time; + if (!ci->last_irq_cnt) { + ci->last_irq_cnt = ci->irq_cnt; + ci->last_irq_time = ci->irq_time; + } + + if (is_empty_irq) { + ci->empty_irq_cnt++; + ci->empty_irq_time = time; + if (!ci->last_empty_irq_cnt) { + ci->last_empty_irq_cnt = ci->empty_irq_cnt; + ci->last_empty_irq_time = ci->empty_irq_time; + } + } + + if (time_after(ci->irq_time, ci->last_irq_time + HZ)) { + freq = ci->irq_cnt - ci->last_irq_cnt; + freq = freq * HZ / (ci->irq_time - ci->last_irq_time); + pr_debug("irq_cnt = %lu, irq_last_cnt = %lu, freq = %d\n", + ci->irq_cnt, + ci->last_irq_cnt, + freq); + + ci->last_irq_cnt = ci->irq_cnt; + ci->last_irq_time = ci->irq_time; + } + + if (is_empty_irq && + time_after(ci->empty_irq_time, ci->last_empty_irq_time + HZ)) { + + freq = ci->empty_irq_cnt - ci->last_empty_irq_cnt; + freq *= HZ; + freq /= (ci->empty_irq_time - ci->last_empty_irq_time); + pr_debug("empty_irq_cnt = %lu, freq = %d\n", + ci->empty_irq_cnt, + freq); + + ci->last_empty_irq_cnt = ci->empty_irq_cnt; + ci->last_empty_irq_time = ci->empty_irq_time; + } +} +#endif + static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev) { u32 pmovsr; @@ -757,17 +933,45 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev) struct pt_regs *regs; int idx; +#ifdef CONFIG_AMLOGIC_MODIFY + int cpu; + int is_empty_irq = 0; + struct amlpmu_fixup_cpuinfo *ci; + + ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo); + ci->irq_num = irq_num; + amlpmu_fixup_ctx.dev = dev; + cpu = smp_processor_id(); +#endif + /* * Get and reset the IRQ flags */ pmovsr = armv8pmu_getreset_flags(); +#ifdef CONFIG_AMLOGIC_MODIFY + ci->fix_done = 1; +#endif /* * Did an overflow occur? */ +#ifdef CONFIG_AMLOGIC_MODIFY + if (!armv8pmu_has_overflowed(pmovsr)) { + is_empty_irq = 1; + + if (cpu == 0) + is_empty_irq = aml_pmu_fix(); + } + + aml_pmu_fix_stat_account(is_empty_irq); + + /* txlx have some empty pmu irqs, so return IRQ_HANDLED */ + if (is_empty_irq) + return IRQ_HANDLED; +#else if (!armv8pmu_has_overflowed(pmovsr)) return IRQ_NONE; - +#endif /* * Handle the counter(s) overflow(s) */ @@ -810,6 +1014,9 @@ static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev) return IRQ_HANDLED; } + + + static void armv8pmu_start(struct arm_pmu *cpu_pmu) { unsigned long flags; @@ -1100,8 +1307,88 @@ static const struct pmu_probe_info armv8_pmu_probe_table[] = { { /* sentinel value */ } }; + +#ifdef CONFIG_AMLOGIC_MODIFY +static int amlpmu_fixup_init(struct platform_device *pdev) +{ + int ret; + void __iomem *base; + + amlpmu_fixup_ctx.cpuinfo = __alloc_percpu( + sizeof(struct amlpmu_fixup_cpuinfo), 2 * sizeof(void *)); + if (!amlpmu_fixup_ctx.cpuinfo) { + pr_err("alloc percpu failed\n"); + return -ENOMEM; + } + + base = of_iomap(pdev->dev.of_node, 0); + if (IS_ERR(base)) { + pr_err("of_iomap() failed, base = %p\n", base); + return PTR_ERR(base); + } + + ret = of_property_read_u32(pdev->dev.of_node, + "sys_cpu_status0_offset", + &amlpmu_fixup_ctx.sys_cpu_status0_offset); + if (ret) { + pr_err("read sys_cpu_status0_offset failed, ret = %d\n", ret); + return 1; + } + pr_debug("sys_cpu_status0_offset = 0x%0x\n", + amlpmu_fixup_ctx.sys_cpu_status0_offset); + + ret = of_property_read_u32(pdev->dev.of_node, + "sys_cpu_status0_pmuirq_mask", + &amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask); + if (ret) { + pr_err("read sys_cpu_status0_pmuirq_mask failed, ret = %d\n", + ret); + return 1; + } + pr_debug("sys_cpu_status0_pmuirq_mask = 0x%0x\n", + amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask); + + + ret = of_property_read_u32(pdev->dev.of_node, + "relax_timer_ns", + &amlpmu_fixup_ctx.relax_timer_ns); + if (ret) { + pr_err("read prop relax_timer_ns failed, ret = %d\n", ret); + return 1; + } + pr_debug("relax_timer_ns = %u\n", amlpmu_fixup_ctx.relax_timer_ns); + + + ret = of_property_read_u32(pdev->dev.of_node, + "max_wait_cnt", + &amlpmu_fixup_ctx.max_wait_cnt); + if (ret) { + pr_err("read prop max_wait_cnt failed, ret = %d\n", ret); + return 1; + } + pr_debug("max_wait_cnt = %u\n", amlpmu_fixup_ctx.max_wait_cnt); + + + base += (amlpmu_fixup_ctx.sys_cpu_status0_offset << 2); + amlpmu_fixup_ctx.sys_cpu_status0 = base; + pr_debug("sys_cpu_status0 = %p\n", amlpmu_fixup_ctx.sys_cpu_status0); + + + hrtimer_init(&amlpmu_fixup_ctx.relax_timer, + CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + amlpmu_fixup_ctx.relax_timer.function = amlpmu_relax_timer_func; + + return 0; +} +#endif + static int armv8_pmu_device_probe(struct platform_device *pdev) { +#ifdef CONFIG_AMLOGIC_MODIFY + if (amlpmu_fixup_init(pdev)) + return 1; +#endif if (acpi_disabled) return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids, NULL); diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 9870bad..4097031 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -55,6 +55,10 @@ #include #include +#ifdef CONFIG_AMLOGIC_MODIFY +#include +#endif + #define CREATE_TRACE_POINTS #include @@ -76,7 +80,12 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_TIMER, IPI_IRQ_WORK, +#ifdef CONFIG_AMLOGIC_MODIFY + IPI_WAKEUP, + IPI_AML_PMU +#else IPI_WAKEUP +#endif }; #ifdef CONFIG_ARM64_VHE @@ -756,6 +765,9 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"), S(IPI_WAKEUP, "CPU wake-up interrupts"), +#ifdef CONFIG_AMLOGIC_MODIFY + S(IPI_AML_PMU, "AML pmu cross interrupts"), +#endif }; static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) @@ -764,6 +776,13 @@ static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) __smp_cross_call(target, ipinr); } +#ifdef CONFIG_AMLOGIC_MODIFY +void smp_send_aml_pmu(int cpu) +{ + smp_cross_call(cpumask_of(cpu), IPI_AML_PMU); +} +#endif + void show_ipi_list(struct seq_file *p, int prec) { unsigned int cpu, i; @@ -827,6 +846,7 @@ static void ipi_cpu_stop(unsigned int cpu) cpu_relax(); } + /* * Main handler for inter-processor interrupts */ @@ -881,6 +901,12 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; #endif +#ifdef CONFIG_AMLOGIC_MODIFY + case IPI_AML_PMU: + armv8pmu_handle_irq_ipi(); + break; +#endif + default: pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); break; diff --git a/include/linux/smp.h b/include/linux/smp.h index 8e0cb7a..c9b5fb3 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -73,6 +73,12 @@ extern void smp_send_stop(void); */ extern void smp_send_reschedule(int cpu); +#ifdef CONFIG_AMLOGIC_MODIFY +/* + * sends a 'aml pmu' event to another CPU: + */ +extern void smp_send_aml_pmu(int cpu); +#endif /* * Prepare machine for booting other CPUs. -- 2.7.4