bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,cortex-a15-pmu";
- interrupts = <0 137 4>;
- reg = <0xff634400 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xff634680 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
-
gic: interrupt-controller@2c001000 {
compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic";
#interrupt-cells = <3>;
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,cortex-a15-pmu";
- interrupts = <0 137 4>;
- reg = <0xff634400 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xff634680 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
};
timer {
- compatible = "arm,armv8-timer";
+ compatible = "arm,armv7-timer";
interrupts = <GIC_PPI 13 0xff08>,
<GIC_PPI 14 0xff08>,
<GIC_PPI 11 0xff08>,
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
- compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
+ compatible = "arm,cortex-a15-pmu";
+ clusterb-enabled;
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xff634680 0x4>,
+ <0xff6347c0 0x04>;
+ cpumasks = <0x3 0x3C>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
arm_pmu {
compatible = "arm,cortex-a15-pmu";
- interrupts = <0 137 4>;
- reg = <0xc8834400 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xc8834400 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
};
timer {
- compatible = "arm,armv8-timer";
+ compatible = "arm,armv7-timer";
interrupts = <GIC_PPI 13 0xff08>,
<GIC_PPI 14 0xff08>,
<GIC_PPI 11 0xff08>,
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
- compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>,
- <0 138 4>,
- <0 153 4>,
- <0 154 4>;
+ compatible = "arm,cortex-a15-pmu";
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xc8834400 0x4>;
+ cpumasks = <0xf>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
};
timer {
- compatible = "arm,armv8-timer";
+ compatible = "arm,armv7-timer";
interrupts = <GIC_PPI 13 0xff08>,
<GIC_PPI 14 0xff08>,
<GIC_PPI 11 0xff08>,
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
- compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>,
- <0 138 4>,
- <0 153 4>,
- <0 154 4>;
+ compatible = "arm,cortex-a15-pmu";
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xc8834400 0x4>;
+ cpumasks = <0xf>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
arm_pmu {
compatible = "arm,cortex-a15-pmu";
+ /* clusterb-enabled; */
interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
- reg = <0xff634400 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ reg = <0xff634680 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
bit_resolution = <0>;
};
- pmu {
+ arm_pmu {
compatible = "arm,cortex-a15-pmu";
- interrupts = <0 137 4>;
- reg = <0xc8834400 0x1000>;
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
- sys_cpu_status0_pmuirq_mask = <0xf>;
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xc8834400 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
psci {
arm_pmu {
compatible = "arm,cortex-a15-pmu";
- interrupts = <0 137 4>;
- reg = <0xff634400 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xff634680 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
#include <linux/threads.h>
#include <asm/irq.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#define NR_IPI 8
-#else
#define NR_IPI 7
-#endif
typedef struct {
unsigned int __softirq_pending;
(regs)->ARM_sp = current_stack_pointer; \
(regs)->ARM_cpsr = SVC_MODE; \
}
-
-#ifdef CONFIG_AMLOGIC_MODIFY
-
-extern void armv8pmu_handle_irq_ipi(void);
-
-struct amlpmu_fixup_cpuinfo {
- int irq_num;
-
- int fix_done;
-
- unsigned long irq_cnt;
- unsigned long empty_irq_cnt;
-
- unsigned long irq_time;
- unsigned long empty_irq_time;
-
- unsigned long last_irq_cnt;
- unsigned long last_empty_irq_cnt;
-
- unsigned long last_irq_time;
- unsigned long last_empty_irq_time;
-};
-
-struct amlpmu_fixup_context {
- struct amlpmu_fixup_cpuinfo __percpu *cpuinfo;
-
- /* struct arm_pmu */
- void *dev;
-
- /* sys_cpu_status0 reg */
- unsigned int *sys_cpu_status0;
-
- /*
- * In main pmu irq route wait for other cpu fix done may cause lockup,
- * when lockup we disable main irq for a while.
- * relax_timer will enable main irq again.
- */
- struct hrtimer relax_timer;
-
- /* dts prop */
- unsigned int sys_cpu_status0_offset;
- unsigned int sys_cpu_status0_pmuirq_mask;
- unsigned int relax_timer_ns;
- unsigned int max_wait_cnt;
-};
-#endif
-
#endif /* __ARM_PERF_EVENT_H__ */
#ifdef CONFIG_CPU_V7
-#ifdef CONFIG_AMLOGIC_MODIFY
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#endif
-
#include <asm/cp15.h>
#include <asm/cputype.h>
#include <asm/irq_regs.h>
#include <linux/perf/arm_pmu.h>
#include <linux/platform_device.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#include <linux/of.h>
-#include <linux/of_irq.h>
-#include <linux/of_device.h>
-#include <linux/irq.h>
-#include <asm/irq.h>
-#include <linux/interrupt.h>
-#include <linux/irqdesc.h>
-#include <linux/of_address.h>
-#include <linux/delay.h>
-#endif
-
-
/*
* Common ARMv7 event types
*
}
#ifdef CONFIG_AMLOGIC_MODIFY
-static struct amlpmu_fixup_context amlpmu_fixup_ctx;
-
-static enum hrtimer_restart amlpmu_relax_timer_func(struct hrtimer *timer)
-{
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
-
- pr_alert("enable cpu0_irq %d again, irq cnt = %lu\n",
- ci->irq_num,
- ci->irq_cnt);
- enable_irq(ci->irq_num);
-
- return HRTIMER_NORESTART;
-}
-
-
-static void amlpmu_relax_timer_start(int other_cpu)
-{
- struct amlpmu_fixup_cpuinfo *ci;
- int cpu;
-
- cpu = smp_processor_id();
- WARN_ON(cpu != 0);
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
-
- pr_alert("wait cpu %d fixup done timeout, main cpu irq cnt = %lu\n",
- other_cpu,
- ci->irq_cnt);
-
- if (hrtimer_active(&amlpmu_fixup_ctx.relax_timer)) {
- pr_alert("relax_timer already active, return!\n");
- return;
- }
-
- disable_irq_nosync(ci->irq_num);
-
- hrtimer_start(&amlpmu_fixup_ctx.relax_timer,
- ns_to_ktime(amlpmu_fixup_ctx.relax_timer_ns),
- HRTIMER_MODE_REL);
-}
+#include <linux/perf/arm_pmu.h>
static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev);
-void armv8pmu_handle_irq_ipi(void)
+void amlpmu_handle_irq_ipi(void *arg)
{
- int cpu = smp_processor_id();
-
- WARN_ON(cpu == 0);
- WARN_ON(!amlpmu_fixup_ctx.dev);
-
- armv7pmu_handle_irq(-1, amlpmu_fixup_ctx.dev);
-}
-
-static int aml_pmu_fix(void)
-{
- int i;
- int cpu;
- int pmuirq_val;
- struct amlpmu_fixup_cpuinfo *ci;
-
- int max_wait_cnt = amlpmu_fixup_ctx.max_wait_cnt;
-
- pmuirq_val = readl(amlpmu_fixup_ctx.sys_cpu_status0);
- pmuirq_val &= amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask;
-
- for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
- if (pmuirq_val & (1<<cpu)) {
- if (cpu == 0) {
- pr_debug("cpu0 shouldn't fix pmuirq = 0x%x\n",
- pmuirq_val);
- } else {
- pr_debug("fix pmu irq cpu %d, pmuirq = 0x%x\n",
- cpu,
- pmuirq_val);
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo,
- cpu);
-
- ci->fix_done = 0;
-
- /* aml pmu IPI will set fix_done to 1 */
- mb();
-
- smp_send_aml_pmu(cpu);
-
- for (i = 0; i < max_wait_cnt; i++) {
- if (READ_ONCE(ci->fix_done))
- break;
-
- udelay(1);
- }
-
- if (i == amlpmu_fixup_ctx.max_wait_cnt)
- amlpmu_relax_timer_start(cpu);
-
- return 0;
- }
- }
- }
-
- return 1;
-}
-
-static void aml_pmu_fix_stat_account(int is_empty_irq)
-{
- int freq;
- unsigned long time = jiffies;
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
-
- ci->irq_cnt++;
- ci->irq_time = time;
- if (!ci->last_irq_cnt) {
- ci->last_irq_cnt = ci->irq_cnt;
- ci->last_irq_time = ci->irq_time;
- }
-
- if (is_empty_irq) {
- ci->empty_irq_cnt++;
- ci->empty_irq_time = time;
- if (!ci->last_empty_irq_cnt) {
- ci->last_empty_irq_cnt = ci->empty_irq_cnt;
- ci->last_empty_irq_time = ci->empty_irq_time;
- }
- }
-
- if (time_after(ci->irq_time, ci->last_irq_time + HZ)) {
- freq = ci->irq_cnt - ci->last_irq_cnt;
- freq = freq * HZ / (ci->irq_time - ci->last_irq_time);
- pr_debug("irq_cnt = %lu, irq_last_cnt = %lu, freq = %d\n",
- ci->irq_cnt,
- ci->last_irq_cnt,
- freq);
-
- ci->last_irq_cnt = ci->irq_cnt;
- ci->last_irq_time = ci->irq_time;
- }
-
- if (is_empty_irq &&
- time_after(ci->empty_irq_time, ci->last_empty_irq_time + HZ)) {
-
- freq = ci->empty_irq_cnt - ci->last_empty_irq_cnt;
- freq *= HZ;
- freq /= (ci->empty_irq_time - ci->last_empty_irq_time);
- pr_debug("empty_irq_cnt = %lu, freq = %d\n",
- ci->empty_irq_cnt,
- freq);
-
- ci->last_empty_irq_cnt = ci->empty_irq_cnt;
- ci->last_empty_irq_time = ci->empty_irq_time;
- }
+ armv7pmu_handle_irq(-1, amlpmu_ctx.pmu);
}
#endif
struct pt_regs *regs;
int idx;
-#ifdef CONFIG_AMLOGIC_MODIFY
- int cpu;
- int is_empty_irq = 0;
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
- ci->irq_num = irq_num;
- amlpmu_fixup_ctx.dev = dev;
- cpu = smp_processor_id();
-#endif
-
/*
* Get and reset the IRQ flags
*/
pmnc = armv7_pmnc_getreset_flags();
#ifdef CONFIG_AMLOGIC_MODIFY
- ci->fix_done = 1;
-#endif
-
+ /* amlpmu have routed the interrupt successfully, return IRQ_HANDLED */
+ if (amlpmu_handle_irq(cpu_pmu,
+ irq_num,
+ armv7_pmnc_has_overflowed(pmnc)))
+ return IRQ_HANDLED;
+#else
/*
* Did an overflow occur?
*/
-#ifdef CONFIG_AMLOGIC_MODIFY
- if (!armv7_pmnc_has_overflowed(pmnc)) {
- is_empty_irq = 1;
-
- if (cpu == 0)
- is_empty_irq = aml_pmu_fix();
- }
-
- aml_pmu_fix_stat_account(is_empty_irq);
-
- /* txlx have some empty pmu irqs, so return IRQ_HANDLED */
- if (is_empty_irq)
- return IRQ_HANDLED;
-#else
if (!armv7_pmnc_has_overflowed(pmnc))
return IRQ_NONE;
-
#endif
+
/*
* Handle the counter(s) overflow(s)
*/
{ /* sentinel value */ }
};
-#ifdef CONFIG_AMLOGIC_MODIFY
-static int amlpmu_fixup_init(struct platform_device *pdev)
-{
- int ret;
- void __iomem *base;
-
- amlpmu_fixup_ctx.cpuinfo = __alloc_percpu(
- sizeof(struct amlpmu_fixup_cpuinfo), 2 * sizeof(void *));
- if (!amlpmu_fixup_ctx.cpuinfo) {
- pr_err("alloc percpu failed\n");
- return -ENOMEM;
- }
-
- base = of_iomap(pdev->dev.of_node, 0);
- if (IS_ERR(base)) {
- pr_err("of_iomap() failed, base = %p\n", base);
- return PTR_ERR(base);
- }
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "sys_cpu_status0_offset",
- &amlpmu_fixup_ctx.sys_cpu_status0_offset);
- if (ret) {
- pr_err("read sys_cpu_status0_offset failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("sys_cpu_status0_offset = 0x%0x\n",
- amlpmu_fixup_ctx.sys_cpu_status0_offset);
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "sys_cpu_status0_pmuirq_mask",
- &amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
- if (ret) {
- pr_err("read sys_cpu_status0_pmuirq_mask failed, ret = %d\n",
- ret);
- return 1;
- }
- pr_debug("sys_cpu_status0_pmuirq_mask = 0x%0x\n",
- amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
-
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "relax_timer_ns",
- &amlpmu_fixup_ctx.relax_timer_ns);
- if (ret) {
- pr_err("read prop relax_timer_ns failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("relax_timer_ns = %u\n", amlpmu_fixup_ctx.relax_timer_ns);
-
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "max_wait_cnt",
- &amlpmu_fixup_ctx.max_wait_cnt);
- if (ret) {
- pr_err("read prop max_wait_cnt failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("max_wait_cnt = %u\n", amlpmu_fixup_ctx.max_wait_cnt);
-
-
- base += (amlpmu_fixup_ctx.sys_cpu_status0_offset << 2);
- amlpmu_fixup_ctx.sys_cpu_status0 = base;
- pr_debug("sys_cpu_status0 = %p\n", amlpmu_fixup_ctx.sys_cpu_status0);
-
-
- hrtimer_init(&amlpmu_fixup_ctx.relax_timer,
- CLOCK_MONOTONIC,
- HRTIMER_MODE_REL);
- amlpmu_fixup_ctx.relax_timer.function = amlpmu_relax_timer_func;
-
- return 0;
-}
-#endif
-
static int armv7_pmu_device_probe(struct platform_device *pdev)
{
-#ifdef CONFIG_AMLOGIC_MODIFY
- if (amlpmu_fixup_init(pdev))
- return 1;
-#endif
-
return arm_pmu_device_probe(pdev, armv7_pmu_of_device_ids,
armv7_pmu_probe_table);
}
#include <asm/mach/arch.h>
#include <asm/mpu.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#include <asm/perf_event.h>
-#endif
-
#define CREATE_TRACE_POINTS
#include <trace/events/ipi.h>
IPI_CPU_STOP,
IPI_IRQ_WORK,
IPI_COMPLETION,
- #ifdef CONFIG_AMLOGIC_MODIFY
- IPI_AML_PMU,
- #endif
IPI_CPU_BACKTRACE,
/*
* SGI8-15 can be reserved by secure firmware, and thus may
S(IPI_CPU_STOP, "CPU stop interrupts"),
S(IPI_IRQ_WORK, "IRQ work interrupts"),
S(IPI_COMPLETION, "completion interrupts"),
-#ifdef CONFIG_AMLOGIC_MODIFY
- S(IPI_AML_PMU, "AML pmu cross interrupts"),
-#endif
};
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
__smp_cross_call(target, ipinr);
}
-#ifdef CONFIG_AMLOGIC_MODIFY
-void smp_send_aml_pmu(int cpu)
-{
- smp_cross_call(cpumask_of(cpu), IPI_AML_PMU);
-}
-#endif
-
void show_ipi_list(struct seq_file *p, int prec)
{
unsigned int cpu, i;
printk_nmi_exit();
break;
-#ifdef CONFIG_AMLOGIC_MODIFY
- case IPI_AML_PMU:
- armv8pmu_handle_irq_ipi();
- break;
-#endif
-
default:
pr_crit("CPU%u: Unknown IPI message 0x%x\n",
cpu, ipinr);
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
- reg = <0x0 0xff634400 0 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xff634680 0x0 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
-
gic: interrupt-controller@2c001000 {
compatible = "arm,cortex-a15-gic", "arm,cortex-a9-gic";
#interrupt-cells = <3>;
bit_mode=<12>;
bit_resolution=<0>;
};
- arm_pmu {
- compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
- reg = <0x0 0xff634400 0 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
+ arm_pmu {
+ compatible = "arm,cortex-a15-pmu";
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0xff634680 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
max-duty-cycle = <1250>;
/* Voltage Duty-Cycle */
voltage-table = <1022000 0>,
- <1011000 3>,
+ <1011000 3>,
<1001000 6>,
<991000 10>,
<981000 13>,
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
+ clusterb-enabled;
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+ <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xff634680 0x0 0x4>,
+ <0x0 0xff6347c0 0x0 0x04>;
+ cpumasks = <0x3 0x3C>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
- reg = <0x0 0xc8834400 0 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xc8834400 0x0 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>,
- <0 138 4>,
- <0 153 4>,
- <0 154 4>;
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xc8834400 0x0 0x4>;
+ cpumasks = <0xf>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
bit_mode=<12>;
bit_resolution=<0>;
};
+
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>,
- <0 138 4>,
- <0 153 4>,
- <0 154 4>;
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xc8834400 0x0 0x4>;
+ cpumasks = <0xf>;
+ /* default 10ms */
+ relax-timer-ns = <10000000>;
+ /* default 10000us */
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
arm_pmu {
compatible = "arm,armv8-pmuv3";
+ /* clusterb-enabled; */
interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
- reg = <0x0 0xff634400 0x0 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ reg = <0x0 0xff634680 0x0 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
- reg = <0x0 0xc8834400 0 0x1000>;
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
- sys_cpu_status0_pmuirq_mask = <0xf>;
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xc8834400 0x0 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
psci {
arm_pmu {
compatible = "arm,armv8-pmuv3";
- interrupts = <0 137 4>;
- reg = <0x0 0xff634400 0 0x1000>;
-
- /* addr = base + offset << 2 */
- sys_cpu_status0_offset = <0xa0>;
-
- sys_cpu_status0_pmuirq_mask = <0xf>;
-
+ /* clusterb-enabled; */
+ interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
+ reg = <0x0 0xff634680 0x0 0x4>;
+ cpumasks = <0xf>;
/* default 10ms */
- relax_timer_ns = <10000000>;
-
+ relax-timer-ns = <10000000>;
/* default 10000us */
- max_wait_cnt = <10000>;
+ max-wait-cnt = <10000>;
};
gic: interrupt-controller@2c001000 {
#include <linux/threads.h>
#include <asm/irq.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#define NR_IPI 7
-#else
#define NR_IPI 6
-#endif
typedef struct {
unsigned int __softirq_pending;
}
-#ifdef CONFIG_AMLOGIC_MODIFY
-
-extern void armv8pmu_handle_irq_ipi(void);
-
-struct amlpmu_fixup_cpuinfo {
- int irq_num;
-
- int fix_done;
-
- unsigned long irq_cnt;
- unsigned long empty_irq_cnt;
-
- unsigned long irq_time;
- unsigned long empty_irq_time;
-
- unsigned long last_irq_cnt;
- unsigned long last_empty_irq_cnt;
- unsigned long last_irq_time;
- unsigned long last_empty_irq_time;
-};
-
-struct amlpmu_fixup_context {
- struct amlpmu_fixup_cpuinfo __percpu *cpuinfo;
-
- /* struct arm_pmu */
- void *dev;
-
- /* sys_cpu_status0 reg */
- unsigned int *sys_cpu_status0;
-
- /*
- * In main pmu irq route wait for other cpu fix done may cause lockup,
- * when lockup we disable main irq for a while.
- * relax_timer will enable main irq again.
- */
- struct hrtimer relax_timer;
-
- /* dts prop */
- unsigned int sys_cpu_status0_offset;
- unsigned int sys_cpu_status0_pmuirq_mask;
- unsigned int relax_timer_ns;
- unsigned int max_wait_cnt;
-};
-#endif
#endif
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-#ifdef CONFIG_AMLOGIC_MODIFY
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#endif
-
#include <asm/irq_regs.h>
#include <asm/perf_event.h>
#include <asm/sysreg.h>
#include <linux/platform_device.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#include <linux/of.h>
-#include <linux/of_irq.h>
-#include <linux/of_device.h>
-#include <linux/irq.h>
-#include <asm/irq.h>
-#include <linux/interrupt.h>
-#include <linux/irqdesc.h>
-#include <linux/of_address.h>
-#include <linux/delay.h>
-#endif
-
/*
* ARMv8 PMUv3 Performance Events handling code.
* Common event types (some are defined in asm/perf_event.h).
}
#ifdef CONFIG_AMLOGIC_MODIFY
-
-static struct amlpmu_fixup_context amlpmu_fixup_ctx;
-
-static enum hrtimer_restart amlpmu_relax_timer_func(struct hrtimer *timer)
-{
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
-
- pr_alert("enable cpu0_irq %d again, irq cnt = %lu\n",
- ci->irq_num,
- ci->irq_cnt);
- enable_irq(ci->irq_num);
-
- return HRTIMER_NORESTART;
-}
-
-
-static void amlpmu_relax_timer_start(int other_cpu)
-{
- struct amlpmu_fixup_cpuinfo *ci;
- int cpu;
-
- cpu = smp_processor_id();
- WARN_ON(cpu != 0);
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo, 0);
-
- pr_alert("wait cpu %d fixup done timeout, main cpu irq cnt = %lu\n",
- other_cpu,
- ci->irq_cnt);
-
- if (hrtimer_active(&amlpmu_fixup_ctx.relax_timer)) {
- pr_alert("relax_timer already active, return!\n");
- return;
- }
-
- disable_irq_nosync(ci->irq_num);
-
- hrtimer_start(&amlpmu_fixup_ctx.relax_timer,
- ns_to_ktime(amlpmu_fixup_ctx.relax_timer_ns),
- HRTIMER_MODE_REL);
-}
+#include <linux/perf/arm_pmu.h>
static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev);
-void armv8pmu_handle_irq_ipi(void)
+void amlpmu_handle_irq_ipi(void *arg)
{
- int cpu = smp_processor_id();
-
- WARN_ON(cpu == 0);
- WARN_ON(!amlpmu_fixup_ctx.dev);
-
- armv8pmu_handle_irq(-1, amlpmu_fixup_ctx.dev);
-}
-
-static int aml_pmu_fix(void)
-{
- int i;
- int cpu;
- int pmuirq_val;
- struct amlpmu_fixup_cpuinfo *ci;
-
- int max_wait_cnt = amlpmu_fixup_ctx.max_wait_cnt;
-
- pmuirq_val = readl(amlpmu_fixup_ctx.sys_cpu_status0);
- pmuirq_val &= amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask;
-
- for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
- if (pmuirq_val & (1<<cpu)) {
- if (cpu == 0) {
- pr_debug("cpu0 shouldn't fix pmuirq = 0x%x\n",
- pmuirq_val);
- } else {
- pr_debug("fix pmu irq cpu %d, pmuirq = 0x%x\n",
- cpu,
- pmuirq_val);
-
- ci = per_cpu_ptr(amlpmu_fixup_ctx.cpuinfo,
- cpu);
-
- ci->fix_done = 0;
-
- /* aml pmu IPI will set fix_done to 1 */
- mb();
-
- smp_send_aml_pmu(cpu);
-
- for (i = 0; i < max_wait_cnt; i++) {
- if (READ_ONCE(ci->fix_done))
- break;
-
- udelay(1);
- }
-
- if (i == amlpmu_fixup_ctx.max_wait_cnt)
- amlpmu_relax_timer_start(cpu);
-
- return 0;
- }
- }
- }
-
- return 1;
-}
-
-static void aml_pmu_fix_stat_account(int is_empty_irq)
-{
- int freq;
- unsigned long time = jiffies;
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
-
- ci->irq_cnt++;
- ci->irq_time = time;
- if (!ci->last_irq_cnt) {
- ci->last_irq_cnt = ci->irq_cnt;
- ci->last_irq_time = ci->irq_time;
- }
-
- if (is_empty_irq) {
- ci->empty_irq_cnt++;
- ci->empty_irq_time = time;
- if (!ci->last_empty_irq_cnt) {
- ci->last_empty_irq_cnt = ci->empty_irq_cnt;
- ci->last_empty_irq_time = ci->empty_irq_time;
- }
- }
-
- if (time_after(ci->irq_time, ci->last_irq_time + HZ)) {
- freq = ci->irq_cnt - ci->last_irq_cnt;
- freq = freq * HZ / (ci->irq_time - ci->last_irq_time);
- pr_debug("irq_cnt = %lu, irq_last_cnt = %lu, freq = %d\n",
- ci->irq_cnt,
- ci->last_irq_cnt,
- freq);
-
- ci->last_irq_cnt = ci->irq_cnt;
- ci->last_irq_time = ci->irq_time;
- }
-
- if (is_empty_irq &&
- time_after(ci->empty_irq_time, ci->last_empty_irq_time + HZ)) {
-
- freq = ci->empty_irq_cnt - ci->last_empty_irq_cnt;
- freq *= HZ;
- freq /= (ci->empty_irq_time - ci->last_empty_irq_time);
- pr_debug("empty_irq_cnt = %lu, freq = %d\n",
- ci->empty_irq_cnt,
- freq);
-
- ci->last_empty_irq_cnt = ci->empty_irq_cnt;
- ci->last_empty_irq_time = ci->empty_irq_time;
- }
+ armv8pmu_handle_irq(-1, amlpmu_ctx.pmu);
}
#endif
struct pt_regs *regs;
int idx;
-#ifdef CONFIG_AMLOGIC_MODIFY
- int cpu;
- int is_empty_irq = 0;
- struct amlpmu_fixup_cpuinfo *ci;
-
- ci = this_cpu_ptr(amlpmu_fixup_ctx.cpuinfo);
- ci->irq_num = irq_num;
- amlpmu_fixup_ctx.dev = dev;
- cpu = smp_processor_id();
-#endif
-
/*
* Get and reset the IRQ flags
*/
pmovsr = armv8pmu_getreset_flags();
#ifdef CONFIG_AMLOGIC_MODIFY
- ci->fix_done = 1;
-#endif
+ /* amlpmu have routed the interrupt already, so return IRQ_HANDLED */
+ if (amlpmu_handle_irq(cpu_pmu,
+ irq_num,
+ armv8pmu_has_overflowed(pmovsr)))
+ return IRQ_HANDLED;
+#else
/*
* Did an overflow occur?
*/
-#ifdef CONFIG_AMLOGIC_MODIFY
- if (!armv8pmu_has_overflowed(pmovsr)) {
- is_empty_irq = 1;
-
- if (cpu == 0)
- is_empty_irq = aml_pmu_fix();
- }
-
- aml_pmu_fix_stat_account(is_empty_irq);
-
- /* txlx have some empty pmu irqs, so return IRQ_HANDLED */
- if (is_empty_irq)
- return IRQ_HANDLED;
-#else
if (!armv8pmu_has_overflowed(pmovsr))
return IRQ_NONE;
#endif
return IRQ_HANDLED;
}
-
-
-
static void armv8pmu_start(struct arm_pmu *cpu_pmu)
{
unsigned long flags;
};
-#ifdef CONFIG_AMLOGIC_MODIFY
-static int amlpmu_fixup_init(struct platform_device *pdev)
-{
- int ret;
- void __iomem *base;
-
- amlpmu_fixup_ctx.cpuinfo = __alloc_percpu(
- sizeof(struct amlpmu_fixup_cpuinfo), 2 * sizeof(void *));
- if (!amlpmu_fixup_ctx.cpuinfo) {
- pr_err("alloc percpu failed\n");
- return -ENOMEM;
- }
-
- base = of_iomap(pdev->dev.of_node, 0);
- if (IS_ERR(base)) {
- pr_err("of_iomap() failed, base = %p\n", base);
- return PTR_ERR(base);
- }
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "sys_cpu_status0_offset",
- &amlpmu_fixup_ctx.sys_cpu_status0_offset);
- if (ret) {
- pr_err("read sys_cpu_status0_offset failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("sys_cpu_status0_offset = 0x%0x\n",
- amlpmu_fixup_ctx.sys_cpu_status0_offset);
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "sys_cpu_status0_pmuirq_mask",
- &amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
- if (ret) {
- pr_err("read sys_cpu_status0_pmuirq_mask failed, ret = %d\n",
- ret);
- return 1;
- }
- pr_debug("sys_cpu_status0_pmuirq_mask = 0x%0x\n",
- amlpmu_fixup_ctx.sys_cpu_status0_pmuirq_mask);
-
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "relax_timer_ns",
- &amlpmu_fixup_ctx.relax_timer_ns);
- if (ret) {
- pr_err("read prop relax_timer_ns failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("relax_timer_ns = %u\n", amlpmu_fixup_ctx.relax_timer_ns);
-
-
- ret = of_property_read_u32(pdev->dev.of_node,
- "max_wait_cnt",
- &amlpmu_fixup_ctx.max_wait_cnt);
- if (ret) {
- pr_err("read prop max_wait_cnt failed, ret = %d\n", ret);
- return 1;
- }
- pr_debug("max_wait_cnt = %u\n", amlpmu_fixup_ctx.max_wait_cnt);
-
-
- base += (amlpmu_fixup_ctx.sys_cpu_status0_offset << 2);
- amlpmu_fixup_ctx.sys_cpu_status0 = base;
- pr_debug("sys_cpu_status0 = %p\n", amlpmu_fixup_ctx.sys_cpu_status0);
-
-
- hrtimer_init(&amlpmu_fixup_ctx.relax_timer,
- CLOCK_MONOTONIC,
- HRTIMER_MODE_REL);
- amlpmu_fixup_ctx.relax_timer.function = amlpmu_relax_timer_func;
-
- return 0;
-}
-#endif
-
static int armv8_pmu_device_probe(struct platform_device *pdev)
{
-#ifdef CONFIG_AMLOGIC_MODIFY
- if (amlpmu_fixup_init(pdev))
- return 1;
-#endif
if (acpi_disabled)
return arm_pmu_device_probe(pdev, armv8_pmu_of_device_ids,
NULL);
#include <asm/ptrace.h>
#include <asm/virt.h>
-#ifdef CONFIG_AMLOGIC_MODIFY
-#include <asm/perf_event.h>
-#endif
-
#ifdef CONFIG_AMLOGIC_VMAP
#include <linux/amlogic/vmap_stack.h>
#endif
IPI_CPU_STOP,
IPI_TIMER,
IPI_IRQ_WORK,
-#ifdef CONFIG_AMLOGIC_MODIFY
- IPI_WAKEUP,
- IPI_AML_PMU
-#else
IPI_WAKEUP
-#endif
};
#ifdef CONFIG_ARM64_VHE
S(IPI_TIMER, "Timer broadcast interrupts"),
S(IPI_IRQ_WORK, "IRQ work interrupts"),
S(IPI_WAKEUP, "CPU wake-up interrupts"),
-#ifdef CONFIG_AMLOGIC_MODIFY
- S(IPI_AML_PMU, "AML pmu cross interrupts"),
-#endif
};
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
__smp_cross_call(target, ipinr);
}
-#ifdef CONFIG_AMLOGIC_MODIFY
-void smp_send_aml_pmu(int cpu)
-{
- smp_cross_call(cpumask_of(cpu), IPI_AML_PMU);
-}
-#endif
-
void show_ipi_list(struct seq_file *p, int prec)
{
unsigned int cpu, i;
break;
#endif
-#ifdef CONFIG_AMLOGIC_MODIFY
- case IPI_AML_PMU:
- armv8pmu_handle_irq_ipi();
- break;
-#endif
-
default:
pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
break;
return 0;
}
+
+#ifdef CONFIG_AMLOGIC_MODIFY
+#include <linux/of_address.h>
+#include <linux/delay.h>
+
+struct amlpmu_context amlpmu_ctx;
+
+static enum hrtimer_restart amlpmu_relax_timer_func(struct hrtimer *timer)
+{
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+ struct amlpmu_cpuinfo *ci;
+
+ ci = per_cpu_ptr(ctx->cpuinfo, 0);
+
+ pr_info("enable cpu0_irq %d again, irq cnt = %lu\n",
+ ci->irq_num,
+ ci->valid_irq_cnt);
+ enable_irq(ci->irq_num);
+
+ return HRTIMER_NORESTART;
+}
+
+void amlpmu_relax_timer_start(int other_cpu)
+{
+ struct amlpmu_cpuinfo *ci;
+ int cpu;
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+
+ cpu = smp_processor_id();
+ WARN_ON(cpu != 0);
+
+ ci = per_cpu_ptr(ctx->cpuinfo, 0);
+
+ pr_warn("wait cpu %d fixup done timeout, main cpu irq cnt = %lu\n",
+ other_cpu,
+ ci->valid_irq_cnt);
+
+ if (hrtimer_active(&ctx->relax_timer)) {
+ pr_alert("relax_timer already active, return!\n");
+ return;
+ }
+
+ disable_irq_nosync(ci->irq_num);
+
+ hrtimer_start(&ctx->relax_timer,
+ ns_to_ktime(ctx->relax_timer_ns),
+ HRTIMER_MODE_REL);
+}
+
+static void amlpmu_fix_setup_affinity(int irq)
+{
+ int cluster_index = 0;
+ int cpu;
+ int affinity_cpu = -1;
+ struct amlpmu_cpuinfo *ci = NULL;
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+ s64 latest_next_stamp = S64_MAX;
+
+ if (irq == ctx->irqs[0])
+ cluster_index = 0;
+ else if (ctx->clusterb_enabled && irq ==
+ ctx->irqs[1])
+ cluster_index = 1;
+ else {
+ pr_err("amlpmu_fix_setup_affinity() bad irq = %d\n", irq);
+ return;
+ }
+
+ /*
+ * find latest next_predicted_stamp cpu for affinity cpu
+ * if no cpu have predict time, select first cpu of cpumask
+ * todo:
+ * - if a cpu predict failed for continuous N times,
+ * try add some punishment.
+ * - if no cpu have predicted time, try recently most used cpu
+ * for affinity
+ * - try to keep and promote prediction accuracy
+ */
+ for_each_cpu_and(cpu,
+ &ctx->cpumasks[cluster_index],
+ cpu_possible_mask) {
+ ci = per_cpu_ptr(ctx->cpuinfo, cpu);
+ //pr_info("cpu = %d, ci->next_predicted_stamp.tv64 = %lld\n",
+ // cpu, ci->next_predicted_stamp.tv64);
+ if (ci->next_predicted_stamp.tv64 &&
+ ci->next_predicted_stamp.tv64 < latest_next_stamp) {
+ latest_next_stamp = ci->next_predicted_stamp.tv64;
+ affinity_cpu = cpu;
+ }
+ }
+
+ if (affinity_cpu == -1) {
+ affinity_cpu = cpumask_first(&ctx->cpumasks[cluster_index]);
+ pr_debug("used first cpu: %d, cluster: 0x%lx\n",
+ affinity_cpu,
+ *cpumask_bits(&ctx->cpumasks[cluster_index]));
+ } else
+ pr_debug("find affinity cpu: %d, next_predicted_stamp: %lld\n",
+ affinity_cpu,
+ latest_next_stamp);
+
+ if (irq_set_affinity(irq, cpumask_of(affinity_cpu)))
+ pr_err("irq_set_affinity() failed irq: %d, affinity_cpu: %d\n",
+ irq,
+ affinity_cpu);
+}
+
+/*
+ * on pmu interrupt generated cpu, @irq_num is valid
+ * on other cpus(called by AML_PMU_IPI), @irq_num is -1
+ */
+static int amlpmu_irq_fix(int irq_num)
+{
+ int i;
+ int cpu;
+ int cur_cpu;
+ int pmuirq_val;
+ int cluster_index = 0;
+ int fix_success = 0;
+ struct amlpmu_cpuinfo *ci;
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+ struct call_single_data csd_stack;
+ int max_wait_cnt = ctx->max_wait_cnt;
+
+ csd_stack.func = amlpmu_handle_irq_ipi;
+ csd_stack.info = NULL;
+
+ cur_cpu = smp_processor_id();
+
+ if (irq_num == ctx->irqs[0])
+ cluster_index = 0;
+ else if (ctx->clusterb_enabled && irq_num == ctx->irqs[1])
+ cluster_index = 1;
+ else {
+ pr_err("amlpmu_irq_fix() bad irq = %d\n", irq_num);
+ return fix_success;
+ }
+
+ if (!cpumask_test_cpu(cur_cpu, &ctx->cpumasks[cluster_index])) {
+ pr_warn("amlpmu_irq_fix() cur_cpu %d not in cluster: 0x%lx\n",
+ cur_cpu,
+ *cpumask_bits(&ctx->cpumasks[cluster_index]));
+ }
+
+ pmuirq_val = readl(ctx->regs[cluster_index]);
+ pmuirq_val &= 0xf;
+ pmuirq_val <<= ctx->first_cpus[cluster_index];
+
+ pr_debug("amlpmu_irq_fix() val=0x%0x, first_cpu=%d, cluster=0x%lx\n",
+ readl(ctx->regs[cluster_index]),
+ ctx->first_cpus[cluster_index],
+ *cpumask_bits(&ctx->cpumasks[cluster_index]));
+
+ for_each_cpu_and(cpu,
+ &ctx->cpumasks[cluster_index],
+ cpu_possible_mask) {
+ if (pmuirq_val & (1<<cpu)) {
+ if (cpu == cur_cpu) {
+ pr_info("ownercpu %d in pmuirq = 0x%x\n",
+ cur_cpu, pmuirq_val);
+ continue;
+ }
+ pr_debug("fix pmu irq cpu %d, pmuirq = 0x%x\n",
+ cpu,
+ pmuirq_val);
+
+ ci = per_cpu_ptr(ctx->cpuinfo, cpu);
+ WRITE_ONCE(ci->fix_done, 0);
+ WRITE_ONCE(ci->fix_overflowed, 0);
+
+ csd_stack.flags = 0;
+ smp_call_function_single_async(cpu, &csd_stack);
+
+ for (i = 0; i < max_wait_cnt; i++) {
+ if (READ_ONCE(ci->fix_done))
+ break;
+
+ udelay(1);
+ }
+
+ if (i == ctx->max_wait_cnt) {
+ pr_err("wait for cpu %d done timeout\n",
+ cpu);
+ //amlpmu_relax_timer_start(cpu);
+ }
+
+ if (READ_ONCE(ci->fix_overflowed))
+ fix_success++;
+ }
+ }
+
+ return fix_success;
+}
+
+static void amlpmu_update_stats(int irq_num,
+ int has_overflowed,
+ int fix_success)
+{
+ int freq;
+ int i;
+ ktime_t stamp;
+ unsigned long time = jiffies;
+ struct amlpmu_cpuinfo *ci;
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+
+ ci = this_cpu_ptr(ctx->cpuinfo);
+
+ if (!has_overflowed && !fix_success) {
+ pr_debug("empty_irq_cnt: %lu\n", ci->empty_irq_cnt);
+ ci->empty_irq_cnt++;
+ ci->empty_irq_time = time;
+ }
+
+ if (fix_success) {
+ /* send IPI success */
+ pr_debug("fix_irq_cnt: %lu, fix_success = %d\n",
+ ci->fix_irq_cnt,
+ fix_success);
+ ci->fix_irq_cnt++;
+ ci->fix_irq_time = time;
+ }
+
+ if (has_overflowed) {
+ ci->valid_irq_cnt++;
+ ci->valid_irq_time = time;
+
+ stamp = ktime_get();
+ ci->stamp_deltas[ci->valid_irq_cnt % MAX_DELTA_CNT] =
+ stamp.tv64 - ci->last_stamp.tv64;
+ ci->last_stamp = stamp;
+
+ /* update avg_delta if it's valid */
+ ci->avg_delta = 0;
+ for (i = 0; i < MAX_DELTA_CNT; i++)
+ ci->avg_delta += ci->stamp_deltas[i];
+
+ ci->avg_delta /= MAX_DELTA_CNT;
+ for (i = 0; i < MAX_DELTA_CNT; i++) {
+ if (ci->stamp_deltas[i] > ci->avg_delta * 3/2 ||
+ ci->stamp_deltas[i] < ci->avg_delta / 2) {
+ ci->avg_delta = 0;
+ break;
+ }
+ }
+ if (ci->avg_delta)
+ ci->next_predicted_stamp.tv64 =
+ ci->last_stamp.tv64 + ci->avg_delta;
+ else
+ ci->next_predicted_stamp.tv64 = 0;
+
+ pr_debug("irq_num = %d, valid_irq_cnt = %lu\n",
+ irq_num,
+ ci->valid_irq_cnt);
+ pr_debug("cur_delta = %lld, avg_delta = %lld, next = %lld\n",
+ ci->stamp_deltas[ci->valid_irq_cnt % MAX_DELTA_CNT],
+ ci->avg_delta,
+ ci->next_predicted_stamp.tv64);
+ }
+
+ if (time_after(ci->valid_irq_time, ci->last_valid_irq_time + 2*HZ)) {
+ freq = ci->empty_irq_cnt - ci->last_empty_irq_cnt;
+ freq *= HZ;
+ freq /= (ci->empty_irq_time - ci->last_empty_irq_time);
+ pr_info("######## empty_irq_cnt: %lu - %lu = %lu, freq = %d\n",
+ ci->empty_irq_cnt,
+ ci->last_empty_irq_cnt,
+ ci->empty_irq_cnt - ci->last_empty_irq_cnt,
+ freq);
+
+ ci->last_empty_irq_cnt = ci->empty_irq_cnt;
+ ci->last_empty_irq_time = ci->empty_irq_time;
+
+
+ freq = ci->fix_irq_cnt - ci->last_fix_irq_cnt;
+ freq *= HZ;
+ freq /= (ci->fix_irq_time - ci->last_fix_irq_time);
+ pr_info("######## fix_irq_cnt: %lu - %lu = %lu, freq = %d\n",
+ ci->fix_irq_cnt,
+ ci->last_fix_irq_cnt,
+ ci->fix_irq_cnt - ci->last_fix_irq_cnt,
+ freq);
+
+ ci->last_fix_irq_cnt = ci->fix_irq_cnt;
+ ci->last_fix_irq_time = ci->fix_irq_time;
+
+
+ freq = ci->valid_irq_cnt - ci->last_valid_irq_cnt;
+ freq *= HZ;
+ freq /= (ci->valid_irq_time - ci->last_valid_irq_time);
+ pr_info("######## valid_irq_cnt: %lu - %lu = %lu, freq = %d\n",
+ ci->valid_irq_cnt,
+ ci->last_valid_irq_cnt,
+ ci->valid_irq_cnt - ci->last_valid_irq_cnt,
+ freq);
+
+ ci->last_valid_irq_cnt = ci->valid_irq_cnt;
+ ci->last_valid_irq_time = ci->valid_irq_time;
+ }
+}
+
+int amlpmu_handle_irq(struct arm_pmu *cpu_pmu, int irq_num, int has_overflowed)
+{
+ int cpu;
+ int fix_success = 0;
+ struct amlpmu_cpuinfo *ci;
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+
+ ci = this_cpu_ptr(ctx->cpuinfo);
+ ci->irq_num = irq_num;
+ cpu = smp_processor_id();
+
+ pr_debug("amlpmu_handle_irq() irq_num = %d, overflowed = %d\n",
+ irq_num, has_overflowed);
+
+ /*
+ * if current cpu is not overflowed, it's possible some other
+ * cpus caused the pmu interrupt.
+ * so if current cpu is interrupt generated cpu(irq_num != -1),
+ * call aml_pmu_fix() try to send IPI to other cpus and waiting
+ * for fix_done.
+ */
+ if (!has_overflowed && irq_num != -1)
+ fix_success = amlpmu_irq_fix(irq_num);
+
+ /*
+ * valid_irq, fix_irq and empty_irq status
+ * avg_delta time account to predict next interrupt time
+ */
+ amlpmu_update_stats(irq_num, has_overflowed, fix_success);
+
+ /*
+ * armv*pmu_getreset_flags() will clear interrupt. If current
+ * interrupt is IPI fix(irq_num = -1), interrupt generated cpu
+ * now is waiting for ci->fix_done=1(clear interrupt).
+ * we must set ci->fix_done to 1 after amlpmu_stat_account(),
+ * because interrupt generated cpu need this predict time info
+ * to setup interrupt affinity.
+ */
+ if (irq_num == -1) {
+ WRITE_ONCE(ci->fix_overflowed, has_overflowed);
+ /* fix_overflowed must before fix_done */
+ mb();
+ WRITE_ONCE(ci->fix_done, 1);
+ }
+
+ /* only interrupt generated cpu need setup affinity */
+ if (irq_num != -1)
+ amlpmu_fix_setup_affinity(irq_num);
+
+ /*
+ * when a pmu interrupt generated, if current cpu is not
+ * overflowed and some other cpus succeed in handling the
+ * interrupt by IPIs return true.
+ */
+ return !has_overflowed && fix_success;
+}
+
+static int amlpmu_init(struct platform_device *pdev, struct arm_pmu *pmu)
+{
+ int cpu;
+ int ret = 0;
+ int irq;
+ u32 cpumasks[MAX_CLUSTER_NR] = {0};
+ struct amlpmu_context *ctx = &amlpmu_ctx;
+ struct amlpmu_cpuinfo *ci;
+
+ memset(ctx, 0, sizeof(*ctx));
+
+ ctx->cpuinfo = __alloc_percpu_gfp(
+ sizeof(struct amlpmu_cpuinfo),
+ SMP_CACHE_BYTES,
+ GFP_KERNEL | __GFP_ZERO);
+ if (!ctx->cpuinfo) {
+ pr_err("alloc percpu failed\n");
+ ret = -ENOMEM;
+ goto free;
+ }
+
+ for_each_possible_cpu(cpu) {
+ ci = per_cpu_ptr(ctx->cpuinfo, cpu);
+ ci->last_valid_irq_time = INITIAL_JIFFIES;
+ ci->last_fix_irq_time = INITIAL_JIFFIES;
+ ci->last_empty_irq_time = INITIAL_JIFFIES;
+ }
+
+ ctx->pmu = pmu;
+
+ if (of_property_read_bool(pdev->dev.of_node, "clusterb-enabled"))
+ ctx->clusterb_enabled = 1;
+
+ pr_info("clusterb_enabled = %d\n", ctx->clusterb_enabled);
+
+ ret = of_property_read_u32_array(pdev->dev.of_node,
+ "cpumasks",
+ cpumasks,
+ ctx->clusterb_enabled ? MAX_CLUSTER_NR : 1);
+ if (ret) {
+ pr_err("read prop cpumasks failed, ret = %d\n", ret);
+ ret = -EINVAL;
+ goto free;
+ }
+ pr_info("cpumasks 0x%0x, 0x%0x\n", cpumasks[0], cpumasks[1]);
+
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "relax-timer-ns",
+ &ctx->relax_timer_ns);
+ if (ret) {
+ pr_err("read prop relax-timer-ns failed, ret = %d\n", ret);
+ ret = -EINVAL;
+ goto free;
+ }
+
+ ret = of_property_read_u32(pdev->dev.of_node,
+ "max-wait-cnt",
+ &ctx->max_wait_cnt);
+ if (ret) {
+ pr_err("read prop max-wait-cnt failed, ret = %d\n", ret);
+ ret = -EINVAL;
+ goto free;
+ }
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0) {
+ pr_err("get clusterA irq failed, %d\n", irq);
+ ret = -EINVAL;
+ goto free;
+ }
+ ctx->irqs[0] = irq;
+ pr_info("cluster A irq = %d\n", irq);
+
+ ctx->regs[0] = of_iomap(pdev->dev.of_node, 0);
+ if (IS_ERR(ctx->regs[0])) {
+ pr_err("of_iomap() clusterA failed, base = %p\n", ctx->regs[0]);
+ ret = PTR_ERR(ctx->regs[0]);
+ goto free;
+ }
+
+ cpumask_clear(&ctx->cpumasks[0]);
+ memcpy(cpumask_bits(&ctx->cpumasks[0]),
+ &cpumasks[0],
+ sizeof(cpumasks[0]));
+ if (!cpumask_intersects(&ctx->cpumasks[0], cpu_possible_mask)) {
+ pr_err("bad cpumasks[0] 0x%x\n", cpumasks[0]);
+ ret = -EINVAL;
+ goto free;
+ }
+ ctx->first_cpus[0] = cpumask_first(&ctx->cpumasks[0]);
+
+ amlpmu_fix_setup_affinity(ctx->irqs[0]);
+
+ hrtimer_init(&ctx->relax_timer,
+ CLOCK_MONOTONIC,
+ HRTIMER_MODE_REL);
+ ctx->relax_timer.function = amlpmu_relax_timer_func;
+
+ if (!ctx->clusterb_enabled)
+ return 0;
+
+ irq = platform_get_irq(pdev, 1);
+ if (irq < 0) {
+ pr_err("get clusterB irq failed, %d\n", irq);
+ ret = -EINVAL;
+ goto free;
+ }
+ ctx->irqs[1] = irq;
+ pr_info("cluster B irq = %d\n", irq);
+
+
+ ctx->regs[1] = of_iomap(pdev->dev.of_node, 1);
+ if (IS_ERR(ctx->regs[1])) {
+ pr_err("of_iomap() clusterA failed, base = %p\n", ctx->regs[1]);
+ ret = PTR_ERR(ctx->regs[1]);
+ goto free;
+ }
+
+ cpumask_clear(&ctx->cpumasks[1]);
+ memcpy(cpumask_bits(&ctx->cpumasks[1]),
+ &cpumasks[1],
+ sizeof(cpumasks[1]));
+ if (!cpumask_intersects(&ctx->cpumasks[1], cpu_possible_mask)) {
+ pr_err("bad cpumasks[1] 0x%x\n", cpumasks[1]);
+ ret = -EINVAL;
+ goto free;
+ } else if (cpumask_intersects(&ctx->cpumasks[0], &ctx->cpumasks[1])) {
+ pr_err("cpumasks intersect 0x%x : 0x%x\n",
+ cpumasks[0],
+ cpumasks[1]);
+ ret = -EINVAL;
+ goto free;
+ }
+ ctx->first_cpus[1] = cpumask_first(&ctx->cpumasks[1]);
+
+ amlpmu_fix_setup_affinity(ctx->irqs[1]);
+
+ return 0;
+
+free:
+ if (ctx->cpuinfo)
+ free_percpu(ctx->cpuinfo);
+
+ if (ctx->regs[0])
+ iounmap(ctx->regs[0]);
+
+ if (ctx->regs[1])
+ iounmap(ctx->regs[1]);
+
+ return ret;
+}
+
+#endif
+
int arm_pmu_device_probe(struct platform_device *pdev,
const struct of_device_id *of_table,
const struct pmu_probe_info *probe_table)
return -ENOMEM;
}
+#ifdef CONFIG_AMLOGIC_MODIFY
+ if (amlpmu_init(pdev, pmu)) {
+ pr_err("amlpmu_init() failed\n");
+ return 1;
+ }
+#endif
+
armpmu_init(pmu);
pmu->plat_device = pdev;
#define ARMV8_PMU_PDEV_NAME "armv8-pmu"
+#ifdef CONFIG_AMLOGIC_MODIFY
+#define MAX_DELTA_CNT 4
+struct amlpmu_cpuinfo {
+ int irq_num;
+
+ /*
+ * In interrupt generated cpu(affinity cpu)
+ * If pmu no overflowed, then we need to send IPI to some other cpus to
+ * fix it. And before send IPI, set corresponding cpu's fix_done and
+ * fix_overflowed to zero, in corresponding cpu's IPI interrupt will set
+ * fix_done to inform source cpu and if indeed pmu overflowed then also
+ * set fix_overflowed to 1, then inerrupt generated cpu can feel that.
+ */
+ int fix_done;
+ int fix_overflowed;
+
+ /* for interrupt affinity prediction */
+ ktime_t last_stamp;
+ s64 stamp_deltas[MAX_DELTA_CNT];
+ s64 avg_delta;
+ ktime_t next_predicted_stamp;
+
+ /*
+ * irq state account of this cpu
+ *
+ * - valid_irq_cnt:
+ * valid irq cnt.(pmu overflow happened)
+ * - fix_irq_cnt:
+ * when this cpu is pmu interrupt generated affinity cpu, a pmu
+ * interrupt if cpu affinity predict failed so no pmu overflow
+ * happened and succeeded send IPI to other cpu, then it's a send
+ * fix irq. So the lower is better.
+ * - empty_irq_cnt:
+ * when this cpu is pmu interrupt generated affinity cpu, a pmu
+ * interrupt that no overflow happened and also no fix IPI sended to
+ * other cpus, then it's a empty irq.
+ * when this cpu is not affinity cpu, a IPI interrupt(pmu fix from
+ * affinity cpu) that no pmu overflow happened, it's a empty irq.
+ *
+ * attention:
+ * A interrupt can be a valid_irq and also a fix_irq.
+ */
+ unsigned long valid_irq_cnt;
+ unsigned long fix_irq_cnt;
+ unsigned long empty_irq_cnt;
+
+ unsigned long valid_irq_time;
+ unsigned long fix_irq_time;
+ unsigned long empty_irq_time;
+
+ unsigned long last_valid_irq_cnt;
+ unsigned long last_fix_irq_cnt;
+ unsigned long last_empty_irq_cnt;
+
+ unsigned long last_valid_irq_time;
+ unsigned long last_fix_irq_time;
+ unsigned long last_empty_irq_time;
+};
+
+
+#define MAX_CLUSTER_NR 2
+struct amlpmu_context {
+ struct amlpmu_cpuinfo __percpu *cpuinfo;
+
+ /* struct arm_pmu */
+ struct arm_pmu *pmu;
+
+ int clusterb_enabled;
+
+ unsigned int __iomem *regs[MAX_CLUSTER_NR];
+ int irqs[MAX_CLUSTER_NR];
+ struct cpumask cpumasks[MAX_CLUSTER_NR];
+ int first_cpus[MAX_CLUSTER_NR];
+
+ /*
+ * In main pmu irq route wait for other cpu fix done may cause lockup,
+ * when lockup we disable main irq for a while.
+ * relax_timer will enable main irq again.
+ */
+ struct hrtimer relax_timer;
+
+ unsigned int relax_timer_ns;
+ unsigned int max_wait_cnt;
+};
+
+extern struct amlpmu_context amlpmu_ctx;
+
+int amlpmu_handle_irq(struct arm_pmu *cpu_pmu, int irq_num, int has_overflowed);
+
+/* defined int arch/arm(64)/kernel/perf_event(_v7).c */
+void amlpmu_handle_irq_ipi(void *arg);
+#endif
+
#endif /* CONFIG_ARM_PMU */
#endif /* __ARM_PMU_H__ */
*/
extern void smp_send_reschedule(int cpu);
-#ifdef CONFIG_AMLOGIC_MODIFY
-/*
- * sends a 'aml pmu' event to another CPU:
- */
-extern void smp_send_aml_pmu(int cpu);
-#endif
-
/*
* Prepare machine for booting other CPUs.
*/
projects / platform / kernel / linux-amlogic.git / commitdiff