ARM: arch_timer: ipipe: enable interrupt pipelining

diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h
index 9327e3a101dc4852e530a9ef6a6aaeb2e83e92f2..e9e7656bc1370ea92a96801c65e430e6f83cc671 100644 (file)
--- a/arch/arm/include/asm/arch_timer.h
+++ b/arch/arm/include/asm/arch_timer.h
@@ -106,6 +106,10 @@ static inline u32 arch_timer_get_cntkctl(void)
 
 static inline void arch_timer_set_cntkctl(u32 cntkctl)
 {
+#ifdef CONFIG_IPIPE
+       /* Enable access to user-space (may not be needed) */
+       cntkctl |= ARCH_TIMER_USR_PCT_ACCESS_EN;
+#endif
        asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl));
 }
 

diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index ea2f47e2f0cba46f030e354b61c8b464bcaddab4..4861c0cabcc68bac1701f44e6edfc5e88f49c47b 100644 (file)
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -20,6 +20,8 @@
 #include <linux/clockchips.h>
 #include <linux/clocksource.h>
 #include <linux/interrupt.h>
+#include <linux/ipipe.h>
+#include <linux/ipipe_tickdev.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/io.h>
@@ -548,8 +550,7 @@ static bool arch_timer_this_cpu_has_cntvct_wa(void)
 #define arch_timer_this_cpu_has_cntvct_wa()            ({false;})
 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 
-static __always_inline irqreturn_t timer_handler(const int access,
-                                       struct clock_event_device *evt)
+static int arch_timer_ack(const int access, struct clock_event_device *evt)
 {
        unsigned long ctrl;
 
@@ -557,6 +558,52 @@ static __always_inline irqreturn_t timer_handler(const int access,
        if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
                ctrl |= ARCH_TIMER_CTRL_IT_MASK;
                arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
+               return 1;
+       }
+       return 0;
+}
+
+#ifdef CONFIG_IPIPE
+static DEFINE_PER_CPU(struct ipipe_timer, arch_itimer);
+static struct __ipipe_tscinfo tsc_info = {
+       .type = IPIPE_TSC_TYPE_FREERUNNING_ARCH,
+       .u = {
+               {
+                       .mask = 0xffffffffffffffff,
+               },
+       },
+};
+
+static void arch_itimer_ack_phys(void)
+{
+       struct clock_event_device *evt = this_cpu_ptr(arch_timer_evt);
+       arch_timer_ack(ARCH_TIMER_PHYS_ACCESS, evt);
+}
+
+static void arch_itimer_ack_virt(void)
+{
+       struct clock_event_device *evt = this_cpu_ptr(arch_timer_evt);
+       arch_timer_ack(ARCH_TIMER_VIRT_ACCESS, evt);
+}
+#endif /* CONFIG_IPIPE */
+
+static inline irqreturn_t timer_handler(int irq, const int access,
+                                       struct clock_event_device *evt)
+{
+       if (clockevent_ipipe_stolen(evt))
+               goto stolen;
+
+       if (arch_timer_ack(access, evt)) {
+#ifdef CONFIG_IPIPE
+               struct ipipe_timer *itimer = raw_cpu_ptr(&arch_itimer);
+               if (itimer->irq != irq)
+                       itimer->irq = irq;
+#endif /* CONFIG_IPIPE */
+         stolen:
+               /*
+                * This is a 64bit clock source, no need for TSC
+                * update.
+                */
                evt->event_handler(evt);
                return IRQ_HANDLED;
        }
@@ -568,28 +615,28 @@ static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 {
        struct clock_event_device *evt = dev_id;
 
-       return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
+       return timer_handler(irq, ARCH_TIMER_VIRT_ACCESS, evt);
 }
 
 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 {
        struct clock_event_device *evt = dev_id;
 
-       return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
+       return timer_handler(irq, ARCH_TIMER_PHYS_ACCESS, evt);
 }
 
 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 {
        struct clock_event_device *evt = dev_id;
 
-       return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
+       return timer_handler(irq, ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 }
 
 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 {
        struct clock_event_device *evt = dev_id;
 
-       return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
+       return timer_handler(irq, ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 }
 
 static __always_inline int timer_shutdown(const int access,
@@ -703,6 +750,17 @@ static void __arch_timer_setup(unsigned type,
                }
 
                arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
+#ifdef CONFIG_IPIPE
+               clk->ipipe_timer = raw_cpu_ptr(&arch_itimer);
+               if (arch_timer_mem_use_virtual) {
+                       clk->ipipe_timer->irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
+                       clk->ipipe_timer->ack = arch_itimer_ack_virt;
+               } else {
+                       clk->ipipe_timer->irq = arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI];
+                       clk->ipipe_timer->ack = arch_itimer_ack_phys;
+               }
+               clk->ipipe_timer->freq = arch_timer_rate;
+#endif
        } else {
                clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
                clk->name = "arch_mem_timer";
@@ -776,6 +834,9 @@ static void arch_counter_set_user_access(void)
        else
                cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN | ARCH_TIMER_USR_PCT_ACCESS_EN;
 
+#ifdef CONFIG_IPIPE
+       cntkctl |= ARCH_TIMER_USR_PCT_ACCESS_EN;
+#endif
        arch_timer_set_cntkctl(cntkctl);
 }
 
@@ -899,6 +960,10 @@ static void __init arch_counter_register(unsigned type)
                arch_timer_read_counter = arch_counter_get_cntvct_mem;
        }
 
+#ifdef CONFIG_IPIPE
+       tsc_info.freq = arch_timer_rate;
+       __ipipe_tsc_register(&tsc_info);
+#endif /* CONFIG_IPIPE */
        if (!arch_counter_suspend_stop)
                clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
        start_count = arch_timer_read_counter();