x86/entry: Use idtentry for interrupts

Replace the extra interrupt handling code and reuse the existing idtentry
machinery. This moves the irq stack switching on 64-bit from ASM to C code;
32-bit already does the stack switching in C.

This requires to remove HAVE_IRQ_EXIT_ON_IRQ_STACK as the stack switch is
not longer in the low level entry code.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Andy Lutomirski <luto@kernel.org>
Link: https://lore.kernel.org/r/20200521202119.078690991@linutronix.de

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 10dae8b..a16c454 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -181,7 +181,6 @@ config X86
        select HAVE_HW_BREAKPOINT
        select HAVE_IDE
        select HAVE_IOREMAP_PROT
-       select HAVE_IRQ_EXIT_ON_IRQ_STACK       if X86_64
        select HAVE_IRQ_TIME_ACCOUNTING
        select HAVE_KERNEL_BZIP2
        select HAVE_KERNEL_GZIP

diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index ba2a70d..b47b7b2 100644 (file)
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -1229,37 +1229,6 @@ SYM_FUNC_END(entry_INT80_32)
 #endif
 .endm
 
-#ifdef CONFIG_X86_LOCAL_APIC
-SYM_CODE_START_LOCAL(common_spurious)
-       ASM_CLAC
-       SAVE_ALL switch_stacks=1
-       ENCODE_FRAME_POINTER
-       TRACE_IRQS_OFF
-       movl    %esp, %eax
-       movl    PT_ORIG_EAX(%esp), %edx         /* get the vector from stack */
-       movl    $-1, PT_ORIG_EAX(%esp)          /* no syscall to restart */
-       call    smp_spurious_interrupt
-       jmp     ret_from_intr
-SYM_CODE_END(common_spurious)
-#endif
-
-/*
- * the CPU automatically disables interrupts when executing an IRQ vector,
- * so IRQ-flags tracing has to follow that:
- */
-       .p2align CONFIG_X86_L1_CACHE_SHIFT
-SYM_CODE_START_LOCAL(common_interrupt)
-       ASM_CLAC
-       SAVE_ALL switch_stacks=1
-       ENCODE_FRAME_POINTER
-       TRACE_IRQS_OFF
-       movl    %esp, %eax
-       movl    PT_ORIG_EAX(%esp), %edx         /* get the vector from stack */
-       movl    $-1, PT_ORIG_EAX(%esp)          /* no syscall to restart */
-       call    do_IRQ
-       jmp     ret_from_intr
-SYM_CODE_END(common_interrupt)
-
 #define BUILD_INTERRUPT3(name, nr, fn)                 \
 SYM_FUNC_START(name)                                   \
        ASM_CLAC;                                       \

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 9162a07..e54bcd3 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -737,32 +737,7 @@ SYM_CODE_START(interrupt_entry)
 SYM_CODE_END(interrupt_entry)
 _ASM_NOKPROBE(interrupt_entry)
 
-
-/* Interrupt entry/exit. */
-
-/*
- * The interrupt stubs push vector onto the stack and
- * then jump to common_spurious/interrupt.
- */
-SYM_CODE_START_LOCAL(common_spurious)
-       call    interrupt_entry
-       UNWIND_HINT_REGS indirect=1
-       movq    ORIG_RAX(%rdi), %rsi            /* get vector from stack */
-       movq    $-1, ORIG_RAX(%rdi)             /* no syscall to restart */
-       call    smp_spurious_interrupt          /* rdi points to pt_regs */
-       jmp     ret_from_intr
-SYM_CODE_END(common_spurious)
-_ASM_NOKPROBE(common_spurious)
-
-/* common_interrupt is a hotpath. Align it */
-       .p2align CONFIG_X86_L1_CACHE_SHIFT
-SYM_CODE_START_LOCAL(common_interrupt)
-       call    interrupt_entry
-       UNWIND_HINT_REGS indirect=1
-       movq    ORIG_RAX(%rdi), %rsi            /* get vector from stack */
-       movq    $-1, ORIG_RAX(%rdi)             /* no syscall to restart */
-       call    do_IRQ                          /* rdi points to pt_regs */
-       /* 0(%rsp): old RSP */
+SYM_CODE_START_LOCAL(common_interrupt_return)
 ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_ANY)
        TRACE_IRQS_OFF
@@ -945,8 +920,8 @@ native_irq_return_ldt:
         */
        jmp     native_irq_return_iret
 #endif
-SYM_CODE_END(common_interrupt)
-_ASM_NOKPROBE(common_interrupt)
+SYM_CODE_END(common_interrupt_return)
+_ASM_NOKPROBE(common_interrupt_return)
 
 /*
  * APIC interrupts.

diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 0ffe807..3213d36 100644 (file)
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -38,7 +38,6 @@ extern asmlinkage void error_interrupt(void);
 extern asmlinkage void irq_work_interrupt(void);
 extern asmlinkage void uv_bau_message_intr1(void);
 
-extern asmlinkage void spurious_interrupt(void);
 extern asmlinkage void spurious_apic_interrupt(void);
 extern asmlinkage void thermal_interrupt(void);
 extern asmlinkage void reschedule_interrupt(void);

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index eaee48b..3418881 100644 (file)
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -417,7 +417,7 @@ SYM_CODE_START(irq_entries_start)
     .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
        UNWIND_HINT_IRET_REGS
        .byte   0x6a, vector
-       jmp     common_interrupt
+       jmp     asm_common_interrupt
        nop
        /* Ensure that the above is 8 bytes max */
        . = pos + 8
@@ -434,7 +434,7 @@ SYM_CODE_START(spurious_entries_start)
     .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
        UNWIND_HINT_IRET_REGS
        .byte   0x6a, vector
-       jmp     common_spurious
+       jmp     asm_spurious_interrupt
        nop
        /* Ensure that the above is 8 bytes max */
        . = pos + 8
@@ -506,6 +506,12 @@ DECLARE_IDTENTRY_DF(X86_TRAP_DF,   exc_double_fault);
 DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback);
 #endif
 
+/* Device interrupts common/spurious */
+DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,   common_interrupt);
+#ifdef CONFIG_X86_LOCAL_APIC
+DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER,   spurious_interrupt);
+#endif
+
 #undef X86_TRAP_OTHER
 
 #endif

diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 32b2bec..97e6945 100644 (file)
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -43,7 +43,6 @@ asmlinkage void smp_deferred_error_interrupt(struct pt_regs *regs);
 void smp_apic_timer_interrupt(struct pt_regs *regs);
 void smp_error_interrupt(struct pt_regs *regs);
 void smp_spurious_apic_interrupt(struct pt_regs *regs);
-void smp_spurious_interrupt(struct pt_regs *regs, unsigned long vector);
 asmlinkage void smp_irq_move_cleanup_interrupt(void);
 
 #ifdef CONFIG_VMAP_STACK

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 6c2b807..b7bfd3a 100644 (file)
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -2121,9 +2121,9 @@ void __init register_lapic_address(unsigned long address)
  */
 
 /**
- * smp_spurious_interrupt - Catch all for interrupts raised on unused vectors
+ * spurious_interrupt - Catch all for interrupts raised on unused vectors
  * @regs:      Pointer to pt_regs on stack
- * @error_code:        The vector number is in the lower 8 bits
+ * @vector:    The vector number
  *
  * This is invoked from ASM entry code to catch all interrupts which
  * trigger on an entry which is routed to the common_spurious idtentry
@@ -2131,18 +2131,10 @@ void __init register_lapic_address(unsigned long address)
  *
  * Also called from smp_spurious_apic_interrupt().
  */
-__visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs,
-                                                 unsigned long vector)
+DEFINE_IDTENTRY_IRQ(spurious_interrupt)
 {
        u32 v;
 
-       entering_irq();
-       /*
-        * The push in the entry ASM code which stores the vector number on
-        * the stack in the error code slot is sign expanding. Just use the
-        * lower 8 bits.
-        */
-       vector &= 0xFF;
        trace_spurious_apic_entry(vector);
 
        inc_irq_stat(irq_spurious_count);
@@ -2163,21 +2155,22 @@ __visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs,
         */
        v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
        if (v & (1 << (vector & 0x1f))) {
-               pr_info("Spurious interrupt (vector 0x%02lx) on CPU#%d. Acked\n",
+               pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
                        vector, smp_processor_id());
                ack_APIC_irq();
        } else {
-               pr_info("Spurious interrupt (vector 0x%02lx) on CPU#%d. Not pending!\n",
+               pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
                        vector, smp_processor_id());
        }
 out:
        trace_spurious_apic_exit(vector);
-       exiting_irq();
 }
 
 __visible void smp_spurious_apic_interrupt(struct pt_regs *regs)
 {
-       smp_spurious_interrupt(regs, SPURIOUS_APIC_VECTOR);
+       entering_irq();
+       __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR);
+       exiting_irq();
 }
 
 /*

diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c
index 159bd0c..5cbaca5 100644 (file)
--- a/arch/x86/kernel/apic/msi.c
+++ b/arch/x86/kernel/apic/msi.c
@@ -115,7 +115,8 @@ msi_set_affinity(struct irq_data *irqd, const struct cpumask *mask, bool force)
         * denote it as spurious which is no harm as this is a rare event
         * and interrupt handlers have to cope with spurious interrupts
         * anyway. If the vector is unused, then it is marked so it won't
-        * trigger the 'No irq handler for vector' warning in do_IRQ().
+        * trigger the 'No irq handler for vector' warning in
+        * common_interrupt().
         *
         * This requires to hold vector lock to prevent concurrent updates to
         * the affected vector.

diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 5495ea4..c449b84 100644 (file)
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -20,6 +20,7 @@
 #include <asm/mce.h>
 #include <asm/hw_irq.h>
 #include <asm/desc.h>
+#include <asm/traps.h>
 
 #define CREATE_TRACE_POINTS
 #include <asm/trace/irq_vectors.h>
@@ -232,37 +233,25 @@ static __always_inline void handle_irq(struct irq_desc *desc,
 }
 
 /*
- * do_IRQ handles all normal device IRQ's (the special
- * SMP cross-CPU interrupts have their own specific
- * handlers).
+ * common_interrupt() handles all normal device IRQ's (the special SMP
+ * cross-CPU interrupts have their own entry points).
  */
-__visible void __irq_entry do_IRQ(struct pt_regs *regs, unsigned long vector)
+DEFINE_IDTENTRY_IRQ(common_interrupt)
 {
        struct pt_regs *old_regs = set_irq_regs(regs);
        struct irq_desc *desc;
 
-       entering_irq();
-       /*
-        * The push in the entry ASM code which stores the vector number on
-        * the stack in the error code slot is sign expanding. Just use the
-        * lower 8 bits.
-        */
-       vector &= 0xFF;
-
-       /* entering_irq() tells RCU that we're not quiescent.  Check it. */
+       /* entry code tells RCU that we're not quiescent.  Check it. */
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
 
        desc = __this_cpu_read(vector_irq[vector]);
        if (likely(!IS_ERR_OR_NULL(desc))) {
-               if (IS_ENABLED(CONFIG_X86_32))
-                       __handle_irq(desc, regs);
-               else
-                       generic_handle_irq_desc(desc);
+               handle_irq(desc, regs);
        } else {
                ack_APIC_irq();
 
                if (desc == VECTOR_UNUSED) {
-                       pr_emerg_ratelimited("%s: %d.%lu No irq handler for vector\n",
+                       pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n",
                                             __func__, smp_processor_id(),
                                             vector);
                } else {
@@ -270,8 +259,6 @@ __visible void __irq_entry do_IRQ(struct pt_regs *regs, unsigned long vector)
                }
        }
 
-       exiting_irq();
-
        set_irq_regs(old_regs);
 }