x86/irq: Convey vector as argument and not in ptregs
authorThomas Gleixner <tglx@linutronix.de>
Thu, 21 May 2020 20:05:34 +0000 (22:05 +0200)
committerThomas Gleixner <tglx@linutronix.de>
Thu, 11 Jun 2020 13:15:11 +0000 (15:15 +0200)
Device interrupts which go through do_IRQ() or the spurious interrupt
handler have their separate entry code on 64 bit for no good reason.

Both 32 and 64 bit transport the vector number through ORIG_[RE]AX in
pt_regs. Further the vector number is forced to fit into an u8 and is
complemented and offset by 0x80 so it's in the signed character
range. Otherwise GAS would expand the pushq to a 5 byte instruction for any
vector > 0x7F.

Treat the vector number like an error code and hand it to the C function as
argument. This allows to get rid of the extra entry code in a later step.

Simplify the error code push magic by implementing the pushq imm8 via a
'.byte 0x6a, vector' sequence so GAS is not able to screw it up. As the
pushq imm8 is sign extending the resulting error code needs to be truncated
to 8 bits in C code.

Originally-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Andy Lutomirski <luto@kernel.org>
Link: https://lore.kernel.org/r/20200521202118.796915981@linutronix.de
arch/x86/entry/calling.h
arch/x86/entry/entry_32.S
arch/x86/entry/entry_64.S
arch/x86/include/asm/entry_arch.h
arch/x86/include/asm/hw_irq.h
arch/x86/include/asm/idtentry.h
arch/x86/include/asm/irq.h
arch/x86/include/asm/traps.h
arch/x86/kernel/apic/apic.c
arch/x86/kernel/idt.c
arch/x86/kernel/irq.c

index 1c7f13bb67286238cb555ff1238557b6458b179e..98da0d3c0b1a76b0d008e8d067ce6781f858e6af 100644 (file)
@@ -341,7 +341,10 @@ For 32-bit we have the following conventions - kernel is built with
 #endif
 .endm
 
-#endif /* CONFIG_X86_64 */
+#else /* CONFIG_X86_64 */
+# undef                UNWIND_HINT_IRET_REGS
+# define       UNWIND_HINT_IRET_REGS
+#endif /* !CONFIG_X86_64 */
 
 .macro STACKLEAK_ERASE
 #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
index 158a5250ebc57fb090c2013fcaa318fd8dc9ce3f..40092c81dcb81f1eb2842a92f3150899ed2f566f 100644 (file)
@@ -1215,40 +1215,15 @@ SYM_FUNC_END(entry_INT80_32)
 #endif
 .endm
 
-/*
- * Build the entry stubs with some assembler magic.
- * We pack 1 stub into every 8-byte block.
- */
-       .align 8
-SYM_CODE_START(irq_entries_start)
-    vector=FIRST_EXTERNAL_VECTOR
-    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
-       pushl   $(~vector+0x80)                 /* Note: always in signed byte range */
-    vector=vector+1
-       jmp     common_interrupt
-       .align  8
-    .endr
-SYM_CODE_END(irq_entries_start)
-
 #ifdef CONFIG_X86_LOCAL_APIC
-       .align 8
-SYM_CODE_START(spurious_entries_start)
-    vector=FIRST_SYSTEM_VECTOR
-    .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
-       pushl   $(~vector+0x80)                 /* Note: always in signed byte range */
-    vector=vector+1
-       jmp     common_spurious
-       .align  8
-    .endr
-SYM_CODE_END(spurious_entries_start)
-
 SYM_CODE_START_LOCAL(common_spurious)
        ASM_CLAC
-       addl    $-0x80, (%esp)                  /* Adjust vector into the [-256, -1] range */
        SAVE_ALL switch_stacks=1
        ENCODE_FRAME_POINTER
        TRACE_IRQS_OFF
        movl    %esp, %eax
+       movl    PT_ORIG_EAX(%esp), %edx         /* get the vector from stack */
+       movl    $-1, PT_ORIG_EAX(%esp)          /* no syscall to restart */
        call    smp_spurious_interrupt
        jmp     ret_from_intr
 SYM_CODE_END(common_spurious)
@@ -1261,12 +1236,12 @@ SYM_CODE_END(common_spurious)
        .p2align CONFIG_X86_L1_CACHE_SHIFT
 SYM_CODE_START_LOCAL(common_interrupt)
        ASM_CLAC
-       addl    $-0x80, (%esp)                  /* Adjust vector into the [-256, -1] range */
-
        SAVE_ALL switch_stacks=1
        ENCODE_FRAME_POINTER
        TRACE_IRQS_OFF
        movl    %esp, %eax
+       movl    PT_ORIG_EAX(%esp), %edx         /* get the vector from stack */
+       movl    $-1, PT_ORIG_EAX(%esp)          /* no syscall to restart */
        call    do_IRQ
        jmp     ret_from_intr
 SYM_CODE_END(common_interrupt)
index 76993591fdf60d69f80cb344c5bb5bc93d5fe37f..e7434cda9a381ed4185d2d43830a9ad1b373908e 100644 (file)
@@ -358,34 +358,6 @@ SYM_CODE_START(ret_from_fork)
 SYM_CODE_END(ret_from_fork)
 .popsection
 
-/*
- * Build the entry stubs with some assembler magic.
- * We pack 1 stub into every 8-byte block.
- */
-       .align 8
-SYM_CODE_START(irq_entries_start)
-    vector=FIRST_EXTERNAL_VECTOR
-    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
-       UNWIND_HINT_IRET_REGS
-       pushq   $(~vector+0x80)                 /* Note: always in signed byte range */
-       jmp     common_interrupt
-       .align  8
-       vector=vector+1
-    .endr
-SYM_CODE_END(irq_entries_start)
-
-       .align 8
-SYM_CODE_START(spurious_entries_start)
-    vector=FIRST_SYSTEM_VECTOR
-    .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
-       UNWIND_HINT_IRET_REGS
-       pushq   $(~vector+0x80)                 /* Note: always in signed byte range */
-       jmp     common_spurious
-       .align  8
-       vector=vector+1
-    .endr
-SYM_CODE_END(spurious_entries_start)
-
 .macro DEBUG_ENTRY_ASSERT_IRQS_OFF
 #ifdef CONFIG_DEBUG_ENTRY
        pushq %rax
@@ -755,13 +727,14 @@ _ASM_NOKPROBE(interrupt_entry)
 /* Interrupt entry/exit. */
 
 /*
- * The interrupt stubs push (~vector+0x80) onto the stack and
+ * The interrupt stubs push vector onto the stack and
  * then jump to common_spurious/interrupt.
  */
 SYM_CODE_START_LOCAL(common_spurious)
-       addq    $-0x80, (%rsp)                  /* Adjust vector to [-256, -1] range */
        call    interrupt_entry
        UNWIND_HINT_REGS indirect=1
+       movq    ORIG_RAX(%rdi), %rsi            /* get vector from stack */
+       movq    $-1, ORIG_RAX(%rdi)             /* no syscall to restart */
        call    smp_spurious_interrupt          /* rdi points to pt_regs */
        jmp     ret_from_intr
 SYM_CODE_END(common_spurious)
@@ -770,10 +743,11 @@ _ASM_NOKPROBE(common_spurious)
 /* common_interrupt is a hotpath. Align it */
        .p2align CONFIG_X86_L1_CACHE_SHIFT
 SYM_CODE_START_LOCAL(common_interrupt)
-       addq    $-0x80, (%rsp)                  /* Adjust vector to [-256, -1] range */
        call    interrupt_entry
        UNWIND_HINT_REGS indirect=1
-       call    do_IRQ  /* rdi points to pt_regs */
+       movq    ORIG_RAX(%rdi), %rsi            /* get vector from stack */
+       movq    $-1, ORIG_RAX(%rdi)             /* no syscall to restart */
+       call    do_IRQ                          /* rdi points to pt_regs */
        /* 0(%rsp): old RSP */
 ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_ANY)
@@ -1022,7 +996,7 @@ apicinterrupt RESCHEDULE_VECTOR                    reschedule_interrupt            smp_reschedule_interrupt
 #endif
 
 apicinterrupt ERROR_APIC_VECTOR                        error_interrupt                 smp_error_interrupt
-apicinterrupt SPURIOUS_APIC_VECTOR             spurious_interrupt              smp_spurious_interrupt
+apicinterrupt SPURIOUS_APIC_VECTOR             spurious_apic_interrupt         smp_spurious_apic_interrupt
 
 #ifdef CONFIG_IRQ_WORK
 apicinterrupt IRQ_WORK_VECTOR                  irq_work_interrupt              smp_irq_work_interrupt
index 416422762845b3f036c3a11a03102cf282388e15..cd57ce6134c9121de59afd319543512664db9b45 100644 (file)
@@ -35,7 +35,7 @@ BUILD_INTERRUPT(kvm_posted_intr_nested_ipi, POSTED_INTR_NESTED_VECTOR)
 
 BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR)
 BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
-BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
+BUILD_INTERRUPT(spurious_apic_interrupt,SPURIOUS_APIC_VECTOR)
 BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR)
 
 #ifdef CONFIG_IRQ_WORK
index 4154bc5f6a4ed075952e60d6042889d964649a1c..0ffe80792b2d3eb0df171304d1fe4f2d0441af8f 100644 (file)
@@ -39,6 +39,7 @@ extern asmlinkage void irq_work_interrupt(void);
 extern asmlinkage void uv_bau_message_intr1(void);
 
 extern asmlinkage void spurious_interrupt(void);
+extern asmlinkage void spurious_apic_interrupt(void);
 extern asmlinkage void thermal_interrupt(void);
 extern asmlinkage void reschedule_interrupt(void);
 
index 36e5b929389b6cb9774fa03989e8cf86869c9619..2fc0dc8af2a4c45e5c6970dc04b1510a3f784a51 100644 (file)
@@ -347,6 +347,54 @@ __visible noinstr void func(struct pt_regs *regs,                  \
 #define DECLARE_IDTENTRY_XEN(vector, func)                             \
        idtentry vector asm_exc_xen##func exc_##func has_error_code=0
 
+/*
+ * ASM code to emit the common vector entry stubs where each stub is
+ * packed into 8 bytes.
+ *
+ * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because
+ * GCC treats the local vector variable as unsigned int and would expand
+ * all vectors above 0x7F to a 5 byte push. The original code did an
+ * adjustment of the vector number to be in the signed byte range to avoid
+ * this. While clever it's mindboggling counterintuitive and requires the
+ * odd conversion back to a real vector number in the C entry points. Using
+ * .byte achieves the same thing and the only fixup needed in the C entry
+ * point is to mask off the bits above bit 7 because the push is sign
+ * extending.
+ */
+       .align 8
+SYM_CODE_START(irq_entries_start)
+    vector=FIRST_EXTERNAL_VECTOR
+    pos = .
+    .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
+       UNWIND_HINT_IRET_REGS
+       .byte   0x6a, vector
+       jmp     common_interrupt
+       nop
+       /* Ensure that the above is 8 bytes max */
+       . = pos + 8
+    pos=pos+8
+    vector=vector+1
+    .endr
+SYM_CODE_END(irq_entries_start)
+
+#ifdef CONFIG_X86_LOCAL_APIC
+       .align 8
+SYM_CODE_START(spurious_entries_start)
+    vector=FIRST_SYSTEM_VECTOR
+    pos = .
+    .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR)
+       UNWIND_HINT_IRET_REGS
+       .byte   0x6a, vector
+       jmp     common_spurious
+       nop
+       /* Ensure that the above is 8 bytes max */
+       . = pos + 8
+    pos=pos+8
+    vector=vector+1
+    .endr
+SYM_CODE_END(spurious_entries_start)
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 /*
index 72fba0eeeb3074efc6d043881b68409f93247555..74690a373c58797666fe09628159fa48c2d2003f 100644 (file)
@@ -36,7 +36,7 @@ extern void native_init_IRQ(void);
 
 extern void handle_irq(struct irq_desc *desc, struct pt_regs *regs);
 
-extern __visible void do_IRQ(struct pt_regs *regs);
+extern __visible void do_IRQ(struct pt_regs *regs, unsigned long vector);
 
 extern void init_ISA_irqs(void);
 
index d7de360eec74cda8f251875f74d19a0ac5f24f51..32b2becf7806ccb2568a4d709bb6208c160fd2f1 100644 (file)
@@ -41,8 +41,9 @@ asmlinkage void smp_deferred_error_interrupt(struct pt_regs *regs);
 #endif
 
 void smp_apic_timer_interrupt(struct pt_regs *regs);
-void smp_spurious_interrupt(struct pt_regs *regs);
 void smp_error_interrupt(struct pt_regs *regs);
+void smp_spurious_apic_interrupt(struct pt_regs *regs);
+void smp_spurious_interrupt(struct pt_regs *regs, unsigned long vector);
 asmlinkage void smp_irq_move_cleanup_interrupt(void);
 
 #ifdef CONFIG_VMAP_STACK
index 4b1d31be50b4a1aba2816c2a51bc89affbe96f14..6c2b807a7eaead43609d4b84e5f20253d1b0c76d 100644 (file)
@@ -2120,15 +2120,29 @@ void __init register_lapic_address(unsigned long address)
  * Local APIC interrupts
  */
 
-/*
- * This interrupt should _never_ happen with our APIC/SMP architecture
+/**
+ * smp_spurious_interrupt - Catch all for interrupts raised on unused vectors
+ * @regs:      Pointer to pt_regs on stack
+ * @error_code:        The vector number is in the lower 8 bits
+ *
+ * This is invoked from ASM entry code to catch all interrupts which
+ * trigger on an entry which is routed to the common_spurious idtentry
+ * point.
+ *
+ * Also called from smp_spurious_apic_interrupt().
  */
-__visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs)
+__visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs,
+                                                 unsigned long vector)
 {
-       u8 vector = ~regs->orig_ax;
        u32 v;
 
        entering_irq();
+       /*
+        * The push in the entry ASM code which stores the vector number on
+        * the stack in the error code slot is sign expanding. Just use the
+        * lower 8 bits.
+        */
+       vector &= 0xFF;
        trace_spurious_apic_entry(vector);
 
        inc_irq_stat(irq_spurious_count);
@@ -2149,11 +2163,11 @@ __visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs)
         */
        v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
        if (v & (1 << (vector & 0x1f))) {
-               pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
+               pr_info("Spurious interrupt (vector 0x%02lx) on CPU#%d. Acked\n",
                        vector, smp_processor_id());
                ack_APIC_irq();
        } else {
-               pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
+               pr_info("Spurious interrupt (vector 0x%02lx) on CPU#%d. Not pending!\n",
                        vector, smp_processor_id());
        }
 out:
@@ -2161,6 +2175,11 @@ out:
        exiting_irq();
 }
 
+__visible void smp_spurious_apic_interrupt(struct pt_regs *regs)
+{
+       smp_spurious_interrupt(regs, SPURIOUS_APIC_VECTOR);
+}
+
 /*
  * This interrupt should never happen with our APIC/SMP architecture
  */
index ddb11154aeeeb3de4408f6daab51cf4629c80e98..20408e31c18de3b11c9c757067751ce2bf8f27b8 100644 (file)
@@ -145,7 +145,7 @@ static const __initconst struct idt_data apic_idts[] = {
 #ifdef CONFIG_X86_UV
        INTG(UV_BAU_MESSAGE,            uv_bau_message_intr1),
 #endif
-       INTG(SPURIOUS_APIC_VECTOR,      spurious_interrupt),
+       INTG(SPURIOUS_APIC_VECTOR,      spurious_apic_interrupt),
        INTG(ERROR_APIC_VECTOR,         error_interrupt),
 #endif
 };
index 252065d32ab53497672aeb50644be0401944413e..c7669363251a2bae994ec10967d5d8a691144815 100644 (file)
@@ -227,14 +227,18 @@ u64 arch_irq_stat(void)
  * SMP cross-CPU interrupts have their own specific
  * handlers).
  */
-__visible void __irq_entry do_IRQ(struct pt_regs *regs)
+__visible void __irq_entry do_IRQ(struct pt_regs *regs, unsigned long vector)
 {
        struct pt_regs *old_regs = set_irq_regs(regs);
-       struct irq_desc * desc;
-       /* high bit used in ret_from_ code  */
-       unsigned vector = ~regs->orig_ax;
+       struct irq_desc *desc;
 
        entering_irq();
+       /*
+        * The push in the entry ASM code which stores the vector number on
+        * the stack in the error code slot is sign expanding. Just use the
+        * lower 8 bits.
+        */
+       vector &= 0xFF;
 
        /* entering_irq() tells RCU that we're not quiescent.  Check it. */
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
@@ -249,7 +253,7 @@ __visible void __irq_entry do_IRQ(struct pt_regs *regs)
                ack_APIC_irq();
 
                if (desc == VECTOR_UNUSED) {
-                       pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n",
+                       pr_emerg_ratelimited("%s: %d.%lu No irq handler for vector\n",
                                             __func__, smp_processor_id(),
                                             vector);
                } else {