Merge tag 'x86-apic-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git...

Pull x86 exception handling updates from Ingo Molnar:

 - Clean up & simplify AP exception handling setup.

 - Consolidate the disjoint IDT setup code living in idt_setup_traps()
   and idt_setup_ist_traps() into a single idt_setup_traps()
   initialization function and call it before cpu_init().

* tag 'x86-apic-2021-06-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/idt: Rework IDT setup for boot CPU
  x86/cpu: Init AP exception handling from cpu_init_secondary()

diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index 476082a..96021e9 100644 (file)
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -421,10 +421,8 @@ extern bool idt_is_f00f_address(unsigned long address);
 
 #ifdef CONFIG_X86_64
 extern void idt_setup_early_pf(void);
-extern void idt_setup_ist_traps(void);
 #else
 static inline void idt_setup_early_pf(void) { }
-static inline void idt_setup_ist_traps(void) { }
 #endif
 
 extern void idt_invalidate(void *addr);

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 556b2b1..364d0e4 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -663,6 +663,7 @@ extern void load_direct_gdt(int);
 extern void load_fixmap_gdt(int);
 extern void load_percpu_segment(int);
 extern void cpu_init(void);
+extern void cpu_init_secondary(void);
 extern void cpu_init_exception_handling(void);
 extern void cr4_init(void);
 

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a1b756c..212e8bc 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1938,13 +1938,12 @@ void cpu_init_exception_handling(void)
 
 /*
  * cpu_init() initializes state that is per-CPU. Some data is already
- * initialized (naturally) in the bootstrap process, such as the GDT
- * and IDT. We reload them nevertheless, this function acts as a
- * 'CPU state barrier', nothing should get across.
+ * initialized (naturally) in the bootstrap process, such as the GDT.  We
+ * reload it nevertheless, this function acts as a 'CPU state barrier',
+ * nothing should get across.
  */
 void cpu_init(void)
 {
-       struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
        struct task_struct *cur = current;
        int cpu = raw_smp_processor_id();
 
@@ -1957,8 +1956,6 @@ void cpu_init(void)
            early_cpu_to_node(cpu) != NUMA_NO_NODE)
                set_numa_node(early_cpu_to_node(cpu));
 #endif
-       setup_getcpu(cpu);
-
        pr_debug("Initializing CPU#%d\n", cpu);
 
        if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
@@ -1970,7 +1967,6 @@ void cpu_init(void)
         * and set up the GDT descriptor:
         */
        switch_to_new_gdt(cpu);
-       load_current_idt();
 
        if (IS_ENABLED(CONFIG_X86_64)) {
                loadsegment(fs, 0);
@@ -1990,12 +1986,6 @@ void cpu_init(void)
        initialize_tlbstate_and_flush();
        enter_lazy_tlb(&init_mm, cur);
 
-       /* Initialize the TSS. */
-       tss_setup_ist(tss);
-       tss_setup_io_bitmap(tss);
-       set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
-
-       load_TR_desc();
        /*
         * sp0 points to the entry trampoline stack regardless of what task
         * is running.
@@ -2017,6 +2007,18 @@ void cpu_init(void)
        load_fixmap_gdt(cpu);
 }
 
+#ifdef CONFIG_SMP
+void cpu_init_secondary(void)
+{
+       /*
+        * Relies on the BP having set-up the IDT tables, which are loaded
+        * on this CPU in cpu_init_exception_handling().
+        */
+       cpu_init_exception_handling();
+       cpu_init();
+}
+#endif
+
 /*
  * The microcode loader calls this upon late microcode load to recheck features,
  * only when microcode has been updated. Caller holds microcode_mutex and CPU

diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index d552f17..6cce604 100644 (file)
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -35,12 +35,16 @@
 #define SYSG(_vector, _addr)                           \
        G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
 
+#ifdef CONFIG_X86_64
 /*
  * Interrupt gate with interrupt stack. The _ist index is the index in
  * the tss.ist[] array, but for the descriptor it needs to start at 1.
  */
 #define ISTG(_vector, _addr, _ist)                     \
        G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
+#else
+#define ISTG(_vector, _addr, _ist)     INTG(_vector, _addr)
+#endif
 
 /* Task gate */
 #define TSKG(_vector, _gdt)                            \
@@ -74,7 +78,7 @@ static const __initconst struct idt_data early_idts[] = {
  */
 static const __initconst struct idt_data def_idts[] = {
        INTG(X86_TRAP_DE,               asm_exc_divide_error),
-       INTG(X86_TRAP_NMI,              asm_exc_nmi),
+       ISTG(X86_TRAP_NMI,              asm_exc_nmi, IST_INDEX_NMI),
        INTG(X86_TRAP_BR,               asm_exc_bounds),
        INTG(X86_TRAP_UD,               asm_exc_invalid_op),
        INTG(X86_TRAP_NM,               asm_exc_device_not_available),
@@ -91,12 +95,16 @@ static const __initconst struct idt_data def_idts[] = {
 #ifdef CONFIG_X86_32
        TSKG(X86_TRAP_DF,               GDT_ENTRY_DOUBLEFAULT_TSS),
 #else
-       INTG(X86_TRAP_DF,               asm_exc_double_fault),
+       ISTG(X86_TRAP_DF,               asm_exc_double_fault, IST_INDEX_DF),
 #endif
-       INTG(X86_TRAP_DB,               asm_exc_debug),
+       ISTG(X86_TRAP_DB,               asm_exc_debug, IST_INDEX_DB),
 
 #ifdef CONFIG_X86_MCE
-       INTG(X86_TRAP_MC,               asm_exc_machine_check),
+       ISTG(X86_TRAP_MC,               asm_exc_machine_check, IST_INDEX_MCE),
+#endif
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+       ISTG(X86_TRAP_VC,               asm_exc_vmm_communication, IST_INDEX_VC),
 #endif
 
        SYSG(X86_TRAP_OF,               asm_exc_overflow),
@@ -221,22 +229,6 @@ static const __initconst struct idt_data early_pf_idts[] = {
        INTG(X86_TRAP_PF,               asm_exc_page_fault),
 };
 
-/*
- * The exceptions which use Interrupt stacks. They are setup after
- * cpu_init() when the TSS has been initialized.
- */
-static const __initconst struct idt_data ist_idts[] = {
-       ISTG(X86_TRAP_DB,       asm_exc_debug,                  IST_INDEX_DB),
-       ISTG(X86_TRAP_NMI,      asm_exc_nmi,                    IST_INDEX_NMI),
-       ISTG(X86_TRAP_DF,       asm_exc_double_fault,           IST_INDEX_DF),
-#ifdef CONFIG_X86_MCE
-       ISTG(X86_TRAP_MC,       asm_exc_machine_check,          IST_INDEX_MCE),
-#endif
-#ifdef CONFIG_AMD_MEM_ENCRYPT
-       ISTG(X86_TRAP_VC,       asm_exc_vmm_communication,      IST_INDEX_VC),
-#endif
-};
-
 /**
  * idt_setup_early_pf - Initialize the idt table with early pagefault handler
  *
@@ -254,14 +246,6 @@ void __init idt_setup_early_pf(void)
        idt_setup_from_table(idt_table, early_pf_idts,
                             ARRAY_SIZE(early_pf_idts), true);
 }
-
-/**
- * idt_setup_ist_traps - Initialize the idt table with traps using IST
- */
-void __init idt_setup_ist_traps(void)
-{
-       idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
-}
 #endif
 
 static void __init idt_map_in_cea(void)

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ec2d64a..9320285 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -232,8 +232,7 @@ static void notrace start_secondary(void *unused)
        load_cr3(swapper_pg_dir);
        __flush_tlb_all();
 #endif
-       cpu_init_exception_handling();
-       cpu_init();
+       cpu_init_secondary();
        rcu_cpu_starting(raw_smp_processor_id());
        x86_cpuinit.early_percpu_clock_init();
        smp_callin();

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 853ea7a..ed540e0 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -1160,12 +1160,9 @@ void __init trap_init(void)
        /* Init GHCB memory pages when running as an SEV-ES guest */
        sev_es_init_vc_handling();
 
+       /* Initialize TSS before setting up traps so ISTs work */
+       cpu_init_exception_handling();
+       /* Setup traps as cpu_init() might #GP */
        idt_setup_traps();
-
-       /*
-        * Should be a barrier for any external CPU state:
-        */
        cpu_init();
-
-       idt_setup_ist_traps();
 }