1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/sched.h>
4 #include <linux/sched/debug.h>
5 #include <linux/init_task.h>
8 #include <linux/uaccess.h>
9 #include <asm/processor.h>
11 #include <asm/traps.h>
13 extern void double_fault(void);
14 #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM)
16 #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x)
18 static void set_df_gdt_entry(unsigned int cpu);
21 * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks
22 * we're running the doublefault task. Cannot return.
24 asmlinkage notrace void __noreturn doublefault_shim(void)
29 BUILD_BUG_ON(sizeof(struct doublefault_stack) != PAGE_SIZE);
31 cr2 = native_read_cr2();
33 /* Reset back to the normal kernel task. */
35 set_df_gdt_entry(smp_processor_id());
40 * Fill in pt_regs. A downside of doing this in C is that the unwinder
41 * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump
42 * won't successfully unwind to the source of the double fault.
43 * The main dump from do_double_fault() is fine, though, since it
44 * uses these regs directly.
46 * If anyone ever cares, this could be moved to asm.
51 regs.flags = TSS(flags);
53 /* We won't go through the entry asm, so we can leave __csh as 0. */
73 do_double_fault(®s, 0, cr2);
76 * x86_32 does not save the original CR3 anywhere on a task switch.
77 * This means that, even if we wanted to return, we would need to find
78 * some way to reconstruct CR3. We could make a credible guess based
79 * on cpu_tlbstate, but that would be racy and would not account for
82 * Instead, don't bother. We can return through
83 * rewind_stack_do_exit() instead.
85 panic("cannot return from double fault\n");
87 NOKPROBE_SYMBOL(doublefault_shim);
89 DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = {
92 * No sp0 or ss0 -- we never run CPL != 0 with this TSS
93 * active. sp is filled in later.
96 .io_bitmap_base = IO_BITMAP_OFFSET_INVALID,
98 .ip = (unsigned long) double_fault,
99 .flags = X86_EFLAGS_FIXED,
104 .fs = __KERNEL_PERCPU,
105 #ifndef CONFIG_X86_32_LAZY_GS
106 .gs = __KERNEL_STACK_CANARY,
109 .__cr3 = __pa_nodebug(swapper_pg_dir),
113 static void set_df_gdt_entry(unsigned int cpu)
115 /* Set up doublefault TSS pointer in the GDT */
116 __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS,
117 &get_cpu_entry_area(cpu)->doublefault_stack.tss);
121 void doublefault_init_cpu_tss(void)
123 unsigned int cpu = smp_processor_id();
124 struct cpu_entry_area *cea = get_cpu_entry_area(cpu);
127 * The linker isn't smart enough to initialize percpu variables that
128 * point to other places in percpu space.
130 this_cpu_write(doublefault_stack.tss.sp,
131 (unsigned long)&cea->doublefault_stack.stack +
132 sizeof(doublefault_stack.stack));
134 set_df_gdt_entry(cpu);