1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 Regents of the University of California
7 #include <linux/kernel.h>
8 #include <linux/init.h>
9 #include <linux/sched.h>
10 #include <linux/sched/debug.h>
11 #include <linux/sched/signal.h>
12 #include <linux/signal.h>
13 #include <linux/kdebug.h>
14 #include <linux/uaccess.h>
15 #include <linux/kprobes.h>
17 #include <linux/module.h>
18 #include <linux/irq.h>
19 #include <linux/kexec.h>
20 #include <linux/entry-common.h>
22 #include <asm/asm-prototypes.h>
26 #include <asm/processor.h>
27 #include <asm/ptrace.h>
28 #include <asm/syscall.h>
29 #include <asm/thread_info.h>
30 #include <asm/vector.h>
31 #include <asm/irq_stack.h>
33 int show_unhandled_signals = 1;
35 static DEFINE_SPINLOCK(die_lock);
37 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs)
39 char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
40 const u16 *insns = (u16 *)instruction_pointer(regs);
45 for (i = -10; i < 2; i++) {
46 bad = get_kernel_nofault(val, &insns[i]);
48 p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
50 printk("%sCode: Unable to access instruction at 0x%px.\n",
55 printk("%sCode: %s\n", loglvl, str);
58 void die(struct pt_regs *regs, const char *str)
60 static int die_counter;
67 spin_lock_irqsave(&die_lock, flags);
71 pr_emerg("%s [#%d]\n", str, ++die_counter);
75 dump_kernel_instr(KERN_EMERG, regs);
78 cause = regs ? regs->cause : -1;
79 ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
81 if (kexec_should_crash(current))
85 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
86 spin_unlock_irqrestore(&die_lock, flags);
90 panic("Fatal exception in interrupt");
92 panic("Fatal exception");
93 if (ret != NOTIFY_STOP)
94 make_task_dead(SIGSEGV);
97 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
99 struct task_struct *tsk = current;
101 if (show_unhandled_signals && unhandled_signal(tsk, signo)
102 && printk_ratelimit()) {
103 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
104 tsk->comm, task_pid_nr(tsk), signo, code, addr);
105 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
110 force_sig_fault(signo, code, (void __user *)addr);
113 static void do_trap_error(struct pt_regs *regs, int signo, int code,
114 unsigned long addr, const char *str)
116 current->thread.bad_cause = regs->cause;
118 if (user_mode(regs)) {
119 do_trap(regs, signo, code, addr);
121 if (!fixup_exception(regs))
126 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
127 #define __trap_section __noinstr_section(".xip.traps")
129 #define __trap_section noinstr
131 #define DO_ERROR_INFO(name, signo, code, str) \
132 asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
134 if (user_mode(regs)) { \
135 irqentry_enter_from_user_mode(regs); \
136 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
137 irqentry_exit_to_user_mode(regs); \
139 irqentry_state_t state = irqentry_nmi_enter(regs); \
140 do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
141 irqentry_nmi_exit(regs, state); \
145 DO_ERROR_INFO(do_trap_unknown,
146 SIGILL, ILL_ILLTRP, "unknown exception");
147 DO_ERROR_INFO(do_trap_insn_misaligned,
148 SIGBUS, BUS_ADRALN, "instruction address misaligned");
149 DO_ERROR_INFO(do_trap_insn_fault,
150 SIGSEGV, SEGV_ACCERR, "instruction access fault");
152 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
156 if (user_mode(regs)) {
157 irqentry_enter_from_user_mode(regs);
161 handled = riscv_v_first_use_handler(regs);
166 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
167 "Oops - illegal instruction");
169 irqentry_exit_to_user_mode(regs);
171 irqentry_state_t state = irqentry_nmi_enter(regs);
173 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
174 "Oops - illegal instruction");
176 irqentry_nmi_exit(regs, state);
180 DO_ERROR_INFO(do_trap_load_fault,
181 SIGSEGV, SEGV_ACCERR, "load access fault");
182 #ifndef CONFIG_RISCV_M_MODE
183 DO_ERROR_INFO(do_trap_load_misaligned,
184 SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
185 DO_ERROR_INFO(do_trap_store_misaligned,
186 SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
188 int handle_misaligned_load(struct pt_regs *regs);
189 int handle_misaligned_store(struct pt_regs *regs);
191 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
193 if (user_mode(regs)) {
194 irqentry_enter_from_user_mode(regs);
196 if (handle_misaligned_load(regs))
197 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
198 "Oops - load address misaligned");
200 irqentry_exit_to_user_mode(regs);
202 irqentry_state_t state = irqentry_nmi_enter(regs);
204 if (handle_misaligned_load(regs))
205 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
206 "Oops - load address misaligned");
208 irqentry_nmi_exit(regs, state);
212 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
214 if (user_mode(regs)) {
215 irqentry_enter_from_user_mode(regs);
217 if (handle_misaligned_store(regs))
218 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
219 "Oops - store (or AMO) address misaligned");
221 irqentry_exit_to_user_mode(regs);
223 irqentry_state_t state = irqentry_nmi_enter(regs);
225 if (handle_misaligned_store(regs))
226 do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
227 "Oops - store (or AMO) address misaligned");
229 irqentry_nmi_exit(regs, state);
233 DO_ERROR_INFO(do_trap_store_fault,
234 SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
235 DO_ERROR_INFO(do_trap_ecall_s,
236 SIGILL, ILL_ILLTRP, "environment call from S-mode");
237 DO_ERROR_INFO(do_trap_ecall_m,
238 SIGILL, ILL_ILLTRP, "environment call from M-mode");
240 static inline unsigned long get_break_insn_length(unsigned long pc)
244 if (get_kernel_nofault(insn, (bug_insn_t *)pc))
247 return GET_INSN_LENGTH(insn);
250 void handle_break(struct pt_regs *regs)
252 #ifdef CONFIG_KPROBES
253 if (kprobe_single_step_handler(regs))
256 if (kprobe_breakpoint_handler(regs))
259 #ifdef CONFIG_UPROBES
260 if (uprobe_single_step_handler(regs))
263 if (uprobe_breakpoint_handler(regs))
266 current->thread.bad_cause = regs->cause;
269 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
271 else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
275 else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
276 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
277 regs->epc += get_break_insn_length(regs->epc);
279 die(regs, "Kernel BUG");
282 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
284 if (user_mode(regs)) {
285 irqentry_enter_from_user_mode(regs);
289 irqentry_exit_to_user_mode(regs);
291 irqentry_state_t state = irqentry_nmi_enter(regs);
295 irqentry_nmi_exit(regs, state);
299 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
301 if (user_mode(regs)) {
302 long syscall = regs->a7;
305 regs->orig_a0 = regs->a0;
307 riscv_v_vstate_discard(regs);
309 syscall = syscall_enter_from_user_mode(regs, syscall);
311 if (syscall >= 0 && syscall < NR_syscalls)
312 syscall_handler(regs, syscall);
313 else if (syscall != -1)
316 syscall_exit_to_user_mode(regs);
318 irqentry_state_t state = irqentry_nmi_enter(regs);
320 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
321 "Oops - environment call from U-mode");
323 irqentry_nmi_exit(regs, state);
329 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
331 irqentry_state_t state = irqentry_enter(regs);
333 handle_page_fault(regs);
337 irqentry_exit(regs, state);
341 static void noinstr handle_riscv_irq(struct pt_regs *regs)
343 struct pt_regs *old_regs;
346 old_regs = set_irq_regs(regs);
347 handle_arch_irq(regs);
348 set_irq_regs(old_regs);
352 asmlinkage void noinstr do_irq(struct pt_regs *regs)
354 irqentry_state_t state = irqentry_enter(regs);
355 #ifdef CONFIG_IRQ_STACKS
356 if (on_thread_stack()) {
357 ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
358 + IRQ_STACK_SIZE/sizeof(ulong);
360 "addi sp, sp, -"RISCV_SZPTR "\n"
362 "addi sp, sp, -"RISCV_SZPTR "\n"
364 "addi s0, sp, 2*"RISCV_SZPTR "\n"
366 "move a0, %[regs] \n"
367 "call handle_riscv_irq \n"
368 "addi sp, s0, -2*"RISCV_SZPTR"\n"
370 "addi sp, sp, "RISCV_SZPTR "\n"
372 "addi sp, sp, "RISCV_SZPTR "\n"
374 : [sp] "r" (sp), [regs] "r" (regs)
375 : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
376 "t0", "t1", "t2", "t3", "t4", "t5", "t6",
377 #ifndef CONFIG_FRAME_POINTER
383 handle_riscv_irq(regs);
385 irqentry_exit(regs, state);
388 #ifdef CONFIG_GENERIC_BUG
389 int is_valid_bugaddr(unsigned long pc)
393 if (pc < VMALLOC_START)
395 if (get_kernel_nofault(insn, (bug_insn_t *)pc))
397 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
398 return (insn == __BUG_INSN_32);
400 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
402 #endif /* CONFIG_GENERIC_BUG */
404 #ifdef CONFIG_VMAP_STACK
406 * Extra stack space that allows us to provide panic messages when the kernel
407 * has overflowed its stack.
409 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
410 overflow_stack)__aligned(16);
412 * A temporary stack for use by handle_kernel_stack_overflow. This is used so
413 * we can call into C code to get the per-hart overflow stack. Usage of this
414 * stack must be protected by spin_shadow_stack.
416 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
419 * A pseudo spinlock to protect the shadow stack from being used by multiple
420 * harts concurrently. This isn't a real spinlock because the lock side must
421 * be taken without a valid stack and only a single register, it's only taken
422 * while in the process of panicing anyway so the performance and error
423 * checking a proper spinlock gives us doesn't matter.
425 unsigned long spin_shadow_stack;
427 asmlinkage unsigned long get_overflow_stack(void)
429 return (unsigned long)this_cpu_ptr(overflow_stack) +
433 asmlinkage void handle_bad_stack(struct pt_regs *regs)
435 unsigned long tsk_stk = (unsigned long)current->stack;
436 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
439 * We're done with the shadow stack by this point, as we're on the
440 * overflow stack. Tell any other concurrent overflowing harts that
441 * they can proceed with panicing by releasing the pseudo-spinlock.
443 * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
445 smp_store_release(&spin_shadow_stack, 0);
449 pr_emerg("Insufficient stack space to handle exception!\n");
450 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
451 tsk_stk, tsk_stk + THREAD_SIZE);
452 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
453 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
456 panic("Kernel stack overflow");
459 wait_for_interrupt();