arch/riscv/kernel/traps.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 Regents of the University of California
   4  */
   5
   6 #include <linux/cpu.h>
   7 #include <linux/kernel.h>
   8 #include <linux/init.h>
   9 #include <linux/sched.h>
  10 #include <linux/sched/debug.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/signal.h>
  13 #include <linux/kdebug.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/kprobes.h>
  16 #include <linux/mm.h>
  17 #include <linux/module.h>
  18 #include <linux/irq.h>
  19 #include <linux/kexec.h>
  20 #include <linux/entry-common.h>
  21
  22 #include <asm/asm-prototypes.h>
  23 #include <asm/bug.h>
  24 #include <asm/csr.h>
  25 #include <asm/processor.h>
  26 #include <asm/ptrace.h>
  27 #include <asm/syscall.h>
  28 #include <asm/thread_info.h>
  29 #include <asm/vector.h>
  30
  31 int show_unhandled_signals = 1;
  32
  33 static DEFINE_SPINLOCK(die_lock);
  34
  35 static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs)
  36 {
  37         char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
  38         const u16 *insns = (u16 *)instruction_pointer(regs);
  39         long bad;
  40         u16 val;
  41         int i;
  42
  43         for (i = -10; i < 2; i++) {
  44                 bad = get_kernel_nofault(val, &insns[i]);
  45                 if (!bad) {
  46                         p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
  47                 } else {
  48                         printk("%sCode: Unable to access instruction at 0x%px.\n",
  49                                loglvl, &insns[i]);
  50                         return;
  51                 }
  52         }
  53         printk("%sCode: %s\n", loglvl, str);
  54 }
  55
  56 void die(struct pt_regs *regs, const char *str)
  57 {
  58         static int die_counter;
  59         int ret;
  60         long cause;
  61         unsigned long flags;
  62
  63         oops_enter();
  64
  65         spin_lock_irqsave(&die_lock, flags);
  66         console_verbose();
  67         bust_spinlocks(1);
  68
  69         pr_emerg("%s [#%d]\n", str, ++die_counter);
  70         print_modules();
  71         if (regs) {
  72                 show_regs(regs);
  73                 dump_kernel_instr(KERN_EMERG, regs);
  74         }
  75
  76         cause = regs ? regs->cause : -1;
  77         ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
  78
  79         if (kexec_should_crash(current))
  80                 crash_kexec(regs);
  81
  82         bust_spinlocks(0);
  83         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
  84         spin_unlock_irqrestore(&die_lock, flags);
  85         oops_exit();
  86
  87         if (in_interrupt())
  88                 panic("Fatal exception in interrupt");
  89         if (panic_on_oops)
  90                 panic("Fatal exception");
  91         if (ret != NOTIFY_STOP)
  92                 make_task_dead(SIGSEGV);
  93 }
  94
  95 void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
  96 {
  97         struct task_struct *tsk = current;
  98
  99         if (show_unhandled_signals && unhandled_signal(tsk, signo)
 100             && printk_ratelimit()) {
 101                 pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
 102                         tsk->comm, task_pid_nr(tsk), signo, code, addr);
 103                 print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
 104                 pr_cont("\n");
 105                 __show_regs(regs);
 106         }
 107
 108         force_sig_fault(signo, code, (void __user *)addr);
 109 }
 110
 111 static void do_trap_error(struct pt_regs *regs, int signo, int code,
 112         unsigned long addr, const char *str)
 113 {
 114         current->thread.bad_cause = regs->cause;
 115
 116         if (user_mode(regs)) {
 117                 do_trap(regs, signo, code, addr);
 118         } else {
 119                 if (!fixup_exception(regs))
 120                         die(regs, str);
 121         }
 122 }
 123
 124 #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
 125 #define __trap_section __noinstr_section(".xip.traps")
 126 #else
 127 #define __trap_section noinstr
 128 #endif
 129 #define DO_ERROR_INFO(name, signo, code, str)                                   \
 130 asmlinkage __visible __trap_section void name(struct pt_regs *regs)             \
 131 {                                                                               \
 132         if (user_mode(regs)) {                                                  \
 133                 irqentry_enter_from_user_mode(regs);                            \
 134                 do_trap_error(regs, signo, code, regs->epc, "Oops - " str);     \
 135                 irqentry_exit_to_user_mode(regs);                               \
 136         } else {                                                                \
 137                 irqentry_state_t state = irqentry_nmi_enter(regs);              \
 138                 do_trap_error(regs, signo, code, regs->epc, "Oops - " str);     \
 139                 irqentry_nmi_exit(regs, state);                                 \
 140         }                                                                       \
 141 }
 142
 143 DO_ERROR_INFO(do_trap_unknown,
 144         SIGILL, ILL_ILLTRP, "unknown exception");
 145 DO_ERROR_INFO(do_trap_insn_misaligned,
 146         SIGBUS, BUS_ADRALN, "instruction address misaligned");
 147 DO_ERROR_INFO(do_trap_insn_fault,
 148         SIGSEGV, SEGV_ACCERR, "instruction access fault");
 149
 150 asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
 151 {
 152         if (user_mode(regs)) {
 153                 irqentry_enter_from_user_mode(regs);
 154
 155                 local_irq_enable();
 156
 157                 if (!riscv_v_first_use_handler(regs))
 158                         do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
 159                                       "Oops - illegal instruction");
 160
 161                 irqentry_exit_to_user_mode(regs);
 162         } else {
 163                 irqentry_state_t state = irqentry_nmi_enter(regs);
 164
 165                 do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
 166                               "Oops - illegal instruction");
 167
 168                 irqentry_nmi_exit(regs, state);
 169         }
 170 }
 171
 172 DO_ERROR_INFO(do_trap_load_fault,
 173         SIGSEGV, SEGV_ACCERR, "load access fault");
 174 #ifndef CONFIG_RISCV_M_MODE
 175 DO_ERROR_INFO(do_trap_load_misaligned,
 176         SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
 177 DO_ERROR_INFO(do_trap_store_misaligned,
 178         SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
 179 #else
 180 int handle_misaligned_load(struct pt_regs *regs);
 181 int handle_misaligned_store(struct pt_regs *regs);
 182
 183 asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
 184 {
 185         if (user_mode(regs)) {
 186                 irqentry_enter_from_user_mode(regs);
 187
 188                 if (handle_misaligned_load(regs))
 189                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 190                               "Oops - load address misaligned");
 191
 192                 irqentry_exit_to_user_mode(regs);
 193         } else {
 194                 irqentry_state_t state = irqentry_nmi_enter(regs);
 195
 196                 if (handle_misaligned_load(regs))
 197                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 198                               "Oops - load address misaligned");
 199
 200                 irqentry_nmi_exit(regs, state);
 201         }
 202 }
 203
 204 asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
 205 {
 206         if (user_mode(regs)) {
 207                 irqentry_enter_from_user_mode(regs);
 208
 209                 if (handle_misaligned_store(regs))
 210                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 211                                 "Oops - store (or AMO) address misaligned");
 212
 213                 irqentry_exit_to_user_mode(regs);
 214         } else {
 215                 irqentry_state_t state = irqentry_nmi_enter(regs);
 216
 217                 if (handle_misaligned_store(regs))
 218                         do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
 219                                 "Oops - store (or AMO) address misaligned");
 220
 221                 irqentry_nmi_exit(regs, state);
 222         }
 223 }
 224 #endif
 225 DO_ERROR_INFO(do_trap_store_fault,
 226         SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
 227 DO_ERROR_INFO(do_trap_ecall_s,
 228         SIGILL, ILL_ILLTRP, "environment call from S-mode");
 229 DO_ERROR_INFO(do_trap_ecall_m,
 230         SIGILL, ILL_ILLTRP, "environment call from M-mode");
 231
 232 static inline unsigned long get_break_insn_length(unsigned long pc)
 233 {
 234         bug_insn_t insn;
 235
 236         if (get_kernel_nofault(insn, (bug_insn_t *)pc))
 237                 return 0;
 238
 239         return GET_INSN_LENGTH(insn);
 240 }
 241
 242 void handle_break(struct pt_regs *regs)
 243 {
 244 #ifdef CONFIG_KPROBES
 245         if (kprobe_single_step_handler(regs))
 246                 return;
 247
 248         if (kprobe_breakpoint_handler(regs))
 249                 return;
 250 #endif
 251 #ifdef CONFIG_UPROBES
 252         if (uprobe_single_step_handler(regs))
 253                 return;
 254
 255         if (uprobe_breakpoint_handler(regs))
 256                 return;
 257 #endif
 258         current->thread.bad_cause = regs->cause;
 259
 260         if (user_mode(regs))
 261                 force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
 262 #ifdef CONFIG_KGDB
 263         else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
 264                                                                 == NOTIFY_STOP)
 265                 return;
 266 #endif
 267         else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN)
 268                 regs->epc += get_break_insn_length(regs->epc);
 269         else
 270                 die(regs, "Kernel BUG");
 271 }
 272
 273 asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
 274 {
 275         if (user_mode(regs)) {
 276                 irqentry_enter_from_user_mode(regs);
 277
 278                 handle_break(regs);
 279
 280                 irqentry_exit_to_user_mode(regs);
 281         } else {
 282                 irqentry_state_t state = irqentry_nmi_enter(regs);
 283
 284                 handle_break(regs);
 285
 286                 irqentry_nmi_exit(regs, state);
 287         }
 288 }
 289
 290 asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
 291 {
 292         if (user_mode(regs)) {
 293                 ulong syscall = regs->a7;
 294
 295                 regs->epc += 4;
 296                 regs->orig_a0 = regs->a0;
 297
 298                 syscall = syscall_enter_from_user_mode(regs, syscall);
 299
 300                 if (syscall < NR_syscalls)
 301                         syscall_handler(regs, syscall);
 302                 else
 303                         regs->a0 = -ENOSYS;
 304
 305                 syscall_exit_to_user_mode(regs);
 306         } else {
 307                 irqentry_state_t state = irqentry_nmi_enter(regs);
 308
 309                 do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
 310                         "Oops - environment call from U-mode");
 311
 312                 irqentry_nmi_exit(regs, state);
 313         }
 314
 315 }
 316
 317 #ifdef CONFIG_MMU
 318 asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
 319 {
 320         irqentry_state_t state = irqentry_enter(regs);
 321
 322         handle_page_fault(regs);
 323
 324         local_irq_disable();
 325
 326         irqentry_exit(regs, state);
 327 }
 328 #endif
 329
 330 asmlinkage __visible noinstr void do_irq(struct pt_regs *regs)
 331 {
 332         struct pt_regs *old_regs;
 333         irqentry_state_t state = irqentry_enter(regs);
 334
 335         irq_enter_rcu();
 336         old_regs = set_irq_regs(regs);
 337         handle_arch_irq(regs);
 338         set_irq_regs(old_regs);
 339         irq_exit_rcu();
 340
 341         irqentry_exit(regs, state);
 342 }
 343
 344 #ifdef CONFIG_GENERIC_BUG
 345 int is_valid_bugaddr(unsigned long pc)
 346 {
 347         bug_insn_t insn;
 348
 349         if (pc < VMALLOC_START)
 350                 return 0;
 351         if (get_kernel_nofault(insn, (bug_insn_t *)pc))
 352                 return 0;
 353         if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
 354                 return (insn == __BUG_INSN_32);
 355         else
 356                 return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
 357 }
 358 #endif /* CONFIG_GENERIC_BUG */
 359
 360 #ifdef CONFIG_VMAP_STACK
 361 /*
 362  * Extra stack space that allows us to provide panic messages when the kernel
 363  * has overflowed its stack.
 364  */
 365 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
 366                 overflow_stack)__aligned(16);
 367 /*
 368  * A temporary stack for use by handle_kernel_stack_overflow.  This is used so
 369  * we can call into C code to get the per-hart overflow stack.  Usage of this
 370  * stack must be protected by spin_shadow_stack.
 371  */
 372 long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
 373
 374 /*
 375  * A pseudo spinlock to protect the shadow stack from being used by multiple
 376  * harts concurrently.  This isn't a real spinlock because the lock side must
 377  * be taken without a valid stack and only a single register, it's only taken
 378  * while in the process of panicing anyway so the performance and error
 379  * checking a proper spinlock gives us doesn't matter.
 380  */
 381 unsigned long spin_shadow_stack;
 382
 383 asmlinkage unsigned long get_overflow_stack(void)
 384 {
 385         return (unsigned long)this_cpu_ptr(overflow_stack) +
 386                 OVERFLOW_STACK_SIZE;
 387 }
 388
 389 asmlinkage void handle_bad_stack(struct pt_regs *regs)
 390 {
 391         unsigned long tsk_stk = (unsigned long)current->stack;
 392         unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 393
 394         /*
 395          * We're done with the shadow stack by this point, as we're on the
 396          * overflow stack.  Tell any other concurrent overflowing harts that
 397          * they can proceed with panicing by releasing the pseudo-spinlock.
 398          *
 399          * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
 400          */
 401         smp_store_release(&spin_shadow_stack, 0);
 402
 403         console_verbose();
 404
 405         pr_emerg("Insufficient stack space to handle exception!\n");
 406         pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
 407                         tsk_stk, tsk_stk + THREAD_SIZE);
 408         pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
 409                         ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
 410
 411         __show_regs(regs);
 412         panic("Kernel stack overflow");
 413
 414         for (;;)
 415                 wait_for_interrupt();
 416 }
 417 #endif