2 * Based on arch/arm/mm/fault.c
4 * Copyright (C) 1995 Linus Torvalds
5 * Copyright (C) 1995-2004 Russell King
6 * Copyright (C) 2012 ARM Ltd.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 #include <linux/extable.h>
22 #include <linux/signal.h>
24 #include <linux/hardirq.h>
25 #include <linux/init.h>
26 #include <linux/kprobes.h>
27 #include <linux/uaccess.h>
28 #include <linux/page-flags.h>
29 #include <linux/sched.h>
30 #include <linux/highmem.h>
31 #include <linux/perf_event.h>
32 #include <linux/preempt.h>
35 #include <asm/cpufeature.h>
36 #include <asm/exception.h>
37 #include <asm/debug-monitors.h>
39 #include <asm/sysreg.h>
40 #include <asm/system_misc.h>
41 #include <asm/pgtable.h>
42 #include <asm/tlbflush.h>
45 int (*fn)(unsigned long addr, unsigned int esr,
46 struct pt_regs *regs);
52 static const struct fault_info fault_info[];
54 static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
56 return fault_info + (esr & 63);
60 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
64 /* kprobe_running() needs smp_processor_id() */
65 if (!user_mode(regs)) {
67 if (kprobe_running() && kprobe_fault_handler(regs, esr))
75 static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
82 * Dump out the page tables associated with 'addr' in the currently active mm.
84 void show_pte(unsigned long addr)
89 if (addr < TASK_SIZE) {
91 mm = current->active_mm;
93 pr_alert("[%016lx] user address but active_mm is swapper\n",
97 } else if (addr >= VA_START) {
101 pr_alert("[%016lx] address between user and kernel address ranges\n",
106 pr_alert("pgd = %p\n", mm->pgd);
107 pgd = pgd_offset(mm, addr);
108 pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd));
115 if (pgd_none(*pgd) || pgd_bad(*pgd))
118 pud = pud_offset(pgd, addr);
119 pr_cont(", *pud=%016llx", pud_val(*pud));
120 if (pud_none(*pud) || pud_bad(*pud))
123 pmd = pmd_offset(pud, addr);
124 pr_cont(", *pmd=%016llx", pmd_val(*pmd));
125 if (pmd_none(*pmd) || pmd_bad(*pmd))
128 pte = pte_offset_map(pmd, addr);
129 pr_cont(", *pte=%016llx", pte_val(*pte));
136 #ifdef CONFIG_ARM64_HW_AFDBM
138 * This function sets the access flags (dirty, accessed), as well as write
139 * permission, and only to a more permissive setting.
141 * It needs to cope with hardware update of the accessed/dirty state by other
142 * agents in the system and can safely skip the __sync_icache_dcache() call as,
143 * like set_pte_at(), the PTE is never changed from no-exec to exec here.
145 * Returns whether or not the PTE actually changed.
147 int ptep_set_access_flags(struct vm_area_struct *vma,
148 unsigned long address, pte_t *ptep,
149 pte_t entry, int dirty)
154 if (pte_same(*ptep, entry))
157 /* only preserve the access flags and write permission */
158 pte_val(entry) &= PTE_AF | PTE_WRITE | PTE_DIRTY;
161 * PTE_RDONLY is cleared by default in the asm below, so set it in
162 * back if necessary (read-only or clean PTE).
164 if (!pte_write(entry) || !pte_sw_dirty(entry))
165 pte_val(entry) |= PTE_RDONLY;
168 * Setting the flags must be done atomically to avoid racing with the
169 * hardware update of the access/dirty state.
171 asm volatile("// ptep_set_access_flags\n"
172 " prfm pstl1strm, %2\n"
174 " and %0, %0, %3 // clear PTE_RDONLY\n"
175 " orr %0, %0, %4 // set flags\n"
176 " stxr %w1, %0, %2\n"
178 : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))
179 : "L" (~PTE_RDONLY), "r" (pte_val(entry)));
181 flush_tlb_fix_spurious_fault(vma, address);
186 #ifdef CONFIG_AMLOGIC_USER_FAULT
187 static long get_user_pfn(struct mm_struct *mm, unsigned long addr)
192 if (!mm || addr >= VMALLOC_START)
195 pgd = pgd_offset(mm, addr);
202 if (pgd_none(*pgd) || pgd_bad(*pgd))
205 pud = pud_offset(pgd, addr);
206 if (pud_none(*pud) || pud_bad(*pud))
209 pmd = pmd_offset(pud, addr);
210 if (pmd_none(*pmd) || pmd_bad(*pmd))
213 pte = pte_offset_map(pmd, addr);
220 #endif /* CONFIG_AMLOGIC_USER_FAULT */
223 static bool is_el1_instruction_abort(unsigned int esr)
225 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
229 * The kernel tried to access some page that wasn't present.
231 static void __do_kernel_fault(unsigned long addr, unsigned int esr,
232 struct pt_regs *regs)
235 * Are we prepared to handle this kernel fault?
236 * We are almost certainly not prepared to handle instruction faults.
238 if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
242 * No handler, we'll have to terminate things with extreme prejudice.
245 pr_alert("Unable to handle kernel %s at virtual address %08lx\n",
246 (addr < PAGE_SIZE) ? "NULL pointer dereference" :
247 "paging request", addr);
250 die("Oops", regs, esr);
255 #ifdef CONFIG_AMLOGIC_USER_FAULT
256 void show_all_pfn(struct task_struct *task, struct pt_regs *regs)
263 if (compat_user_mode(regs))
267 pr_info("reg value pfn ");
268 pr_cont("reg value pfn\n");
269 for (i = 0; i < top; i++) {
270 pfn1 = get_user_pfn(task->mm, regs->regs[i]);
272 sprintf(s1, "%8lx", pfn1);
274 sprintf(s1, "--------");
276 pr_cont("r%-2d: %016llx %s\n", i, regs->regs[i], s1);
278 pr_info("r%-2d: %016llx %s ", i, regs->regs[i], s1);
281 pfn1 = get_user_pfn(task->mm, regs->pc);
283 sprintf(s1, "%8lx", pfn1);
285 sprintf(s1, "--------");
286 pr_info("pc : %016llx %s\n", regs->pc, s1);
287 pfn1 = get_user_pfn(task->mm, regs->sp);
289 sprintf(s1, "%8lx", pfn1);
291 sprintf(s1, "--------");
292 pr_info("sp : %016llx %s\n", regs->sp, s1);
294 pfn1 = get_user_pfn(task->mm, regs->unused);
296 sprintf(s1, "%8lx", pfn1);
298 sprintf(s1, "--------");
299 pr_info("unused : %016llx %s\n", regs->unused, s1);
301 #endif /* CONFIG_AMLOGIC_USER_FAULT */
304 * Something tried to access memory that isn't in our memory map. User mode
305 * accesses just cause a SIGSEGV
307 static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
308 unsigned int esr, unsigned int sig, int code,
309 struct pt_regs *regs)
312 const struct fault_info *inf;
314 if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) {
315 inf = esr_to_fault_info(esr);
316 pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n",
317 tsk->comm, task_pid_nr(tsk), inf->name, sig,
322 tsk->thread.fault_address = addr;
323 tsk->thread.fault_code = esr;
327 si.si_addr = (void __user *)addr;
328 force_sig_info(sig, &si, tsk);
331 static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
333 struct task_struct *tsk = current;
334 const struct fault_info *inf;
337 * If we are in kernel mode at this point, we have no context to
338 * handle this fault with.
340 if (user_mode(regs)) {
341 inf = esr_to_fault_info(esr);
342 __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs);
344 __do_kernel_fault(addr, esr, regs);
347 #define VM_FAULT_BADMAP 0x010000
348 #define VM_FAULT_BADACCESS 0x020000
350 static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
351 unsigned int mm_flags, unsigned long vm_flags,
352 struct task_struct *tsk)
354 struct vm_area_struct *vma;
357 vma = find_vma(mm, addr);
358 fault = VM_FAULT_BADMAP;
361 if (unlikely(vma->vm_start > addr))
365 * Ok, we have a good vm_area for this memory access, so we can handle
370 * Check that the permissions on the VMA allow for the fault which
373 if (!(vma->vm_flags & vm_flags)) {
374 fault = VM_FAULT_BADACCESS;
378 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);
381 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
387 static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs)
389 unsigned int ec = ESR_ELx_EC(esr);
390 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
392 if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
395 if (system_uses_ttbr0_pan())
396 return fsc_type == ESR_ELx_FSC_FAULT &&
397 (regs->pstate & PSR_PAN_BIT);
399 return fsc_type == ESR_ELx_FSC_PERM;
402 static bool is_el0_instruction_abort(unsigned int esr)
404 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
407 static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
408 struct pt_regs *regs)
410 struct task_struct *tsk;
411 struct mm_struct *mm;
412 int fault, sig, code;
413 unsigned long vm_flags = VM_READ | VM_WRITE | VM_EXEC;
414 unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
416 if (notify_page_fault(regs, esr))
423 * If we're in an interrupt or have no user context, we must not take
426 if (faulthandler_disabled() || !mm)
430 mm_flags |= FAULT_FLAG_USER;
432 if (is_el0_instruction_abort(esr)) {
434 } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
436 mm_flags |= FAULT_FLAG_WRITE;
439 if (addr < TASK_SIZE && is_permission_fault(esr, regs)) {
440 /* regs->orig_addr_limit may be 0 if we entered from EL0 */
441 if (regs->orig_addr_limit == KERNEL_DS)
442 die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
444 if (is_el1_instruction_abort(esr))
445 die("Attempting to execute userspace memory", regs, esr);
447 if (!search_exception_tables(regs->pc))
448 die("Accessing user space memory outside uaccess.h routines", regs, esr);
452 * As per x86, we may deadlock here. However, since the kernel only
453 * validly references user space from well defined areas of the code,
454 * we can bug out early if this is from code which shouldn't.
456 if (!down_read_trylock(&mm->mmap_sem)) {
457 if (!user_mode(regs) && !search_exception_tables(regs->pc))
460 down_read(&mm->mmap_sem);
463 * The above down_read_trylock() might have succeeded in which
464 * case, we'll have missed the might_sleep() from down_read().
467 #ifdef CONFIG_DEBUG_VM
468 if (!user_mode(regs) && !search_exception_tables(regs->pc))
473 fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
476 * If we need to retry but a fatal signal is pending, handle the
477 * signal first. We do not need to release the mmap_sem because it
478 * would already be released in __lock_page_or_retry in mm/filemap.c.
480 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
481 if (!user_mode(regs))
487 * Major/minor page fault accounting is only done on the initial
488 * attempt. If we go through a retry, it is extremely likely that the
489 * page will be found in page cache at that point.
492 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
493 if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
494 if (fault & VM_FAULT_MAJOR) {
496 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
500 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
503 if (fault & VM_FAULT_RETRY) {
505 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
508 mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
509 mm_flags |= FAULT_FLAG_TRIED;
514 up_read(&mm->mmap_sem);
517 * Handle the "normal" case first - VM_FAULT_MAJOR
519 if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
520 VM_FAULT_BADACCESS))))
524 * If we are in kernel mode at this point, we have no context to
525 * handle this fault with.
527 if (!user_mode(regs))
530 if (fault & VM_FAULT_OOM) {
532 * We ran out of memory, call the OOM killer, and return to
533 * userspace (which will retry the fault, or kill us if we got
536 pagefault_out_of_memory();
540 if (fault & VM_FAULT_SIGBUS) {
542 * We had some memory, but were unable to successfully fix up
549 * Something tried to access memory that isn't in our memory
553 code = fault == VM_FAULT_BADACCESS ?
554 SEGV_ACCERR : SEGV_MAPERR;
557 __do_user_fault(tsk, addr, esr, sig, code, regs);
561 __do_kernel_fault(addr, esr, regs);
566 * First Level Translation Fault Handler
568 * We enter here because the first level page table doesn't contain a valid
569 * entry for the address.
571 * If the address is in kernel space (>= TASK_SIZE), then we are probably
572 * faulting in the vmalloc() area.
574 * If the init_task's first level page tables contains the relevant entry, we
575 * copy the it to this task. If not, we send the process a signal, fixup the
576 * exception, or oops the kernel.
578 * NOTE! We MUST NOT take any locks for this case. We may be in an interrupt
579 * or a critical region, and should only copy the information from the master
580 * page table, nothing more.
582 static int __kprobes do_translation_fault(unsigned long addr,
584 struct pt_regs *regs)
586 if (addr < TASK_SIZE)
587 return do_page_fault(addr, esr, regs);
589 do_bad_area(addr, esr, regs);
593 static int do_alignment_fault(unsigned long addr, unsigned int esr,
594 struct pt_regs *regs)
596 do_bad_area(addr, esr, regs);
601 * This abort handler always returns "fault".
603 static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
608 static const struct fault_info fault_info[] = {
609 { do_bad, SIGBUS, 0, "ttbr address size fault" },
610 { do_bad, SIGBUS, 0, "level 1 address size fault" },
611 { do_bad, SIGBUS, 0, "level 2 address size fault" },
612 { do_bad, SIGBUS, 0, "level 3 address size fault" },
613 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
614 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
615 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
616 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
617 { do_bad, SIGBUS, 0, "unknown 8" },
618 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
619 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
620 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
621 { do_bad, SIGBUS, 0, "unknown 12" },
622 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
623 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
624 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
625 { do_bad, SIGBUS, 0, "synchronous external abort" },
626 { do_bad, SIGBUS, 0, "unknown 17" },
627 { do_bad, SIGBUS, 0, "unknown 18" },
628 { do_bad, SIGBUS, 0, "unknown 19" },
629 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
630 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
631 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
632 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
633 { do_bad, SIGBUS, 0, "synchronous parity error" },
634 { do_bad, SIGBUS, 0, "unknown 25" },
635 { do_bad, SIGBUS, 0, "unknown 26" },
636 { do_bad, SIGBUS, 0, "unknown 27" },
637 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
638 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
639 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
640 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk)" },
641 { do_bad, SIGBUS, 0, "unknown 32" },
642 { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" },
643 { do_bad, SIGBUS, 0, "unknown 34" },
644 { do_bad, SIGBUS, 0, "unknown 35" },
645 { do_bad, SIGBUS, 0, "unknown 36" },
646 { do_bad, SIGBUS, 0, "unknown 37" },
647 { do_bad, SIGBUS, 0, "unknown 38" },
648 { do_bad, SIGBUS, 0, "unknown 39" },
649 { do_bad, SIGBUS, 0, "unknown 40" },
650 { do_bad, SIGBUS, 0, "unknown 41" },
651 { do_bad, SIGBUS, 0, "unknown 42" },
652 { do_bad, SIGBUS, 0, "unknown 43" },
653 { do_bad, SIGBUS, 0, "unknown 44" },
654 { do_bad, SIGBUS, 0, "unknown 45" },
655 { do_bad, SIGBUS, 0, "unknown 46" },
656 { do_bad, SIGBUS, 0, "unknown 47" },
657 { do_bad, SIGBUS, 0, "TLB conflict abort" },
658 { do_bad, SIGBUS, 0, "unknown 49" },
659 { do_bad, SIGBUS, 0, "unknown 50" },
660 { do_bad, SIGBUS, 0, "unknown 51" },
661 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
662 { do_bad, SIGBUS, 0, "implementation fault (unsupported exclusive)" },
663 { do_bad, SIGBUS, 0, "unknown 54" },
664 { do_bad, SIGBUS, 0, "unknown 55" },
665 { do_bad, SIGBUS, 0, "unknown 56" },
666 { do_bad, SIGBUS, 0, "unknown 57" },
667 { do_bad, SIGBUS, 0, "unknown 58" },
668 { do_bad, SIGBUS, 0, "unknown 59" },
669 { do_bad, SIGBUS, 0, "unknown 60" },
670 { do_bad, SIGBUS, 0, "section domain fault" },
671 { do_bad, SIGBUS, 0, "page domain fault" },
672 { do_bad, SIGBUS, 0, "unknown 63" },
675 #ifdef CONFIG_AMLOGIC_VMAP
676 asmlinkage static void die_wrap(const struct fault_info *inf,
677 struct pt_regs *regs, unsigned int esr,
682 info.si_signo = inf->sig;
684 info.si_code = inf->code;
685 info.si_addr = (void __user *)addr;
686 arm64_notify_die("", regs, &info, esr);
690 * Dispatch a data abort to the relevant handler.
692 asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
693 struct pt_regs *regs)
695 const struct fault_info *inf = esr_to_fault_info(esr);
696 #ifndef CONFIG_AMLOGIC_VMAP
700 if (!inf->fn(addr, esr, regs))
703 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
704 inf->name, esr, addr);
706 #ifndef CONFIG_AMLOGIC_VMAP
707 info.si_signo = inf->sig;
709 info.si_code = inf->code;
710 info.si_addr = (void __user *)addr;
711 arm64_notify_die("", regs, &info, esr);
713 die_wrap(inf, regs, esr, addr);
717 asmlinkage void __exception do_el0_irq_bp_hardening(void)
719 /* PC has already been checked in entry.S */
720 arm64_apply_bp_hardening();
723 asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr,
725 struct pt_regs *regs)
728 * We've taken an instruction abort from userspace and not yet
729 * re-enabled IRQs. If the address is a kernel address, apply
730 * BP hardening prior to enabling IRQs and pre-emption.
732 if (addr > TASK_SIZE)
733 arm64_apply_bp_hardening();
736 do_mem_abort(addr, esr, regs);
741 * Handle stack alignment exceptions.
743 asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
745 struct pt_regs *regs)
748 struct task_struct *tsk = current;
750 if (user_mode(regs)) {
751 if (instruction_pointer(regs) > TASK_SIZE)
752 arm64_apply_bp_hardening();
756 if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
757 pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
758 tsk->comm, task_pid_nr(tsk),
759 esr_get_class_string(esr), (void *)regs->pc,
762 info.si_signo = SIGBUS;
764 info.si_code = BUS_ADRALN;
765 info.si_addr = (void __user *)addr;
766 arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr);
769 int __init early_brk64(unsigned long addr, unsigned int esr,
770 struct pt_regs *regs);
773 * __refdata because early_brk64 is __init, but the reference to it is
774 * clobbered at arch_initcall time.
775 * See traps.c and debug-monitors.c:debug_traps_init().
777 static struct fault_info __refdata debug_fault_info[] = {
778 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" },
779 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" },
780 { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" },
781 { do_bad, SIGBUS, 0, "unknown 3" },
782 { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" },
783 { do_bad, SIGTRAP, 0, "aarch32 vector catch" },
784 { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" },
785 { do_bad, SIGBUS, 0, "unknown 7" },
788 void __init hook_debug_fault_code(int nr,
789 int (*fn)(unsigned long, unsigned int, struct pt_regs *),
790 int sig, int code, const char *name)
792 BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));
794 debug_fault_info[nr].fn = fn;
795 debug_fault_info[nr].sig = sig;
796 debug_fault_info[nr].code = code;
797 debug_fault_info[nr].name = name;
800 asmlinkage int __exception do_debug_exception(unsigned long addr_if_watchpoint,
802 struct pt_regs *regs)
804 const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
805 unsigned long pc = instruction_pointer(regs);
810 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were
811 * already disabled to preserve the last enabled/disabled addresses.
813 if (interrupts_enabled(regs))
814 trace_hardirqs_off();
816 if (user_mode(regs) && pc > TASK_SIZE)
817 arm64_apply_bp_hardening();
819 if (!inf->fn(addr_if_watchpoint, esr, regs)) {
822 pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
825 info.si_signo = inf->sig;
827 info.si_code = inf->code;
828 info.si_addr = (void __user *)pc;
829 arm64_notify_die("", regs, &info, 0);
833 if (interrupts_enabled(regs))
838 NOKPROBE_SYMBOL(do_debug_exception);
840 #ifdef CONFIG_ARM64_PAN
841 int cpu_enable_pan(void *__unused)
844 * We modify PSTATE. This won't work from irq context as the PSTATE
845 * is discarded once we return from the exception.
847 WARN_ON_ONCE(in_interrupt());
849 config_sctlr_el1(SCTLR_EL1_SPAN, 0);
850 asm(SET_PSTATE_PAN(1));
853 #endif /* CONFIG_ARM64_PAN */
855 #ifdef CONFIG_ARM64_UAO
857 * Kernel threads have fs=KERNEL_DS by default, and don't need to call
858 * set_fs(), devtmpfs in particular relies on this behaviour.
859 * We need to enable the feature at runtime (instead of adding it to
860 * PSR_MODE_EL1h) as the feature may not be implemented by the cpu.
862 int cpu_enable_uao(void *__unused)
864 asm(SET_PSTATE_UAO(1));
867 #endif /* CONFIG_ARM64_UAO */