arch/arm64/kernel/ptrace.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/ptrace.c
   4  *
   5  * By Ross Biro 1/23/92
   6  * edited by Linus Torvalds
   7  * ARM modifications Copyright (C) 2000 Russell King
   8  * Copyright (C) 2012 ARM Ltd.
   9  */
  10
  11 #include <linux/audit.h>
  12 #include <linux/compat.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/sched/task_stack.h>
  16 #include <linux/mm.h>
  17 #include <linux/nospec.h>
  18 #include <linux/smp.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/user.h>
  21 #include <linux/seccomp.h>
  22 #include <linux/security.h>
  23 #include <linux/init.h>
  24 #include <linux/signal.h>
  25 #include <linux/string.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/perf_event.h>
  28 #include <linux/hw_breakpoint.h>
  29 #include <linux/regset.h>
  30 #include <linux/elf.h>
  31
  32 #include <asm/compat.h>
  33 #include <asm/cpufeature.h>
  34 #include <asm/debug-monitors.h>
  35 #include <asm/fpsimd.h>
  36 #include <asm/mte.h>
  37 #include <asm/pointer_auth.h>
  38 #include <asm/stacktrace.h>
  39 #include <asm/syscall.h>
  40 #include <asm/traps.h>
  41 #include <asm/system_misc.h>
  42
  43 #define CREATE_TRACE_POINTS
  44 #include <trace/events/syscalls.h>
  45
  46 struct pt_regs_offset {
  47         const char *name;
  48         int offset;
  49 };
  50
  51 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)}
  52 #define REG_OFFSET_END {.name = NULL, .offset = 0}
  53 #define GPR_OFFSET_NAME(r) \
  54         {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])}
  55
  56 static const struct pt_regs_offset regoffset_table[] = {
  57         GPR_OFFSET_NAME(0),
  58         GPR_OFFSET_NAME(1),
  59         GPR_OFFSET_NAME(2),
  60         GPR_OFFSET_NAME(3),
  61         GPR_OFFSET_NAME(4),
  62         GPR_OFFSET_NAME(5),
  63         GPR_OFFSET_NAME(6),
  64         GPR_OFFSET_NAME(7),
  65         GPR_OFFSET_NAME(8),
  66         GPR_OFFSET_NAME(9),
  67         GPR_OFFSET_NAME(10),
  68         GPR_OFFSET_NAME(11),
  69         GPR_OFFSET_NAME(12),
  70         GPR_OFFSET_NAME(13),
  71         GPR_OFFSET_NAME(14),
  72         GPR_OFFSET_NAME(15),
  73         GPR_OFFSET_NAME(16),
  74         GPR_OFFSET_NAME(17),
  75         GPR_OFFSET_NAME(18),
  76         GPR_OFFSET_NAME(19),
  77         GPR_OFFSET_NAME(20),
  78         GPR_OFFSET_NAME(21),
  79         GPR_OFFSET_NAME(22),
  80         GPR_OFFSET_NAME(23),
  81         GPR_OFFSET_NAME(24),
  82         GPR_OFFSET_NAME(25),
  83         GPR_OFFSET_NAME(26),
  84         GPR_OFFSET_NAME(27),
  85         GPR_OFFSET_NAME(28),
  86         GPR_OFFSET_NAME(29),
  87         GPR_OFFSET_NAME(30),
  88         {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])},
  89         REG_OFFSET_NAME(sp),
  90         REG_OFFSET_NAME(pc),
  91         REG_OFFSET_NAME(pstate),
  92         REG_OFFSET_END,
  93 };
  94
  95 /**
  96  * regs_query_register_offset() - query register offset from its name
  97  * @name:       the name of a register
  98  *
  99  * regs_query_register_offset() returns the offset of a register in struct
 100  * pt_regs from its name. If the name is invalid, this returns -EINVAL;
 101  */
 102 int regs_query_register_offset(const char *name)
 103 {
 104         const struct pt_regs_offset *roff;
 105
 106         for (roff = regoffset_table; roff->name != NULL; roff++)
 107                 if (!strcmp(roff->name, name))
 108                         return roff->offset;
 109         return -EINVAL;
 110 }
 111
 112 /**
 113  * regs_within_kernel_stack() - check the address in the stack
 114  * @regs:      pt_regs which contains kernel stack pointer.
 115  * @addr:      address which is checked.
 116  *
 117  * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
 118  * If @addr is within the kernel stack, it returns true. If not, returns false.
 119  */
 120 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 121 {
 122         return ((addr & ~(THREAD_SIZE - 1))  ==
 123                 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
 124                 on_irq_stack(addr, sizeof(unsigned long));
 125 }
 126
 127 /**
 128  * regs_get_kernel_stack_nth() - get Nth entry of the stack
 129  * @regs:       pt_regs which contains kernel stack pointer.
 130  * @n:          stack entry number.
 131  *
 132  * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
 133  * is specified by @regs. If the @n th entry is NOT in the kernel stack,
 134  * this returns 0.
 135  */
 136 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
 137 {
 138         unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
 139
 140         addr += n;
 141         if (regs_within_kernel_stack(regs, (unsigned long)addr))
 142                 return *addr;
 143         else
 144                 return 0;
 145 }
 146
 147 /*
 148  * TODO: does not yet catch signals sent when the child dies.
 149  * in exit.c or in signal.c.
 150  */
 151
 152 /*
 153  * Called by kernel/ptrace.c when detaching..
 154  */
 155 void ptrace_disable(struct task_struct *child)
 156 {
 157         /*
 158          * This would be better off in core code, but PTRACE_DETACH has
 159          * grown its fair share of arch-specific worts and changing it
 160          * is likely to cause regressions on obscure architectures.
 161          */
 162         user_disable_single_step(child);
 163 }
 164
 165 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 166 /*
 167  * Handle hitting a HW-breakpoint.
 168  */
 169 static void ptrace_hbptriggered(struct perf_event *bp,
 170                                 struct perf_sample_data *data,
 171                                 struct pt_regs *regs)
 172 {
 173         struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 174         const char *desc = "Hardware breakpoint trap (ptrace)";
 175
 176 #ifdef CONFIG_COMPAT
 177         if (is_compat_task()) {
 178                 int si_errno = 0;
 179                 int i;
 180
 181                 for (i = 0; i < ARM_MAX_BRP; ++i) {
 182                         if (current->thread.debug.hbp_break[i] == bp) {
 183                                 si_errno = (i << 1) + 1;
 184                                 break;
 185                         }
 186                 }
 187
 188                 for (i = 0; i < ARM_MAX_WRP; ++i) {
 189                         if (current->thread.debug.hbp_watch[i] == bp) {
 190                                 si_errno = -((i << 1) + 1);
 191                                 break;
 192                         }
 193                 }
 194                 arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger,
 195                                                   desc);
 196                 return;
 197         }
 198 #endif
 199         arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc);
 200 }
 201
 202 /*
 203  * Unregister breakpoints from this task and reset the pointers in
 204  * the thread_struct.
 205  */
 206 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 207 {
 208         int i;
 209         struct thread_struct *t = &tsk->thread;
 210
 211         for (i = 0; i < ARM_MAX_BRP; i++) {
 212                 if (t->debug.hbp_break[i]) {
 213                         unregister_hw_breakpoint(t->debug.hbp_break[i]);
 214                         t->debug.hbp_break[i] = NULL;
 215                 }
 216         }
 217
 218         for (i = 0; i < ARM_MAX_WRP; i++) {
 219                 if (t->debug.hbp_watch[i]) {
 220                         unregister_hw_breakpoint(t->debug.hbp_watch[i]);
 221                         t->debug.hbp_watch[i] = NULL;
 222                 }
 223         }
 224 }
 225
 226 void ptrace_hw_copy_thread(struct task_struct *tsk)
 227 {
 228         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
 229 }
 230
 231 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type,
 232                                                struct task_struct *tsk,
 233                                                unsigned long idx)
 234 {
 235         struct perf_event *bp = ERR_PTR(-EINVAL);
 236
 237         switch (note_type) {
 238         case NT_ARM_HW_BREAK:
 239                 if (idx >= ARM_MAX_BRP)
 240                         goto out;
 241                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 242                 bp = tsk->thread.debug.hbp_break[idx];
 243                 break;
 244         case NT_ARM_HW_WATCH:
 245                 if (idx >= ARM_MAX_WRP)
 246                         goto out;
 247                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 248                 bp = tsk->thread.debug.hbp_watch[idx];
 249                 break;
 250         }
 251
 252 out:
 253         return bp;
 254 }
 255
 256 static int ptrace_hbp_set_event(unsigned int note_type,
 257                                 struct task_struct *tsk,
 258                                 unsigned long idx,
 259                                 struct perf_event *bp)
 260 {
 261         int err = -EINVAL;
 262
 263         switch (note_type) {
 264         case NT_ARM_HW_BREAK:
 265                 if (idx >= ARM_MAX_BRP)
 266                         goto out;
 267                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 268                 tsk->thread.debug.hbp_break[idx] = bp;
 269                 err = 0;
 270                 break;
 271         case NT_ARM_HW_WATCH:
 272                 if (idx >= ARM_MAX_WRP)
 273                         goto out;
 274                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 275                 tsk->thread.debug.hbp_watch[idx] = bp;
 276                 err = 0;
 277                 break;
 278         }
 279
 280 out:
 281         return err;
 282 }
 283
 284 static struct perf_event *ptrace_hbp_create(unsigned int note_type,
 285                                             struct task_struct *tsk,
 286                                             unsigned long idx)
 287 {
 288         struct perf_event *bp;
 289         struct perf_event_attr attr;
 290         int err, type;
 291
 292         switch (note_type) {
 293         case NT_ARM_HW_BREAK:
 294                 type = HW_BREAKPOINT_X;
 295                 break;
 296         case NT_ARM_HW_WATCH:
 297                 type = HW_BREAKPOINT_RW;
 298                 break;
 299         default:
 300                 return ERR_PTR(-EINVAL);
 301         }
 302
 303         ptrace_breakpoint_init(&attr);
 304
 305         /*
 306          * Initialise fields to sane defaults
 307          * (i.e. values that will pass validation).
 308          */
 309         attr.bp_addr    = 0;
 310         attr.bp_len     = HW_BREAKPOINT_LEN_4;
 311         attr.bp_type    = type;
 312         attr.disabled   = 1;
 313
 314         bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk);
 315         if (IS_ERR(bp))
 316                 return bp;
 317
 318         err = ptrace_hbp_set_event(note_type, tsk, idx, bp);
 319         if (err)
 320                 return ERR_PTR(err);
 321
 322         return bp;
 323 }
 324
 325 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 326                                      struct arch_hw_breakpoint_ctrl ctrl,
 327                                      struct perf_event_attr *attr)
 328 {
 329         int err, len, type, offset, disabled = !ctrl.enabled;
 330
 331         attr->disabled = disabled;
 332         if (disabled)
 333                 return 0;
 334
 335         err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 336         if (err)
 337                 return err;
 338
 339         switch (note_type) {
 340         case NT_ARM_HW_BREAK:
 341                 if ((type & HW_BREAKPOINT_X) != type)
 342                         return -EINVAL;
 343                 break;
 344         case NT_ARM_HW_WATCH:
 345                 if ((type & HW_BREAKPOINT_RW) != type)
 346                         return -EINVAL;
 347                 break;
 348         default:
 349                 return -EINVAL;
 350         }
 351
 352         attr->bp_len    = len;
 353         attr->bp_type   = type;
 354         attr->bp_addr   += offset;
 355
 356         return 0;
 357 }
 358
 359 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info)
 360 {
 361         u8 num;
 362         u32 reg = 0;
 363
 364         switch (note_type) {
 365         case NT_ARM_HW_BREAK:
 366                 num = hw_breakpoint_slots(TYPE_INST);
 367                 break;
 368         case NT_ARM_HW_WATCH:
 369                 num = hw_breakpoint_slots(TYPE_DATA);
 370                 break;
 371         default:
 372                 return -EINVAL;
 373         }
 374
 375         reg |= debug_monitors_arch();
 376         reg <<= 8;
 377         reg |= num;
 378
 379         *info = reg;
 380         return 0;
 381 }
 382
 383 static int ptrace_hbp_get_ctrl(unsigned int note_type,
 384                                struct task_struct *tsk,
 385                                unsigned long idx,
 386                                u32 *ctrl)
 387 {
 388         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 389
 390         if (IS_ERR(bp))
 391                 return PTR_ERR(bp);
 392
 393         *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0;
 394         return 0;
 395 }
 396
 397 static int ptrace_hbp_get_addr(unsigned int note_type,
 398                                struct task_struct *tsk,
 399                                unsigned long idx,
 400                                u64 *addr)
 401 {
 402         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 403
 404         if (IS_ERR(bp))
 405                 return PTR_ERR(bp);
 406
 407         *addr = bp ? counter_arch_bp(bp)->address : 0;
 408         return 0;
 409 }
 410
 411 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type,
 412                                                         struct task_struct *tsk,
 413                                                         unsigned long idx)
 414 {
 415         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 416
 417         if (!bp)
 418                 bp = ptrace_hbp_create(note_type, tsk, idx);
 419
 420         return bp;
 421 }
 422
 423 static int ptrace_hbp_set_ctrl(unsigned int note_type,
 424                                struct task_struct *tsk,
 425                                unsigned long idx,
 426                                u32 uctrl)
 427 {
 428         int err;
 429         struct perf_event *bp;
 430         struct perf_event_attr attr;
 431         struct arch_hw_breakpoint_ctrl ctrl;
 432
 433         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 434         if (IS_ERR(bp)) {
 435                 err = PTR_ERR(bp);
 436                 return err;
 437         }
 438
 439         attr = bp->attr;
 440         decode_ctrl_reg(uctrl, &ctrl);
 441         err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr);
 442         if (err)
 443                 return err;
 444
 445         return modify_user_hw_breakpoint(bp, &attr);
 446 }
 447
 448 static int ptrace_hbp_set_addr(unsigned int note_type,
 449                                struct task_struct *tsk,
 450                                unsigned long idx,
 451                                u64 addr)
 452 {
 453         int err;
 454         struct perf_event *bp;
 455         struct perf_event_attr attr;
 456
 457         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 458         if (IS_ERR(bp)) {
 459                 err = PTR_ERR(bp);
 460                 return err;
 461         }
 462
 463         attr = bp->attr;
 464         attr.bp_addr = addr;
 465         err = modify_user_hw_breakpoint(bp, &attr);
 466         return err;
 467 }
 468
 469 #define PTRACE_HBP_ADDR_SZ      sizeof(u64)
 470 #define PTRACE_HBP_CTRL_SZ      sizeof(u32)
 471 #define PTRACE_HBP_PAD_SZ       sizeof(u32)
 472
 473 static int hw_break_get(struct task_struct *target,
 474                         const struct user_regset *regset,
 475                         struct membuf to)
 476 {
 477         unsigned int note_type = regset->core_note_type;
 478         int ret, idx = 0;
 479         u32 info, ctrl;
 480         u64 addr;
 481
 482         /* Resource info */
 483         ret = ptrace_hbp_get_resource_info(note_type, &info);
 484         if (ret)
 485                 return ret;
 486
 487         membuf_write(&to, &info, sizeof(info));
 488         membuf_zero(&to, sizeof(u32));
 489         /* (address, ctrl) registers */
 490         while (to.left) {
 491                 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
 492                 if (ret)
 493                         return ret;
 494                 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
 495                 if (ret)
 496                         return ret;
 497                 membuf_store(&to, addr);
 498                 membuf_store(&to, ctrl);
 499                 membuf_zero(&to, sizeof(u32));
 500                 idx++;
 501         }
 502         return 0;
 503 }
 504
 505 static int hw_break_set(struct task_struct *target,
 506                         const struct user_regset *regset,
 507                         unsigned int pos, unsigned int count,
 508                         const void *kbuf, const void __user *ubuf)
 509 {
 510         unsigned int note_type = regset->core_note_type;
 511         int ret, idx = 0, offset, limit;
 512         u32 ctrl;
 513         u64 addr;
 514
 515         /* Resource info and pad */
 516         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 517         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
 518
 519         /* (address, ctrl) registers */
 520         limit = regset->n * regset->size;
 521         while (count && offset < limit) {
 522                 if (count < PTRACE_HBP_ADDR_SZ)
 523                         return -EINVAL;
 524                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
 525                                          offset, offset + PTRACE_HBP_ADDR_SZ);
 526                 if (ret)
 527                         return ret;
 528                 ret = ptrace_hbp_set_addr(note_type, target, idx, addr);
 529                 if (ret)
 530                         return ret;
 531                 offset += PTRACE_HBP_ADDR_SZ;
 532
 533                 if (!count)
 534                         break;
 535                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl,
 536                                          offset, offset + PTRACE_HBP_CTRL_SZ);
 537                 if (ret)
 538                         return ret;
 539                 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl);
 540                 if (ret)
 541                         return ret;
 542                 offset += PTRACE_HBP_CTRL_SZ;
 543
 544                 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 545                                           offset, offset + PTRACE_HBP_PAD_SZ);
 546                 offset += PTRACE_HBP_PAD_SZ;
 547                 idx++;
 548         }
 549
 550         return 0;
 551 }
 552 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
 553
 554 static int gpr_get(struct task_struct *target,
 555                    const struct user_regset *regset,
 556                    struct membuf to)
 557 {
 558         struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
 559         return membuf_write(&to, uregs, sizeof(*uregs));
 560 }
 561
 562 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 563                    unsigned int pos, unsigned int count,
 564                    const void *kbuf, const void __user *ubuf)
 565 {
 566         int ret;
 567         struct user_pt_regs newregs = task_pt_regs(target)->user_regs;
 568
 569         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1);
 570         if (ret)
 571                 return ret;
 572
 573         if (!valid_user_regs(&newregs, target))
 574                 return -EINVAL;
 575
 576         task_pt_regs(target)->user_regs = newregs;
 577         return 0;
 578 }
 579
 580 static int fpr_active(struct task_struct *target, const struct user_regset *regset)
 581 {
 582         if (!system_supports_fpsimd())
 583                 return -ENODEV;
 584         return regset->n;
 585 }
 586
 587 /*
 588  * TODO: update fp accessors for lazy context switching (sync/flush hwstate)
 589  */
 590 static int __fpr_get(struct task_struct *target,
 591                      const struct user_regset *regset,
 592                      struct membuf to)
 593 {
 594         struct user_fpsimd_state *uregs;
 595
 596         sve_sync_to_fpsimd(target);
 597
 598         uregs = &target->thread.uw.fpsimd_state;
 599
 600         return membuf_write(&to, uregs, sizeof(*uregs));
 601 }
 602
 603 static int fpr_get(struct task_struct *target, const struct user_regset *regset,
 604                    struct membuf to)
 605 {
 606         if (!system_supports_fpsimd())
 607                 return -EINVAL;
 608
 609         if (target == current)
 610                 fpsimd_preserve_current_state();
 611
 612         return __fpr_get(target, regset, to);
 613 }
 614
 615 static int __fpr_set(struct task_struct *target,
 616                      const struct user_regset *regset,
 617                      unsigned int pos, unsigned int count,
 618                      const void *kbuf, const void __user *ubuf,
 619                      unsigned int start_pos)
 620 {
 621         int ret;
 622         struct user_fpsimd_state newstate;
 623
 624         /*
 625          * Ensure target->thread.uw.fpsimd_state is up to date, so that a
 626          * short copyin can't resurrect stale data.
 627          */
 628         sve_sync_to_fpsimd(target);
 629
 630         newstate = target->thread.uw.fpsimd_state;
 631
 632         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
 633                                  start_pos, start_pos + sizeof(newstate));
 634         if (ret)
 635                 return ret;
 636
 637         target->thread.uw.fpsimd_state = newstate;
 638
 639         return ret;
 640 }
 641
 642 static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 643                    unsigned int pos, unsigned int count,
 644                    const void *kbuf, const void __user *ubuf)
 645 {
 646         int ret;
 647
 648         if (!system_supports_fpsimd())
 649                 return -EINVAL;
 650
 651         ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
 652         if (ret)
 653                 return ret;
 654
 655         sve_sync_from_fpsimd_zeropad(target);
 656         fpsimd_flush_task_state(target);
 657
 658         return ret;
 659 }
 660
 661 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 662                    struct membuf to)
 663 {
 664         int ret;
 665
 666         if (target == current)
 667                 tls_preserve_current_state();
 668
 669         ret = membuf_store(&to, target->thread.uw.tp_value);
 670         if (system_supports_tpidr2())
 671                 ret = membuf_store(&to, target->thread.tpidr2_el0);
 672         else
 673                 ret = membuf_zero(&to, sizeof(u64));
 674
 675         return ret;
 676 }
 677
 678 static int tls_set(struct task_struct *target, const struct user_regset *regset,
 679                    unsigned int pos, unsigned int count,
 680                    const void *kbuf, const void __user *ubuf)
 681 {
 682         int ret;
 683         unsigned long tls[2];
 684
 685         tls[0] = target->thread.uw.tp_value;
 686         if (system_supports_sme())
 687                 tls[1] = target->thread.tpidr2_el0;
 688
 689         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count);
 690         if (ret)
 691                 return ret;
 692
 693         target->thread.uw.tp_value = tls[0];
 694         if (system_supports_sme())
 695                 target->thread.tpidr2_el0 = tls[1];
 696
 697         return ret;
 698 }
 699
 700 static int system_call_get(struct task_struct *target,
 701                            const struct user_regset *regset,
 702                            struct membuf to)
 703 {
 704         return membuf_store(&to, task_pt_regs(target)->syscallno);
 705 }
 706
 707 static int system_call_set(struct task_struct *target,
 708                            const struct user_regset *regset,
 709                            unsigned int pos, unsigned int count,
 710                            const void *kbuf, const void __user *ubuf)
 711 {
 712         int syscallno = task_pt_regs(target)->syscallno;
 713         int ret;
 714
 715         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1);
 716         if (ret)
 717                 return ret;
 718
 719         task_pt_regs(target)->syscallno = syscallno;
 720         return ret;
 721 }
 722
 723 #ifdef CONFIG_ARM64_SVE
 724
 725 static void sve_init_header_from_task(struct user_sve_header *header,
 726                                       struct task_struct *target,
 727                                       enum vec_type type)
 728 {
 729         unsigned int vq;
 730         bool active;
 731         bool fpsimd_only;
 732         enum vec_type task_type;
 733
 734         memset(header, 0, sizeof(*header));
 735
 736         /* Check if the requested registers are active for the task */
 737         if (thread_sm_enabled(&target->thread))
 738                 task_type = ARM64_VEC_SME;
 739         else
 740                 task_type = ARM64_VEC_SVE;
 741         active = (task_type == type);
 742
 743         switch (type) {
 744         case ARM64_VEC_SVE:
 745                 if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
 746                         header->flags |= SVE_PT_VL_INHERIT;
 747                 fpsimd_only = !test_tsk_thread_flag(target, TIF_SVE);
 748                 break;
 749         case ARM64_VEC_SME:
 750                 if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
 751                         header->flags |= SVE_PT_VL_INHERIT;
 752                 fpsimd_only = false;
 753                 break;
 754         default:
 755                 WARN_ON_ONCE(1);
 756                 return;
 757         }
 758
 759         if (active) {
 760                 if (fpsimd_only) {
 761                         header->flags |= SVE_PT_REGS_FPSIMD;
 762                 } else {
 763                         header->flags |= SVE_PT_REGS_SVE;
 764                 }
 765         }
 766
 767         header->vl = task_get_vl(target, type);
 768         vq = sve_vq_from_vl(header->vl);
 769
 770         header->max_vl = vec_max_vl(type);
 771         header->size = SVE_PT_SIZE(vq, header->flags);
 772         header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 773                                       SVE_PT_REGS_SVE);
 774 }
 775
 776 static unsigned int sve_size_from_header(struct user_sve_header const *header)
 777 {
 778         return ALIGN(header->size, SVE_VQ_BYTES);
 779 }
 780
 781 static int sve_get_common(struct task_struct *target,
 782                           const struct user_regset *regset,
 783                           struct membuf to,
 784                           enum vec_type type)
 785 {
 786         struct user_sve_header header;
 787         unsigned int vq;
 788         unsigned long start, end;
 789
 790         /* Header */
 791         sve_init_header_from_task(&header, target, type);
 792         vq = sve_vq_from_vl(header.vl);
 793
 794         membuf_write(&to, &header, sizeof(header));
 795
 796         if (target == current)
 797                 fpsimd_preserve_current_state();
 798
 799         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 800         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 801
 802         switch ((header.flags & SVE_PT_REGS_MASK)) {
 803         case SVE_PT_REGS_FPSIMD:
 804                 return __fpr_get(target, regset, to);
 805
 806         case SVE_PT_REGS_SVE:
 807                 start = SVE_PT_SVE_OFFSET;
 808                 end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 809                 membuf_write(&to, target->thread.sve_state, end - start);
 810
 811                 start = end;
 812                 end = SVE_PT_SVE_FPSR_OFFSET(vq);
 813                 membuf_zero(&to, end - start);
 814
 815                 /*
 816                  * Copy fpsr, and fpcr which must follow contiguously in
 817                  * struct fpsimd_state:
 818                  */
 819                 start = end;
 820                 end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 821                 membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr,
 822                              end - start);
 823
 824                 start = end;
 825                 end = sve_size_from_header(&header);
 826                 return membuf_zero(&to, end - start);
 827
 828         default:
 829                 return 0;
 830         }
 831 }
 832
 833 static int sve_get(struct task_struct *target,
 834                    const struct user_regset *regset,
 835                    struct membuf to)
 836 {
 837         if (!system_supports_sve())
 838                 return -EINVAL;
 839
 840         return sve_get_common(target, regset, to, ARM64_VEC_SVE);
 841 }
 842
 843 static int sve_set_common(struct task_struct *target,
 844                           const struct user_regset *regset,
 845                           unsigned int pos, unsigned int count,
 846                           const void *kbuf, const void __user *ubuf,
 847                           enum vec_type type)
 848 {
 849         int ret;
 850         struct user_sve_header header;
 851         unsigned int vq;
 852         unsigned long start, end;
 853
 854         /* Header */
 855         if (count < sizeof(header))
 856                 return -EINVAL;
 857         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
 858                                  0, sizeof(header));
 859         if (ret)
 860                 goto out;
 861
 862         /*
 863          * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by
 864          * vec_set_vector_length(), which will also validate them for us:
 865          */
 866         ret = vec_set_vector_length(target, type, header.vl,
 867                 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 868         if (ret)
 869                 goto out;
 870
 871         /* Actual VL set may be less than the user asked for: */
 872         vq = sve_vq_from_vl(task_get_vl(target, type));
 873
 874         /* Enter/exit streaming mode */
 875         if (system_supports_sme()) {
 876                 u64 old_svcr = target->thread.svcr;
 877
 878                 switch (type) {
 879                 case ARM64_VEC_SVE:
 880                         target->thread.svcr &= ~SVCR_SM_MASK;
 881                         break;
 882                 case ARM64_VEC_SME:
 883                         target->thread.svcr |= SVCR_SM_MASK;
 884                         break;
 885                 default:
 886                         WARN_ON_ONCE(1);
 887                         return -EINVAL;
 888                 }
 889
 890                 /*
 891                  * If we switched then invalidate any existing SVE
 892                  * state and ensure there's storage.
 893                  */
 894                 if (target->thread.svcr != old_svcr)
 895                         sve_alloc(target, true);
 896         }
 897
 898         /* Registers: FPSIMD-only case */
 899
 900         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 901         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
 902                 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
 903                                 SVE_PT_FPSIMD_OFFSET);
 904                 clear_tsk_thread_flag(target, TIF_SVE);
 905                 target->thread.fp_type = FP_STATE_FPSIMD;
 906                 goto out;
 907         }
 908
 909         /*
 910          * Otherwise: no registers or full SVE case.  For backwards
 911          * compatibility reasons we treat empty flags as SVE registers.
 912          */
 913
 914         /*
 915          * If setting a different VL from the requested VL and there is
 916          * register data, the data layout will be wrong: don't even
 917          * try to set the registers in this case.
 918          */
 919         if (count && vq != sve_vq_from_vl(header.vl)) {
 920                 ret = -EIO;
 921                 goto out;
 922         }
 923
 924         sve_alloc(target, true);
 925         if (!target->thread.sve_state) {
 926                 ret = -ENOMEM;
 927                 clear_tsk_thread_flag(target, TIF_SVE);
 928                 target->thread.fp_type = FP_STATE_FPSIMD;
 929                 goto out;
 930         }
 931
 932         /*
 933          * Ensure target->thread.sve_state is up to date with target's
 934          * FPSIMD regs, so that a short copyin leaves trailing
 935          * registers unmodified.  Always enable SVE even if going into
 936          * streaming mode.
 937          */
 938         fpsimd_sync_to_sve(target);
 939         set_tsk_thread_flag(target, TIF_SVE);
 940         target->thread.fp_type = FP_STATE_SVE;
 941
 942         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 943         start = SVE_PT_SVE_OFFSET;
 944         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 945         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 946                                  target->thread.sve_state,
 947                                  start, end);
 948         if (ret)
 949                 goto out;
 950
 951         start = end;
 952         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 953         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, start, end);
 954
 955         /*
 956          * Copy fpsr, and fpcr which must follow contiguously in
 957          * struct fpsimd_state:
 958          */
 959         start = end;
 960         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 961         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 962                                  &target->thread.uw.fpsimd_state.fpsr,
 963                                  start, end);
 964
 965 out:
 966         fpsimd_flush_task_state(target);
 967         return ret;
 968 }
 969
 970 static int sve_set(struct task_struct *target,
 971                    const struct user_regset *regset,
 972                    unsigned int pos, unsigned int count,
 973                    const void *kbuf, const void __user *ubuf)
 974 {
 975         if (!system_supports_sve())
 976                 return -EINVAL;
 977
 978         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
 979                               ARM64_VEC_SVE);
 980 }
 981
 982 #endif /* CONFIG_ARM64_SVE */
 983
 984 #ifdef CONFIG_ARM64_SME
 985
 986 static int ssve_get(struct task_struct *target,
 987                    const struct user_regset *regset,
 988                    struct membuf to)
 989 {
 990         if (!system_supports_sme())
 991                 return -EINVAL;
 992
 993         return sve_get_common(target, regset, to, ARM64_VEC_SME);
 994 }
 995
 996 static int ssve_set(struct task_struct *target,
 997                     const struct user_regset *regset,
 998                     unsigned int pos, unsigned int count,
 999                     const void *kbuf, const void __user *ubuf)
1000 {
1001         if (!system_supports_sme())
1002                 return -EINVAL;
1003
1004         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
1005                               ARM64_VEC_SME);
1006 }
1007
1008 static int za_get(struct task_struct *target,
1009                   const struct user_regset *regset,
1010                   struct membuf to)
1011 {
1012         struct user_za_header header;
1013         unsigned int vq;
1014         unsigned long start, end;
1015
1016         if (!system_supports_sme())
1017                 return -EINVAL;
1018
1019         /* Header */
1020         memset(&header, 0, sizeof(header));
1021
1022         if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
1023                 header.flags |= ZA_PT_VL_INHERIT;
1024
1025         header.vl = task_get_sme_vl(target);
1026         vq = sve_vq_from_vl(header.vl);
1027         header.max_vl = sme_max_vl();
1028         header.max_size = ZA_PT_SIZE(vq);
1029
1030         /* If ZA is not active there is only the header */
1031         if (thread_za_enabled(&target->thread))
1032                 header.size = ZA_PT_SIZE(vq);
1033         else
1034                 header.size = ZA_PT_ZA_OFFSET;
1035
1036         membuf_write(&to, &header, sizeof(header));
1037
1038         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1039         end = ZA_PT_ZA_OFFSET;
1040
1041         if (target == current)
1042                 fpsimd_preserve_current_state();
1043
1044         /* Any register data to include? */
1045         if (thread_za_enabled(&target->thread)) {
1046                 start = end;
1047                 end = ZA_PT_SIZE(vq);
1048                 membuf_write(&to, target->thread.za_state, end - start);
1049         }
1050
1051         /* Zero any trailing padding */
1052         start = end;
1053         end = ALIGN(header.size, SVE_VQ_BYTES);
1054         return membuf_zero(&to, end - start);
1055 }
1056
1057 static int za_set(struct task_struct *target,
1058                   const struct user_regset *regset,
1059                   unsigned int pos, unsigned int count,
1060                   const void *kbuf, const void __user *ubuf)
1061 {
1062         int ret;
1063         struct user_za_header header;
1064         unsigned int vq;
1065         unsigned long start, end;
1066
1067         if (!system_supports_sme())
1068                 return -EINVAL;
1069
1070         /* Header */
1071         if (count < sizeof(header))
1072                 return -EINVAL;
1073         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
1074                                  0, sizeof(header));
1075         if (ret)
1076                 goto out;
1077
1078         /*
1079          * All current ZA_PT_* flags are consumed by
1080          * vec_set_vector_length(), which will also validate them for
1081          * us:
1082          */
1083         ret = vec_set_vector_length(target, ARM64_VEC_SME, header.vl,
1084                 ((unsigned long)header.flags) << 16);
1085         if (ret)
1086                 goto out;
1087
1088         /* Actual VL set may be less than the user asked for: */
1089         vq = sve_vq_from_vl(task_get_sme_vl(target));
1090
1091         /* Ensure there is some SVE storage for streaming mode */
1092         if (!target->thread.sve_state) {
1093                 sve_alloc(target, false);
1094                 if (!target->thread.sve_state) {
1095                         ret = -ENOMEM;
1096                         goto out;
1097                 }
1098         }
1099
1100         /* Allocate/reinit ZA storage */
1101         sme_alloc(target);
1102         if (!target->thread.za_state) {
1103                 ret = -ENOMEM;
1104                 goto out;
1105         }
1106
1107         /* If there is no data then disable ZA */
1108         if (!count) {
1109                 target->thread.svcr &= ~SVCR_ZA_MASK;
1110                 goto out;
1111         }
1112
1113         /*
1114          * If setting a different VL from the requested VL and there is
1115          * register data, the data layout will be wrong: don't even
1116          * try to set the registers in this case.
1117          */
1118         if (vq != sve_vq_from_vl(header.vl)) {
1119                 ret = -EIO;
1120                 goto out;
1121         }
1122
1123         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1124         start = ZA_PT_ZA_OFFSET;
1125         end = ZA_PT_SIZE(vq);
1126         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1127                                  target->thread.za_state,
1128                                  start, end);
1129         if (ret)
1130                 goto out;
1131
1132         /* Mark ZA as active and let userspace use it */
1133         set_tsk_thread_flag(target, TIF_SME);
1134         target->thread.svcr |= SVCR_ZA_MASK;
1135
1136 out:
1137         fpsimd_flush_task_state(target);
1138         return ret;
1139 }
1140
1141 #endif /* CONFIG_ARM64_SME */
1142
1143 #ifdef CONFIG_ARM64_PTR_AUTH
1144 static int pac_mask_get(struct task_struct *target,
1145                         const struct user_regset *regset,
1146                         struct membuf to)
1147 {
1148         /*
1149          * The PAC bits can differ across data and instruction pointers
1150          * depending on TCR_EL1.TBID*, which we may make use of in future, so
1151          * we expose separate masks.
1152          */
1153         unsigned long mask = ptrauth_user_pac_mask();
1154         struct user_pac_mask uregs = {
1155                 .data_mask = mask,
1156                 .insn_mask = mask,
1157         };
1158
1159         if (!system_supports_address_auth())
1160                 return -EINVAL;
1161
1162         return membuf_write(&to, &uregs, sizeof(uregs));
1163 }
1164
1165 static int pac_enabled_keys_get(struct task_struct *target,
1166                                 const struct user_regset *regset,
1167                                 struct membuf to)
1168 {
1169         long enabled_keys = ptrauth_get_enabled_keys(target);
1170
1171         if (IS_ERR_VALUE(enabled_keys))
1172                 return enabled_keys;
1173
1174         return membuf_write(&to, &enabled_keys, sizeof(enabled_keys));
1175 }
1176
1177 static int pac_enabled_keys_set(struct task_struct *target,
1178                                 const struct user_regset *regset,
1179                                 unsigned int pos, unsigned int count,
1180                                 const void *kbuf, const void __user *ubuf)
1181 {
1182         int ret;
1183         long enabled_keys = ptrauth_get_enabled_keys(target);
1184
1185         if (IS_ERR_VALUE(enabled_keys))
1186                 return enabled_keys;
1187
1188         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0,
1189                                  sizeof(long));
1190         if (ret)
1191                 return ret;
1192
1193         return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK,
1194                                         enabled_keys);
1195 }
1196
1197 #ifdef CONFIG_CHECKPOINT_RESTORE
1198 static __uint128_t pac_key_to_user(const struct ptrauth_key *key)
1199 {
1200         return (__uint128_t)key->hi << 64 | key->lo;
1201 }
1202
1203 static struct ptrauth_key pac_key_from_user(__uint128_t ukey)
1204 {
1205         struct ptrauth_key key = {
1206                 .lo = (unsigned long)ukey,
1207                 .hi = (unsigned long)(ukey >> 64),
1208         };
1209
1210         return key;
1211 }
1212
1213 static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys,
1214                                      const struct ptrauth_keys_user *keys)
1215 {
1216         ukeys->apiakey = pac_key_to_user(&keys->apia);
1217         ukeys->apibkey = pac_key_to_user(&keys->apib);
1218         ukeys->apdakey = pac_key_to_user(&keys->apda);
1219         ukeys->apdbkey = pac_key_to_user(&keys->apdb);
1220 }
1221
1222 static void pac_address_keys_from_user(struct ptrauth_keys_user *keys,
1223                                        const struct user_pac_address_keys *ukeys)
1224 {
1225         keys->apia = pac_key_from_user(ukeys->apiakey);
1226         keys->apib = pac_key_from_user(ukeys->apibkey);
1227         keys->apda = pac_key_from_user(ukeys->apdakey);
1228         keys->apdb = pac_key_from_user(ukeys->apdbkey);
1229 }
1230
1231 static int pac_address_keys_get(struct task_struct *target,
1232                                 const struct user_regset *regset,
1233                                 struct membuf to)
1234 {
1235         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1236         struct user_pac_address_keys user_keys;
1237
1238         if (!system_supports_address_auth())
1239                 return -EINVAL;
1240
1241         pac_address_keys_to_user(&user_keys, keys);
1242
1243         return membuf_write(&to, &user_keys, sizeof(user_keys));
1244 }
1245
1246 static int pac_address_keys_set(struct task_struct *target,
1247                                 const struct user_regset *regset,
1248                                 unsigned int pos, unsigned int count,
1249                                 const void *kbuf, const void __user *ubuf)
1250 {
1251         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1252         struct user_pac_address_keys user_keys;
1253         int ret;
1254
1255         if (!system_supports_address_auth())
1256                 return -EINVAL;
1257
1258         pac_address_keys_to_user(&user_keys, keys);
1259         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1260                                  &user_keys, 0, -1);
1261         if (ret)
1262                 return ret;
1263         pac_address_keys_from_user(keys, &user_keys);
1264
1265         return 0;
1266 }
1267
1268 static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys,
1269                                      const struct ptrauth_keys_user *keys)
1270 {
1271         ukeys->apgakey = pac_key_to_user(&keys->apga);
1272 }
1273
1274 static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys,
1275                                        const struct user_pac_generic_keys *ukeys)
1276 {
1277         keys->apga = pac_key_from_user(ukeys->apgakey);
1278 }
1279
1280 static int pac_generic_keys_get(struct task_struct *target,
1281                                 const struct user_regset *regset,
1282                                 struct membuf to)
1283 {
1284         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1285         struct user_pac_generic_keys user_keys;
1286
1287         if (!system_supports_generic_auth())
1288                 return -EINVAL;
1289
1290         pac_generic_keys_to_user(&user_keys, keys);
1291
1292         return membuf_write(&to, &user_keys, sizeof(user_keys));
1293 }
1294
1295 static int pac_generic_keys_set(struct task_struct *target,
1296                                 const struct user_regset *regset,
1297                                 unsigned int pos, unsigned int count,
1298                                 const void *kbuf, const void __user *ubuf)
1299 {
1300         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1301         struct user_pac_generic_keys user_keys;
1302         int ret;
1303
1304         if (!system_supports_generic_auth())
1305                 return -EINVAL;
1306
1307         pac_generic_keys_to_user(&user_keys, keys);
1308         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1309                                  &user_keys, 0, -1);
1310         if (ret)
1311                 return ret;
1312         pac_generic_keys_from_user(keys, &user_keys);
1313
1314         return 0;
1315 }
1316 #endif /* CONFIG_CHECKPOINT_RESTORE */
1317 #endif /* CONFIG_ARM64_PTR_AUTH */
1318
1319 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1320 static int tagged_addr_ctrl_get(struct task_struct *target,
1321                                 const struct user_regset *regset,
1322                                 struct membuf to)
1323 {
1324         long ctrl = get_tagged_addr_ctrl(target);
1325
1326         if (IS_ERR_VALUE(ctrl))
1327                 return ctrl;
1328
1329         return membuf_write(&to, &ctrl, sizeof(ctrl));
1330 }
1331
1332 static int tagged_addr_ctrl_set(struct task_struct *target, const struct
1333                                 user_regset *regset, unsigned int pos,
1334                                 unsigned int count, const void *kbuf, const
1335                                 void __user *ubuf)
1336 {
1337         int ret;
1338         long ctrl;
1339
1340         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
1341         if (ret)
1342                 return ret;
1343
1344         return set_tagged_addr_ctrl(target, ctrl);
1345 }
1346 #endif
1347
1348 enum aarch64_regset {
1349         REGSET_GPR,
1350         REGSET_FPR,
1351         REGSET_TLS,
1352 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1353         REGSET_HW_BREAK,
1354         REGSET_HW_WATCH,
1355 #endif
1356         REGSET_SYSTEM_CALL,
1357 #ifdef CONFIG_ARM64_SVE
1358         REGSET_SVE,
1359 #endif
1360 #ifdef CONFIG_ARM64_SVE
1361         REGSET_SSVE,
1362         REGSET_ZA,
1363 #endif
1364 #ifdef CONFIG_ARM64_PTR_AUTH
1365         REGSET_PAC_MASK,
1366         REGSET_PAC_ENABLED_KEYS,
1367 #ifdef CONFIG_CHECKPOINT_RESTORE
1368         REGSET_PACA_KEYS,
1369         REGSET_PACG_KEYS,
1370 #endif
1371 #endif
1372 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1373         REGSET_TAGGED_ADDR_CTRL,
1374 #endif
1375 };
1376
1377 static const struct user_regset aarch64_regsets[] = {
1378         [REGSET_GPR] = {
1379                 .core_note_type = NT_PRSTATUS,
1380                 .n = sizeof(struct user_pt_regs) / sizeof(u64),
1381                 .size = sizeof(u64),
1382                 .align = sizeof(u64),
1383                 .regset_get = gpr_get,
1384                 .set = gpr_set
1385         },
1386         [REGSET_FPR] = {
1387                 .core_note_type = NT_PRFPREG,
1388                 .n = sizeof(struct user_fpsimd_state) / sizeof(u32),
1389                 /*
1390                  * We pretend we have 32-bit registers because the fpsr and
1391                  * fpcr are 32-bits wide.
1392                  */
1393                 .size = sizeof(u32),
1394                 .align = sizeof(u32),
1395                 .active = fpr_active,
1396                 .regset_get = fpr_get,
1397                 .set = fpr_set
1398         },
1399         [REGSET_TLS] = {
1400                 .core_note_type = NT_ARM_TLS,
1401                 .n = 2,
1402                 .size = sizeof(void *),
1403                 .align = sizeof(void *),
1404                 .regset_get = tls_get,
1405                 .set = tls_set,
1406         },
1407 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1408         [REGSET_HW_BREAK] = {
1409                 .core_note_type = NT_ARM_HW_BREAK,
1410                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1411                 .size = sizeof(u32),
1412                 .align = sizeof(u32),
1413                 .regset_get = hw_break_get,
1414                 .set = hw_break_set,
1415         },
1416         [REGSET_HW_WATCH] = {
1417                 .core_note_type = NT_ARM_HW_WATCH,
1418                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1419                 .size = sizeof(u32),
1420                 .align = sizeof(u32),
1421                 .regset_get = hw_break_get,
1422                 .set = hw_break_set,
1423         },
1424 #endif
1425         [REGSET_SYSTEM_CALL] = {
1426                 .core_note_type = NT_ARM_SYSTEM_CALL,
1427                 .n = 1,
1428                 .size = sizeof(int),
1429                 .align = sizeof(int),
1430                 .regset_get = system_call_get,
1431                 .set = system_call_set,
1432         },
1433 #ifdef CONFIG_ARM64_SVE
1434         [REGSET_SVE] = { /* Scalable Vector Extension */
1435                 .core_note_type = NT_ARM_SVE,
1436                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE),
1437                                   SVE_VQ_BYTES),
1438                 .size = SVE_VQ_BYTES,
1439                 .align = SVE_VQ_BYTES,
1440                 .regset_get = sve_get,
1441                 .set = sve_set,
1442         },
1443 #endif
1444 #ifdef CONFIG_ARM64_SME
1445         [REGSET_SSVE] = { /* Streaming mode SVE */
1446                 .core_note_type = NT_ARM_SSVE,
1447                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SME_VQ_MAX, SVE_PT_REGS_SVE),
1448                                   SVE_VQ_BYTES),
1449                 .size = SVE_VQ_BYTES,
1450                 .align = SVE_VQ_BYTES,
1451                 .regset_get = ssve_get,
1452                 .set = ssve_set,
1453         },
1454         [REGSET_ZA] = { /* SME ZA */
1455                 .core_note_type = NT_ARM_ZA,
1456                 /*
1457                  * ZA is a single register but it's variably sized and
1458                  * the ptrace core requires that the size of any data
1459                  * be an exact multiple of the configured register
1460                  * size so report as though we had SVE_VQ_BYTES
1461                  * registers. These values aren't exposed to
1462                  * userspace.
1463                  */
1464                 .n = DIV_ROUND_UP(ZA_PT_SIZE(SME_VQ_MAX), SVE_VQ_BYTES),
1465                 .size = SVE_VQ_BYTES,
1466                 .align = SVE_VQ_BYTES,
1467                 .regset_get = za_get,
1468                 .set = za_set,
1469         },
1470 #endif
1471 #ifdef CONFIG_ARM64_PTR_AUTH
1472         [REGSET_PAC_MASK] = {
1473                 .core_note_type = NT_ARM_PAC_MASK,
1474                 .n = sizeof(struct user_pac_mask) / sizeof(u64),
1475                 .size = sizeof(u64),
1476                 .align = sizeof(u64),
1477                 .regset_get = pac_mask_get,
1478                 /* this cannot be set dynamically */
1479         },
1480         [REGSET_PAC_ENABLED_KEYS] = {
1481                 .core_note_type = NT_ARM_PAC_ENABLED_KEYS,
1482                 .n = 1,
1483                 .size = sizeof(long),
1484                 .align = sizeof(long),
1485                 .regset_get = pac_enabled_keys_get,
1486                 .set = pac_enabled_keys_set,
1487         },
1488 #ifdef CONFIG_CHECKPOINT_RESTORE
1489         [REGSET_PACA_KEYS] = {
1490                 .core_note_type = NT_ARM_PACA_KEYS,
1491                 .n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t),
1492                 .size = sizeof(__uint128_t),
1493                 .align = sizeof(__uint128_t),
1494                 .regset_get = pac_address_keys_get,
1495                 .set = pac_address_keys_set,
1496         },
1497         [REGSET_PACG_KEYS] = {
1498                 .core_note_type = NT_ARM_PACG_KEYS,
1499                 .n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t),
1500                 .size = sizeof(__uint128_t),
1501                 .align = sizeof(__uint128_t),
1502                 .regset_get = pac_generic_keys_get,
1503                 .set = pac_generic_keys_set,
1504         },
1505 #endif
1506 #endif
1507 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1508         [REGSET_TAGGED_ADDR_CTRL] = {
1509                 .core_note_type = NT_ARM_TAGGED_ADDR_CTRL,
1510                 .n = 1,
1511                 .size = sizeof(long),
1512                 .align = sizeof(long),
1513                 .regset_get = tagged_addr_ctrl_get,
1514                 .set = tagged_addr_ctrl_set,
1515         },
1516 #endif
1517 };
1518
1519 static const struct user_regset_view user_aarch64_view = {
1520         .name = "aarch64", .e_machine = EM_AARCH64,
1521         .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets)
1522 };
1523
1524 #ifdef CONFIG_COMPAT
1525 enum compat_regset {
1526         REGSET_COMPAT_GPR,
1527         REGSET_COMPAT_VFP,
1528 };
1529
1530 static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx)
1531 {
1532         struct pt_regs *regs = task_pt_regs(task);
1533
1534         switch (idx) {
1535         case 15:
1536                 return regs->pc;
1537         case 16:
1538                 return pstate_to_compat_psr(regs->pstate);
1539         case 17:
1540                 return regs->orig_x0;
1541         default:
1542                 return regs->regs[idx];
1543         }
1544 }
1545
1546 static int compat_gpr_get(struct task_struct *target,
1547                           const struct user_regset *regset,
1548                           struct membuf to)
1549 {
1550         int i = 0;
1551
1552         while (to.left)
1553                 membuf_store(&to, compat_get_user_reg(target, i++));
1554         return 0;
1555 }
1556
1557 static int compat_gpr_set(struct task_struct *target,
1558                           const struct user_regset *regset,
1559                           unsigned int pos, unsigned int count,
1560                           const void *kbuf, const void __user *ubuf)
1561 {
1562         struct pt_regs newregs;
1563         int ret = 0;
1564         unsigned int i, start, num_regs;
1565
1566         /* Calculate the number of AArch32 registers contained in count */
1567         num_regs = count / regset->size;
1568
1569         /* Convert pos into an register number */
1570         start = pos / regset->size;
1571
1572         if (start + num_regs > regset->n)
1573                 return -EIO;
1574
1575         newregs = *task_pt_regs(target);
1576
1577         for (i = 0; i < num_regs; ++i) {
1578                 unsigned int idx = start + i;
1579                 compat_ulong_t reg;
1580
1581                 if (kbuf) {
1582                         memcpy(&reg, kbuf, sizeof(reg));
1583                         kbuf += sizeof(reg);
1584                 } else {
1585                         ret = copy_from_user(&reg, ubuf, sizeof(reg));
1586                         if (ret) {
1587                                 ret = -EFAULT;
1588                                 break;
1589                         }
1590
1591                         ubuf += sizeof(reg);
1592                 }
1593
1594                 switch (idx) {
1595                 case 15:
1596                         newregs.pc = reg;
1597                         break;
1598                 case 16:
1599                         reg = compat_psr_to_pstate(reg);
1600                         newregs.pstate = reg;
1601                         break;
1602                 case 17:
1603                         newregs.orig_x0 = reg;
1604                         break;
1605                 default:
1606                         newregs.regs[idx] = reg;
1607                 }
1608
1609         }
1610
1611         if (valid_user_regs(&newregs.user_regs, target))
1612                 *task_pt_regs(target) = newregs;
1613         else
1614                 ret = -EINVAL;
1615
1616         return ret;
1617 }
1618
1619 static int compat_vfp_get(struct task_struct *target,
1620                           const struct user_regset *regset,
1621                           struct membuf to)
1622 {
1623         struct user_fpsimd_state *uregs;
1624         compat_ulong_t fpscr;
1625
1626         if (!system_supports_fpsimd())
1627                 return -EINVAL;
1628
1629         uregs = &target->thread.uw.fpsimd_state;
1630
1631         if (target == current)
1632                 fpsimd_preserve_current_state();
1633
1634         /*
1635          * The VFP registers are packed into the fpsimd_state, so they all sit
1636          * nicely together for us. We just need to create the fpscr separately.
1637          */
1638         membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t));
1639         fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
1640                 (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
1641         return membuf_store(&to, fpscr);
1642 }
1643
1644 static int compat_vfp_set(struct task_struct *target,
1645                           const struct user_regset *regset,
1646                           unsigned int pos, unsigned int count,
1647                           const void *kbuf, const void __user *ubuf)
1648 {
1649         struct user_fpsimd_state *uregs;
1650         compat_ulong_t fpscr;
1651         int ret, vregs_end_pos;
1652
1653         if (!system_supports_fpsimd())
1654                 return -EINVAL;
1655
1656         uregs = &target->thread.uw.fpsimd_state;
1657
1658         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1659         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0,
1660                                  vregs_end_pos);
1661
1662         if (count && !ret) {
1663                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr,
1664                                          vregs_end_pos, VFP_STATE_SIZE);
1665                 if (!ret) {
1666                         uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK;
1667                         uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
1668                 }
1669         }
1670
1671         fpsimd_flush_task_state(target);
1672         return ret;
1673 }
1674
1675 static int compat_tls_get(struct task_struct *target,
1676                           const struct user_regset *regset,
1677                           struct membuf to)
1678 {
1679         return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value);
1680 }
1681
1682 static int compat_tls_set(struct task_struct *target,
1683                           const struct user_regset *regset, unsigned int pos,
1684                           unsigned int count, const void *kbuf,
1685                           const void __user *ubuf)
1686 {
1687         int ret;
1688         compat_ulong_t tls = target->thread.uw.tp_value;
1689
1690         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1691         if (ret)
1692                 return ret;
1693
1694         target->thread.uw.tp_value = tls;
1695         return ret;
1696 }
1697
1698 static const struct user_regset aarch32_regsets[] = {
1699         [REGSET_COMPAT_GPR] = {
1700                 .core_note_type = NT_PRSTATUS,
1701                 .n = COMPAT_ELF_NGREG,
1702                 .size = sizeof(compat_elf_greg_t),
1703                 .align = sizeof(compat_elf_greg_t),
1704                 .regset_get = compat_gpr_get,
1705                 .set = compat_gpr_set
1706         },
1707         [REGSET_COMPAT_VFP] = {
1708                 .core_note_type = NT_ARM_VFP,
1709                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1710                 .size = sizeof(compat_ulong_t),
1711                 .align = sizeof(compat_ulong_t),
1712                 .active = fpr_active,
1713                 .regset_get = compat_vfp_get,
1714                 .set = compat_vfp_set
1715         },
1716 };
1717
1718 static const struct user_regset_view user_aarch32_view = {
1719         .name = "aarch32", .e_machine = EM_ARM,
1720         .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
1721 };
1722
1723 static const struct user_regset aarch32_ptrace_regsets[] = {
1724         [REGSET_GPR] = {
1725                 .core_note_type = NT_PRSTATUS,
1726                 .n = COMPAT_ELF_NGREG,
1727                 .size = sizeof(compat_elf_greg_t),
1728                 .align = sizeof(compat_elf_greg_t),
1729                 .regset_get = compat_gpr_get,
1730                 .set = compat_gpr_set
1731         },
1732         [REGSET_FPR] = {
1733                 .core_note_type = NT_ARM_VFP,
1734                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1735                 .size = sizeof(compat_ulong_t),
1736                 .align = sizeof(compat_ulong_t),
1737                 .regset_get = compat_vfp_get,
1738                 .set = compat_vfp_set
1739         },
1740         [REGSET_TLS] = {
1741                 .core_note_type = NT_ARM_TLS,
1742                 .n = 1,
1743                 .size = sizeof(compat_ulong_t),
1744                 .align = sizeof(compat_ulong_t),
1745                 .regset_get = compat_tls_get,
1746                 .set = compat_tls_set,
1747         },
1748 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1749         [REGSET_HW_BREAK] = {
1750                 .core_note_type = NT_ARM_HW_BREAK,
1751                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1752                 .size = sizeof(u32),
1753                 .align = sizeof(u32),
1754                 .regset_get = hw_break_get,
1755                 .set = hw_break_set,
1756         },
1757         [REGSET_HW_WATCH] = {
1758                 .core_note_type = NT_ARM_HW_WATCH,
1759                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1760                 .size = sizeof(u32),
1761                 .align = sizeof(u32),
1762                 .regset_get = hw_break_get,
1763                 .set = hw_break_set,
1764         },
1765 #endif
1766         [REGSET_SYSTEM_CALL] = {
1767                 .core_note_type = NT_ARM_SYSTEM_CALL,
1768                 .n = 1,
1769                 .size = sizeof(int),
1770                 .align = sizeof(int),
1771                 .regset_get = system_call_get,
1772                 .set = system_call_set,
1773         },
1774 };
1775
1776 static const struct user_regset_view user_aarch32_ptrace_view = {
1777         .name = "aarch32", .e_machine = EM_ARM,
1778         .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
1779 };
1780
1781 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
1782                                    compat_ulong_t __user *ret)
1783 {
1784         compat_ulong_t tmp;
1785
1786         if (off & 3)
1787                 return -EIO;
1788
1789         if (off == COMPAT_PT_TEXT_ADDR)
1790                 tmp = tsk->mm->start_code;
1791         else if (off == COMPAT_PT_DATA_ADDR)
1792                 tmp = tsk->mm->start_data;
1793         else if (off == COMPAT_PT_TEXT_END_ADDR)
1794                 tmp = tsk->mm->end_code;
1795         else if (off < sizeof(compat_elf_gregset_t))
1796                 tmp = compat_get_user_reg(tsk, off >> 2);
1797         else if (off >= COMPAT_USER_SZ)
1798                 return -EIO;
1799         else
1800                 tmp = 0;
1801
1802         return put_user(tmp, ret);
1803 }
1804
1805 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
1806                                     compat_ulong_t val)
1807 {
1808         struct pt_regs newregs = *task_pt_regs(tsk);
1809         unsigned int idx = off / 4;
1810
1811         if (off & 3 || off >= COMPAT_USER_SZ)
1812                 return -EIO;
1813
1814         if (off >= sizeof(compat_elf_gregset_t))
1815                 return 0;
1816
1817         switch (idx) {
1818         case 15:
1819                 newregs.pc = val;
1820                 break;
1821         case 16:
1822                 newregs.pstate = compat_psr_to_pstate(val);
1823                 break;
1824         case 17:
1825                 newregs.orig_x0 = val;
1826                 break;
1827         default:
1828                 newregs.regs[idx] = val;
1829         }
1830
1831         if (!valid_user_regs(&newregs.user_regs, tsk))
1832                 return -EINVAL;
1833
1834         *task_pt_regs(tsk) = newregs;
1835         return 0;
1836 }
1837
1838 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1839
1840 /*
1841  * Convert a virtual register number into an index for a thread_info
1842  * breakpoint array. Breakpoints are identified using positive numbers
1843  * whilst watchpoints are negative. The registers are laid out as pairs
1844  * of (address, control), each pair mapping to a unique hw_breakpoint struct.
1845  * Register 0 is reserved for describing resource information.
1846  */
1847 static int compat_ptrace_hbp_num_to_idx(compat_long_t num)
1848 {
1849         return (abs(num) - 1) >> 1;
1850 }
1851
1852 static int compat_ptrace_hbp_get_resource_info(u32 *kdata)
1853 {
1854         u8 num_brps, num_wrps, debug_arch, wp_len;
1855         u32 reg = 0;
1856
1857         num_brps        = hw_breakpoint_slots(TYPE_INST);
1858         num_wrps        = hw_breakpoint_slots(TYPE_DATA);
1859
1860         debug_arch      = debug_monitors_arch();
1861         wp_len          = 8;
1862         reg             |= debug_arch;
1863         reg             <<= 8;
1864         reg             |= wp_len;
1865         reg             <<= 8;
1866         reg             |= num_wrps;
1867         reg             <<= 8;
1868         reg             |= num_brps;
1869
1870         *kdata = reg;
1871         return 0;
1872 }
1873
1874 static int compat_ptrace_hbp_get(unsigned int note_type,
1875                                  struct task_struct *tsk,
1876                                  compat_long_t num,
1877                                  u32 *kdata)
1878 {
1879         u64 addr = 0;
1880         u32 ctrl = 0;
1881
1882         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1883
1884         if (num & 1) {
1885                 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
1886                 *kdata = (u32)addr;
1887         } else {
1888                 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl);
1889                 *kdata = ctrl;
1890         }
1891
1892         return err;
1893 }
1894
1895 static int compat_ptrace_hbp_set(unsigned int note_type,
1896                                  struct task_struct *tsk,
1897                                  compat_long_t num,
1898                                  u32 *kdata)
1899 {
1900         u64 addr;
1901         u32 ctrl;
1902
1903         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1904
1905         if (num & 1) {
1906                 addr = *kdata;
1907                 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr);
1908         } else {
1909                 ctrl = *kdata;
1910                 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl);
1911         }
1912
1913         return err;
1914 }
1915
1916 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num,
1917                                     compat_ulong_t __user *data)
1918 {
1919         int ret;
1920         u32 kdata;
1921
1922         /* Watchpoint */
1923         if (num < 0) {
1924                 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata);
1925         /* Resource info */
1926         } else if (num == 0) {
1927                 ret = compat_ptrace_hbp_get_resource_info(&kdata);
1928         /* Breakpoint */
1929         } else {
1930                 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata);
1931         }
1932
1933         if (!ret)
1934                 ret = put_user(kdata, data);
1935
1936         return ret;
1937 }
1938
1939 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num,
1940                                     compat_ulong_t __user *data)
1941 {
1942         int ret;
1943         u32 kdata = 0;
1944
1945         if (num == 0)
1946                 return 0;
1947
1948         ret = get_user(kdata, data);
1949         if (ret)
1950                 return ret;
1951
1952         if (num < 0)
1953                 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata);
1954         else
1955                 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata);
1956
1957         return ret;
1958 }
1959 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
1960
1961 long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
1962                         compat_ulong_t caddr, compat_ulong_t cdata)
1963 {
1964         unsigned long addr = caddr;
1965         unsigned long data = cdata;
1966         void __user *datap = compat_ptr(data);
1967         int ret;
1968
1969         switch (request) {
1970                 case PTRACE_PEEKUSR:
1971                         ret = compat_ptrace_read_user(child, addr, datap);
1972                         break;
1973
1974                 case PTRACE_POKEUSR:
1975                         ret = compat_ptrace_write_user(child, addr, data);
1976                         break;
1977
1978                 case COMPAT_PTRACE_GETREGS:
1979                         ret = copy_regset_to_user(child,
1980                                                   &user_aarch32_view,
1981                                                   REGSET_COMPAT_GPR,
1982                                                   0, sizeof(compat_elf_gregset_t),
1983                                                   datap);
1984                         break;
1985
1986                 case COMPAT_PTRACE_SETREGS:
1987                         ret = copy_regset_from_user(child,
1988                                                     &user_aarch32_view,
1989                                                     REGSET_COMPAT_GPR,
1990                                                     0, sizeof(compat_elf_gregset_t),
1991                                                     datap);
1992                         break;
1993
1994                 case COMPAT_PTRACE_GET_THREAD_AREA:
1995                         ret = put_user((compat_ulong_t)child->thread.uw.tp_value,
1996                                        (compat_ulong_t __user *)datap);
1997                         break;
1998
1999                 case COMPAT_PTRACE_SET_SYSCALL:
2000                         task_pt_regs(child)->syscallno = data;
2001                         ret = 0;
2002                         break;
2003
2004                 case COMPAT_PTRACE_GETVFPREGS:
2005                         ret = copy_regset_to_user(child,
2006                                                   &user_aarch32_view,
2007                                                   REGSET_COMPAT_VFP,
2008                                                   0, VFP_STATE_SIZE,
2009                                                   datap);
2010                         break;
2011
2012                 case COMPAT_PTRACE_SETVFPREGS:
2013                         ret = copy_regset_from_user(child,
2014                                                     &user_aarch32_view,
2015                                                     REGSET_COMPAT_VFP,
2016                                                     0, VFP_STATE_SIZE,
2017                                                     datap);
2018                         break;
2019
2020 #ifdef CONFIG_HAVE_HW_BREAKPOINT
2021                 case COMPAT_PTRACE_GETHBPREGS:
2022                         ret = compat_ptrace_gethbpregs(child, addr, datap);
2023                         break;
2024
2025                 case COMPAT_PTRACE_SETHBPREGS:
2026                         ret = compat_ptrace_sethbpregs(child, addr, datap);
2027                         break;
2028 #endif
2029
2030                 default:
2031                         ret = compat_ptrace_request(child, request, addr,
2032                                                     data);
2033                         break;
2034         }
2035
2036         return ret;
2037 }
2038 #endif /* CONFIG_COMPAT */
2039
2040 const struct user_regset_view *task_user_regset_view(struct task_struct *task)
2041 {
2042 #ifdef CONFIG_COMPAT
2043         /*
2044          * Core dumping of 32-bit tasks or compat ptrace requests must use the
2045          * user_aarch32_view compatible with arm32. Native ptrace requests on
2046          * 32-bit children use an extended user_aarch32_ptrace_view to allow
2047          * access to the TLS register.
2048          */
2049         if (is_compat_task())
2050                 return &user_aarch32_view;
2051         else if (is_compat_thread(task_thread_info(task)))
2052                 return &user_aarch32_ptrace_view;
2053 #endif
2054         return &user_aarch64_view;
2055 }
2056
2057 long arch_ptrace(struct task_struct *child, long request,
2058                  unsigned long addr, unsigned long data)
2059 {
2060         switch (request) {
2061         case PTRACE_PEEKMTETAGS:
2062         case PTRACE_POKEMTETAGS:
2063                 return mte_ptrace_copy_tags(child, request, addr, data);
2064         }
2065
2066         return ptrace_request(child, request, addr, data);
2067 }
2068
2069 enum ptrace_syscall_dir {
2070         PTRACE_SYSCALL_ENTER = 0,
2071         PTRACE_SYSCALL_EXIT,
2072 };
2073
2074 static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir)
2075 {
2076         int regno;
2077         unsigned long saved_reg;
2078
2079         /*
2080          * We have some ABI weirdness here in the way that we handle syscall
2081          * exit stops because we indicate whether or not the stop has been
2082          * signalled from syscall entry or syscall exit by clobbering a general
2083          * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
2084          * and restoring its old value after the stop. This means that:
2085          *
2086          * - Any writes by the tracer to this register during the stop are
2087          *   ignored/discarded.
2088          *
2089          * - The actual value of the register is not available during the stop,
2090          *   so the tracer cannot save it and restore it later.
2091          *
2092          * - Syscall stops behave differently to seccomp and pseudo-step traps
2093          *   (the latter do not nobble any registers).
2094          */
2095         regno = (is_compat_task() ? 12 : 7);
2096         saved_reg = regs->regs[regno];
2097         regs->regs[regno] = dir;
2098
2099         if (dir == PTRACE_SYSCALL_ENTER) {
2100                 if (ptrace_report_syscall_entry(regs))
2101                         forget_syscall(regs);
2102                 regs->regs[regno] = saved_reg;
2103         } else if (!test_thread_flag(TIF_SINGLESTEP)) {
2104                 ptrace_report_syscall_exit(regs, 0);
2105                 regs->regs[regno] = saved_reg;
2106         } else {
2107                 regs->regs[regno] = saved_reg;
2108
2109                 /*
2110                  * Signal a pseudo-step exception since we are stepping but
2111                  * tracer modifications to the registers may have rewound the
2112                  * state machine.
2113                  */
2114                 ptrace_report_syscall_exit(regs, 1);
2115         }
2116 }
2117
2118 int syscall_trace_enter(struct pt_regs *regs)
2119 {
2120         unsigned long flags = read_thread_flags();
2121
2122         if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
2123                 report_syscall(regs, PTRACE_SYSCALL_ENTER);
2124                 if (flags & _TIF_SYSCALL_EMU)
2125                         return NO_SYSCALL;
2126         }
2127
2128         /* Do the secure computing after ptrace; failures should be fast. */
2129         if (secure_computing() == -1)
2130                 return NO_SYSCALL;
2131
2132         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
2133                 trace_sys_enter(regs, regs->syscallno);
2134
2135         audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
2136                             regs->regs[2], regs->regs[3]);
2137
2138         return regs->syscallno;
2139 }
2140
2141 void syscall_trace_exit(struct pt_regs *regs)
2142 {
2143         unsigned long flags = read_thread_flags();
2144
2145         audit_syscall_exit(regs);
2146
2147         if (flags & _TIF_SYSCALL_TRACEPOINT)
2148                 trace_sys_exit(regs, syscall_get_return_value(current, regs));
2149
2150         if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
2151                 report_syscall(regs, PTRACE_SYSCALL_EXIT);
2152
2153         rseq_syscall(regs);
2154 }
2155
2156 /*
2157  * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
2158  * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
2159  * not described in ARM DDI 0487D.a.
2160  * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may
2161  * be allocated an EL0 meaning in future.
2162  * Userspace cannot use these until they have an architectural meaning.
2163  * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
2164  * We also reserve IL for the kernel; SS is handled dynamically.
2165  */
2166 #define SPSR_EL1_AARCH64_RES0_BITS \
2167         (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \
2168          GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
2169 #define SPSR_EL1_AARCH32_RES0_BITS \
2170         (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
2171
2172 static int valid_compat_regs(struct user_pt_regs *regs)
2173 {
2174         regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
2175
2176         if (!system_supports_mixed_endian_el0()) {
2177                 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
2178                         regs->pstate |= PSR_AA32_E_BIT;
2179                 else
2180                         regs->pstate &= ~PSR_AA32_E_BIT;
2181         }
2182
2183         if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
2184             (regs->pstate & PSR_AA32_A_BIT) == 0 &&
2185             (regs->pstate & PSR_AA32_I_BIT) == 0 &&
2186             (regs->pstate & PSR_AA32_F_BIT) == 0) {
2187                 return 1;
2188         }
2189
2190         /*
2191          * Force PSR to a valid 32-bit EL0t, preserving the same bits as
2192          * arch/arm.
2193          */
2194         regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT |
2195                         PSR_AA32_C_BIT | PSR_AA32_V_BIT |
2196                         PSR_AA32_Q_BIT | PSR_AA32_IT_MASK |
2197                         PSR_AA32_GE_MASK | PSR_AA32_E_BIT |
2198                         PSR_AA32_T_BIT;
2199         regs->pstate |= PSR_MODE32_BIT;
2200
2201         return 0;
2202 }
2203
2204 static int valid_native_regs(struct user_pt_regs *regs)
2205 {
2206         regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
2207
2208         if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
2209             (regs->pstate & PSR_D_BIT) == 0 &&
2210             (regs->pstate & PSR_A_BIT) == 0 &&
2211             (regs->pstate & PSR_I_BIT) == 0 &&
2212             (regs->pstate & PSR_F_BIT) == 0) {
2213                 return 1;
2214         }
2215
2216         /* Force PSR to a valid 64-bit EL0t */
2217         regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
2218
2219         return 0;
2220 }
2221
2222 /*
2223  * Are the current registers suitable for user mode? (used to maintain
2224  * security in signal handlers)
2225  */
2226 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
2227 {
2228         /* https://lore.kernel.org/lkml/20191118131525.GA4180@willie-the-truck */
2229         user_regs_reset_single_step(regs, task);
2230
2231         if (is_compat_thread(task_thread_info(task)))
2232                 return valid_compat_regs(regs);
2233         else
2234                 return valid_native_regs(regs);
2235 }