arch/arm64/kernel/ptrace.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/ptrace.c
   4  *
   5  * By Ross Biro 1/23/92
   6  * edited by Linus Torvalds
   7  * ARM modifications Copyright (C) 2000 Russell King
   8  * Copyright (C) 2012 ARM Ltd.
   9  */
  10
  11 #include <linux/audit.h>
  12 #include <linux/compat.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched/signal.h>
  15 #include <linux/sched/task_stack.h>
  16 #include <linux/mm.h>
  17 #include <linux/nospec.h>
  18 #include <linux/smp.h>
  19 #include <linux/ptrace.h>
  20 #include <linux/user.h>
  21 #include <linux/seccomp.h>
  22 #include <linux/security.h>
  23 #include <linux/init.h>
  24 #include <linux/signal.h>
  25 #include <linux/string.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/perf_event.h>
  28 #include <linux/hw_breakpoint.h>
  29 #include <linux/regset.h>
  30 #include <linux/elf.h>
  31 #include <linux/rseq.h>
  32
  33 #include <asm/compat.h>
  34 #include <asm/cpufeature.h>
  35 #include <asm/debug-monitors.h>
  36 #include <asm/fpsimd.h>
  37 #include <asm/mte.h>
  38 #include <asm/pointer_auth.h>
  39 #include <asm/stacktrace.h>
  40 #include <asm/syscall.h>
  41 #include <asm/traps.h>
  42 #include <asm/system_misc.h>
  43
  44 #define CREATE_TRACE_POINTS
  45 #include <trace/events/syscalls.h>
  46
  47 struct pt_regs_offset {
  48         const char *name;
  49         int offset;
  50 };
  51
  52 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)}
  53 #define REG_OFFSET_END {.name = NULL, .offset = 0}
  54 #define GPR_OFFSET_NAME(r) \
  55         {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])}
  56
  57 static const struct pt_regs_offset regoffset_table[] = {
  58         GPR_OFFSET_NAME(0),
  59         GPR_OFFSET_NAME(1),
  60         GPR_OFFSET_NAME(2),
  61         GPR_OFFSET_NAME(3),
  62         GPR_OFFSET_NAME(4),
  63         GPR_OFFSET_NAME(5),
  64         GPR_OFFSET_NAME(6),
  65         GPR_OFFSET_NAME(7),
  66         GPR_OFFSET_NAME(8),
  67         GPR_OFFSET_NAME(9),
  68         GPR_OFFSET_NAME(10),
  69         GPR_OFFSET_NAME(11),
  70         GPR_OFFSET_NAME(12),
  71         GPR_OFFSET_NAME(13),
  72         GPR_OFFSET_NAME(14),
  73         GPR_OFFSET_NAME(15),
  74         GPR_OFFSET_NAME(16),
  75         GPR_OFFSET_NAME(17),
  76         GPR_OFFSET_NAME(18),
  77         GPR_OFFSET_NAME(19),
  78         GPR_OFFSET_NAME(20),
  79         GPR_OFFSET_NAME(21),
  80         GPR_OFFSET_NAME(22),
  81         GPR_OFFSET_NAME(23),
  82         GPR_OFFSET_NAME(24),
  83         GPR_OFFSET_NAME(25),
  84         GPR_OFFSET_NAME(26),
  85         GPR_OFFSET_NAME(27),
  86         GPR_OFFSET_NAME(28),
  87         GPR_OFFSET_NAME(29),
  88         GPR_OFFSET_NAME(30),
  89         {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])},
  90         REG_OFFSET_NAME(sp),
  91         REG_OFFSET_NAME(pc),
  92         REG_OFFSET_NAME(pstate),
  93         REG_OFFSET_END,
  94 };
  95
  96 /**
  97  * regs_query_register_offset() - query register offset from its name
  98  * @name:       the name of a register
  99  *
 100  * regs_query_register_offset() returns the offset of a register in struct
 101  * pt_regs from its name. If the name is invalid, this returns -EINVAL;
 102  */
 103 int regs_query_register_offset(const char *name)
 104 {
 105         const struct pt_regs_offset *roff;
 106
 107         for (roff = regoffset_table; roff->name != NULL; roff++)
 108                 if (!strcmp(roff->name, name))
 109                         return roff->offset;
 110         return -EINVAL;
 111 }
 112
 113 /**
 114  * regs_within_kernel_stack() - check the address in the stack
 115  * @regs:      pt_regs which contains kernel stack pointer.
 116  * @addr:      address which is checked.
 117  *
 118  * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
 119  * If @addr is within the kernel stack, it returns true. If not, returns false.
 120  */
 121 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 122 {
 123         return ((addr & ~(THREAD_SIZE - 1))  ==
 124                 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
 125                 on_irq_stack(addr, sizeof(unsigned long));
 126 }
 127
 128 /**
 129  * regs_get_kernel_stack_nth() - get Nth entry of the stack
 130  * @regs:       pt_regs which contains kernel stack pointer.
 131  * @n:          stack entry number.
 132  *
 133  * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
 134  * is specified by @regs. If the @n th entry is NOT in the kernel stack,
 135  * this returns 0.
 136  */
 137 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
 138 {
 139         unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
 140
 141         addr += n;
 142         if (regs_within_kernel_stack(regs, (unsigned long)addr))
 143                 return *addr;
 144         else
 145                 return 0;
 146 }
 147
 148 /*
 149  * TODO: does not yet catch signals sent when the child dies.
 150  * in exit.c or in signal.c.
 151  */
 152
 153 /*
 154  * Called by kernel/ptrace.c when detaching..
 155  */
 156 void ptrace_disable(struct task_struct *child)
 157 {
 158         /*
 159          * This would be better off in core code, but PTRACE_DETACH has
 160          * grown its fair share of arch-specific worts and changing it
 161          * is likely to cause regressions on obscure architectures.
 162          */
 163         user_disable_single_step(child);
 164 }
 165
 166 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 167 /*
 168  * Handle hitting a HW-breakpoint.
 169  */
 170 static void ptrace_hbptriggered(struct perf_event *bp,
 171                                 struct perf_sample_data *data,
 172                                 struct pt_regs *regs)
 173 {
 174         struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 175         const char *desc = "Hardware breakpoint trap (ptrace)";
 176
 177         if (is_compat_task()) {
 178                 int si_errno = 0;
 179                 int i;
 180
 181                 for (i = 0; i < ARM_MAX_BRP; ++i) {
 182                         if (current->thread.debug.hbp_break[i] == bp) {
 183                                 si_errno = (i << 1) + 1;
 184                                 break;
 185                         }
 186                 }
 187
 188                 for (i = 0; i < ARM_MAX_WRP; ++i) {
 189                         if (current->thread.debug.hbp_watch[i] == bp) {
 190                                 si_errno = -((i << 1) + 1);
 191                                 break;
 192                         }
 193                 }
 194                 arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger,
 195                                                   desc);
 196                 return;
 197         }
 198
 199         arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc);
 200 }
 201
 202 /*
 203  * Unregister breakpoints from this task and reset the pointers in
 204  * the thread_struct.
 205  */
 206 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 207 {
 208         int i;
 209         struct thread_struct *t = &tsk->thread;
 210
 211         for (i = 0; i < ARM_MAX_BRP; i++) {
 212                 if (t->debug.hbp_break[i]) {
 213                         unregister_hw_breakpoint(t->debug.hbp_break[i]);
 214                         t->debug.hbp_break[i] = NULL;
 215                 }
 216         }
 217
 218         for (i = 0; i < ARM_MAX_WRP; i++) {
 219                 if (t->debug.hbp_watch[i]) {
 220                         unregister_hw_breakpoint(t->debug.hbp_watch[i]);
 221                         t->debug.hbp_watch[i] = NULL;
 222                 }
 223         }
 224 }
 225
 226 void ptrace_hw_copy_thread(struct task_struct *tsk)
 227 {
 228         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
 229 }
 230
 231 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type,
 232                                                struct task_struct *tsk,
 233                                                unsigned long idx)
 234 {
 235         struct perf_event *bp = ERR_PTR(-EINVAL);
 236
 237         switch (note_type) {
 238         case NT_ARM_HW_BREAK:
 239                 if (idx >= ARM_MAX_BRP)
 240                         goto out;
 241                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 242                 bp = tsk->thread.debug.hbp_break[idx];
 243                 break;
 244         case NT_ARM_HW_WATCH:
 245                 if (idx >= ARM_MAX_WRP)
 246                         goto out;
 247                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 248                 bp = tsk->thread.debug.hbp_watch[idx];
 249                 break;
 250         }
 251
 252 out:
 253         return bp;
 254 }
 255
 256 static int ptrace_hbp_set_event(unsigned int note_type,
 257                                 struct task_struct *tsk,
 258                                 unsigned long idx,
 259                                 struct perf_event *bp)
 260 {
 261         int err = -EINVAL;
 262
 263         switch (note_type) {
 264         case NT_ARM_HW_BREAK:
 265                 if (idx >= ARM_MAX_BRP)
 266                         goto out;
 267                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 268                 tsk->thread.debug.hbp_break[idx] = bp;
 269                 err = 0;
 270                 break;
 271         case NT_ARM_HW_WATCH:
 272                 if (idx >= ARM_MAX_WRP)
 273                         goto out;
 274                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 275                 tsk->thread.debug.hbp_watch[idx] = bp;
 276                 err = 0;
 277                 break;
 278         }
 279
 280 out:
 281         return err;
 282 }
 283
 284 static struct perf_event *ptrace_hbp_create(unsigned int note_type,
 285                                             struct task_struct *tsk,
 286                                             unsigned long idx)
 287 {
 288         struct perf_event *bp;
 289         struct perf_event_attr attr;
 290         int err, type;
 291
 292         switch (note_type) {
 293         case NT_ARM_HW_BREAK:
 294                 type = HW_BREAKPOINT_X;
 295                 break;
 296         case NT_ARM_HW_WATCH:
 297                 type = HW_BREAKPOINT_RW;
 298                 break;
 299         default:
 300                 return ERR_PTR(-EINVAL);
 301         }
 302
 303         ptrace_breakpoint_init(&attr);
 304
 305         /*
 306          * Initialise fields to sane defaults
 307          * (i.e. values that will pass validation).
 308          */
 309         attr.bp_addr    = 0;
 310         attr.bp_len     = HW_BREAKPOINT_LEN_4;
 311         attr.bp_type    = type;
 312         attr.disabled   = 1;
 313
 314         bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk);
 315         if (IS_ERR(bp))
 316                 return bp;
 317
 318         err = ptrace_hbp_set_event(note_type, tsk, idx, bp);
 319         if (err)
 320                 return ERR_PTR(err);
 321
 322         return bp;
 323 }
 324
 325 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 326                                      struct arch_hw_breakpoint_ctrl ctrl,
 327                                      struct perf_event_attr *attr)
 328 {
 329         int err, len, type, offset, disabled = !ctrl.enabled;
 330
 331         attr->disabled = disabled;
 332         if (disabled)
 333                 return 0;
 334
 335         err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 336         if (err)
 337                 return err;
 338
 339         switch (note_type) {
 340         case NT_ARM_HW_BREAK:
 341                 if ((type & HW_BREAKPOINT_X) != type)
 342                         return -EINVAL;
 343                 break;
 344         case NT_ARM_HW_WATCH:
 345                 if ((type & HW_BREAKPOINT_RW) != type)
 346                         return -EINVAL;
 347                 break;
 348         default:
 349                 return -EINVAL;
 350         }
 351
 352         attr->bp_len    = len;
 353         attr->bp_type   = type;
 354         attr->bp_addr   += offset;
 355
 356         return 0;
 357 }
 358
 359 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info)
 360 {
 361         u8 num;
 362         u32 reg = 0;
 363
 364         switch (note_type) {
 365         case NT_ARM_HW_BREAK:
 366                 num = hw_breakpoint_slots(TYPE_INST);
 367                 break;
 368         case NT_ARM_HW_WATCH:
 369                 num = hw_breakpoint_slots(TYPE_DATA);
 370                 break;
 371         default:
 372                 return -EINVAL;
 373         }
 374
 375         reg |= debug_monitors_arch();
 376         reg <<= 8;
 377         reg |= num;
 378
 379         *info = reg;
 380         return 0;
 381 }
 382
 383 static int ptrace_hbp_get_ctrl(unsigned int note_type,
 384                                struct task_struct *tsk,
 385                                unsigned long idx,
 386                                u32 *ctrl)
 387 {
 388         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 389
 390         if (IS_ERR(bp))
 391                 return PTR_ERR(bp);
 392
 393         *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0;
 394         return 0;
 395 }
 396
 397 static int ptrace_hbp_get_addr(unsigned int note_type,
 398                                struct task_struct *tsk,
 399                                unsigned long idx,
 400                                u64 *addr)
 401 {
 402         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 403
 404         if (IS_ERR(bp))
 405                 return PTR_ERR(bp);
 406
 407         *addr = bp ? counter_arch_bp(bp)->address : 0;
 408         return 0;
 409 }
 410
 411 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type,
 412                                                         struct task_struct *tsk,
 413                                                         unsigned long idx)
 414 {
 415         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 416
 417         if (!bp)
 418                 bp = ptrace_hbp_create(note_type, tsk, idx);
 419
 420         return bp;
 421 }
 422
 423 static int ptrace_hbp_set_ctrl(unsigned int note_type,
 424                                struct task_struct *tsk,
 425                                unsigned long idx,
 426                                u32 uctrl)
 427 {
 428         int err;
 429         struct perf_event *bp;
 430         struct perf_event_attr attr;
 431         struct arch_hw_breakpoint_ctrl ctrl;
 432
 433         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 434         if (IS_ERR(bp)) {
 435                 err = PTR_ERR(bp);
 436                 return err;
 437         }
 438
 439         attr = bp->attr;
 440         decode_ctrl_reg(uctrl, &ctrl);
 441         err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr);
 442         if (err)
 443                 return err;
 444
 445         return modify_user_hw_breakpoint(bp, &attr);
 446 }
 447
 448 static int ptrace_hbp_set_addr(unsigned int note_type,
 449                                struct task_struct *tsk,
 450                                unsigned long idx,
 451                                u64 addr)
 452 {
 453         int err;
 454         struct perf_event *bp;
 455         struct perf_event_attr attr;
 456
 457         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 458         if (IS_ERR(bp)) {
 459                 err = PTR_ERR(bp);
 460                 return err;
 461         }
 462
 463         attr = bp->attr;
 464         attr.bp_addr = addr;
 465         err = modify_user_hw_breakpoint(bp, &attr);
 466         return err;
 467 }
 468
 469 #define PTRACE_HBP_ADDR_SZ      sizeof(u64)
 470 #define PTRACE_HBP_CTRL_SZ      sizeof(u32)
 471 #define PTRACE_HBP_PAD_SZ       sizeof(u32)
 472
 473 static int hw_break_get(struct task_struct *target,
 474                         const struct user_regset *regset,
 475                         struct membuf to)
 476 {
 477         unsigned int note_type = regset->core_note_type;
 478         int ret, idx = 0;
 479         u32 info, ctrl;
 480         u64 addr;
 481
 482         /* Resource info */
 483         ret = ptrace_hbp_get_resource_info(note_type, &info);
 484         if (ret)
 485                 return ret;
 486
 487         membuf_write(&to, &info, sizeof(info));
 488         membuf_zero(&to, sizeof(u32));
 489         /* (address, ctrl) registers */
 490         while (to.left) {
 491                 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
 492                 if (ret)
 493                         return ret;
 494                 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
 495                 if (ret)
 496                         return ret;
 497                 membuf_store(&to, addr);
 498                 membuf_store(&to, ctrl);
 499                 membuf_zero(&to, sizeof(u32));
 500                 idx++;
 501         }
 502         return 0;
 503 }
 504
 505 static int hw_break_set(struct task_struct *target,
 506                         const struct user_regset *regset,
 507                         unsigned int pos, unsigned int count,
 508                         const void *kbuf, const void __user *ubuf)
 509 {
 510         unsigned int note_type = regset->core_note_type;
 511         int ret, idx = 0, offset, limit;
 512         u32 ctrl;
 513         u64 addr;
 514
 515         /* Resource info and pad */
 516         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 517         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
 518
 519         /* (address, ctrl) registers */
 520         limit = regset->n * regset->size;
 521         while (count && offset < limit) {
 522                 if (count < PTRACE_HBP_ADDR_SZ)
 523                         return -EINVAL;
 524                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
 525                                          offset, offset + PTRACE_HBP_ADDR_SZ);
 526                 if (ret)
 527                         return ret;
 528                 ret = ptrace_hbp_set_addr(note_type, target, idx, addr);
 529                 if (ret)
 530                         return ret;
 531                 offset += PTRACE_HBP_ADDR_SZ;
 532
 533                 if (!count)
 534                         break;
 535                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl,
 536                                          offset, offset + PTRACE_HBP_CTRL_SZ);
 537                 if (ret)
 538                         return ret;
 539                 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl);
 540                 if (ret)
 541                         return ret;
 542                 offset += PTRACE_HBP_CTRL_SZ;
 543
 544                 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 545                                           offset, offset + PTRACE_HBP_PAD_SZ);
 546                 offset += PTRACE_HBP_PAD_SZ;
 547                 idx++;
 548         }
 549
 550         return 0;
 551 }
 552 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
 553
 554 static int gpr_get(struct task_struct *target,
 555                    const struct user_regset *regset,
 556                    struct membuf to)
 557 {
 558         struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
 559         return membuf_write(&to, uregs, sizeof(*uregs));
 560 }
 561
 562 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 563                    unsigned int pos, unsigned int count,
 564                    const void *kbuf, const void __user *ubuf)
 565 {
 566         int ret;
 567         struct user_pt_regs newregs = task_pt_regs(target)->user_regs;
 568
 569         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1);
 570         if (ret)
 571                 return ret;
 572
 573         if (!valid_user_regs(&newregs, target))
 574                 return -EINVAL;
 575
 576         task_pt_regs(target)->user_regs = newregs;
 577         return 0;
 578 }
 579
 580 static int fpr_active(struct task_struct *target, const struct user_regset *regset)
 581 {
 582         if (!system_supports_fpsimd())
 583                 return -ENODEV;
 584         return regset->n;
 585 }
 586
 587 /*
 588  * TODO: update fp accessors for lazy context switching (sync/flush hwstate)
 589  */
 590 static int __fpr_get(struct task_struct *target,
 591                      const struct user_regset *regset,
 592                      struct membuf to)
 593 {
 594         struct user_fpsimd_state *uregs;
 595
 596         sve_sync_to_fpsimd(target);
 597
 598         uregs = &target->thread.uw.fpsimd_state;
 599
 600         return membuf_write(&to, uregs, sizeof(*uregs));
 601 }
 602
 603 static int fpr_get(struct task_struct *target, const struct user_regset *regset,
 604                    struct membuf to)
 605 {
 606         if (!system_supports_fpsimd())
 607                 return -EINVAL;
 608
 609         if (target == current)
 610                 fpsimd_preserve_current_state();
 611
 612         return __fpr_get(target, regset, to);
 613 }
 614
 615 static int __fpr_set(struct task_struct *target,
 616                      const struct user_regset *regset,
 617                      unsigned int pos, unsigned int count,
 618                      const void *kbuf, const void __user *ubuf,
 619                      unsigned int start_pos)
 620 {
 621         int ret;
 622         struct user_fpsimd_state newstate;
 623
 624         /*
 625          * Ensure target->thread.uw.fpsimd_state is up to date, so that a
 626          * short copyin can't resurrect stale data.
 627          */
 628         sve_sync_to_fpsimd(target);
 629
 630         newstate = target->thread.uw.fpsimd_state;
 631
 632         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
 633                                  start_pos, start_pos + sizeof(newstate));
 634         if (ret)
 635                 return ret;
 636
 637         target->thread.uw.fpsimd_state = newstate;
 638
 639         return ret;
 640 }
 641
 642 static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 643                    unsigned int pos, unsigned int count,
 644                    const void *kbuf, const void __user *ubuf)
 645 {
 646         int ret;
 647
 648         if (!system_supports_fpsimd())
 649                 return -EINVAL;
 650
 651         ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
 652         if (ret)
 653                 return ret;
 654
 655         sve_sync_from_fpsimd_zeropad(target);
 656         fpsimd_flush_task_state(target);
 657
 658         return ret;
 659 }
 660
 661 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 662                    struct membuf to)
 663 {
 664         int ret;
 665
 666         if (target == current)
 667                 tls_preserve_current_state();
 668
 669         ret = membuf_store(&to, target->thread.uw.tp_value);
 670         if (system_supports_tpidr2())
 671                 ret = membuf_store(&to, target->thread.tpidr2_el0);
 672         else
 673                 ret = membuf_zero(&to, sizeof(u64));
 674
 675         return ret;
 676 }
 677
 678 static int tls_set(struct task_struct *target, const struct user_regset *regset,
 679                    unsigned int pos, unsigned int count,
 680                    const void *kbuf, const void __user *ubuf)
 681 {
 682         int ret;
 683         unsigned long tls[2];
 684
 685         tls[0] = target->thread.uw.tp_value;
 686         if (system_supports_tpidr2())
 687                 tls[1] = target->thread.tpidr2_el0;
 688
 689         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count);
 690         if (ret)
 691                 return ret;
 692
 693         target->thread.uw.tp_value = tls[0];
 694         if (system_supports_tpidr2())
 695                 target->thread.tpidr2_el0 = tls[1];
 696
 697         return ret;
 698 }
 699
 700 static int fpmr_get(struct task_struct *target, const struct user_regset *regset,
 701                    struct membuf to)
 702 {
 703         if (!system_supports_fpmr())
 704                 return -EINVAL;
 705
 706         if (target == current)
 707                 fpsimd_preserve_current_state();
 708
 709         return membuf_store(&to, target->thread.uw.fpmr);
 710 }
 711
 712 static int fpmr_set(struct task_struct *target, const struct user_regset *regset,
 713                    unsigned int pos, unsigned int count,
 714                    const void *kbuf, const void __user *ubuf)
 715 {
 716         int ret;
 717         unsigned long fpmr;
 718
 719         if (!system_supports_fpmr())
 720                 return -EINVAL;
 721
 722         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpmr, 0, count);
 723         if (ret)
 724                 return ret;
 725
 726         target->thread.uw.fpmr = fpmr;
 727
 728         fpsimd_flush_task_state(target);
 729
 730         return 0;
 731 }
 732
 733 static int system_call_get(struct task_struct *target,
 734                            const struct user_regset *regset,
 735                            struct membuf to)
 736 {
 737         return membuf_store(&to, task_pt_regs(target)->syscallno);
 738 }
 739
 740 static int system_call_set(struct task_struct *target,
 741                            const struct user_regset *regset,
 742                            unsigned int pos, unsigned int count,
 743                            const void *kbuf, const void __user *ubuf)
 744 {
 745         int syscallno = task_pt_regs(target)->syscallno;
 746         int ret;
 747
 748         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1);
 749         if (ret)
 750                 return ret;
 751
 752         task_pt_regs(target)->syscallno = syscallno;
 753         return ret;
 754 }
 755
 756 #ifdef CONFIG_ARM64_SVE
 757
 758 static void sve_init_header_from_task(struct user_sve_header *header,
 759                                       struct task_struct *target,
 760                                       enum vec_type type)
 761 {
 762         unsigned int vq;
 763         bool active;
 764         bool fpsimd_only;
 765         enum vec_type task_type;
 766
 767         memset(header, 0, sizeof(*header));
 768
 769         /* Check if the requested registers are active for the task */
 770         if (thread_sm_enabled(&target->thread))
 771                 task_type = ARM64_VEC_SME;
 772         else
 773                 task_type = ARM64_VEC_SVE;
 774         active = (task_type == type);
 775
 776         switch (type) {
 777         case ARM64_VEC_SVE:
 778                 if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
 779                         header->flags |= SVE_PT_VL_INHERIT;
 780                 fpsimd_only = !test_tsk_thread_flag(target, TIF_SVE);
 781                 break;
 782         case ARM64_VEC_SME:
 783                 if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
 784                         header->flags |= SVE_PT_VL_INHERIT;
 785                 fpsimd_only = false;
 786                 break;
 787         default:
 788                 WARN_ON_ONCE(1);
 789                 return;
 790         }
 791
 792         if (active) {
 793                 if (fpsimd_only) {
 794                         header->flags |= SVE_PT_REGS_FPSIMD;
 795                 } else {
 796                         header->flags |= SVE_PT_REGS_SVE;
 797                 }
 798         }
 799
 800         header->vl = task_get_vl(target, type);
 801         vq = sve_vq_from_vl(header->vl);
 802
 803         header->max_vl = vec_max_vl(type);
 804         header->size = SVE_PT_SIZE(vq, header->flags);
 805         header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 806                                       SVE_PT_REGS_SVE);
 807 }
 808
 809 static unsigned int sve_size_from_header(struct user_sve_header const *header)
 810 {
 811         return ALIGN(header->size, SVE_VQ_BYTES);
 812 }
 813
 814 static int sve_get_common(struct task_struct *target,
 815                           const struct user_regset *regset,
 816                           struct membuf to,
 817                           enum vec_type type)
 818 {
 819         struct user_sve_header header;
 820         unsigned int vq;
 821         unsigned long start, end;
 822
 823         /* Header */
 824         sve_init_header_from_task(&header, target, type);
 825         vq = sve_vq_from_vl(header.vl);
 826
 827         membuf_write(&to, &header, sizeof(header));
 828
 829         if (target == current)
 830                 fpsimd_preserve_current_state();
 831
 832         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 833         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 834
 835         switch ((header.flags & SVE_PT_REGS_MASK)) {
 836         case SVE_PT_REGS_FPSIMD:
 837                 return __fpr_get(target, regset, to);
 838
 839         case SVE_PT_REGS_SVE:
 840                 start = SVE_PT_SVE_OFFSET;
 841                 end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 842                 membuf_write(&to, target->thread.sve_state, end - start);
 843
 844                 start = end;
 845                 end = SVE_PT_SVE_FPSR_OFFSET(vq);
 846                 membuf_zero(&to, end - start);
 847
 848                 /*
 849                  * Copy fpsr, and fpcr which must follow contiguously in
 850                  * struct fpsimd_state:
 851                  */
 852                 start = end;
 853                 end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 854                 membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr,
 855                              end - start);
 856
 857                 start = end;
 858                 end = sve_size_from_header(&header);
 859                 return membuf_zero(&to, end - start);
 860
 861         default:
 862                 return 0;
 863         }
 864 }
 865
 866 static int sve_get(struct task_struct *target,
 867                    const struct user_regset *regset,
 868                    struct membuf to)
 869 {
 870         if (!system_supports_sve())
 871                 return -EINVAL;
 872
 873         return sve_get_common(target, regset, to, ARM64_VEC_SVE);
 874 }
 875
 876 static int sve_set_common(struct task_struct *target,
 877                           const struct user_regset *regset,
 878                           unsigned int pos, unsigned int count,
 879                           const void *kbuf, const void __user *ubuf,
 880                           enum vec_type type)
 881 {
 882         int ret;
 883         struct user_sve_header header;
 884         unsigned int vq;
 885         unsigned long start, end;
 886
 887         /* Header */
 888         if (count < sizeof(header))
 889                 return -EINVAL;
 890         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
 891                                  0, sizeof(header));
 892         if (ret)
 893                 goto out;
 894
 895         /*
 896          * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are consumed by
 897          * vec_set_vector_length(), which will also validate them for us:
 898          */
 899         ret = vec_set_vector_length(target, type, header.vl,
 900                 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 901         if (ret)
 902                 goto out;
 903
 904         /* Actual VL set may be less than the user asked for: */
 905         vq = sve_vq_from_vl(task_get_vl(target, type));
 906
 907         /* Enter/exit streaming mode */
 908         if (system_supports_sme()) {
 909                 u64 old_svcr = target->thread.svcr;
 910
 911                 switch (type) {
 912                 case ARM64_VEC_SVE:
 913                         target->thread.svcr &= ~SVCR_SM_MASK;
 914                         break;
 915                 case ARM64_VEC_SME:
 916                         target->thread.svcr |= SVCR_SM_MASK;
 917
 918                         /*
 919                          * Disable traps and ensure there is SME storage but
 920                          * preserve any currently set values in ZA/ZT.
 921                          */
 922                         sme_alloc(target, false);
 923                         set_tsk_thread_flag(target, TIF_SME);
 924                         break;
 925                 default:
 926                         WARN_ON_ONCE(1);
 927                         ret = -EINVAL;
 928                         goto out;
 929                 }
 930
 931                 /*
 932                  * If we switched then invalidate any existing SVE
 933                  * state and ensure there's storage.
 934                  */
 935                 if (target->thread.svcr != old_svcr)
 936                         sve_alloc(target, true);
 937         }
 938
 939         /* Registers: FPSIMD-only case */
 940
 941         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 942         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
 943                 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
 944                                 SVE_PT_FPSIMD_OFFSET);
 945                 clear_tsk_thread_flag(target, TIF_SVE);
 946                 target->thread.fp_type = FP_STATE_FPSIMD;
 947                 goto out;
 948         }
 949
 950         /*
 951          * Otherwise: no registers or full SVE case.  For backwards
 952          * compatibility reasons we treat empty flags as SVE registers.
 953          */
 954
 955         /*
 956          * If setting a different VL from the requested VL and there is
 957          * register data, the data layout will be wrong: don't even
 958          * try to set the registers in this case.
 959          */
 960         if (count && vq != sve_vq_from_vl(header.vl)) {
 961                 ret = -EIO;
 962                 goto out;
 963         }
 964
 965         sve_alloc(target, true);
 966         if (!target->thread.sve_state) {
 967                 ret = -ENOMEM;
 968                 clear_tsk_thread_flag(target, TIF_SVE);
 969                 target->thread.fp_type = FP_STATE_FPSIMD;
 970                 goto out;
 971         }
 972
 973         /*
 974          * Ensure target->thread.sve_state is up to date with target's
 975          * FPSIMD regs, so that a short copyin leaves trailing
 976          * registers unmodified.  Only enable SVE if we are
 977          * configuring normal SVE, a system with streaming SVE may not
 978          * have normal SVE.
 979          */
 980         fpsimd_sync_to_sve(target);
 981         if (type == ARM64_VEC_SVE)
 982                 set_tsk_thread_flag(target, TIF_SVE);
 983         target->thread.fp_type = FP_STATE_SVE;
 984
 985         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 986         start = SVE_PT_SVE_OFFSET;
 987         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 988         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 989                                  target->thread.sve_state,
 990                                  start, end);
 991         if (ret)
 992                 goto out;
 993
 994         start = end;
 995         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 996         user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, start, end);
 997
 998         /*
 999          * Copy fpsr, and fpcr which must follow contiguously in
1000          * struct fpsimd_state:
1001          */
1002         start = end;
1003         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
1004         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1005                                  &target->thread.uw.fpsimd_state.fpsr,
1006                                  start, end);
1007
1008 out:
1009         fpsimd_flush_task_state(target);
1010         return ret;
1011 }
1012
1013 static int sve_set(struct task_struct *target,
1014                    const struct user_regset *regset,
1015                    unsigned int pos, unsigned int count,
1016                    const void *kbuf, const void __user *ubuf)
1017 {
1018         if (!system_supports_sve())
1019                 return -EINVAL;
1020
1021         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
1022                               ARM64_VEC_SVE);
1023 }
1024
1025 #endif /* CONFIG_ARM64_SVE */
1026
1027 #ifdef CONFIG_ARM64_SME
1028
1029 static int ssve_get(struct task_struct *target,
1030                    const struct user_regset *regset,
1031                    struct membuf to)
1032 {
1033         if (!system_supports_sme())
1034                 return -EINVAL;
1035
1036         return sve_get_common(target, regset, to, ARM64_VEC_SME);
1037 }
1038
1039 static int ssve_set(struct task_struct *target,
1040                     const struct user_regset *regset,
1041                     unsigned int pos, unsigned int count,
1042                     const void *kbuf, const void __user *ubuf)
1043 {
1044         if (!system_supports_sme())
1045                 return -EINVAL;
1046
1047         return sve_set_common(target, regset, pos, count, kbuf, ubuf,
1048                               ARM64_VEC_SME);
1049 }
1050
1051 static int za_get(struct task_struct *target,
1052                   const struct user_regset *regset,
1053                   struct membuf to)
1054 {
1055         struct user_za_header header;
1056         unsigned int vq;
1057         unsigned long start, end;
1058
1059         if (!system_supports_sme())
1060                 return -EINVAL;
1061
1062         /* Header */
1063         memset(&header, 0, sizeof(header));
1064
1065         if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT))
1066                 header.flags |= ZA_PT_VL_INHERIT;
1067
1068         header.vl = task_get_sme_vl(target);
1069         vq = sve_vq_from_vl(header.vl);
1070         header.max_vl = sme_max_vl();
1071         header.max_size = ZA_PT_SIZE(vq);
1072
1073         /* If ZA is not active there is only the header */
1074         if (thread_za_enabled(&target->thread))
1075                 header.size = ZA_PT_SIZE(vq);
1076         else
1077                 header.size = ZA_PT_ZA_OFFSET;
1078
1079         membuf_write(&to, &header, sizeof(header));
1080
1081         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1082         end = ZA_PT_ZA_OFFSET;
1083
1084         if (target == current)
1085                 fpsimd_preserve_current_state();
1086
1087         /* Any register data to include? */
1088         if (thread_za_enabled(&target->thread)) {
1089                 start = end;
1090                 end = ZA_PT_SIZE(vq);
1091                 membuf_write(&to, target->thread.sme_state, end - start);
1092         }
1093
1094         /* Zero any trailing padding */
1095         start = end;
1096         end = ALIGN(header.size, SVE_VQ_BYTES);
1097         return membuf_zero(&to, end - start);
1098 }
1099
1100 static int za_set(struct task_struct *target,
1101                   const struct user_regset *regset,
1102                   unsigned int pos, unsigned int count,
1103                   const void *kbuf, const void __user *ubuf)
1104 {
1105         int ret;
1106         struct user_za_header header;
1107         unsigned int vq;
1108         unsigned long start, end;
1109
1110         if (!system_supports_sme())
1111                 return -EINVAL;
1112
1113         /* Header */
1114         if (count < sizeof(header))
1115                 return -EINVAL;
1116         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
1117                                  0, sizeof(header));
1118         if (ret)
1119                 goto out;
1120
1121         /*
1122          * All current ZA_PT_* flags are consumed by
1123          * vec_set_vector_length(), which will also validate them for
1124          * us:
1125          */
1126         ret = vec_set_vector_length(target, ARM64_VEC_SME, header.vl,
1127                 ((unsigned long)header.flags) << 16);
1128         if (ret)
1129                 goto out;
1130
1131         /* Actual VL set may be less than the user asked for: */
1132         vq = sve_vq_from_vl(task_get_sme_vl(target));
1133
1134         /* Ensure there is some SVE storage for streaming mode */
1135         if (!target->thread.sve_state) {
1136                 sve_alloc(target, false);
1137                 if (!target->thread.sve_state) {
1138                         ret = -ENOMEM;
1139                         goto out;
1140                 }
1141         }
1142
1143         /*
1144          * Only flush the storage if PSTATE.ZA was not already set,
1145          * otherwise preserve any existing data.
1146          */
1147         sme_alloc(target, !thread_za_enabled(&target->thread));
1148         if (!target->thread.sme_state)
1149                 return -ENOMEM;
1150
1151         /* If there is no data then disable ZA */
1152         if (!count) {
1153                 target->thread.svcr &= ~SVCR_ZA_MASK;
1154                 goto out;
1155         }
1156
1157         /*
1158          * If setting a different VL from the requested VL and there is
1159          * register data, the data layout will be wrong: don't even
1160          * try to set the registers in this case.
1161          */
1162         if (vq != sve_vq_from_vl(header.vl)) {
1163                 ret = -EIO;
1164                 goto out;
1165         }
1166
1167         BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header));
1168         start = ZA_PT_ZA_OFFSET;
1169         end = ZA_PT_SIZE(vq);
1170         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1171                                  target->thread.sme_state,
1172                                  start, end);
1173         if (ret)
1174                 goto out;
1175
1176         /* Mark ZA as active and let userspace use it */
1177         set_tsk_thread_flag(target, TIF_SME);
1178         target->thread.svcr |= SVCR_ZA_MASK;
1179
1180 out:
1181         fpsimd_flush_task_state(target);
1182         return ret;
1183 }
1184
1185 static int zt_get(struct task_struct *target,
1186                   const struct user_regset *regset,
1187                   struct membuf to)
1188 {
1189         if (!system_supports_sme2())
1190                 return -EINVAL;
1191
1192         /*
1193          * If PSTATE.ZA is not set then ZT will be zeroed when it is
1194          * enabled so report the current register value as zero.
1195          */
1196         if (thread_za_enabled(&target->thread))
1197                 membuf_write(&to, thread_zt_state(&target->thread),
1198                              ZT_SIG_REG_BYTES);
1199         else
1200                 membuf_zero(&to, ZT_SIG_REG_BYTES);
1201
1202         return 0;
1203 }
1204
1205 static int zt_set(struct task_struct *target,
1206                   const struct user_regset *regset,
1207                   unsigned int pos, unsigned int count,
1208                   const void *kbuf, const void __user *ubuf)
1209 {
1210         int ret;
1211
1212         if (!system_supports_sme2())
1213                 return -EINVAL;
1214
1215         /* Ensure SVE storage in case this is first use of SME */
1216         sve_alloc(target, false);
1217         if (!target->thread.sve_state)
1218                 return -ENOMEM;
1219
1220         if (!thread_za_enabled(&target->thread)) {
1221                 sme_alloc(target, true);
1222                 if (!target->thread.sme_state)
1223                         return -ENOMEM;
1224         }
1225
1226         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1227                                  thread_zt_state(&target->thread),
1228                                  0, ZT_SIG_REG_BYTES);
1229         if (ret == 0) {
1230                 target->thread.svcr |= SVCR_ZA_MASK;
1231                 set_tsk_thread_flag(target, TIF_SME);
1232         }
1233
1234         fpsimd_flush_task_state(target);
1235
1236         return ret;
1237 }
1238
1239 #endif /* CONFIG_ARM64_SME */
1240
1241 #ifdef CONFIG_ARM64_PTR_AUTH
1242 static int pac_mask_get(struct task_struct *target,
1243                         const struct user_regset *regset,
1244                         struct membuf to)
1245 {
1246         /*
1247          * The PAC bits can differ across data and instruction pointers
1248          * depending on TCR_EL1.TBID*, which we may make use of in future, so
1249          * we expose separate masks.
1250          */
1251         unsigned long mask = ptrauth_user_pac_mask();
1252         struct user_pac_mask uregs = {
1253                 .data_mask = mask,
1254                 .insn_mask = mask,
1255         };
1256
1257         if (!system_supports_address_auth())
1258                 return -EINVAL;
1259
1260         return membuf_write(&to, &uregs, sizeof(uregs));
1261 }
1262
1263 static int pac_enabled_keys_get(struct task_struct *target,
1264                                 const struct user_regset *regset,
1265                                 struct membuf to)
1266 {
1267         long enabled_keys = ptrauth_get_enabled_keys(target);
1268
1269         if (IS_ERR_VALUE(enabled_keys))
1270                 return enabled_keys;
1271
1272         return membuf_write(&to, &enabled_keys, sizeof(enabled_keys));
1273 }
1274
1275 static int pac_enabled_keys_set(struct task_struct *target,
1276                                 const struct user_regset *regset,
1277                                 unsigned int pos, unsigned int count,
1278                                 const void *kbuf, const void __user *ubuf)
1279 {
1280         int ret;
1281         long enabled_keys = ptrauth_get_enabled_keys(target);
1282
1283         if (IS_ERR_VALUE(enabled_keys))
1284                 return enabled_keys;
1285
1286         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0,
1287                                  sizeof(long));
1288         if (ret)
1289                 return ret;
1290
1291         return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK,
1292                                         enabled_keys);
1293 }
1294
1295 #ifdef CONFIG_CHECKPOINT_RESTORE
1296 static __uint128_t pac_key_to_user(const struct ptrauth_key *key)
1297 {
1298         return (__uint128_t)key->hi << 64 | key->lo;
1299 }
1300
1301 static struct ptrauth_key pac_key_from_user(__uint128_t ukey)
1302 {
1303         struct ptrauth_key key = {
1304                 .lo = (unsigned long)ukey,
1305                 .hi = (unsigned long)(ukey >> 64),
1306         };
1307
1308         return key;
1309 }
1310
1311 static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys,
1312                                      const struct ptrauth_keys_user *keys)
1313 {
1314         ukeys->apiakey = pac_key_to_user(&keys->apia);
1315         ukeys->apibkey = pac_key_to_user(&keys->apib);
1316         ukeys->apdakey = pac_key_to_user(&keys->apda);
1317         ukeys->apdbkey = pac_key_to_user(&keys->apdb);
1318 }
1319
1320 static void pac_address_keys_from_user(struct ptrauth_keys_user *keys,
1321                                        const struct user_pac_address_keys *ukeys)
1322 {
1323         keys->apia = pac_key_from_user(ukeys->apiakey);
1324         keys->apib = pac_key_from_user(ukeys->apibkey);
1325         keys->apda = pac_key_from_user(ukeys->apdakey);
1326         keys->apdb = pac_key_from_user(ukeys->apdbkey);
1327 }
1328
1329 static int pac_address_keys_get(struct task_struct *target,
1330                                 const struct user_regset *regset,
1331                                 struct membuf to)
1332 {
1333         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1334         struct user_pac_address_keys user_keys;
1335
1336         if (!system_supports_address_auth())
1337                 return -EINVAL;
1338
1339         pac_address_keys_to_user(&user_keys, keys);
1340
1341         return membuf_write(&to, &user_keys, sizeof(user_keys));
1342 }
1343
1344 static int pac_address_keys_set(struct task_struct *target,
1345                                 const struct user_regset *regset,
1346                                 unsigned int pos, unsigned int count,
1347                                 const void *kbuf, const void __user *ubuf)
1348 {
1349         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1350         struct user_pac_address_keys user_keys;
1351         int ret;
1352
1353         if (!system_supports_address_auth())
1354                 return -EINVAL;
1355
1356         pac_address_keys_to_user(&user_keys, keys);
1357         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1358                                  &user_keys, 0, -1);
1359         if (ret)
1360                 return ret;
1361         pac_address_keys_from_user(keys, &user_keys);
1362
1363         return 0;
1364 }
1365
1366 static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys,
1367                                      const struct ptrauth_keys_user *keys)
1368 {
1369         ukeys->apgakey = pac_key_to_user(&keys->apga);
1370 }
1371
1372 static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys,
1373                                        const struct user_pac_generic_keys *ukeys)
1374 {
1375         keys->apga = pac_key_from_user(ukeys->apgakey);
1376 }
1377
1378 static int pac_generic_keys_get(struct task_struct *target,
1379                                 const struct user_regset *regset,
1380                                 struct membuf to)
1381 {
1382         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1383         struct user_pac_generic_keys user_keys;
1384
1385         if (!system_supports_generic_auth())
1386                 return -EINVAL;
1387
1388         pac_generic_keys_to_user(&user_keys, keys);
1389
1390         return membuf_write(&to, &user_keys, sizeof(user_keys));
1391 }
1392
1393 static int pac_generic_keys_set(struct task_struct *target,
1394                                 const struct user_regset *regset,
1395                                 unsigned int pos, unsigned int count,
1396                                 const void *kbuf, const void __user *ubuf)
1397 {
1398         struct ptrauth_keys_user *keys = &target->thread.keys_user;
1399         struct user_pac_generic_keys user_keys;
1400         int ret;
1401
1402         if (!system_supports_generic_auth())
1403                 return -EINVAL;
1404
1405         pac_generic_keys_to_user(&user_keys, keys);
1406         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
1407                                  &user_keys, 0, -1);
1408         if (ret)
1409                 return ret;
1410         pac_generic_keys_from_user(keys, &user_keys);
1411
1412         return 0;
1413 }
1414 #endif /* CONFIG_CHECKPOINT_RESTORE */
1415 #endif /* CONFIG_ARM64_PTR_AUTH */
1416
1417 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1418 static int tagged_addr_ctrl_get(struct task_struct *target,
1419                                 const struct user_regset *regset,
1420                                 struct membuf to)
1421 {
1422         long ctrl = get_tagged_addr_ctrl(target);
1423
1424         if (IS_ERR_VALUE(ctrl))
1425                 return ctrl;
1426
1427         return membuf_write(&to, &ctrl, sizeof(ctrl));
1428 }
1429
1430 static int tagged_addr_ctrl_set(struct task_struct *target, const struct
1431                                 user_regset *regset, unsigned int pos,
1432                                 unsigned int count, const void *kbuf, const
1433                                 void __user *ubuf)
1434 {
1435         int ret;
1436         long ctrl;
1437
1438         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
1439         if (ret)
1440                 return ret;
1441
1442         return set_tagged_addr_ctrl(target, ctrl);
1443 }
1444 #endif
1445
1446 enum aarch64_regset {
1447         REGSET_GPR,
1448         REGSET_FPR,
1449         REGSET_TLS,
1450 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1451         REGSET_HW_BREAK,
1452         REGSET_HW_WATCH,
1453 #endif
1454         REGSET_FPMR,
1455         REGSET_SYSTEM_CALL,
1456 #ifdef CONFIG_ARM64_SVE
1457         REGSET_SVE,
1458 #endif
1459 #ifdef CONFIG_ARM64_SME
1460         REGSET_SSVE,
1461         REGSET_ZA,
1462         REGSET_ZT,
1463 #endif
1464 #ifdef CONFIG_ARM64_PTR_AUTH
1465         REGSET_PAC_MASK,
1466         REGSET_PAC_ENABLED_KEYS,
1467 #ifdef CONFIG_CHECKPOINT_RESTORE
1468         REGSET_PACA_KEYS,
1469         REGSET_PACG_KEYS,
1470 #endif
1471 #endif
1472 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1473         REGSET_TAGGED_ADDR_CTRL,
1474 #endif
1475 };
1476
1477 static const struct user_regset aarch64_regsets[] = {
1478         [REGSET_GPR] = {
1479                 .core_note_type = NT_PRSTATUS,
1480                 .n = sizeof(struct user_pt_regs) / sizeof(u64),
1481                 .size = sizeof(u64),
1482                 .align = sizeof(u64),
1483                 .regset_get = gpr_get,
1484                 .set = gpr_set
1485         },
1486         [REGSET_FPR] = {
1487                 .core_note_type = NT_PRFPREG,
1488                 .n = sizeof(struct user_fpsimd_state) / sizeof(u32),
1489                 /*
1490                  * We pretend we have 32-bit registers because the fpsr and
1491                  * fpcr are 32-bits wide.
1492                  */
1493                 .size = sizeof(u32),
1494                 .align = sizeof(u32),
1495                 .active = fpr_active,
1496                 .regset_get = fpr_get,
1497                 .set = fpr_set
1498         },
1499         [REGSET_TLS] = {
1500                 .core_note_type = NT_ARM_TLS,
1501                 .n = 2,
1502                 .size = sizeof(void *),
1503                 .align = sizeof(void *),
1504                 .regset_get = tls_get,
1505                 .set = tls_set,
1506         },
1507 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1508         [REGSET_HW_BREAK] = {
1509                 .core_note_type = NT_ARM_HW_BREAK,
1510                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1511                 .size = sizeof(u32),
1512                 .align = sizeof(u32),
1513                 .regset_get = hw_break_get,
1514                 .set = hw_break_set,
1515         },
1516         [REGSET_HW_WATCH] = {
1517                 .core_note_type = NT_ARM_HW_WATCH,
1518                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1519                 .size = sizeof(u32),
1520                 .align = sizeof(u32),
1521                 .regset_get = hw_break_get,
1522                 .set = hw_break_set,
1523         },
1524 #endif
1525         [REGSET_SYSTEM_CALL] = {
1526                 .core_note_type = NT_ARM_SYSTEM_CALL,
1527                 .n = 1,
1528                 .size = sizeof(int),
1529                 .align = sizeof(int),
1530                 .regset_get = system_call_get,
1531                 .set = system_call_set,
1532         },
1533         [REGSET_FPMR] = {
1534                 .core_note_type = NT_ARM_FPMR,
1535                 .n = 1,
1536                 .size = sizeof(u64),
1537                 .align = sizeof(u64),
1538                 .regset_get = fpmr_get,
1539                 .set = fpmr_set,
1540         },
1541 #ifdef CONFIG_ARM64_SVE
1542         [REGSET_SVE] = { /* Scalable Vector Extension */
1543                 .core_note_type = NT_ARM_SVE,
1544                 .n = DIV_ROUND_UP(SVE_PT_SIZE(ARCH_SVE_VQ_MAX,
1545                                               SVE_PT_REGS_SVE),
1546                                   SVE_VQ_BYTES),
1547                 .size = SVE_VQ_BYTES,
1548                 .align = SVE_VQ_BYTES,
1549                 .regset_get = sve_get,
1550                 .set = sve_set,
1551         },
1552 #endif
1553 #ifdef CONFIG_ARM64_SME
1554         [REGSET_SSVE] = { /* Streaming mode SVE */
1555                 .core_note_type = NT_ARM_SSVE,
1556                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SME_VQ_MAX, SVE_PT_REGS_SVE),
1557                                   SVE_VQ_BYTES),
1558                 .size = SVE_VQ_BYTES,
1559                 .align = SVE_VQ_BYTES,
1560                 .regset_get = ssve_get,
1561                 .set = ssve_set,
1562         },
1563         [REGSET_ZA] = { /* SME ZA */
1564                 .core_note_type = NT_ARM_ZA,
1565                 /*
1566                  * ZA is a single register but it's variably sized and
1567                  * the ptrace core requires that the size of any data
1568                  * be an exact multiple of the configured register
1569                  * size so report as though we had SVE_VQ_BYTES
1570                  * registers. These values aren't exposed to
1571                  * userspace.
1572                  */
1573                 .n = DIV_ROUND_UP(ZA_PT_SIZE(SME_VQ_MAX), SVE_VQ_BYTES),
1574                 .size = SVE_VQ_BYTES,
1575                 .align = SVE_VQ_BYTES,
1576                 .regset_get = za_get,
1577                 .set = za_set,
1578         },
1579         [REGSET_ZT] = { /* SME ZT */
1580                 .core_note_type = NT_ARM_ZT,
1581                 .n = 1,
1582                 .size = ZT_SIG_REG_BYTES,
1583                 .align = sizeof(u64),
1584                 .regset_get = zt_get,
1585                 .set = zt_set,
1586         },
1587 #endif
1588 #ifdef CONFIG_ARM64_PTR_AUTH
1589         [REGSET_PAC_MASK] = {
1590                 .core_note_type = NT_ARM_PAC_MASK,
1591                 .n = sizeof(struct user_pac_mask) / sizeof(u64),
1592                 .size = sizeof(u64),
1593                 .align = sizeof(u64),
1594                 .regset_get = pac_mask_get,
1595                 /* this cannot be set dynamically */
1596         },
1597         [REGSET_PAC_ENABLED_KEYS] = {
1598                 .core_note_type = NT_ARM_PAC_ENABLED_KEYS,
1599                 .n = 1,
1600                 .size = sizeof(long),
1601                 .align = sizeof(long),
1602                 .regset_get = pac_enabled_keys_get,
1603                 .set = pac_enabled_keys_set,
1604         },
1605 #ifdef CONFIG_CHECKPOINT_RESTORE
1606         [REGSET_PACA_KEYS] = {
1607                 .core_note_type = NT_ARM_PACA_KEYS,
1608                 .n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t),
1609                 .size = sizeof(__uint128_t),
1610                 .align = sizeof(__uint128_t),
1611                 .regset_get = pac_address_keys_get,
1612                 .set = pac_address_keys_set,
1613         },
1614         [REGSET_PACG_KEYS] = {
1615                 .core_note_type = NT_ARM_PACG_KEYS,
1616                 .n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t),
1617                 .size = sizeof(__uint128_t),
1618                 .align = sizeof(__uint128_t),
1619                 .regset_get = pac_generic_keys_get,
1620                 .set = pac_generic_keys_set,
1621         },
1622 #endif
1623 #endif
1624 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
1625         [REGSET_TAGGED_ADDR_CTRL] = {
1626                 .core_note_type = NT_ARM_TAGGED_ADDR_CTRL,
1627                 .n = 1,
1628                 .size = sizeof(long),
1629                 .align = sizeof(long),
1630                 .regset_get = tagged_addr_ctrl_get,
1631                 .set = tagged_addr_ctrl_set,
1632         },
1633 #endif
1634 };
1635
1636 static const struct user_regset_view user_aarch64_view = {
1637         .name = "aarch64", .e_machine = EM_AARCH64,
1638         .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets)
1639 };
1640
1641 enum compat_regset {
1642         REGSET_COMPAT_GPR,
1643         REGSET_COMPAT_VFP,
1644 };
1645
1646 static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx)
1647 {
1648         struct pt_regs *regs = task_pt_regs(task);
1649
1650         switch (idx) {
1651         case 15:
1652                 return regs->pc;
1653         case 16:
1654                 return pstate_to_compat_psr(regs->pstate);
1655         case 17:
1656                 return regs->orig_x0;
1657         default:
1658                 return regs->regs[idx];
1659         }
1660 }
1661
1662 static int compat_gpr_get(struct task_struct *target,
1663                           const struct user_regset *regset,
1664                           struct membuf to)
1665 {
1666         int i = 0;
1667
1668         while (to.left)
1669                 membuf_store(&to, compat_get_user_reg(target, i++));
1670         return 0;
1671 }
1672
1673 static int compat_gpr_set(struct task_struct *target,
1674                           const struct user_regset *regset,
1675                           unsigned int pos, unsigned int count,
1676                           const void *kbuf, const void __user *ubuf)
1677 {
1678         struct pt_regs newregs;
1679         int ret = 0;
1680         unsigned int i, start, num_regs;
1681
1682         /* Calculate the number of AArch32 registers contained in count */
1683         num_regs = count / regset->size;
1684
1685         /* Convert pos into an register number */
1686         start = pos / regset->size;
1687
1688         if (start + num_regs > regset->n)
1689                 return -EIO;
1690
1691         newregs = *task_pt_regs(target);
1692
1693         for (i = 0; i < num_regs; ++i) {
1694                 unsigned int idx = start + i;
1695                 compat_ulong_t reg;
1696
1697                 if (kbuf) {
1698                         memcpy(&reg, kbuf, sizeof(reg));
1699                         kbuf += sizeof(reg);
1700                 } else {
1701                         ret = copy_from_user(&reg, ubuf, sizeof(reg));
1702                         if (ret) {
1703                                 ret = -EFAULT;
1704                                 break;
1705                         }
1706
1707                         ubuf += sizeof(reg);
1708                 }
1709
1710                 switch (idx) {
1711                 case 15:
1712                         newregs.pc = reg;
1713                         break;
1714                 case 16:
1715                         reg = compat_psr_to_pstate(reg);
1716                         newregs.pstate = reg;
1717                         break;
1718                 case 17:
1719                         newregs.orig_x0 = reg;
1720                         break;
1721                 default:
1722                         newregs.regs[idx] = reg;
1723                 }
1724
1725         }
1726
1727         if (valid_user_regs(&newregs.user_regs, target))
1728                 *task_pt_regs(target) = newregs;
1729         else
1730                 ret = -EINVAL;
1731
1732         return ret;
1733 }
1734
1735 static int compat_vfp_get(struct task_struct *target,
1736                           const struct user_regset *regset,
1737                           struct membuf to)
1738 {
1739         struct user_fpsimd_state *uregs;
1740         compat_ulong_t fpscr;
1741
1742         if (!system_supports_fpsimd())
1743                 return -EINVAL;
1744
1745         uregs = &target->thread.uw.fpsimd_state;
1746
1747         if (target == current)
1748                 fpsimd_preserve_current_state();
1749
1750         /*
1751          * The VFP registers are packed into the fpsimd_state, so they all sit
1752          * nicely together for us. We just need to create the fpscr separately.
1753          */
1754         membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t));
1755         fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
1756                 (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
1757         return membuf_store(&to, fpscr);
1758 }
1759
1760 static int compat_vfp_set(struct task_struct *target,
1761                           const struct user_regset *regset,
1762                           unsigned int pos, unsigned int count,
1763                           const void *kbuf, const void __user *ubuf)
1764 {
1765         struct user_fpsimd_state *uregs;
1766         compat_ulong_t fpscr;
1767         int ret, vregs_end_pos;
1768
1769         if (!system_supports_fpsimd())
1770                 return -EINVAL;
1771
1772         uregs = &target->thread.uw.fpsimd_state;
1773
1774         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1775         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0,
1776                                  vregs_end_pos);
1777
1778         if (count && !ret) {
1779                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr,
1780                                          vregs_end_pos, VFP_STATE_SIZE);
1781                 if (!ret) {
1782                         uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK;
1783                         uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
1784                 }
1785         }
1786
1787         fpsimd_flush_task_state(target);
1788         return ret;
1789 }
1790
1791 static int compat_tls_get(struct task_struct *target,
1792                           const struct user_regset *regset,
1793                           struct membuf to)
1794 {
1795         return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value);
1796 }
1797
1798 static int compat_tls_set(struct task_struct *target,
1799                           const struct user_regset *regset, unsigned int pos,
1800                           unsigned int count, const void *kbuf,
1801                           const void __user *ubuf)
1802 {
1803         int ret;
1804         compat_ulong_t tls = target->thread.uw.tp_value;
1805
1806         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1807         if (ret)
1808                 return ret;
1809
1810         target->thread.uw.tp_value = tls;
1811         return ret;
1812 }
1813
1814 static const struct user_regset aarch32_regsets[] = {
1815         [REGSET_COMPAT_GPR] = {
1816                 .core_note_type = NT_PRSTATUS,
1817                 .n = COMPAT_ELF_NGREG,
1818                 .size = sizeof(compat_elf_greg_t),
1819                 .align = sizeof(compat_elf_greg_t),
1820                 .regset_get = compat_gpr_get,
1821                 .set = compat_gpr_set
1822         },
1823         [REGSET_COMPAT_VFP] = {
1824                 .core_note_type = NT_ARM_VFP,
1825                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1826                 .size = sizeof(compat_ulong_t),
1827                 .align = sizeof(compat_ulong_t),
1828                 .active = fpr_active,
1829                 .regset_get = compat_vfp_get,
1830                 .set = compat_vfp_set
1831         },
1832 };
1833
1834 static const struct user_regset_view user_aarch32_view = {
1835         .name = "aarch32", .e_machine = EM_ARM,
1836         .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
1837 };
1838
1839 static const struct user_regset aarch32_ptrace_regsets[] = {
1840         [REGSET_GPR] = {
1841                 .core_note_type = NT_PRSTATUS,
1842                 .n = COMPAT_ELF_NGREG,
1843                 .size = sizeof(compat_elf_greg_t),
1844                 .align = sizeof(compat_elf_greg_t),
1845                 .regset_get = compat_gpr_get,
1846                 .set = compat_gpr_set
1847         },
1848         [REGSET_FPR] = {
1849                 .core_note_type = NT_ARM_VFP,
1850                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1851                 .size = sizeof(compat_ulong_t),
1852                 .align = sizeof(compat_ulong_t),
1853                 .regset_get = compat_vfp_get,
1854                 .set = compat_vfp_set
1855         },
1856         [REGSET_TLS] = {
1857                 .core_note_type = NT_ARM_TLS,
1858                 .n = 1,
1859                 .size = sizeof(compat_ulong_t),
1860                 .align = sizeof(compat_ulong_t),
1861                 .regset_get = compat_tls_get,
1862                 .set = compat_tls_set,
1863         },
1864 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1865         [REGSET_HW_BREAK] = {
1866                 .core_note_type = NT_ARM_HW_BREAK,
1867                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1868                 .size = sizeof(u32),
1869                 .align = sizeof(u32),
1870                 .regset_get = hw_break_get,
1871                 .set = hw_break_set,
1872         },
1873         [REGSET_HW_WATCH] = {
1874                 .core_note_type = NT_ARM_HW_WATCH,
1875                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1876                 .size = sizeof(u32),
1877                 .align = sizeof(u32),
1878                 .regset_get = hw_break_get,
1879                 .set = hw_break_set,
1880         },
1881 #endif
1882         [REGSET_SYSTEM_CALL] = {
1883                 .core_note_type = NT_ARM_SYSTEM_CALL,
1884                 .n = 1,
1885                 .size = sizeof(int),
1886                 .align = sizeof(int),
1887                 .regset_get = system_call_get,
1888                 .set = system_call_set,
1889         },
1890 };
1891
1892 static const struct user_regset_view user_aarch32_ptrace_view = {
1893         .name = "aarch32", .e_machine = EM_ARM,
1894         .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
1895 };
1896
1897 #ifdef CONFIG_COMPAT
1898 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
1899                                    compat_ulong_t __user *ret)
1900 {
1901         compat_ulong_t tmp;
1902
1903         if (off & 3)
1904                 return -EIO;
1905
1906         if (off == COMPAT_PT_TEXT_ADDR)
1907                 tmp = tsk->mm->start_code;
1908         else if (off == COMPAT_PT_DATA_ADDR)
1909                 tmp = tsk->mm->start_data;
1910         else if (off == COMPAT_PT_TEXT_END_ADDR)
1911                 tmp = tsk->mm->end_code;
1912         else if (off < sizeof(compat_elf_gregset_t))
1913                 tmp = compat_get_user_reg(tsk, off >> 2);
1914         else if (off >= COMPAT_USER_SZ)
1915                 return -EIO;
1916         else
1917                 tmp = 0;
1918
1919         return put_user(tmp, ret);
1920 }
1921
1922 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
1923                                     compat_ulong_t val)
1924 {
1925         struct pt_regs newregs = *task_pt_regs(tsk);
1926         unsigned int idx = off / 4;
1927
1928         if (off & 3 || off >= COMPAT_USER_SZ)
1929                 return -EIO;
1930
1931         if (off >= sizeof(compat_elf_gregset_t))
1932                 return 0;
1933
1934         switch (idx) {
1935         case 15:
1936                 newregs.pc = val;
1937                 break;
1938         case 16:
1939                 newregs.pstate = compat_psr_to_pstate(val);
1940                 break;
1941         case 17:
1942                 newregs.orig_x0 = val;
1943                 break;
1944         default:
1945                 newregs.regs[idx] = val;
1946         }
1947
1948         if (!valid_user_regs(&newregs.user_regs, tsk))
1949                 return -EINVAL;
1950
1951         *task_pt_regs(tsk) = newregs;
1952         return 0;
1953 }
1954
1955 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1956
1957 /*
1958  * Convert a virtual register number into an index for a thread_info
1959  * breakpoint array. Breakpoints are identified using positive numbers
1960  * whilst watchpoints are negative. The registers are laid out as pairs
1961  * of (address, control), each pair mapping to a unique hw_breakpoint struct.
1962  * Register 0 is reserved for describing resource information.
1963  */
1964 static int compat_ptrace_hbp_num_to_idx(compat_long_t num)
1965 {
1966         return (abs(num) - 1) >> 1;
1967 }
1968
1969 static int compat_ptrace_hbp_get_resource_info(u32 *kdata)
1970 {
1971         u8 num_brps, num_wrps, debug_arch, wp_len;
1972         u32 reg = 0;
1973
1974         num_brps        = hw_breakpoint_slots(TYPE_INST);
1975         num_wrps        = hw_breakpoint_slots(TYPE_DATA);
1976
1977         debug_arch      = debug_monitors_arch();
1978         wp_len          = 8;
1979         reg             |= debug_arch;
1980         reg             <<= 8;
1981         reg             |= wp_len;
1982         reg             <<= 8;
1983         reg             |= num_wrps;
1984         reg             <<= 8;
1985         reg             |= num_brps;
1986
1987         *kdata = reg;
1988         return 0;
1989 }
1990
1991 static int compat_ptrace_hbp_get(unsigned int note_type,
1992                                  struct task_struct *tsk,
1993                                  compat_long_t num,
1994                                  u32 *kdata)
1995 {
1996         u64 addr = 0;
1997         u32 ctrl = 0;
1998
1999         int err, idx = compat_ptrace_hbp_num_to_idx(num);
2000
2001         if (num & 1) {
2002                 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
2003                 *kdata = (u32)addr;
2004         } else {
2005                 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl);
2006                 *kdata = ctrl;
2007         }
2008
2009         return err;
2010 }
2011
2012 static int compat_ptrace_hbp_set(unsigned int note_type,
2013                                  struct task_struct *tsk,
2014                                  compat_long_t num,
2015                                  u32 *kdata)
2016 {
2017         u64 addr;
2018         u32 ctrl;
2019
2020         int err, idx = compat_ptrace_hbp_num_to_idx(num);
2021
2022         if (num & 1) {
2023                 addr = *kdata;
2024                 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr);
2025         } else {
2026                 ctrl = *kdata;
2027                 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl);
2028         }
2029
2030         return err;
2031 }
2032
2033 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num,
2034                                     compat_ulong_t __user *data)
2035 {
2036         int ret;
2037         u32 kdata;
2038
2039         /* Watchpoint */
2040         if (num < 0) {
2041                 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata);
2042         /* Resource info */
2043         } else if (num == 0) {
2044                 ret = compat_ptrace_hbp_get_resource_info(&kdata);
2045         /* Breakpoint */
2046         } else {
2047                 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata);
2048         }
2049
2050         if (!ret)
2051                 ret = put_user(kdata, data);
2052
2053         return ret;
2054 }
2055
2056 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num,
2057                                     compat_ulong_t __user *data)
2058 {
2059         int ret;
2060         u32 kdata = 0;
2061
2062         if (num == 0)
2063                 return 0;
2064
2065         ret = get_user(kdata, data);
2066         if (ret)
2067                 return ret;
2068
2069         if (num < 0)
2070                 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata);
2071         else
2072                 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata);
2073
2074         return ret;
2075 }
2076 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
2077
2078 long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
2079                         compat_ulong_t caddr, compat_ulong_t cdata)
2080 {
2081         unsigned long addr = caddr;
2082         unsigned long data = cdata;
2083         void __user *datap = compat_ptr(data);
2084         int ret;
2085
2086         switch (request) {
2087                 case PTRACE_PEEKUSR:
2088                         ret = compat_ptrace_read_user(child, addr, datap);
2089                         break;
2090
2091                 case PTRACE_POKEUSR:
2092                         ret = compat_ptrace_write_user(child, addr, data);
2093                         break;
2094
2095                 case COMPAT_PTRACE_GETREGS:
2096                         ret = copy_regset_to_user(child,
2097                                                   &user_aarch32_view,
2098                                                   REGSET_COMPAT_GPR,
2099                                                   0, sizeof(compat_elf_gregset_t),
2100                                                   datap);
2101                         break;
2102
2103                 case COMPAT_PTRACE_SETREGS:
2104                         ret = copy_regset_from_user(child,
2105                                                     &user_aarch32_view,
2106                                                     REGSET_COMPAT_GPR,
2107                                                     0, sizeof(compat_elf_gregset_t),
2108                                                     datap);
2109                         break;
2110
2111                 case COMPAT_PTRACE_GET_THREAD_AREA:
2112                         ret = put_user((compat_ulong_t)child->thread.uw.tp_value,
2113                                        (compat_ulong_t __user *)datap);
2114                         break;
2115
2116                 case COMPAT_PTRACE_SET_SYSCALL:
2117                         task_pt_regs(child)->syscallno = data;
2118                         ret = 0;
2119                         break;
2120
2121                 case COMPAT_PTRACE_GETVFPREGS:
2122                         ret = copy_regset_to_user(child,
2123                                                   &user_aarch32_view,
2124                                                   REGSET_COMPAT_VFP,
2125                                                   0, VFP_STATE_SIZE,
2126                                                   datap);
2127                         break;
2128
2129                 case COMPAT_PTRACE_SETVFPREGS:
2130                         ret = copy_regset_from_user(child,
2131                                                     &user_aarch32_view,
2132                                                     REGSET_COMPAT_VFP,
2133                                                     0, VFP_STATE_SIZE,
2134                                                     datap);
2135                         break;
2136
2137 #ifdef CONFIG_HAVE_HW_BREAKPOINT
2138                 case COMPAT_PTRACE_GETHBPREGS:
2139                         ret = compat_ptrace_gethbpregs(child, addr, datap);
2140                         break;
2141
2142                 case COMPAT_PTRACE_SETHBPREGS:
2143                         ret = compat_ptrace_sethbpregs(child, addr, datap);
2144                         break;
2145 #endif
2146
2147                 default:
2148                         ret = compat_ptrace_request(child, request, addr,
2149                                                     data);
2150                         break;
2151         }
2152
2153         return ret;
2154 }
2155 #endif /* CONFIG_COMPAT */
2156
2157 const struct user_regset_view *task_user_regset_view(struct task_struct *task)
2158 {
2159         /*
2160          * Core dumping of 32-bit tasks or compat ptrace requests must use the
2161          * user_aarch32_view compatible with arm32. Native ptrace requests on
2162          * 32-bit children use an extended user_aarch32_ptrace_view to allow
2163          * access to the TLS register.
2164          */
2165         if (is_compat_task())
2166                 return &user_aarch32_view;
2167         else if (is_compat_thread(task_thread_info(task)))
2168                 return &user_aarch32_ptrace_view;
2169
2170         return &user_aarch64_view;
2171 }
2172
2173 long arch_ptrace(struct task_struct *child, long request,
2174                  unsigned long addr, unsigned long data)
2175 {
2176         switch (request) {
2177         case PTRACE_PEEKMTETAGS:
2178         case PTRACE_POKEMTETAGS:
2179                 return mte_ptrace_copy_tags(child, request, addr, data);
2180         }
2181
2182         return ptrace_request(child, request, addr, data);
2183 }
2184
2185 enum ptrace_syscall_dir {
2186         PTRACE_SYSCALL_ENTER = 0,
2187         PTRACE_SYSCALL_EXIT,
2188 };
2189
2190 static void report_syscall(struct pt_regs *regs, enum ptrace_syscall_dir dir)
2191 {
2192         int regno;
2193         unsigned long saved_reg;
2194
2195         /*
2196          * We have some ABI weirdness here in the way that we handle syscall
2197          * exit stops because we indicate whether or not the stop has been
2198          * signalled from syscall entry or syscall exit by clobbering a general
2199          * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee
2200          * and restoring its old value after the stop. This means that:
2201          *
2202          * - Any writes by the tracer to this register during the stop are
2203          *   ignored/discarded.
2204          *
2205          * - The actual value of the register is not available during the stop,
2206          *   so the tracer cannot save it and restore it later.
2207          *
2208          * - Syscall stops behave differently to seccomp and pseudo-step traps
2209          *   (the latter do not nobble any registers).
2210          */
2211         regno = (is_compat_task() ? 12 : 7);
2212         saved_reg = regs->regs[regno];
2213         regs->regs[regno] = dir;
2214
2215         if (dir == PTRACE_SYSCALL_ENTER) {
2216                 if (ptrace_report_syscall_entry(regs))
2217                         forget_syscall(regs);
2218                 regs->regs[regno] = saved_reg;
2219         } else if (!test_thread_flag(TIF_SINGLESTEP)) {
2220                 ptrace_report_syscall_exit(regs, 0);
2221                 regs->regs[regno] = saved_reg;
2222         } else {
2223                 regs->regs[regno] = saved_reg;
2224
2225                 /*
2226                  * Signal a pseudo-step exception since we are stepping but
2227                  * tracer modifications to the registers may have rewound the
2228                  * state machine.
2229                  */
2230                 ptrace_report_syscall_exit(regs, 1);
2231         }
2232 }
2233
2234 int syscall_trace_enter(struct pt_regs *regs)
2235 {
2236         unsigned long flags = read_thread_flags();
2237
2238         if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) {
2239                 report_syscall(regs, PTRACE_SYSCALL_ENTER);
2240                 if (flags & _TIF_SYSCALL_EMU)
2241                         return NO_SYSCALL;
2242         }
2243
2244         /* Do the secure computing after ptrace; failures should be fast. */
2245         if (secure_computing() == -1)
2246                 return NO_SYSCALL;
2247
2248         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
2249                 trace_sys_enter(regs, regs->syscallno);
2250
2251         audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
2252                             regs->regs[2], regs->regs[3]);
2253
2254         return regs->syscallno;
2255 }
2256
2257 void syscall_trace_exit(struct pt_regs *regs)
2258 {
2259         unsigned long flags = read_thread_flags();
2260
2261         audit_syscall_exit(regs);
2262
2263         if (flags & _TIF_SYSCALL_TRACEPOINT)
2264                 trace_sys_exit(regs, syscall_get_return_value(current, regs));
2265
2266         if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP))
2267                 report_syscall(regs, PTRACE_SYSCALL_EXIT);
2268
2269         rseq_syscall(regs);
2270 }
2271
2272 /*
2273  * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
2274  * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
2275  * not described in ARM DDI 0487D.a.
2276  * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may
2277  * be allocated an EL0 meaning in future.
2278  * Userspace cannot use these until they have an architectural meaning.
2279  * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
2280  * We also reserve IL for the kernel; SS is handled dynamically.
2281  */
2282 #define SPSR_EL1_AARCH64_RES0_BITS \
2283         (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \
2284          GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5))
2285 #define SPSR_EL1_AARCH32_RES0_BITS \
2286         (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
2287
2288 static int valid_compat_regs(struct user_pt_regs *regs)
2289 {
2290         regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
2291
2292         if (!system_supports_mixed_endian_el0()) {
2293                 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
2294                         regs->pstate |= PSR_AA32_E_BIT;
2295                 else
2296                         regs->pstate &= ~PSR_AA32_E_BIT;
2297         }
2298
2299         if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
2300             (regs->pstate & PSR_AA32_A_BIT) == 0 &&
2301             (regs->pstate & PSR_AA32_I_BIT) == 0 &&
2302             (regs->pstate & PSR_AA32_F_BIT) == 0) {
2303                 return 1;
2304         }
2305
2306         /*
2307          * Force PSR to a valid 32-bit EL0t, preserving the same bits as
2308          * arch/arm.
2309          */
2310         regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT |
2311                         PSR_AA32_C_BIT | PSR_AA32_V_BIT |
2312                         PSR_AA32_Q_BIT | PSR_AA32_IT_MASK |
2313                         PSR_AA32_GE_MASK | PSR_AA32_E_BIT |
2314                         PSR_AA32_T_BIT;
2315         regs->pstate |= PSR_MODE32_BIT;
2316
2317         return 0;
2318 }
2319
2320 static int valid_native_regs(struct user_pt_regs *regs)
2321 {
2322         regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
2323
2324         if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
2325             (regs->pstate & PSR_D_BIT) == 0 &&
2326             (regs->pstate & PSR_A_BIT) == 0 &&
2327             (regs->pstate & PSR_I_BIT) == 0 &&
2328             (regs->pstate & PSR_F_BIT) == 0) {
2329                 return 1;
2330         }
2331
2332         /* Force PSR to a valid 64-bit EL0t */
2333         regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
2334
2335         return 0;
2336 }
2337
2338 /*
2339  * Are the current registers suitable for user mode? (used to maintain
2340  * security in signal handlers)
2341  */
2342 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
2343 {
2344         /* https://lore.kernel.org/lkml/20191118131525.GA4180@willie-the-truck */
2345         user_regs_reset_single_step(regs, task);
2346
2347         if (is_compat_thread(task_thread_info(task)))
2348                 return valid_compat_regs(regs);
2349         else
2350                 return valid_native_regs(regs);
2351 }