arch/arm64/kernel/ptrace.c

   1 /*
   2  * Based on arch/arm/kernel/ptrace.c
   3  *
   4  * By Ross Biro 1/23/92
   5  * edited by Linus Torvalds
   6  * ARM modifications Copyright (C) 2000 Russell King
   7  * Copyright (C) 2012 ARM Ltd.
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20  */
  21
  22 #include <linux/audit.h>
  23 #include <linux/compat.h>
  24 #include <linux/kernel.h>
  25 #include <linux/sched/signal.h>
  26 #include <linux/sched/task_stack.h>
  27 #include <linux/mm.h>
  28 #include <linux/nospec.h>
  29 #include <linux/smp.h>
  30 #include <linux/ptrace.h>
  31 #include <linux/user.h>
  32 #include <linux/seccomp.h>
  33 #include <linux/security.h>
  34 #include <linux/init.h>
  35 #include <linux/signal.h>
  36 #include <linux/string.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/perf_event.h>
  39 #include <linux/hw_breakpoint.h>
  40 #include <linux/regset.h>
  41 #include <linux/tracehook.h>
  42 #include <linux/elf.h>
  43
  44 #include <asm/compat.h>
  45 #include <asm/cpufeature.h>
  46 #include <asm/debug-monitors.h>
  47 #include <asm/fpsimd.h>
  48 #include <asm/pgtable.h>
  49 #include <asm/pointer_auth.h>
  50 #include <asm/stacktrace.h>
  51 #include <asm/syscall.h>
  52 #include <asm/traps.h>
  53 #include <asm/system_misc.h>
  54
  55 #define CREATE_TRACE_POINTS
  56 #include <trace/events/syscalls.h>
  57
  58 struct pt_regs_offset {
  59         const char *name;
  60         int offset;
  61 };
  62
  63 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)}
  64 #define REG_OFFSET_END {.name = NULL, .offset = 0}
  65 #define GPR_OFFSET_NAME(r) \
  66         {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])}
  67
  68 static const struct pt_regs_offset regoffset_table[] = {
  69         GPR_OFFSET_NAME(0),
  70         GPR_OFFSET_NAME(1),
  71         GPR_OFFSET_NAME(2),
  72         GPR_OFFSET_NAME(3),
  73         GPR_OFFSET_NAME(4),
  74         GPR_OFFSET_NAME(5),
  75         GPR_OFFSET_NAME(6),
  76         GPR_OFFSET_NAME(7),
  77         GPR_OFFSET_NAME(8),
  78         GPR_OFFSET_NAME(9),
  79         GPR_OFFSET_NAME(10),
  80         GPR_OFFSET_NAME(11),
  81         GPR_OFFSET_NAME(12),
  82         GPR_OFFSET_NAME(13),
  83         GPR_OFFSET_NAME(14),
  84         GPR_OFFSET_NAME(15),
  85         GPR_OFFSET_NAME(16),
  86         GPR_OFFSET_NAME(17),
  87         GPR_OFFSET_NAME(18),
  88         GPR_OFFSET_NAME(19),
  89         GPR_OFFSET_NAME(20),
  90         GPR_OFFSET_NAME(21),
  91         GPR_OFFSET_NAME(22),
  92         GPR_OFFSET_NAME(23),
  93         GPR_OFFSET_NAME(24),
  94         GPR_OFFSET_NAME(25),
  95         GPR_OFFSET_NAME(26),
  96         GPR_OFFSET_NAME(27),
  97         GPR_OFFSET_NAME(28),
  98         GPR_OFFSET_NAME(29),
  99         GPR_OFFSET_NAME(30),
 100         {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])},
 101         REG_OFFSET_NAME(sp),
 102         REG_OFFSET_NAME(pc),
 103         REG_OFFSET_NAME(pstate),
 104         REG_OFFSET_END,
 105 };
 106
 107 /**
 108  * regs_query_register_offset() - query register offset from its name
 109  * @name:       the name of a register
 110  *
 111  * regs_query_register_offset() returns the offset of a register in struct
 112  * pt_regs from its name. If the name is invalid, this returns -EINVAL;
 113  */
 114 int regs_query_register_offset(const char *name)
 115 {
 116         const struct pt_regs_offset *roff;
 117
 118         for (roff = regoffset_table; roff->name != NULL; roff++)
 119                 if (!strcmp(roff->name, name))
 120                         return roff->offset;
 121         return -EINVAL;
 122 }
 123
 124 /**
 125  * regs_within_kernel_stack() - check the address in the stack
 126  * @regs:      pt_regs which contains kernel stack pointer.
 127  * @addr:      address which is checked.
 128  *
 129  * regs_within_kernel_stack() checks @addr is within the kernel stack page(s).
 130  * If @addr is within the kernel stack, it returns true. If not, returns false.
 131  */
 132 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 133 {
 134         return ((addr & ~(THREAD_SIZE - 1))  ==
 135                 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
 136                 on_irq_stack(addr, NULL);
 137 }
 138
 139 /**
 140  * regs_get_kernel_stack_nth() - get Nth entry of the stack
 141  * @regs:       pt_regs which contains kernel stack pointer.
 142  * @n:          stack entry number.
 143  *
 144  * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which
 145  * is specified by @regs. If the @n th entry is NOT in the kernel stack,
 146  * this returns 0.
 147  */
 148 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n)
 149 {
 150         unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs);
 151
 152         addr += n;
 153         if (regs_within_kernel_stack(regs, (unsigned long)addr))
 154                 return *addr;
 155         else
 156                 return 0;
 157 }
 158
 159 /*
 160  * TODO: does not yet catch signals sent when the child dies.
 161  * in exit.c or in signal.c.
 162  */
 163
 164 /*
 165  * Called by kernel/ptrace.c when detaching..
 166  */
 167 void ptrace_disable(struct task_struct *child)
 168 {
 169         /*
 170          * This would be better off in core code, but PTRACE_DETACH has
 171          * grown its fair share of arch-specific worts and changing it
 172          * is likely to cause regressions on obscure architectures.
 173          */
 174         user_disable_single_step(child);
 175 }
 176
 177 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 178 /*
 179  * Handle hitting a HW-breakpoint.
 180  */
 181 static void ptrace_hbptriggered(struct perf_event *bp,
 182                                 struct perf_sample_data *data,
 183                                 struct pt_regs *regs)
 184 {
 185         struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 186         const char *desc = "Hardware breakpoint trap (ptrace)";
 187
 188 #ifdef CONFIG_COMPAT
 189         if (is_compat_task()) {
 190                 int si_errno = 0;
 191                 int i;
 192
 193                 for (i = 0; i < ARM_MAX_BRP; ++i) {
 194                         if (current->thread.debug.hbp_break[i] == bp) {
 195                                 si_errno = (i << 1) + 1;
 196                                 break;
 197                         }
 198                 }
 199
 200                 for (i = 0; i < ARM_MAX_WRP; ++i) {
 201                         if (current->thread.debug.hbp_watch[i] == bp) {
 202                                 si_errno = -((i << 1) + 1);
 203                                 break;
 204                         }
 205                 }
 206                 arm64_force_sig_ptrace_errno_trap(si_errno,
 207                                                   (void __user *)bkpt->trigger,
 208                                                   desc);
 209         }
 210 #endif
 211         arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT,
 212                               (void __user *)(bkpt->trigger),
 213                               desc);
 214 }
 215
 216 /*
 217  * Unregister breakpoints from this task and reset the pointers in
 218  * the thread_struct.
 219  */
 220 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
 221 {
 222         int i;
 223         struct thread_struct *t = &tsk->thread;
 224
 225         for (i = 0; i < ARM_MAX_BRP; i++) {
 226                 if (t->debug.hbp_break[i]) {
 227                         unregister_hw_breakpoint(t->debug.hbp_break[i]);
 228                         t->debug.hbp_break[i] = NULL;
 229                 }
 230         }
 231
 232         for (i = 0; i < ARM_MAX_WRP; i++) {
 233                 if (t->debug.hbp_watch[i]) {
 234                         unregister_hw_breakpoint(t->debug.hbp_watch[i]);
 235                         t->debug.hbp_watch[i] = NULL;
 236                 }
 237         }
 238 }
 239
 240 void ptrace_hw_copy_thread(struct task_struct *tsk)
 241 {
 242         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
 243 }
 244
 245 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type,
 246                                                struct task_struct *tsk,
 247                                                unsigned long idx)
 248 {
 249         struct perf_event *bp = ERR_PTR(-EINVAL);
 250
 251         switch (note_type) {
 252         case NT_ARM_HW_BREAK:
 253                 if (idx >= ARM_MAX_BRP)
 254                         goto out;
 255                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 256                 bp = tsk->thread.debug.hbp_break[idx];
 257                 break;
 258         case NT_ARM_HW_WATCH:
 259                 if (idx >= ARM_MAX_WRP)
 260                         goto out;
 261                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 262                 bp = tsk->thread.debug.hbp_watch[idx];
 263                 break;
 264         }
 265
 266 out:
 267         return bp;
 268 }
 269
 270 static int ptrace_hbp_set_event(unsigned int note_type,
 271                                 struct task_struct *tsk,
 272                                 unsigned long idx,
 273                                 struct perf_event *bp)
 274 {
 275         int err = -EINVAL;
 276
 277         switch (note_type) {
 278         case NT_ARM_HW_BREAK:
 279                 if (idx >= ARM_MAX_BRP)
 280                         goto out;
 281                 idx = array_index_nospec(idx, ARM_MAX_BRP);
 282                 tsk->thread.debug.hbp_break[idx] = bp;
 283                 err = 0;
 284                 break;
 285         case NT_ARM_HW_WATCH:
 286                 if (idx >= ARM_MAX_WRP)
 287                         goto out;
 288                 idx = array_index_nospec(idx, ARM_MAX_WRP);
 289                 tsk->thread.debug.hbp_watch[idx] = bp;
 290                 err = 0;
 291                 break;
 292         }
 293
 294 out:
 295         return err;
 296 }
 297
 298 static struct perf_event *ptrace_hbp_create(unsigned int note_type,
 299                                             struct task_struct *tsk,
 300                                             unsigned long idx)
 301 {
 302         struct perf_event *bp;
 303         struct perf_event_attr attr;
 304         int err, type;
 305
 306         switch (note_type) {
 307         case NT_ARM_HW_BREAK:
 308                 type = HW_BREAKPOINT_X;
 309                 break;
 310         case NT_ARM_HW_WATCH:
 311                 type = HW_BREAKPOINT_RW;
 312                 break;
 313         default:
 314                 return ERR_PTR(-EINVAL);
 315         }
 316
 317         ptrace_breakpoint_init(&attr);
 318
 319         /*
 320          * Initialise fields to sane defaults
 321          * (i.e. values that will pass validation).
 322          */
 323         attr.bp_addr    = 0;
 324         attr.bp_len     = HW_BREAKPOINT_LEN_4;
 325         attr.bp_type    = type;
 326         attr.disabled   = 1;
 327
 328         bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk);
 329         if (IS_ERR(bp))
 330                 return bp;
 331
 332         err = ptrace_hbp_set_event(note_type, tsk, idx, bp);
 333         if (err)
 334                 return ERR_PTR(err);
 335
 336         return bp;
 337 }
 338
 339 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type,
 340                                      struct arch_hw_breakpoint_ctrl ctrl,
 341                                      struct perf_event_attr *attr)
 342 {
 343         int err, len, type, offset, disabled = !ctrl.enabled;
 344
 345         attr->disabled = disabled;
 346         if (disabled)
 347                 return 0;
 348
 349         err = arch_bp_generic_fields(ctrl, &len, &type, &offset);
 350         if (err)
 351                 return err;
 352
 353         switch (note_type) {
 354         case NT_ARM_HW_BREAK:
 355                 if ((type & HW_BREAKPOINT_X) != type)
 356                         return -EINVAL;
 357                 break;
 358         case NT_ARM_HW_WATCH:
 359                 if ((type & HW_BREAKPOINT_RW) != type)
 360                         return -EINVAL;
 361                 break;
 362         default:
 363                 return -EINVAL;
 364         }
 365
 366         attr->bp_len    = len;
 367         attr->bp_type   = type;
 368         attr->bp_addr   += offset;
 369
 370         return 0;
 371 }
 372
 373 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info)
 374 {
 375         u8 num;
 376         u32 reg = 0;
 377
 378         switch (note_type) {
 379         case NT_ARM_HW_BREAK:
 380                 num = hw_breakpoint_slots(TYPE_INST);
 381                 break;
 382         case NT_ARM_HW_WATCH:
 383                 num = hw_breakpoint_slots(TYPE_DATA);
 384                 break;
 385         default:
 386                 return -EINVAL;
 387         }
 388
 389         reg |= debug_monitors_arch();
 390         reg <<= 8;
 391         reg |= num;
 392
 393         *info = reg;
 394         return 0;
 395 }
 396
 397 static int ptrace_hbp_get_ctrl(unsigned int note_type,
 398                                struct task_struct *tsk,
 399                                unsigned long idx,
 400                                u32 *ctrl)
 401 {
 402         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 403
 404         if (IS_ERR(bp))
 405                 return PTR_ERR(bp);
 406
 407         *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0;
 408         return 0;
 409 }
 410
 411 static int ptrace_hbp_get_addr(unsigned int note_type,
 412                                struct task_struct *tsk,
 413                                unsigned long idx,
 414                                u64 *addr)
 415 {
 416         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 417
 418         if (IS_ERR(bp))
 419                 return PTR_ERR(bp);
 420
 421         *addr = bp ? counter_arch_bp(bp)->address : 0;
 422         return 0;
 423 }
 424
 425 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type,
 426                                                         struct task_struct *tsk,
 427                                                         unsigned long idx)
 428 {
 429         struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx);
 430
 431         if (!bp)
 432                 bp = ptrace_hbp_create(note_type, tsk, idx);
 433
 434         return bp;
 435 }
 436
 437 static int ptrace_hbp_set_ctrl(unsigned int note_type,
 438                                struct task_struct *tsk,
 439                                unsigned long idx,
 440                                u32 uctrl)
 441 {
 442         int err;
 443         struct perf_event *bp;
 444         struct perf_event_attr attr;
 445         struct arch_hw_breakpoint_ctrl ctrl;
 446
 447         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 448         if (IS_ERR(bp)) {
 449                 err = PTR_ERR(bp);
 450                 return err;
 451         }
 452
 453         attr = bp->attr;
 454         decode_ctrl_reg(uctrl, &ctrl);
 455         err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr);
 456         if (err)
 457                 return err;
 458
 459         return modify_user_hw_breakpoint(bp, &attr);
 460 }
 461
 462 static int ptrace_hbp_set_addr(unsigned int note_type,
 463                                struct task_struct *tsk,
 464                                unsigned long idx,
 465                                u64 addr)
 466 {
 467         int err;
 468         struct perf_event *bp;
 469         struct perf_event_attr attr;
 470
 471         bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx);
 472         if (IS_ERR(bp)) {
 473                 err = PTR_ERR(bp);
 474                 return err;
 475         }
 476
 477         attr = bp->attr;
 478         attr.bp_addr = addr;
 479         err = modify_user_hw_breakpoint(bp, &attr);
 480         return err;
 481 }
 482
 483 #define PTRACE_HBP_ADDR_SZ      sizeof(u64)
 484 #define PTRACE_HBP_CTRL_SZ      sizeof(u32)
 485 #define PTRACE_HBP_PAD_SZ       sizeof(u32)
 486
 487 static int hw_break_get(struct task_struct *target,
 488                         const struct user_regset *regset,
 489                         unsigned int pos, unsigned int count,
 490                         void *kbuf, void __user *ubuf)
 491 {
 492         unsigned int note_type = regset->core_note_type;
 493         int ret, idx = 0, offset, limit;
 494         u32 info, ctrl;
 495         u64 addr;
 496
 497         /* Resource info */
 498         ret = ptrace_hbp_get_resource_info(note_type, &info);
 499         if (ret)
 500                 return ret;
 501
 502         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &info, 0,
 503                                   sizeof(info));
 504         if (ret)
 505                 return ret;
 506
 507         /* Pad */
 508         offset = offsetof(struct user_hwdebug_state, pad);
 509         ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, offset,
 510                                        offset + PTRACE_HBP_PAD_SZ);
 511         if (ret)
 512                 return ret;
 513
 514         /* (address, ctrl) registers */
 515         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 516         limit = regset->n * regset->size;
 517         while (count && offset < limit) {
 518                 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr);
 519                 if (ret)
 520                         return ret;
 521                 ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &addr,
 522                                           offset, offset + PTRACE_HBP_ADDR_SZ);
 523                 if (ret)
 524                         return ret;
 525                 offset += PTRACE_HBP_ADDR_SZ;
 526
 527                 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl);
 528                 if (ret)
 529                         return ret;
 530                 ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &ctrl,
 531                                           offset, offset + PTRACE_HBP_CTRL_SZ);
 532                 if (ret)
 533                         return ret;
 534                 offset += PTRACE_HBP_CTRL_SZ;
 535
 536                 ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
 537                                                offset,
 538                                                offset + PTRACE_HBP_PAD_SZ);
 539                 if (ret)
 540                         return ret;
 541                 offset += PTRACE_HBP_PAD_SZ;
 542                 idx++;
 543         }
 544
 545         return 0;
 546 }
 547
 548 static int hw_break_set(struct task_struct *target,
 549                         const struct user_regset *regset,
 550                         unsigned int pos, unsigned int count,
 551                         const void *kbuf, const void __user *ubuf)
 552 {
 553         unsigned int note_type = regset->core_note_type;
 554         int ret, idx = 0, offset, limit;
 555         u32 ctrl;
 556         u64 addr;
 557
 558         /* Resource info and pad */
 559         offset = offsetof(struct user_hwdebug_state, dbg_regs);
 560         ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset);
 561         if (ret)
 562                 return ret;
 563
 564         /* (address, ctrl) registers */
 565         limit = regset->n * regset->size;
 566         while (count && offset < limit) {
 567                 if (count < PTRACE_HBP_ADDR_SZ)
 568                         return -EINVAL;
 569                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr,
 570                                          offset, offset + PTRACE_HBP_ADDR_SZ);
 571                 if (ret)
 572                         return ret;
 573                 ret = ptrace_hbp_set_addr(note_type, target, idx, addr);
 574                 if (ret)
 575                         return ret;
 576                 offset += PTRACE_HBP_ADDR_SZ;
 577
 578                 if (!count)
 579                         break;
 580                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl,
 581                                          offset, offset + PTRACE_HBP_CTRL_SZ);
 582                 if (ret)
 583                         return ret;
 584                 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl);
 585                 if (ret)
 586                         return ret;
 587                 offset += PTRACE_HBP_CTRL_SZ;
 588
 589                 ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 590                                                 offset,
 591                                                 offset + PTRACE_HBP_PAD_SZ);
 592                 if (ret)
 593                         return ret;
 594                 offset += PTRACE_HBP_PAD_SZ;
 595                 idx++;
 596         }
 597
 598         return 0;
 599 }
 600 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
 601
 602 static int gpr_get(struct task_struct *target,
 603                    const struct user_regset *regset,
 604                    unsigned int pos, unsigned int count,
 605                    void *kbuf, void __user *ubuf)
 606 {
 607         struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs;
 608         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs, 0, -1);
 609 }
 610
 611 static int gpr_set(struct task_struct *target, const struct user_regset *regset,
 612                    unsigned int pos, unsigned int count,
 613                    const void *kbuf, const void __user *ubuf)
 614 {
 615         int ret;
 616         struct user_pt_regs newregs = task_pt_regs(target)->user_regs;
 617
 618         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1);
 619         if (ret)
 620                 return ret;
 621
 622         if (!valid_user_regs(&newregs, target))
 623                 return -EINVAL;
 624
 625         task_pt_regs(target)->user_regs = newregs;
 626         return 0;
 627 }
 628
 629 /*
 630  * TODO: update fp accessors for lazy context switching (sync/flush hwstate)
 631  */
 632 static int __fpr_get(struct task_struct *target,
 633                      const struct user_regset *regset,
 634                      unsigned int pos, unsigned int count,
 635                      void *kbuf, void __user *ubuf, unsigned int start_pos)
 636 {
 637         struct user_fpsimd_state *uregs;
 638
 639         sve_sync_to_fpsimd(target);
 640
 641         uregs = &target->thread.uw.fpsimd_state;
 642
 643         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
 644                                    start_pos, start_pos + sizeof(*uregs));
 645 }
 646
 647 static int fpr_get(struct task_struct *target, const struct user_regset *regset,
 648                    unsigned int pos, unsigned int count,
 649                    void *kbuf, void __user *ubuf)
 650 {
 651         if (target == current)
 652                 fpsimd_preserve_current_state();
 653
 654         return __fpr_get(target, regset, pos, count, kbuf, ubuf, 0);
 655 }
 656
 657 static int __fpr_set(struct task_struct *target,
 658                      const struct user_regset *regset,
 659                      unsigned int pos, unsigned int count,
 660                      const void *kbuf, const void __user *ubuf,
 661                      unsigned int start_pos)
 662 {
 663         int ret;
 664         struct user_fpsimd_state newstate;
 665
 666         /*
 667          * Ensure target->thread.uw.fpsimd_state is up to date, so that a
 668          * short copyin can't resurrect stale data.
 669          */
 670         sve_sync_to_fpsimd(target);
 671
 672         newstate = target->thread.uw.fpsimd_state;
 673
 674         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate,
 675                                  start_pos, start_pos + sizeof(newstate));
 676         if (ret)
 677                 return ret;
 678
 679         target->thread.uw.fpsimd_state = newstate;
 680
 681         return ret;
 682 }
 683
 684 static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 685                    unsigned int pos, unsigned int count,
 686                    const void *kbuf, const void __user *ubuf)
 687 {
 688         int ret;
 689
 690         ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0);
 691         if (ret)
 692                 return ret;
 693
 694         sve_sync_from_fpsimd_zeropad(target);
 695         fpsimd_flush_task_state(target);
 696
 697         return ret;
 698 }
 699
 700 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 701                    unsigned int pos, unsigned int count,
 702                    void *kbuf, void __user *ubuf)
 703 {
 704         unsigned long *tls = &target->thread.uw.tp_value;
 705
 706         if (target == current)
 707                 tls_preserve_current_state();
 708
 709         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, tls, 0, -1);
 710 }
 711
 712 static int tls_set(struct task_struct *target, const struct user_regset *regset,
 713                    unsigned int pos, unsigned int count,
 714                    const void *kbuf, const void __user *ubuf)
 715 {
 716         int ret;
 717         unsigned long tls = target->thread.uw.tp_value;
 718
 719         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
 720         if (ret)
 721                 return ret;
 722
 723         target->thread.uw.tp_value = tls;
 724         return ret;
 725 }
 726
 727 static int system_call_get(struct task_struct *target,
 728                            const struct user_regset *regset,
 729                            unsigned int pos, unsigned int count,
 730                            void *kbuf, void __user *ubuf)
 731 {
 732         int syscallno = task_pt_regs(target)->syscallno;
 733
 734         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 735                                    &syscallno, 0, -1);
 736 }
 737
 738 static int system_call_set(struct task_struct *target,
 739                            const struct user_regset *regset,
 740                            unsigned int pos, unsigned int count,
 741                            const void *kbuf, const void __user *ubuf)
 742 {
 743         int syscallno = task_pt_regs(target)->syscallno;
 744         int ret;
 745
 746         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1);
 747         if (ret)
 748                 return ret;
 749
 750         task_pt_regs(target)->syscallno = syscallno;
 751         return ret;
 752 }
 753
 754 #ifdef CONFIG_ARM64_SVE
 755
 756 static void sve_init_header_from_task(struct user_sve_header *header,
 757                                       struct task_struct *target)
 758 {
 759         unsigned int vq;
 760
 761         memset(header, 0, sizeof(*header));
 762
 763         header->flags = test_tsk_thread_flag(target, TIF_SVE) ?
 764                 SVE_PT_REGS_SVE : SVE_PT_REGS_FPSIMD;
 765         if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT))
 766                 header->flags |= SVE_PT_VL_INHERIT;
 767
 768         header->vl = target->thread.sve_vl;
 769         vq = sve_vq_from_vl(header->vl);
 770
 771         header->max_vl = sve_max_vl;
 772         header->size = SVE_PT_SIZE(vq, header->flags);
 773         header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl),
 774                                       SVE_PT_REGS_SVE);
 775 }
 776
 777 static unsigned int sve_size_from_header(struct user_sve_header const *header)
 778 {
 779         return ALIGN(header->size, SVE_VQ_BYTES);
 780 }
 781
 782 static unsigned int sve_get_size(struct task_struct *target,
 783                                  const struct user_regset *regset)
 784 {
 785         struct user_sve_header header;
 786
 787         if (!system_supports_sve())
 788                 return 0;
 789
 790         sve_init_header_from_task(&header, target);
 791         return sve_size_from_header(&header);
 792 }
 793
 794 static int sve_get(struct task_struct *target,
 795                    const struct user_regset *regset,
 796                    unsigned int pos, unsigned int count,
 797                    void *kbuf, void __user *ubuf)
 798 {
 799         int ret;
 800         struct user_sve_header header;
 801         unsigned int vq;
 802         unsigned long start, end;
 803
 804         if (!system_supports_sve())
 805                 return -EINVAL;
 806
 807         /* Header */
 808         sve_init_header_from_task(&header, target);
 809         vq = sve_vq_from_vl(header.vl);
 810
 811         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &header,
 812                                   0, sizeof(header));
 813         if (ret)
 814                 return ret;
 815
 816         if (target == current)
 817                 fpsimd_preserve_current_state();
 818
 819         /* Registers: FPSIMD-only case */
 820
 821         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 822         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD)
 823                 return __fpr_get(target, regset, pos, count, kbuf, ubuf,
 824                                  SVE_PT_FPSIMD_OFFSET);
 825
 826         /* Otherwise: full SVE case */
 827
 828         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 829         start = SVE_PT_SVE_OFFSET;
 830         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 831         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 832                                   target->thread.sve_state,
 833                                   start, end);
 834         if (ret)
 835                 return ret;
 836
 837         start = end;
 838         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 839         ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
 840                                        start, end);
 841         if (ret)
 842                 return ret;
 843
 844         /*
 845          * Copy fpsr, and fpcr which must follow contiguously in
 846          * struct fpsimd_state:
 847          */
 848         start = end;
 849         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 850         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 851                                   &target->thread.uw.fpsimd_state.fpsr,
 852                                   start, end);
 853         if (ret)
 854                 return ret;
 855
 856         start = end;
 857         end = sve_size_from_header(&header);
 858         return user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
 859                                         start, end);
 860 }
 861
 862 static int sve_set(struct task_struct *target,
 863                    const struct user_regset *regset,
 864                    unsigned int pos, unsigned int count,
 865                    const void *kbuf, const void __user *ubuf)
 866 {
 867         int ret;
 868         struct user_sve_header header;
 869         unsigned int vq;
 870         unsigned long start, end;
 871
 872         if (!system_supports_sve())
 873                 return -EINVAL;
 874
 875         /* Header */
 876         if (count < sizeof(header))
 877                 return -EINVAL;
 878         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header,
 879                                  0, sizeof(header));
 880         if (ret)
 881                 goto out;
 882
 883         /*
 884          * Apart from PT_SVE_REGS_MASK, all PT_SVE_* flags are consumed by
 885          * sve_set_vector_length(), which will also validate them for us:
 886          */
 887         ret = sve_set_vector_length(target, header.vl,
 888                 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16);
 889         if (ret)
 890                 goto out;
 891
 892         /* Actual VL set may be less than the user asked for: */
 893         vq = sve_vq_from_vl(target->thread.sve_vl);
 894
 895         /* Registers: FPSIMD-only case */
 896
 897         BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header));
 898         if ((header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD) {
 899                 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf,
 900                                 SVE_PT_FPSIMD_OFFSET);
 901                 clear_tsk_thread_flag(target, TIF_SVE);
 902                 goto out;
 903         }
 904
 905         /* Otherwise: full SVE case */
 906
 907         /*
 908          * If setting a different VL from the requested VL and there is
 909          * register data, the data layout will be wrong: don't even
 910          * try to set the registers in this case.
 911          */
 912         if (count && vq != sve_vq_from_vl(header.vl)) {
 913                 ret = -EIO;
 914                 goto out;
 915         }
 916
 917         sve_alloc(target);
 918
 919         /*
 920          * Ensure target->thread.sve_state is up to date with target's
 921          * FPSIMD regs, so that a short copyin leaves trailing registers
 922          * unmodified.
 923          */
 924         fpsimd_sync_to_sve(target);
 925         set_tsk_thread_flag(target, TIF_SVE);
 926
 927         BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header));
 928         start = SVE_PT_SVE_OFFSET;
 929         end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq);
 930         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 931                                  target->thread.sve_state,
 932                                  start, end);
 933         if (ret)
 934                 goto out;
 935
 936         start = end;
 937         end = SVE_PT_SVE_FPSR_OFFSET(vq);
 938         ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
 939                                         start, end);
 940         if (ret)
 941                 goto out;
 942
 943         /*
 944          * Copy fpsr, and fpcr which must follow contiguously in
 945          * struct fpsimd_state:
 946          */
 947         start = end;
 948         end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE;
 949         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 950                                  &target->thread.uw.fpsimd_state.fpsr,
 951                                  start, end);
 952
 953 out:
 954         fpsimd_flush_task_state(target);
 955         return ret;
 956 }
 957
 958 #endif /* CONFIG_ARM64_SVE */
 959
 960 #ifdef CONFIG_ARM64_PTR_AUTH
 961 static int pac_mask_get(struct task_struct *target,
 962                         const struct user_regset *regset,
 963                         unsigned int pos, unsigned int count,
 964                         void *kbuf, void __user *ubuf)
 965 {
 966         /*
 967          * The PAC bits can differ across data and instruction pointers
 968          * depending on TCR_EL1.TBID*, which we may make use of in future, so
 969          * we expose separate masks.
 970          */
 971         unsigned long mask = ptrauth_user_pac_mask();
 972         struct user_pac_mask uregs = {
 973                 .data_mask = mask,
 974                 .insn_mask = mask,
 975         };
 976
 977         if (!system_supports_address_auth())
 978                 return -EINVAL;
 979
 980         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &uregs, 0, -1);
 981 }
 982 #endif /* CONFIG_ARM64_PTR_AUTH */
 983
 984 enum aarch64_regset {
 985         REGSET_GPR,
 986         REGSET_FPR,
 987         REGSET_TLS,
 988 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 989         REGSET_HW_BREAK,
 990         REGSET_HW_WATCH,
 991 #endif
 992         REGSET_SYSTEM_CALL,
 993 #ifdef CONFIG_ARM64_SVE
 994         REGSET_SVE,
 995 #endif
 996 #ifdef CONFIG_ARM64_PTR_AUTH
 997         REGSET_PAC_MASK,
 998 #endif
 999 };
1000
1001 static const struct user_regset aarch64_regsets[] = {
1002         [REGSET_GPR] = {
1003                 .core_note_type = NT_PRSTATUS,
1004                 .n = sizeof(struct user_pt_regs) / sizeof(u64),
1005                 .size = sizeof(u64),
1006                 .align = sizeof(u64),
1007                 .get = gpr_get,
1008                 .set = gpr_set
1009         },
1010         [REGSET_FPR] = {
1011                 .core_note_type = NT_PRFPREG,
1012                 .n = sizeof(struct user_fpsimd_state) / sizeof(u32),
1013                 /*
1014                  * We pretend we have 32-bit registers because the fpsr and
1015                  * fpcr are 32-bits wide.
1016                  */
1017                 .size = sizeof(u32),
1018                 .align = sizeof(u32),
1019                 .get = fpr_get,
1020                 .set = fpr_set
1021         },
1022         [REGSET_TLS] = {
1023                 .core_note_type = NT_ARM_TLS,
1024                 .n = 1,
1025                 .size = sizeof(void *),
1026                 .align = sizeof(void *),
1027                 .get = tls_get,
1028                 .set = tls_set,
1029         },
1030 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1031         [REGSET_HW_BREAK] = {
1032                 .core_note_type = NT_ARM_HW_BREAK,
1033                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1034                 .size = sizeof(u32),
1035                 .align = sizeof(u32),
1036                 .get = hw_break_get,
1037                 .set = hw_break_set,
1038         },
1039         [REGSET_HW_WATCH] = {
1040                 .core_note_type = NT_ARM_HW_WATCH,
1041                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1042                 .size = sizeof(u32),
1043                 .align = sizeof(u32),
1044                 .get = hw_break_get,
1045                 .set = hw_break_set,
1046         },
1047 #endif
1048         [REGSET_SYSTEM_CALL] = {
1049                 .core_note_type = NT_ARM_SYSTEM_CALL,
1050                 .n = 1,
1051                 .size = sizeof(int),
1052                 .align = sizeof(int),
1053                 .get = system_call_get,
1054                 .set = system_call_set,
1055         },
1056 #ifdef CONFIG_ARM64_SVE
1057         [REGSET_SVE] = { /* Scalable Vector Extension */
1058                 .core_note_type = NT_ARM_SVE,
1059                 .n = DIV_ROUND_UP(SVE_PT_SIZE(SVE_VQ_MAX, SVE_PT_REGS_SVE),
1060                                   SVE_VQ_BYTES),
1061                 .size = SVE_VQ_BYTES,
1062                 .align = SVE_VQ_BYTES,
1063                 .get = sve_get,
1064                 .set = sve_set,
1065                 .get_size = sve_get_size,
1066         },
1067 #endif
1068 #ifdef CONFIG_ARM64_PTR_AUTH
1069         [REGSET_PAC_MASK] = {
1070                 .core_note_type = NT_ARM_PAC_MASK,
1071                 .n = sizeof(struct user_pac_mask) / sizeof(u64),
1072                 .size = sizeof(u64),
1073                 .align = sizeof(u64),
1074                 .get = pac_mask_get,
1075                 /* this cannot be set dynamically */
1076         },
1077 #endif
1078 };
1079
1080 static const struct user_regset_view user_aarch64_view = {
1081         .name = "aarch64", .e_machine = EM_AARCH64,
1082         .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets)
1083 };
1084
1085 #ifdef CONFIG_COMPAT
1086 enum compat_regset {
1087         REGSET_COMPAT_GPR,
1088         REGSET_COMPAT_VFP,
1089 };
1090
1091 static int compat_gpr_get(struct task_struct *target,
1092                           const struct user_regset *regset,
1093                           unsigned int pos, unsigned int count,
1094                           void *kbuf, void __user *ubuf)
1095 {
1096         int ret = 0;
1097         unsigned int i, start, num_regs;
1098
1099         /* Calculate the number of AArch32 registers contained in count */
1100         num_regs = count / regset->size;
1101
1102         /* Convert pos into an register number */
1103         start = pos / regset->size;
1104
1105         if (start + num_regs > regset->n)
1106                 return -EIO;
1107
1108         for (i = 0; i < num_regs; ++i) {
1109                 unsigned int idx = start + i;
1110                 compat_ulong_t reg;
1111
1112                 switch (idx) {
1113                 case 15:
1114                         reg = task_pt_regs(target)->pc;
1115                         break;
1116                 case 16:
1117                         reg = task_pt_regs(target)->pstate;
1118                         reg = pstate_to_compat_psr(reg);
1119                         break;
1120                 case 17:
1121                         reg = task_pt_regs(target)->orig_x0;
1122                         break;
1123                 default:
1124                         reg = task_pt_regs(target)->regs[idx];
1125                 }
1126
1127                 if (kbuf) {
1128                         memcpy(kbuf, &reg, sizeof(reg));
1129                         kbuf += sizeof(reg);
1130                 } else {
1131                         ret = copy_to_user(ubuf, &reg, sizeof(reg));
1132                         if (ret) {
1133                                 ret = -EFAULT;
1134                                 break;
1135                         }
1136
1137                         ubuf += sizeof(reg);
1138                 }
1139         }
1140
1141         return ret;
1142 }
1143
1144 static int compat_gpr_set(struct task_struct *target,
1145                           const struct user_regset *regset,
1146                           unsigned int pos, unsigned int count,
1147                           const void *kbuf, const void __user *ubuf)
1148 {
1149         struct pt_regs newregs;
1150         int ret = 0;
1151         unsigned int i, start, num_regs;
1152
1153         /* Calculate the number of AArch32 registers contained in count */
1154         num_regs = count / regset->size;
1155
1156         /* Convert pos into an register number */
1157         start = pos / regset->size;
1158
1159         if (start + num_regs > regset->n)
1160                 return -EIO;
1161
1162         newregs = *task_pt_regs(target);
1163
1164         for (i = 0; i < num_regs; ++i) {
1165                 unsigned int idx = start + i;
1166                 compat_ulong_t reg;
1167
1168                 if (kbuf) {
1169                         memcpy(&reg, kbuf, sizeof(reg));
1170                         kbuf += sizeof(reg);
1171                 } else {
1172                         ret = copy_from_user(&reg, ubuf, sizeof(reg));
1173                         if (ret) {
1174                                 ret = -EFAULT;
1175                                 break;
1176                         }
1177
1178                         ubuf += sizeof(reg);
1179                 }
1180
1181                 switch (idx) {
1182                 case 15:
1183                         newregs.pc = reg;
1184                         break;
1185                 case 16:
1186                         reg = compat_psr_to_pstate(reg);
1187                         newregs.pstate = reg;
1188                         break;
1189                 case 17:
1190                         newregs.orig_x0 = reg;
1191                         break;
1192                 default:
1193                         newregs.regs[idx] = reg;
1194                 }
1195
1196         }
1197
1198         if (valid_user_regs(&newregs.user_regs, target))
1199                 *task_pt_regs(target) = newregs;
1200         else
1201                 ret = -EINVAL;
1202
1203         return ret;
1204 }
1205
1206 static int compat_vfp_get(struct task_struct *target,
1207                           const struct user_regset *regset,
1208                           unsigned int pos, unsigned int count,
1209                           void *kbuf, void __user *ubuf)
1210 {
1211         struct user_fpsimd_state *uregs;
1212         compat_ulong_t fpscr;
1213         int ret, vregs_end_pos;
1214
1215         uregs = &target->thread.uw.fpsimd_state;
1216
1217         if (target == current)
1218                 fpsimd_preserve_current_state();
1219
1220         /*
1221          * The VFP registers are packed into the fpsimd_state, so they all sit
1222          * nicely together for us. We just need to create the fpscr separately.
1223          */
1224         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1225         ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, uregs,
1226                                   0, vregs_end_pos);
1227
1228         if (count && !ret) {
1229                 fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) |
1230                         (uregs->fpcr & VFP_FPSCR_CTRL_MASK);
1231
1232                 ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &fpscr,
1233                                           vregs_end_pos, VFP_STATE_SIZE);
1234         }
1235
1236         return ret;
1237 }
1238
1239 static int compat_vfp_set(struct task_struct *target,
1240                           const struct user_regset *regset,
1241                           unsigned int pos, unsigned int count,
1242                           const void *kbuf, const void __user *ubuf)
1243 {
1244         struct user_fpsimd_state *uregs;
1245         compat_ulong_t fpscr;
1246         int ret, vregs_end_pos;
1247
1248         uregs = &target->thread.uw.fpsimd_state;
1249
1250         vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t);
1251         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0,
1252                                  vregs_end_pos);
1253
1254         if (count && !ret) {
1255                 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr,
1256                                          vregs_end_pos, VFP_STATE_SIZE);
1257                 if (!ret) {
1258                         uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK;
1259                         uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
1260                 }
1261         }
1262
1263         fpsimd_flush_task_state(target);
1264         return ret;
1265 }
1266
1267 static int compat_tls_get(struct task_struct *target,
1268                           const struct user_regset *regset, unsigned int pos,
1269                           unsigned int count, void *kbuf, void __user *ubuf)
1270 {
1271         compat_ulong_t tls = (compat_ulong_t)target->thread.uw.tp_value;
1272         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1273 }
1274
1275 static int compat_tls_set(struct task_struct *target,
1276                           const struct user_regset *regset, unsigned int pos,
1277                           unsigned int count, const void *kbuf,
1278                           const void __user *ubuf)
1279 {
1280         int ret;
1281         compat_ulong_t tls = target->thread.uw.tp_value;
1282
1283         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
1284         if (ret)
1285                 return ret;
1286
1287         target->thread.uw.tp_value = tls;
1288         return ret;
1289 }
1290
1291 static const struct user_regset aarch32_regsets[] = {
1292         [REGSET_COMPAT_GPR] = {
1293                 .core_note_type = NT_PRSTATUS,
1294                 .n = COMPAT_ELF_NGREG,
1295                 .size = sizeof(compat_elf_greg_t),
1296                 .align = sizeof(compat_elf_greg_t),
1297                 .get = compat_gpr_get,
1298                 .set = compat_gpr_set
1299         },
1300         [REGSET_COMPAT_VFP] = {
1301                 .core_note_type = NT_ARM_VFP,
1302                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1303                 .size = sizeof(compat_ulong_t),
1304                 .align = sizeof(compat_ulong_t),
1305                 .get = compat_vfp_get,
1306                 .set = compat_vfp_set
1307         },
1308 };
1309
1310 static const struct user_regset_view user_aarch32_view = {
1311         .name = "aarch32", .e_machine = EM_ARM,
1312         .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets)
1313 };
1314
1315 static const struct user_regset aarch32_ptrace_regsets[] = {
1316         [REGSET_GPR] = {
1317                 .core_note_type = NT_PRSTATUS,
1318                 .n = COMPAT_ELF_NGREG,
1319                 .size = sizeof(compat_elf_greg_t),
1320                 .align = sizeof(compat_elf_greg_t),
1321                 .get = compat_gpr_get,
1322                 .set = compat_gpr_set
1323         },
1324         [REGSET_FPR] = {
1325                 .core_note_type = NT_ARM_VFP,
1326                 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t),
1327                 .size = sizeof(compat_ulong_t),
1328                 .align = sizeof(compat_ulong_t),
1329                 .get = compat_vfp_get,
1330                 .set = compat_vfp_set
1331         },
1332         [REGSET_TLS] = {
1333                 .core_note_type = NT_ARM_TLS,
1334                 .n = 1,
1335                 .size = sizeof(compat_ulong_t),
1336                 .align = sizeof(compat_ulong_t),
1337                 .get = compat_tls_get,
1338                 .set = compat_tls_set,
1339         },
1340 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1341         [REGSET_HW_BREAK] = {
1342                 .core_note_type = NT_ARM_HW_BREAK,
1343                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1344                 .size = sizeof(u32),
1345                 .align = sizeof(u32),
1346                 .get = hw_break_get,
1347                 .set = hw_break_set,
1348         },
1349         [REGSET_HW_WATCH] = {
1350                 .core_note_type = NT_ARM_HW_WATCH,
1351                 .n = sizeof(struct user_hwdebug_state) / sizeof(u32),
1352                 .size = sizeof(u32),
1353                 .align = sizeof(u32),
1354                 .get = hw_break_get,
1355                 .set = hw_break_set,
1356         },
1357 #endif
1358         [REGSET_SYSTEM_CALL] = {
1359                 .core_note_type = NT_ARM_SYSTEM_CALL,
1360                 .n = 1,
1361                 .size = sizeof(int),
1362                 .align = sizeof(int),
1363                 .get = system_call_get,
1364                 .set = system_call_set,
1365         },
1366 };
1367
1368 static const struct user_regset_view user_aarch32_ptrace_view = {
1369         .name = "aarch32", .e_machine = EM_ARM,
1370         .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets)
1371 };
1372
1373 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off,
1374                                    compat_ulong_t __user *ret)
1375 {
1376         compat_ulong_t tmp;
1377
1378         if (off & 3)
1379                 return -EIO;
1380
1381         if (off == COMPAT_PT_TEXT_ADDR)
1382                 tmp = tsk->mm->start_code;
1383         else if (off == COMPAT_PT_DATA_ADDR)
1384                 tmp = tsk->mm->start_data;
1385         else if (off == COMPAT_PT_TEXT_END_ADDR)
1386                 tmp = tsk->mm->end_code;
1387         else if (off < sizeof(compat_elf_gregset_t))
1388                 return copy_regset_to_user(tsk, &user_aarch32_view,
1389                                            REGSET_COMPAT_GPR, off,
1390                                            sizeof(compat_ulong_t), ret);
1391         else if (off >= COMPAT_USER_SZ)
1392                 return -EIO;
1393         else
1394                 tmp = 0;
1395
1396         return put_user(tmp, ret);
1397 }
1398
1399 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off,
1400                                     compat_ulong_t val)
1401 {
1402         int ret;
1403         mm_segment_t old_fs = get_fs();
1404
1405         if (off & 3 || off >= COMPAT_USER_SZ)
1406                 return -EIO;
1407
1408         if (off >= sizeof(compat_elf_gregset_t))
1409                 return 0;
1410
1411         set_fs(KERNEL_DS);
1412         ret = copy_regset_from_user(tsk, &user_aarch32_view,
1413                                     REGSET_COMPAT_GPR, off,
1414                                     sizeof(compat_ulong_t),
1415                                     &val);
1416         set_fs(old_fs);
1417
1418         return ret;
1419 }
1420
1421 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1422
1423 /*
1424  * Convert a virtual register number into an index for a thread_info
1425  * breakpoint array. Breakpoints are identified using positive numbers
1426  * whilst watchpoints are negative. The registers are laid out as pairs
1427  * of (address, control), each pair mapping to a unique hw_breakpoint struct.
1428  * Register 0 is reserved for describing resource information.
1429  */
1430 static int compat_ptrace_hbp_num_to_idx(compat_long_t num)
1431 {
1432         return (abs(num) - 1) >> 1;
1433 }
1434
1435 static int compat_ptrace_hbp_get_resource_info(u32 *kdata)
1436 {
1437         u8 num_brps, num_wrps, debug_arch, wp_len;
1438         u32 reg = 0;
1439
1440         num_brps        = hw_breakpoint_slots(TYPE_INST);
1441         num_wrps        = hw_breakpoint_slots(TYPE_DATA);
1442
1443         debug_arch      = debug_monitors_arch();
1444         wp_len          = 8;
1445         reg             |= debug_arch;
1446         reg             <<= 8;
1447         reg             |= wp_len;
1448         reg             <<= 8;
1449         reg             |= num_wrps;
1450         reg             <<= 8;
1451         reg             |= num_brps;
1452
1453         *kdata = reg;
1454         return 0;
1455 }
1456
1457 static int compat_ptrace_hbp_get(unsigned int note_type,
1458                                  struct task_struct *tsk,
1459                                  compat_long_t num,
1460                                  u32 *kdata)
1461 {
1462         u64 addr = 0;
1463         u32 ctrl = 0;
1464
1465         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1466
1467         if (num & 1) {
1468                 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
1469                 *kdata = (u32)addr;
1470         } else {
1471                 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl);
1472                 *kdata = ctrl;
1473         }
1474
1475         return err;
1476 }
1477
1478 static int compat_ptrace_hbp_set(unsigned int note_type,
1479                                  struct task_struct *tsk,
1480                                  compat_long_t num,
1481                                  u32 *kdata)
1482 {
1483         u64 addr;
1484         u32 ctrl;
1485
1486         int err, idx = compat_ptrace_hbp_num_to_idx(num);
1487
1488         if (num & 1) {
1489                 addr = *kdata;
1490                 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr);
1491         } else {
1492                 ctrl = *kdata;
1493                 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl);
1494         }
1495
1496         return err;
1497 }
1498
1499 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num,
1500                                     compat_ulong_t __user *data)
1501 {
1502         int ret;
1503         u32 kdata;
1504
1505         /* Watchpoint */
1506         if (num < 0) {
1507                 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata);
1508         /* Resource info */
1509         } else if (num == 0) {
1510                 ret = compat_ptrace_hbp_get_resource_info(&kdata);
1511         /* Breakpoint */
1512         } else {
1513                 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata);
1514         }
1515
1516         if (!ret)
1517                 ret = put_user(kdata, data);
1518
1519         return ret;
1520 }
1521
1522 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num,
1523                                     compat_ulong_t __user *data)
1524 {
1525         int ret;
1526         u32 kdata = 0;
1527
1528         if (num == 0)
1529                 return 0;
1530
1531         ret = get_user(kdata, data);
1532         if (ret)
1533                 return ret;
1534
1535         if (num < 0)
1536                 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata);
1537         else
1538                 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata);
1539
1540         return ret;
1541 }
1542 #endif  /* CONFIG_HAVE_HW_BREAKPOINT */
1543
1544 long compat_arch_ptrace(struct task_struct *child, compat_long_t request,
1545                         compat_ulong_t caddr, compat_ulong_t cdata)
1546 {
1547         unsigned long addr = caddr;
1548         unsigned long data = cdata;
1549         void __user *datap = compat_ptr(data);
1550         int ret;
1551
1552         switch (request) {
1553                 case PTRACE_PEEKUSR:
1554                         ret = compat_ptrace_read_user(child, addr, datap);
1555                         break;
1556
1557                 case PTRACE_POKEUSR:
1558                         ret = compat_ptrace_write_user(child, addr, data);
1559                         break;
1560
1561                 case COMPAT_PTRACE_GETREGS:
1562                         ret = copy_regset_to_user(child,
1563                                                   &user_aarch32_view,
1564                                                   REGSET_COMPAT_GPR,
1565                                                   0, sizeof(compat_elf_gregset_t),
1566                                                   datap);
1567                         break;
1568
1569                 case COMPAT_PTRACE_SETREGS:
1570                         ret = copy_regset_from_user(child,
1571                                                     &user_aarch32_view,
1572                                                     REGSET_COMPAT_GPR,
1573                                                     0, sizeof(compat_elf_gregset_t),
1574                                                     datap);
1575                         break;
1576
1577                 case COMPAT_PTRACE_GET_THREAD_AREA:
1578                         ret = put_user((compat_ulong_t)child->thread.uw.tp_value,
1579                                        (compat_ulong_t __user *)datap);
1580                         break;
1581
1582                 case COMPAT_PTRACE_SET_SYSCALL:
1583                         task_pt_regs(child)->syscallno = data;
1584                         ret = 0;
1585                         break;
1586
1587                 case COMPAT_PTRACE_GETVFPREGS:
1588                         ret = copy_regset_to_user(child,
1589                                                   &user_aarch32_view,
1590                                                   REGSET_COMPAT_VFP,
1591                                                   0, VFP_STATE_SIZE,
1592                                                   datap);
1593                         break;
1594
1595                 case COMPAT_PTRACE_SETVFPREGS:
1596                         ret = copy_regset_from_user(child,
1597                                                     &user_aarch32_view,
1598                                                     REGSET_COMPAT_VFP,
1599                                                     0, VFP_STATE_SIZE,
1600                                                     datap);
1601                         break;
1602
1603 #ifdef CONFIG_HAVE_HW_BREAKPOINT
1604                 case COMPAT_PTRACE_GETHBPREGS:
1605                         ret = compat_ptrace_gethbpregs(child, addr, datap);
1606                         break;
1607
1608                 case COMPAT_PTRACE_SETHBPREGS:
1609                         ret = compat_ptrace_sethbpregs(child, addr, datap);
1610                         break;
1611 #endif
1612
1613                 default:
1614                         ret = compat_ptrace_request(child, request, addr,
1615                                                     data);
1616                         break;
1617         }
1618
1619         return ret;
1620 }
1621 #endif /* CONFIG_COMPAT */
1622
1623 const struct user_regset_view *task_user_regset_view(struct task_struct *task)
1624 {
1625 #ifdef CONFIG_COMPAT
1626         /*
1627          * Core dumping of 32-bit tasks or compat ptrace requests must use the
1628          * user_aarch32_view compatible with arm32. Native ptrace requests on
1629          * 32-bit children use an extended user_aarch32_ptrace_view to allow
1630          * access to the TLS register.
1631          */
1632         if (is_compat_task())
1633                 return &user_aarch32_view;
1634         else if (is_compat_thread(task_thread_info(task)))
1635                 return &user_aarch32_ptrace_view;
1636 #endif
1637         return &user_aarch64_view;
1638 }
1639
1640 long arch_ptrace(struct task_struct *child, long request,
1641                  unsigned long addr, unsigned long data)
1642 {
1643         return ptrace_request(child, request, addr, data);
1644 }
1645
1646 enum ptrace_syscall_dir {
1647         PTRACE_SYSCALL_ENTER = 0,
1648         PTRACE_SYSCALL_EXIT,
1649 };
1650
1651 static void tracehook_report_syscall(struct pt_regs *regs,
1652                                      enum ptrace_syscall_dir dir)
1653 {
1654         int regno;
1655         unsigned long saved_reg;
1656
1657         /*
1658          * A scratch register (ip(r12) on AArch32, x7 on AArch64) is
1659          * used to denote syscall entry/exit:
1660          */
1661         regno = (is_compat_task() ? 12 : 7);
1662         saved_reg = regs->regs[regno];
1663         regs->regs[regno] = dir;
1664
1665         if (dir == PTRACE_SYSCALL_EXIT)
1666                 tracehook_report_syscall_exit(regs, 0);
1667         else if (tracehook_report_syscall_entry(regs))
1668                 forget_syscall(regs);
1669
1670         regs->regs[regno] = saved_reg;
1671 }
1672
1673 int syscall_trace_enter(struct pt_regs *regs)
1674 {
1675         if (test_thread_flag(TIF_SYSCALL_TRACE))
1676                 tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER);
1677
1678         /* Do the secure computing after ptrace; failures should be fast. */
1679         if (secure_computing(NULL) == -1)
1680                 return -1;
1681
1682         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
1683                 trace_sys_enter(regs, regs->syscallno);
1684
1685         audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1],
1686                             regs->regs[2], regs->regs[3]);
1687
1688         return regs->syscallno;
1689 }
1690
1691 void syscall_trace_exit(struct pt_regs *regs)
1692 {
1693         audit_syscall_exit(regs);
1694
1695         if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
1696                 trace_sys_exit(regs, regs_return_value(regs));
1697
1698         if (test_thread_flag(TIF_SYSCALL_TRACE))
1699                 tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT);
1700
1701         rseq_syscall(regs);
1702 }
1703
1704 /*
1705  * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a.
1706  * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is
1707  * not described in ARM DDI 0487D.a.
1708  * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may
1709  * be allocated an EL0 meaning in future.
1710  * Userspace cannot use these until they have an architectural meaning.
1711  * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
1712  * We also reserve IL for the kernel; SS is handled dynamically.
1713  */
1714 #define SPSR_EL1_AARCH64_RES0_BITS \
1715         (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 25) | GENMASK_ULL(23, 22) | \
1716          GENMASK_ULL(20, 13) | GENMASK_ULL(11, 10) | GENMASK_ULL(5, 5))
1717 #define SPSR_EL1_AARCH32_RES0_BITS \
1718         (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20))
1719
1720 static int valid_compat_regs(struct user_pt_regs *regs)
1721 {
1722         regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS;
1723
1724         if (!system_supports_mixed_endian_el0()) {
1725                 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1726                         regs->pstate |= PSR_AA32_E_BIT;
1727                 else
1728                         regs->pstate &= ~PSR_AA32_E_BIT;
1729         }
1730
1731         if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
1732             (regs->pstate & PSR_AA32_A_BIT) == 0 &&
1733             (regs->pstate & PSR_AA32_I_BIT) == 0 &&
1734             (regs->pstate & PSR_AA32_F_BIT) == 0) {
1735                 return 1;
1736         }
1737
1738         /*
1739          * Force PSR to a valid 32-bit EL0t, preserving the same bits as
1740          * arch/arm.
1741          */
1742         regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT |
1743                         PSR_AA32_C_BIT | PSR_AA32_V_BIT |
1744                         PSR_AA32_Q_BIT | PSR_AA32_IT_MASK |
1745                         PSR_AA32_GE_MASK | PSR_AA32_E_BIT |
1746                         PSR_AA32_T_BIT;
1747         regs->pstate |= PSR_MODE32_BIT;
1748
1749         return 0;
1750 }
1751
1752 static int valid_native_regs(struct user_pt_regs *regs)
1753 {
1754         regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS;
1755
1756         if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) &&
1757             (regs->pstate & PSR_D_BIT) == 0 &&
1758             (regs->pstate & PSR_A_BIT) == 0 &&
1759             (regs->pstate & PSR_I_BIT) == 0 &&
1760             (regs->pstate & PSR_F_BIT) == 0) {
1761                 return 1;
1762         }
1763
1764         /* Force PSR to a valid 64-bit EL0t */
1765         regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT;
1766
1767         return 0;
1768 }
1769
1770 /*
1771  * Are the current registers suitable for user mode? (used to maintain
1772  * security in signal handlers)
1773  */
1774 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task)
1775 {
1776         if (!test_tsk_thread_flag(task, TIF_SINGLESTEP))
1777                 regs->pstate &= ~DBG_SPSR_SS;
1778
1779         if (is_compat_thread(task_thread_info(task)))
1780                 return valid_compat_regs(regs);
1781         else
1782                 return valid_native_regs(regs);
1783 }