arch/arm64/kernel/traps.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/kernel/traps.c
   4  *
   5  * Copyright (C) 1995-2009 Russell King
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8
   9 #include <linux/bug.h>
  10 #include <linux/context_tracking.h>
  11 #include <linux/signal.h>
  12 #include <linux/kallsyms.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/module.h>
  19 #include <linux/kexec.h>
  20 #include <linux/delay.h>
  21 #include <linux/efi.h>
  22 #include <linux/init.h>
  23 #include <linux/sched/signal.h>
  24 #include <linux/sched/debug.h>
  25 #include <linux/sched/task_stack.h>
  26 #include <linux/sizes.h>
  27 #include <linux/syscalls.h>
  28 #include <linux/mm_types.h>
  29 #include <linux/kasan.h>
  30 #include <linux/ubsan.h>
  31 #include <linux/cfi.h>
  32
  33 #include <asm/atomic.h>
  34 #include <asm/bug.h>
  35 #include <asm/cpufeature.h>
  36 #include <asm/daifflags.h>
  37 #include <asm/debug-monitors.h>
  38 #include <asm/efi.h>
  39 #include <asm/esr.h>
  40 #include <asm/exception.h>
  41 #include <asm/extable.h>
  42 #include <asm/insn.h>
  43 #include <asm/kprobes.h>
  44 #include <asm/text-patching.h>
  45 #include <asm/traps.h>
  46 #include <asm/smp.h>
  47 #include <asm/stack_pointer.h>
  48 #include <asm/stacktrace.h>
  49 #include <asm/system_misc.h>
  50 #include <asm/sysreg.h>
  51
  52 static bool __kprobes __check_eq(unsigned long pstate)
  53 {
  54         return (pstate & PSR_Z_BIT) != 0;
  55 }
  56
  57 static bool __kprobes __check_ne(unsigned long pstate)
  58 {
  59         return (pstate & PSR_Z_BIT) == 0;
  60 }
  61
  62 static bool __kprobes __check_cs(unsigned long pstate)
  63 {
  64         return (pstate & PSR_C_BIT) != 0;
  65 }
  66
  67 static bool __kprobes __check_cc(unsigned long pstate)
  68 {
  69         return (pstate & PSR_C_BIT) == 0;
  70 }
  71
  72 static bool __kprobes __check_mi(unsigned long pstate)
  73 {
  74         return (pstate & PSR_N_BIT) != 0;
  75 }
  76
  77 static bool __kprobes __check_pl(unsigned long pstate)
  78 {
  79         return (pstate & PSR_N_BIT) == 0;
  80 }
  81
  82 static bool __kprobes __check_vs(unsigned long pstate)
  83 {
  84         return (pstate & PSR_V_BIT) != 0;
  85 }
  86
  87 static bool __kprobes __check_vc(unsigned long pstate)
  88 {
  89         return (pstate & PSR_V_BIT) == 0;
  90 }
  91
  92 static bool __kprobes __check_hi(unsigned long pstate)
  93 {
  94         pstate &= ~(pstate >> 1);       /* PSR_C_BIT &= ~PSR_Z_BIT */
  95         return (pstate & PSR_C_BIT) != 0;
  96 }
  97
  98 static bool __kprobes __check_ls(unsigned long pstate)
  99 {
 100         pstate &= ~(pstate >> 1);       /* PSR_C_BIT &= ~PSR_Z_BIT */
 101         return (pstate & PSR_C_BIT) == 0;
 102 }
 103
 104 static bool __kprobes __check_ge(unsigned long pstate)
 105 {
 106         pstate ^= (pstate << 3);        /* PSR_N_BIT ^= PSR_V_BIT */
 107         return (pstate & PSR_N_BIT) == 0;
 108 }
 109
 110 static bool __kprobes __check_lt(unsigned long pstate)
 111 {
 112         pstate ^= (pstate << 3);        /* PSR_N_BIT ^= PSR_V_BIT */
 113         return (pstate & PSR_N_BIT) != 0;
 114 }
 115
 116 static bool __kprobes __check_gt(unsigned long pstate)
 117 {
 118         /*PSR_N_BIT ^= PSR_V_BIT */
 119         unsigned long temp = pstate ^ (pstate << 3);
 120
 121         temp |= (pstate << 1);  /*PSR_N_BIT |= PSR_Z_BIT */
 122         return (temp & PSR_N_BIT) == 0;
 123 }
 124
 125 static bool __kprobes __check_le(unsigned long pstate)
 126 {
 127         /*PSR_N_BIT ^= PSR_V_BIT */
 128         unsigned long temp = pstate ^ (pstate << 3);
 129
 130         temp |= (pstate << 1);  /*PSR_N_BIT |= PSR_Z_BIT */
 131         return (temp & PSR_N_BIT) != 0;
 132 }
 133
 134 static bool __kprobes __check_al(unsigned long pstate)
 135 {
 136         return true;
 137 }
 138
 139 /*
 140  * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
 141  * it behaves identically to 0b1110 ("al").
 142  */
 143 pstate_check_t * const aarch32_opcode_cond_checks[16] = {
 144         __check_eq, __check_ne, __check_cs, __check_cc,
 145         __check_mi, __check_pl, __check_vs, __check_vc,
 146         __check_hi, __check_ls, __check_ge, __check_lt,
 147         __check_gt, __check_le, __check_al, __check_al
 148 };
 149
 150 int show_unhandled_signals = 0;
 151
 152 static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
 153 {
 154         unsigned long addr = instruction_pointer(regs);
 155         char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
 156         int i;
 157
 158         if (user_mode(regs))
 159                 return;
 160
 161         for (i = -4; i < 1; i++) {
 162                 unsigned int val, bad;
 163
 164                 bad = aarch64_insn_read(&((u32 *)addr)[i], &val);
 165
 166                 if (!bad)
 167                         p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
 168                 else
 169                         p += sprintf(p, i == 0 ? "(????????) " : "???????? ");
 170         }
 171
 172         printk("%sCode: %s\n", lvl, str);
 173 }
 174
 175 #ifdef CONFIG_PREEMPT
 176 #define S_PREEMPT " PREEMPT"
 177 #elif defined(CONFIG_PREEMPT_RT)
 178 #define S_PREEMPT " PREEMPT_RT"
 179 #else
 180 #define S_PREEMPT ""
 181 #endif
 182
 183 #define S_SMP " SMP"
 184
 185 static int __die(const char *str, long err, struct pt_regs *regs)
 186 {
 187         static int die_counter;
 188         int ret;
 189
 190         pr_emerg("Internal error: %s: %016lx [#%d]" S_PREEMPT S_SMP "\n",
 191                  str, err, ++die_counter);
 192
 193         /* trap and error numbers are mostly meaningless on ARM */
 194         ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
 195         if (ret == NOTIFY_STOP)
 196                 return ret;
 197
 198         print_modules();
 199         show_regs(regs);
 200
 201         dump_kernel_instr(KERN_EMERG, regs);
 202
 203         return ret;
 204 }
 205
 206 static DEFINE_RAW_SPINLOCK(die_lock);
 207
 208 /*
 209  * This function is protected against re-entrancy.
 210  */
 211 void die(const char *str, struct pt_regs *regs, long err)
 212 {
 213         int ret;
 214         unsigned long flags;
 215
 216         raw_spin_lock_irqsave(&die_lock, flags);
 217
 218         oops_enter();
 219
 220         console_verbose();
 221         bust_spinlocks(1);
 222         ret = __die(str, err, regs);
 223
 224         if (regs && kexec_should_crash(current))
 225                 crash_kexec(regs);
 226
 227         bust_spinlocks(0);
 228         add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
 229         oops_exit();
 230
 231         if (in_interrupt())
 232                 panic("%s: Fatal exception in interrupt", str);
 233         if (panic_on_oops)
 234                 panic("%s: Fatal exception", str);
 235
 236         raw_spin_unlock_irqrestore(&die_lock, flags);
 237
 238         if (ret != NOTIFY_STOP)
 239                 make_task_dead(SIGSEGV);
 240 }
 241
 242 static void arm64_show_signal(int signo, const char *str)
 243 {
 244         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 245                                       DEFAULT_RATELIMIT_BURST);
 246         struct task_struct *tsk = current;
 247         unsigned long esr = tsk->thread.fault_code;
 248         struct pt_regs *regs = task_pt_regs(tsk);
 249
 250         /* Leave if the signal won't be shown */
 251         if (!show_unhandled_signals ||
 252             !unhandled_signal(tsk, signo) ||
 253             !__ratelimit(&rs))
 254                 return;
 255
 256         pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
 257         if (esr)
 258                 pr_cont("%s, ESR 0x%016lx, ", esr_get_class_string(esr), esr);
 259
 260         pr_cont("%s", str);
 261         print_vma_addr(KERN_CONT " in ", regs->pc);
 262         pr_cont("\n");
 263         __show_regs(regs);
 264 }
 265
 266 void arm64_force_sig_fault(int signo, int code, unsigned long far,
 267                            const char *str)
 268 {
 269         arm64_show_signal(signo, str);
 270         if (signo == SIGKILL)
 271                 force_sig(SIGKILL);
 272         else
 273                 force_sig_fault(signo, code, (void __user *)far);
 274 }
 275
 276 void arm64_force_sig_fault_pkey(unsigned long far, const char *str, int pkey)
 277 {
 278         arm64_show_signal(SIGSEGV, str);
 279         force_sig_pkuerr((void __user *)far, pkey);
 280 }
 281
 282 void arm64_force_sig_mceerr(int code, unsigned long far, short lsb,
 283                             const char *str)
 284 {
 285         arm64_show_signal(SIGBUS, str);
 286         force_sig_mceerr(code, (void __user *)far, lsb);
 287 }
 288
 289 void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far,
 290                                        const char *str)
 291 {
 292         arm64_show_signal(SIGTRAP, str);
 293         force_sig_ptrace_errno_trap(errno, (void __user *)far);
 294 }
 295
 296 void arm64_notify_die(const char *str, struct pt_regs *regs,
 297                       int signo, int sicode, unsigned long far,
 298                       unsigned long err)
 299 {
 300         if (user_mode(regs)) {
 301                 WARN_ON(regs != current_pt_regs());
 302                 current->thread.fault_address = 0;
 303                 current->thread.fault_code = err;
 304
 305                 arm64_force_sig_fault(signo, sicode, far, str);
 306         } else {
 307                 die(str, regs, err);
 308         }
 309 }
 310
 311 #ifdef CONFIG_COMPAT
 312 #define PSTATE_IT_1_0_SHIFT     25
 313 #define PSTATE_IT_1_0_MASK      (0x3 << PSTATE_IT_1_0_SHIFT)
 314 #define PSTATE_IT_7_2_SHIFT     10
 315 #define PSTATE_IT_7_2_MASK      (0x3f << PSTATE_IT_7_2_SHIFT)
 316
 317 static u32 compat_get_it_state(struct pt_regs *regs)
 318 {
 319         u32 it, pstate = regs->pstate;
 320
 321         it  = (pstate & PSTATE_IT_1_0_MASK) >> PSTATE_IT_1_0_SHIFT;
 322         it |= ((pstate & PSTATE_IT_7_2_MASK) >> PSTATE_IT_7_2_SHIFT) << 2;
 323
 324         return it;
 325 }
 326
 327 static void compat_set_it_state(struct pt_regs *regs, u32 it)
 328 {
 329         u32 pstate_it;
 330
 331         pstate_it  = (it << PSTATE_IT_1_0_SHIFT) & PSTATE_IT_1_0_MASK;
 332         pstate_it |= ((it >> 2) << PSTATE_IT_7_2_SHIFT) & PSTATE_IT_7_2_MASK;
 333
 334         regs->pstate &= ~PSR_AA32_IT_MASK;
 335         regs->pstate |= pstate_it;
 336 }
 337
 338 static void advance_itstate(struct pt_regs *regs)
 339 {
 340         u32 it;
 341
 342         /* ARM mode */
 343         if (!(regs->pstate & PSR_AA32_T_BIT) ||
 344             !(regs->pstate & PSR_AA32_IT_MASK))
 345                 return;
 346
 347         it  = compat_get_it_state(regs);
 348
 349         /*
 350          * If this is the last instruction of the block, wipe the IT
 351          * state. Otherwise advance it.
 352          */
 353         if (!(it & 7))
 354                 it = 0;
 355         else
 356                 it = (it & 0xe0) | ((it << 1) & 0x1f);
 357
 358         compat_set_it_state(regs, it);
 359 }
 360 #else
 361 static void advance_itstate(struct pt_regs *regs)
 362 {
 363 }
 364 #endif
 365
 366 void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
 367 {
 368         regs->pc += size;
 369
 370         /*
 371          * If we were single stepping, we want to get the step exception after
 372          * we return from the trap.
 373          */
 374         if (user_mode(regs))
 375                 user_fastforward_single_step(current);
 376
 377         if (compat_user_mode(regs))
 378                 advance_itstate(regs);
 379         else
 380                 regs->pstate &= ~PSR_BTYPE_MASK;
 381 }
 382
 383 static int user_insn_read(struct pt_regs *regs, u32 *insnp)
 384 {
 385         u32 instr;
 386         unsigned long pc = instruction_pointer(regs);
 387
 388         if (compat_thumb_mode(regs)) {
 389                 /* 16-bit Thumb instruction */
 390                 __le16 instr_le;
 391                 if (get_user(instr_le, (__le16 __user *)pc))
 392                         return -EFAULT;
 393                 instr = le16_to_cpu(instr_le);
 394                 if (aarch32_insn_is_wide(instr)) {
 395                         u32 instr2;
 396
 397                         if (get_user(instr_le, (__le16 __user *)(pc + 2)))
 398                                 return -EFAULT;
 399                         instr2 = le16_to_cpu(instr_le);
 400                         instr = (instr << 16) | instr2;
 401                 }
 402         } else {
 403                 /* 32-bit ARM instruction */
 404                 __le32 instr_le;
 405                 if (get_user(instr_le, (__le32 __user *)pc))
 406                         return -EFAULT;
 407                 instr = le32_to_cpu(instr_le);
 408         }
 409
 410         *insnp = instr;
 411         return 0;
 412 }
 413
 414 void force_signal_inject(int signal, int code, unsigned long address, unsigned long err)
 415 {
 416         const char *desc;
 417         struct pt_regs *regs = current_pt_regs();
 418
 419         if (WARN_ON(!user_mode(regs)))
 420                 return;
 421
 422         switch (signal) {
 423         case SIGILL:
 424                 desc = "undefined instruction";
 425                 break;
 426         case SIGSEGV:
 427                 desc = "illegal memory access";
 428                 break;
 429         default:
 430                 desc = "unknown or unrecoverable error";
 431                 break;
 432         }
 433
 434         /* Force signals we don't understand to SIGKILL */
 435         if (WARN_ON(signal != SIGKILL &&
 436                     siginfo_layout(signal, code) != SIL_FAULT)) {
 437                 signal = SIGKILL;
 438         }
 439
 440         arm64_notify_die(desc, regs, signal, code, address, err);
 441 }
 442
 443 /*
 444  * Set up process info to signal segmentation fault - called on access error.
 445  */
 446 void arm64_notify_segfault(unsigned long addr)
 447 {
 448         int code;
 449
 450         mmap_read_lock(current->mm);
 451         if (find_vma(current->mm, untagged_addr(addr)) == NULL)
 452                 code = SEGV_MAPERR;
 453         else
 454                 code = SEGV_ACCERR;
 455         mmap_read_unlock(current->mm);
 456
 457         force_signal_inject(SIGSEGV, code, addr, 0);
 458 }
 459
 460 void do_el0_undef(struct pt_regs *regs, unsigned long esr)
 461 {
 462         u32 insn;
 463
 464         /* check for AArch32 breakpoint instructions */
 465         if (!aarch32_break_handler(regs))
 466                 return;
 467
 468         if (user_insn_read(regs, &insn))
 469                 goto out_err;
 470
 471         if (try_emulate_mrs(regs, insn))
 472                 return;
 473
 474         if (try_emulate_armv8_deprecated(regs, insn))
 475                 return;
 476
 477 out_err:
 478         force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 479 }
 480
 481 void do_el1_undef(struct pt_regs *regs, unsigned long esr)
 482 {
 483         u32 insn;
 484
 485         if (aarch64_insn_read((void *)regs->pc, &insn))
 486                 goto out_err;
 487
 488         if (try_emulate_el1_ssbs(regs, insn))
 489                 return;
 490
 491 out_err:
 492         die("Oops - Undefined instruction", regs, esr);
 493 }
 494
 495 void do_el0_bti(struct pt_regs *regs)
 496 {
 497         force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 498 }
 499
 500 void do_el1_bti(struct pt_regs *regs, unsigned long esr)
 501 {
 502         if (efi_runtime_fixup_exception(regs, "BTI violation")) {
 503                 regs->pstate &= ~PSR_BTYPE_MASK;
 504                 return;
 505         }
 506         die("Oops - BTI", regs, esr);
 507 }
 508
 509 void do_el0_gcs(struct pt_regs *regs, unsigned long esr)
 510 {
 511         force_signal_inject(SIGSEGV, SEGV_CPERR, regs->pc, 0);
 512 }
 513
 514 void do_el1_gcs(struct pt_regs *regs, unsigned long esr)
 515 {
 516         die("Oops - GCS", regs, esr);
 517 }
 518
 519 void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
 520 {
 521         force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
 522 }
 523
 524 void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
 525 {
 526         /*
 527          * Unexpected FPAC exception in the kernel: kill the task before it
 528          * does any more harm.
 529          */
 530         die("Oops - FPAC", regs, esr);
 531 }
 532
 533 void do_el0_mops(struct pt_regs *regs, unsigned long esr)
 534 {
 535         arm64_mops_reset_regs(&regs->user_regs, esr);
 536
 537         /*
 538          * If single stepping then finish the step before executing the
 539          * prologue instruction.
 540          */
 541         user_fastforward_single_step(current);
 542 }
 543
 544 void do_el1_mops(struct pt_regs *regs, unsigned long esr)
 545 {
 546         arm64_mops_reset_regs(&regs->user_regs, esr);
 547
 548         kernel_fastforward_single_step(regs);
 549 }
 550
 551 #define __user_cache_maint(insn, address, res)                  \
 552         if (address >= TASK_SIZE_MAX) {                         \
 553                 res = -EFAULT;                                  \
 554         } else {                                                \
 555                 uaccess_ttbr0_enable();                         \
 556                 asm volatile (                                  \
 557                         "1:     " insn ", %1\n"                 \
 558                         "       mov     %w0, #0\n"              \
 559                         "2:\n"                                  \
 560                         _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0)   \
 561                         : "=r" (res)                            \
 562                         : "r" (address));                       \
 563                 uaccess_ttbr0_disable();                        \
 564         }
 565
 566 static void user_cache_maint_handler(unsigned long esr, struct pt_regs *regs)
 567 {
 568         unsigned long tagged_address, address;
 569         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 570         int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
 571         int ret = 0;
 572
 573         tagged_address = pt_regs_read_reg(regs, rt);
 574         address = untagged_addr(tagged_address);
 575
 576         switch (crm) {
 577         case ESR_ELx_SYS64_ISS_CRM_DC_CVAU:     /* DC CVAU, gets promoted */
 578                 __user_cache_maint("dc civac", address, ret);
 579                 break;
 580         case ESR_ELx_SYS64_ISS_CRM_DC_CVAC:     /* DC CVAC, gets promoted */
 581                 __user_cache_maint("dc civac", address, ret);
 582                 break;
 583         case ESR_ELx_SYS64_ISS_CRM_DC_CVADP:    /* DC CVADP */
 584                 __user_cache_maint("sys 3, c7, c13, 1", address, ret);
 585                 break;
 586         case ESR_ELx_SYS64_ISS_CRM_DC_CVAP:     /* DC CVAP */
 587                 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
 588                 break;
 589         case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC:    /* DC CIVAC */
 590                 __user_cache_maint("dc civac", address, ret);
 591                 break;
 592         case ESR_ELx_SYS64_ISS_CRM_IC_IVAU:     /* IC IVAU */
 593                 __user_cache_maint("ic ivau", address, ret);
 594                 break;
 595         default:
 596                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 597                 return;
 598         }
 599
 600         if (ret)
 601                 arm64_notify_segfault(tagged_address);
 602         else
 603                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 604 }
 605
 606 static void ctr_read_handler(unsigned long esr, struct pt_regs *regs)
 607 {
 608         int rt = ESR_ELx_SYS64_ISS_RT(esr);
 609         unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
 610
 611         if (cpus_have_final_cap(ARM64_WORKAROUND_1542419)) {
 612                 /* Hide DIC so that we can trap the unnecessary maintenance...*/
 613                 val &= ~BIT(CTR_EL0_DIC_SHIFT);
 614
 615                 /* ... and fake IminLine to reduce the number of traps. */
 616                 val &= ~CTR_EL0_IminLine_MASK;
 617                 val |= (PAGE_SHIFT - 2) & CTR_EL0_IminLine_MASK;
 618         }
 619
 620         pt_regs_write_reg(regs, rt, val);
 621
 622         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 623 }
 624
 625 static void cntvct_read_handler(unsigned long esr, struct pt_regs *regs)
 626 {
 627         if (test_thread_flag(TIF_TSC_SIGSEGV)) {
 628                 force_sig(SIGSEGV);
 629         } else {
 630                 int rt = ESR_ELx_SYS64_ISS_RT(esr);
 631
 632                 pt_regs_write_reg(regs, rt, arch_timer_read_counter());
 633                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 634         }
 635 }
 636
 637 static void cntfrq_read_handler(unsigned long esr, struct pt_regs *regs)
 638 {
 639         if (test_thread_flag(TIF_TSC_SIGSEGV)) {
 640                 force_sig(SIGSEGV);
 641         } else {
 642                 int rt = ESR_ELx_SYS64_ISS_RT(esr);
 643
 644                 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
 645                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 646         }
 647 }
 648
 649 static void mrs_handler(unsigned long esr, struct pt_regs *regs)
 650 {
 651         u32 sysreg, rt;
 652
 653         rt = ESR_ELx_SYS64_ISS_RT(esr);
 654         sysreg = esr_sys64_to_sysreg(esr);
 655
 656         if (do_emulate_mrs(regs, sysreg, rt) != 0)
 657                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 658 }
 659
 660 static void wfi_handler(unsigned long esr, struct pt_regs *regs)
 661 {
 662         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
 663 }
 664
 665 struct sys64_hook {
 666         unsigned long esr_mask;
 667         unsigned long esr_val;
 668         void (*handler)(unsigned long esr, struct pt_regs *regs);
 669 };
 670
 671 static const struct sys64_hook sys64_hooks[] = {
 672         {
 673                 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
 674                 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
 675                 .handler = user_cache_maint_handler,
 676         },
 677         {
 678                 /* Trap read access to CTR_EL0 */
 679                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 680                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
 681                 .handler = ctr_read_handler,
 682         },
 683         {
 684                 /* Trap read access to CNTVCT_EL0 */
 685                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 686                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
 687                 .handler = cntvct_read_handler,
 688         },
 689         {
 690                 /* Trap read access to CNTVCTSS_EL0 */
 691                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 692                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCTSS,
 693                 .handler = cntvct_read_handler,
 694         },
 695         {
 696                 /* Trap read access to CNTFRQ_EL0 */
 697                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
 698                 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
 699                 .handler = cntfrq_read_handler,
 700         },
 701         {
 702                 /* Trap read access to CPUID registers */
 703                 .esr_mask = ESR_ELx_SYS64_ISS_SYS_MRS_OP_MASK,
 704                 .esr_val = ESR_ELx_SYS64_ISS_SYS_MRS_OP_VAL,
 705                 .handler = mrs_handler,
 706         },
 707         {
 708                 /* Trap WFI instructions executed in userspace */
 709                 .esr_mask = ESR_ELx_WFx_MASK,
 710                 .esr_val = ESR_ELx_WFx_WFI_VAL,
 711                 .handler = wfi_handler,
 712         },
 713         {},
 714 };
 715
 716 #ifdef CONFIG_COMPAT
 717 static bool cp15_cond_valid(unsigned long esr, struct pt_regs *regs)
 718 {
 719         int cond;
 720
 721         /* Only a T32 instruction can trap without CV being set */
 722         if (!(esr & ESR_ELx_CV)) {
 723                 u32 it;
 724
 725                 it = compat_get_it_state(regs);
 726                 if (!it)
 727                         return true;
 728
 729                 cond = it >> 4;
 730         } else {
 731                 cond = (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
 732         }
 733
 734         return aarch32_opcode_cond_checks[cond](regs->pstate);
 735 }
 736
 737 static void compat_cntfrq_read_handler(unsigned long esr, struct pt_regs *regs)
 738 {
 739         int reg = (esr & ESR_ELx_CP15_32_ISS_RT_MASK) >> ESR_ELx_CP15_32_ISS_RT_SHIFT;
 740
 741         pt_regs_write_reg(regs, reg, arch_timer_get_rate());
 742         arm64_skip_faulting_instruction(regs, 4);
 743 }
 744
 745 static const struct sys64_hook cp15_32_hooks[] = {
 746         {
 747                 .esr_mask = ESR_ELx_CP15_32_ISS_SYS_MASK,
 748                 .esr_val = ESR_ELx_CP15_32_ISS_SYS_CNTFRQ,
 749                 .handler = compat_cntfrq_read_handler,
 750         },
 751         {},
 752 };
 753
 754 static void compat_cntvct_read_handler(unsigned long esr, struct pt_regs *regs)
 755 {
 756         int rt = (esr & ESR_ELx_CP15_64_ISS_RT_MASK) >> ESR_ELx_CP15_64_ISS_RT_SHIFT;
 757         int rt2 = (esr & ESR_ELx_CP15_64_ISS_RT2_MASK) >> ESR_ELx_CP15_64_ISS_RT2_SHIFT;
 758         u64 val = arch_timer_read_counter();
 759
 760         pt_regs_write_reg(regs, rt, lower_32_bits(val));
 761         pt_regs_write_reg(regs, rt2, upper_32_bits(val));
 762         arm64_skip_faulting_instruction(regs, 4);
 763 }
 764
 765 static const struct sys64_hook cp15_64_hooks[] = {
 766         {
 767                 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
 768                 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCT,
 769                 .handler = compat_cntvct_read_handler,
 770         },
 771         {
 772                 .esr_mask = ESR_ELx_CP15_64_ISS_SYS_MASK,
 773                 .esr_val = ESR_ELx_CP15_64_ISS_SYS_CNTVCTSS,
 774                 .handler = compat_cntvct_read_handler,
 775         },
 776         {},
 777 };
 778
 779 void do_el0_cp15(unsigned long esr, struct pt_regs *regs)
 780 {
 781         const struct sys64_hook *hook, *hook_base;
 782
 783         if (!cp15_cond_valid(esr, regs)) {
 784                 /*
 785                  * There is no T16 variant of a CP access, so we
 786                  * always advance PC by 4 bytes.
 787                  */
 788                 arm64_skip_faulting_instruction(regs, 4);
 789                 return;
 790         }
 791
 792         switch (ESR_ELx_EC(esr)) {
 793         case ESR_ELx_EC_CP15_32:
 794                 hook_base = cp15_32_hooks;
 795                 break;
 796         case ESR_ELx_EC_CP15_64:
 797                 hook_base = cp15_64_hooks;
 798                 break;
 799         default:
 800                 do_el0_undef(regs, esr);
 801                 return;
 802         }
 803
 804         for (hook = hook_base; hook->handler; hook++)
 805                 if ((hook->esr_mask & esr) == hook->esr_val) {
 806                         hook->handler(esr, regs);
 807                         return;
 808                 }
 809
 810         /*
 811          * New cp15 instructions may previously have been undefined at
 812          * EL0. Fall back to our usual undefined instruction handler
 813          * so that we handle these consistently.
 814          */
 815         do_el0_undef(regs, esr);
 816 }
 817 #endif
 818
 819 void do_el0_sys(unsigned long esr, struct pt_regs *regs)
 820 {
 821         const struct sys64_hook *hook;
 822
 823         for (hook = sys64_hooks; hook->handler; hook++)
 824                 if ((hook->esr_mask & esr) == hook->esr_val) {
 825                         hook->handler(esr, regs);
 826                         return;
 827                 }
 828
 829         /*
 830          * New SYS instructions may previously have been undefined at EL0. Fall
 831          * back to our usual undefined instruction handler so that we handle
 832          * these consistently.
 833          */
 834         do_el0_undef(regs, esr);
 835 }
 836
 837 static const char *esr_class_str[] = {
 838         [0 ... ESR_ELx_EC_MAX]          = "UNRECOGNIZED EC",
 839         [ESR_ELx_EC_UNKNOWN]            = "Unknown/Uncategorized",
 840         [ESR_ELx_EC_WFx]                = "WFI/WFE",
 841         [ESR_ELx_EC_CP15_32]            = "CP15 MCR/MRC",
 842         [ESR_ELx_EC_CP15_64]            = "CP15 MCRR/MRRC",
 843         [ESR_ELx_EC_CP14_MR]            = "CP14 MCR/MRC",
 844         [ESR_ELx_EC_CP14_LS]            = "CP14 LDC/STC",
 845         [ESR_ELx_EC_FP_ASIMD]           = "ASIMD",
 846         [ESR_ELx_EC_CP10_ID]            = "CP10 MRC/VMRS",
 847         [ESR_ELx_EC_PAC]                = "PAC",
 848         [ESR_ELx_EC_CP14_64]            = "CP14 MCRR/MRRC",
 849         [ESR_ELx_EC_BTI]                = "BTI",
 850         [ESR_ELx_EC_ILL]                = "PSTATE.IL",
 851         [ESR_ELx_EC_SVC32]              = "SVC (AArch32)",
 852         [ESR_ELx_EC_HVC32]              = "HVC (AArch32)",
 853         [ESR_ELx_EC_SMC32]              = "SMC (AArch32)",
 854         [ESR_ELx_EC_SVC64]              = "SVC (AArch64)",
 855         [ESR_ELx_EC_HVC64]              = "HVC (AArch64)",
 856         [ESR_ELx_EC_SMC64]              = "SMC (AArch64)",
 857         [ESR_ELx_EC_SYS64]              = "MSR/MRS (AArch64)",
 858         [ESR_ELx_EC_SVE]                = "SVE",
 859         [ESR_ELx_EC_ERET]               = "ERET/ERETAA/ERETAB",
 860         [ESR_ELx_EC_FPAC]               = "FPAC",
 861         [ESR_ELx_EC_SME]                = "SME",
 862         [ESR_ELx_EC_IMP_DEF]            = "EL3 IMP DEF",
 863         [ESR_ELx_EC_IABT_LOW]           = "IABT (lower EL)",
 864         [ESR_ELx_EC_IABT_CUR]           = "IABT (current EL)",
 865         [ESR_ELx_EC_PC_ALIGN]           = "PC Alignment",
 866         [ESR_ELx_EC_DABT_LOW]           = "DABT (lower EL)",
 867         [ESR_ELx_EC_DABT_CUR]           = "DABT (current EL)",
 868         [ESR_ELx_EC_SP_ALIGN]           = "SP Alignment",
 869         [ESR_ELx_EC_MOPS]               = "MOPS",
 870         [ESR_ELx_EC_FP_EXC32]           = "FP (AArch32)",
 871         [ESR_ELx_EC_FP_EXC64]           = "FP (AArch64)",
 872         [ESR_ELx_EC_GCS]                = "Guarded Control Stack",
 873         [ESR_ELx_EC_SERROR]             = "SError",
 874         [ESR_ELx_EC_BREAKPT_LOW]        = "Breakpoint (lower EL)",
 875         [ESR_ELx_EC_BREAKPT_CUR]        = "Breakpoint (current EL)",
 876         [ESR_ELx_EC_SOFTSTP_LOW]        = "Software Step (lower EL)",
 877         [ESR_ELx_EC_SOFTSTP_CUR]        = "Software Step (current EL)",
 878         [ESR_ELx_EC_WATCHPT_LOW]        = "Watchpoint (lower EL)",
 879         [ESR_ELx_EC_WATCHPT_CUR]        = "Watchpoint (current EL)",
 880         [ESR_ELx_EC_BKPT32]             = "BKPT (AArch32)",
 881         [ESR_ELx_EC_VECTOR32]           = "Vector catch (AArch32)",
 882         [ESR_ELx_EC_BRK64]              = "BRK (AArch64)",
 883 };
 884
 885 const char *esr_get_class_string(unsigned long esr)
 886 {
 887         return esr_class_str[ESR_ELx_EC(esr)];
 888 }
 889
 890 /*
 891  * bad_el0_sync handles unexpected, but potentially recoverable synchronous
 892  * exceptions taken from EL0.
 893  */
 894 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr)
 895 {
 896         unsigned long pc = instruction_pointer(regs);
 897
 898         current->thread.fault_address = 0;
 899         current->thread.fault_code = esr;
 900
 901         arm64_force_sig_fault(SIGILL, ILL_ILLOPC, pc,
 902                               "Bad EL0 synchronous exception");
 903 }
 904
 905 #ifdef CONFIG_VMAP_STACK
 906
 907 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
 908         __aligned(16);
 909
 910 void __noreturn panic_bad_stack(struct pt_regs *regs, unsigned long esr, unsigned long far)
 911 {
 912         unsigned long tsk_stk = (unsigned long)current->stack;
 913         unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
 914         unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 915
 916         console_verbose();
 917         pr_emerg("Insufficient stack space to handle exception!");
 918
 919         pr_emerg("ESR: 0x%016lx -- %s\n", esr, esr_get_class_string(esr));
 920         pr_emerg("FAR: 0x%016lx\n", far);
 921
 922         pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
 923                  tsk_stk, tsk_stk + THREAD_SIZE);
 924         pr_emerg("IRQ stack:      [0x%016lx..0x%016lx]\n",
 925                  irq_stk, irq_stk + IRQ_STACK_SIZE);
 926         pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
 927                  ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
 928
 929         __show_regs(regs);
 930
 931         /*
 932          * We use nmi_panic to limit the potential for recusive overflows, and
 933          * to get a better stack trace.
 934          */
 935         nmi_panic(NULL, "kernel stack overflow");
 936         cpu_park_loop();
 937 }
 938 #endif
 939
 940 void __noreturn arm64_serror_panic(struct pt_regs *regs, unsigned long esr)
 941 {
 942         console_verbose();
 943
 944         pr_crit("SError Interrupt on CPU%d, code 0x%016lx -- %s\n",
 945                 smp_processor_id(), esr, esr_get_class_string(esr));
 946         if (regs)
 947                 __show_regs(regs);
 948
 949         nmi_panic(regs, "Asynchronous SError Interrupt");
 950
 951         cpu_park_loop();
 952 }
 953
 954 bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned long esr)
 955 {
 956         unsigned long aet = arm64_ras_serror_get_severity(esr);
 957
 958         switch (aet) {
 959         case ESR_ELx_AET_CE:    /* corrected error */
 960         case ESR_ELx_AET_UEO:   /* restartable, not yet consumed */
 961                 /*
 962                  * The CPU can make progress. We may take UEO again as
 963                  * a more severe error.
 964                  */
 965                 return false;
 966
 967         case ESR_ELx_AET_UEU:   /* Uncorrected Unrecoverable */
 968         case ESR_ELx_AET_UER:   /* Uncorrected Recoverable */
 969                 /*
 970                  * The CPU can't make progress. The exception may have
 971                  * been imprecise.
 972                  *
 973                  * Neoverse-N1 #1349291 means a non-KVM SError reported as
 974                  * Unrecoverable should be treated as Uncontainable. We
 975                  * call arm64_serror_panic() in both cases.
 976                  */
 977                 return true;
 978
 979         case ESR_ELx_AET_UC:    /* Uncontainable or Uncategorized error */
 980         default:
 981                 /* Error has been silently propagated */
 982                 arm64_serror_panic(regs, esr);
 983         }
 984 }
 985
 986 void do_serror(struct pt_regs *regs, unsigned long esr)
 987 {
 988         /* non-RAS errors are not containable */
 989         if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
 990                 arm64_serror_panic(regs, esr);
 991 }
 992
 993 /* GENERIC_BUG traps */
 994 #ifdef CONFIG_GENERIC_BUG
 995 int is_valid_bugaddr(unsigned long addr)
 996 {
 997         /*
 998          * bug_handler() only called for BRK #BUG_BRK_IMM.
 999          * So the answer is trivial -- any spurious instances with no
1000          * bug table entry will be rejected by report_bug() and passed
1001          * back to the debug-monitors code and handled as a fatal
1002          * unexpected debug exception.
1003          */
1004         return 1;
1005 }
1006 #endif
1007
1008 static int bug_handler(struct pt_regs *regs, unsigned long esr)
1009 {
1010         switch (report_bug(regs->pc, regs)) {
1011         case BUG_TRAP_TYPE_BUG:
1012                 die("Oops - BUG", regs, esr);
1013                 break;
1014
1015         case BUG_TRAP_TYPE_WARN:
1016                 break;
1017
1018         default:
1019                 /* unknown/unrecognised bug trap type */
1020                 return DBG_HOOK_ERROR;
1021         }
1022
1023         /* If thread survives, skip over the BUG instruction and continue: */
1024         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1025         return DBG_HOOK_HANDLED;
1026 }
1027
1028 static struct break_hook bug_break_hook = {
1029         .fn = bug_handler,
1030         .imm = BUG_BRK_IMM,
1031 };
1032
1033 #ifdef CONFIG_CFI_CLANG
1034 static int cfi_handler(struct pt_regs *regs, unsigned long esr)
1035 {
1036         unsigned long target;
1037         u32 type;
1038
1039         target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr));
1040         type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr));
1041
1042         switch (report_cfi_failure(regs, regs->pc, &target, type)) {
1043         case BUG_TRAP_TYPE_BUG:
1044                 die("Oops - CFI", regs, esr);
1045                 break;
1046
1047         case BUG_TRAP_TYPE_WARN:
1048                 break;
1049
1050         default:
1051                 return DBG_HOOK_ERROR;
1052         }
1053
1054         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1055         return DBG_HOOK_HANDLED;
1056 }
1057
1058 static struct break_hook cfi_break_hook = {
1059         .fn = cfi_handler,
1060         .imm = CFI_BRK_IMM_BASE,
1061         .mask = CFI_BRK_IMM_MASK,
1062 };
1063 #endif /* CONFIG_CFI_CLANG */
1064
1065 static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr)
1066 {
1067         pr_err("%s generated an invalid instruction at %pS!\n",
1068                 "Kernel text patching",
1069                 (void *)instruction_pointer(regs));
1070
1071         /* We cannot handle this */
1072         return DBG_HOOK_ERROR;
1073 }
1074
1075 static struct break_hook fault_break_hook = {
1076         .fn = reserved_fault_handler,
1077         .imm = FAULT_BRK_IMM,
1078 };
1079
1080 #ifdef CONFIG_KASAN_SW_TAGS
1081
1082 #define KASAN_ESR_RECOVER       0x20
1083 #define KASAN_ESR_WRITE 0x10
1084 #define KASAN_ESR_SIZE_MASK     0x0f
1085 #define KASAN_ESR_SIZE(esr)     (1 << ((esr) & KASAN_ESR_SIZE_MASK))
1086
1087 static int kasan_handler(struct pt_regs *regs, unsigned long esr)
1088 {
1089         bool recover = esr & KASAN_ESR_RECOVER;
1090         bool write = esr & KASAN_ESR_WRITE;
1091         size_t size = KASAN_ESR_SIZE(esr);
1092         void *addr = (void *)regs->regs[0];
1093         u64 pc = regs->pc;
1094
1095         kasan_report(addr, size, write, pc);
1096
1097         /*
1098          * The instrumentation allows to control whether we can proceed after
1099          * a crash was detected. This is done by passing the -recover flag to
1100          * the compiler. Disabling recovery allows to generate more compact
1101          * code.
1102          *
1103          * Unfortunately disabling recovery doesn't work for the kernel right
1104          * now. KASAN reporting is disabled in some contexts (for example when
1105          * the allocator accesses slab object metadata; this is controlled by
1106          * current->kasan_depth). All these accesses are detected by the tool,
1107          * even though the reports for them are not printed.
1108          *
1109          * This is something that might be fixed at some point in the future.
1110          */
1111         if (!recover)
1112                 die("Oops - KASAN", regs, esr);
1113
1114         /* If thread survives, skip over the brk instruction and continue: */
1115         arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
1116         return DBG_HOOK_HANDLED;
1117 }
1118
1119 static struct break_hook kasan_break_hook = {
1120         .fn     = kasan_handler,
1121         .imm    = KASAN_BRK_IMM,
1122         .mask   = KASAN_BRK_MASK,
1123 };
1124 #endif
1125
1126 #ifdef CONFIG_UBSAN_TRAP
1127 static int ubsan_handler(struct pt_regs *regs, unsigned long esr)
1128 {
1129         die(report_ubsan_failure(regs, esr & UBSAN_BRK_MASK), regs, esr);
1130         return DBG_HOOK_HANDLED;
1131 }
1132
1133 static struct break_hook ubsan_break_hook = {
1134         .fn     = ubsan_handler,
1135         .imm    = UBSAN_BRK_IMM,
1136         .mask   = UBSAN_BRK_MASK,
1137 };
1138 #endif
1139
1140 /*
1141  * Initial handler for AArch64 BRK exceptions
1142  * This handler only used until debug_traps_init().
1143  */
1144 int __init early_brk64(unsigned long addr, unsigned long esr,
1145                 struct pt_regs *regs)
1146 {
1147 #ifdef CONFIG_CFI_CLANG
1148         if (esr_is_cfi_brk(esr))
1149                 return cfi_handler(regs, esr) != DBG_HOOK_HANDLED;
1150 #endif
1151 #ifdef CONFIG_KASAN_SW_TAGS
1152         if ((esr_brk_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
1153                 return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
1154 #endif
1155 #ifdef CONFIG_UBSAN_TRAP
1156         if ((esr_brk_comment(esr) & ~UBSAN_BRK_MASK) == UBSAN_BRK_IMM)
1157                 return ubsan_handler(regs, esr) != DBG_HOOK_HANDLED;
1158 #endif
1159         return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
1160 }
1161
1162 void __init trap_init(void)
1163 {
1164         register_kernel_break_hook(&bug_break_hook);
1165 #ifdef CONFIG_CFI_CLANG
1166         register_kernel_break_hook(&cfi_break_hook);
1167 #endif
1168         register_kernel_break_hook(&fault_break_hook);
1169 #ifdef CONFIG_KASAN_SW_TAGS
1170         register_kernel_break_hook(&kasan_break_hook);
1171 #endif
1172 #ifdef CONFIG_UBSAN_TRAP
1173         register_kernel_break_hook(&ubsan_break_hook);
1174 #endif
1175         debug_traps_init();
1176 }