arch/powerpc/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright (C) IBM Corporation, 2002, 2004
   6  *
   7  * 2002-Oct     Created by Vamsi Krishna S <[email protected]> Kernel
   8  *              Probes initial implementation ( includes contributions from
   9  *              Rusty Russell).
  10  * 2004-July    Suparna Bhattacharya <[email protected]> added jumper probes
  11  *              interface to access function arguments.
  12  * 2004-Nov     Ananth N Mavinakayanahalli <[email protected]> kprobes port
  13  *              for PPC64
  14  */
  15
  16 #include <linux/kprobes.h>
  17 #include <linux/ptrace.h>
  18 #include <linux/preempt.h>
  19 #include <linux/extable.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/slab.h>
  22 #include <linux/set_memory.h>
  23 #include <linux/execmem.h>
  24 #include <asm/text-patching.h>
  25 #include <asm/cacheflush.h>
  26 #include <asm/sstep.h>
  27 #include <asm/sections.h>
  28 #include <asm/inst.h>
  29 #include <linux/uaccess.h>
  30
  31 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  32 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  33
  34 struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
  35
  36 bool arch_within_kprobe_blacklist(unsigned long addr)
  37 {
  38         return  (addr >= (unsigned long)__kprobes_text_start &&
  39                  addr < (unsigned long)__kprobes_text_end) ||
  40                 (addr >= (unsigned long)_stext &&
  41                  addr < (unsigned long)__head_end);
  42 }
  43
  44 kprobe_opcode_t *kprobe_lookup_name(const char *name, unsigned int offset)
  45 {
  46         kprobe_opcode_t *addr = NULL;
  47
  48 #ifdef CONFIG_PPC64_ELF_ABI_V2
  49         /* PPC64 ABIv2 needs local entry point */
  50         addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
  51         if (addr && !offset) {
  52 #ifdef CONFIG_KPROBES_ON_FTRACE
  53                 unsigned long faddr;
  54                 /*
  55                  * Per livepatch.h, ftrace location is always within the first
  56                  * 16 bytes of a function on powerpc with -mprofile-kernel.
  57                  */
  58                 faddr = ftrace_location_range((unsigned long)addr,
  59                                               (unsigned long)addr + 16);
  60                 if (faddr)
  61                         addr = (kprobe_opcode_t *)faddr;
  62                 else
  63 #endif
  64                         addr = (kprobe_opcode_t *)ppc_function_entry(addr);
  65         }
  66 #elif defined(CONFIG_PPC64_ELF_ABI_V1)
  67         /*
  68          * 64bit powerpc ABIv1 uses function descriptors:
  69          * - Check for the dot variant of the symbol first.
  70          * - If that fails, try looking up the symbol provided.
  71          *
  72          * This ensures we always get to the actual symbol and not
  73          * the descriptor.
  74          *
  75          * Also handle <module:symbol> format.
  76          */
  77         char dot_name[MODULE_NAME_LEN + 1 + KSYM_NAME_LEN];
  78         bool dot_appended = false;
  79         const char *c;
  80         ssize_t ret = 0;
  81         int len = 0;
  82
  83         if ((c = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
  84                 c++;
  85                 len = c - name;
  86                 memcpy(dot_name, name, len);
  87         } else
  88                 c = name;
  89
  90         if (*c != '\0' && *c != '.') {
  91                 dot_name[len++] = '.';
  92                 dot_appended = true;
  93         }
  94         ret = strscpy(dot_name + len, c, KSYM_NAME_LEN);
  95         if (ret > 0)
  96                 addr = (kprobe_opcode_t *)kallsyms_lookup_name(dot_name);
  97
  98         /* Fallback to the original non-dot symbol lookup */
  99         if (!addr && dot_appended)
 100                 addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
 101 #else
 102         addr = (kprobe_opcode_t *)kallsyms_lookup_name(name);
 103 #endif
 104
 105         return addr;
 106 }
 107
 108 static bool arch_kprobe_on_func_entry(unsigned long addr, unsigned long offset)
 109 {
 110         unsigned long ip = ftrace_location(addr);
 111
 112         if (ip)
 113                 return offset <= (ip - addr);
 114         if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V2) && !IS_ENABLED(CONFIG_PPC_KERNEL_PCREL))
 115                 return offset <= 8;
 116         return !offset;
 117 }
 118
 119 /* XXX try and fold the magic of kprobe_lookup_name() in this */
 120 kprobe_opcode_t *arch_adjust_kprobe_addr(unsigned long addr, unsigned long offset,
 121                                          bool *on_func_entry)
 122 {
 123         *on_func_entry = arch_kprobe_on_func_entry(addr, offset);
 124         return (kprobe_opcode_t *)(addr + offset);
 125 }
 126
 127 int arch_prepare_kprobe(struct kprobe *p)
 128 {
 129         int ret = 0;
 130         struct kprobe *prev;
 131         ppc_inst_t insn = ppc_inst_read(p->addr);
 132
 133         if ((unsigned long)p->addr & 0x03) {
 134                 printk("Attempt to register kprobe at an unaligned address\n");
 135                 ret = -EINVAL;
 136         } else if (!can_single_step(ppc_inst_val(insn))) {
 137                 printk("Cannot register a kprobe on instructions that can't be single stepped\n");
 138                 ret = -EINVAL;
 139         } else if ((unsigned long)p->addr & ~PAGE_MASK &&
 140                    ppc_inst_prefixed(ppc_inst_read(p->addr - 1))) {
 141                 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
 142                 ret = -EINVAL;
 143         }
 144         prev = get_kprobe(p->addr - 1);
 145
 146         /*
 147          * When prev is a ftrace-based kprobe, we don't have an insn, and it
 148          * doesn't probe for prefixed instruction.
 149          */
 150         if (prev && !kprobe_ftrace(prev) &&
 151             ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) {
 152                 printk("Cannot register a kprobe on the second word of prefixed instruction\n");
 153                 ret = -EINVAL;
 154         }
 155
 156         /* insn must be on a special executable page on ppc64.  This is
 157          * not explicitly required on ppc32 (right now), but it doesn't hurt */
 158         if (!ret) {
 159                 p->ainsn.insn = get_insn_slot();
 160                 if (!p->ainsn.insn)
 161                         ret = -ENOMEM;
 162         }
 163
 164         if (!ret) {
 165                 patch_instruction(p->ainsn.insn, insn);
 166                 p->opcode = ppc_inst_val(insn);
 167         }
 168
 169         p->ainsn.boostable = 0;
 170         return ret;
 171 }
 172 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 173
 174 void arch_arm_kprobe(struct kprobe *p)
 175 {
 176         WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(BREAKPOINT_INSTRUCTION)));
 177 }
 178 NOKPROBE_SYMBOL(arch_arm_kprobe);
 179
 180 void arch_disarm_kprobe(struct kprobe *p)
 181 {
 182         WARN_ON_ONCE(patch_instruction(p->addr, ppc_inst(p->opcode)));
 183 }
 184 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 185
 186 void arch_remove_kprobe(struct kprobe *p)
 187 {
 188         if (p->ainsn.insn) {
 189                 free_insn_slot(p->ainsn.insn, 0);
 190                 p->ainsn.insn = NULL;
 191         }
 192 }
 193 NOKPROBE_SYMBOL(arch_remove_kprobe);
 194
 195 static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
 196 {
 197         enable_single_step(regs);
 198
 199         /*
 200          * On powerpc we should single step on the original
 201          * instruction even if the probed insn is a trap
 202          * variant as values in regs could play a part in
 203          * if the trap is taken or not
 204          */
 205         regs_set_return_ip(regs, (unsigned long)p->ainsn.insn);
 206 }
 207
 208 static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
 209 {
 210         kcb->prev_kprobe.kp = kprobe_running();
 211         kcb->prev_kprobe.status = kcb->kprobe_status;
 212         kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
 213 }
 214
 215 static nokprobe_inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
 216 {
 217         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 218         kcb->kprobe_status = kcb->prev_kprobe.status;
 219         kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
 220 }
 221
 222 static nokprobe_inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
 223                                 struct kprobe_ctlblk *kcb)
 224 {
 225         __this_cpu_write(current_kprobe, p);
 226         kcb->kprobe_saved_msr = regs->msr;
 227 }
 228
 229 static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
 230 {
 231         int ret;
 232         ppc_inst_t insn = ppc_inst_read(p->ainsn.insn);
 233
 234         /* regs->nip is also adjusted if emulate_step returns 1 */
 235         ret = emulate_step(regs, insn);
 236         if (ret > 0) {
 237                 /*
 238                  * Once this instruction has been boosted
 239                  * successfully, set the boostable flag
 240                  */
 241                 if (unlikely(p->ainsn.boostable == 0))
 242                         p->ainsn.boostable = 1;
 243         } else if (ret < 0) {
 244                 /*
 245                  * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
 246                  * So, we should never get here... but, its still
 247                  * good to catch them, just in case...
 248                  */
 249                 printk("Can't step on instruction %08lx\n", ppc_inst_as_ulong(insn));
 250                 BUG();
 251         } else {
 252                 /*
 253                  * If we haven't previously emulated this instruction, then it
 254                  * can't be boosted. Note it down so we don't try to do so again.
 255                  *
 256                  * If, however, we had emulated this instruction in the past,
 257                  * then this is just an error with the current run (for
 258                  * instance, exceptions due to a load/store). We return 0 so
 259                  * that this is now single-stepped, but continue to try
 260                  * emulating it in subsequent probe hits.
 261                  */
 262                 if (unlikely(p->ainsn.boostable != 1))
 263                         p->ainsn.boostable = -1;
 264         }
 265
 266         return ret;
 267 }
 268 NOKPROBE_SYMBOL(try_to_emulate);
 269
 270 int kprobe_handler(struct pt_regs *regs)
 271 {
 272         struct kprobe *p;
 273         int ret = 0;
 274         unsigned int *addr = (unsigned int *)regs->nip;
 275         struct kprobe_ctlblk *kcb;
 276
 277         if (user_mode(regs))
 278                 return 0;
 279
 280         if (!IS_ENABLED(CONFIG_BOOKE) &&
 281             (!(regs->msr & MSR_IR) || !(regs->msr & MSR_DR)))
 282                 return 0;
 283
 284         /*
 285          * We don't want to be preempted for the entire
 286          * duration of kprobe processing
 287          */
 288         preempt_disable();
 289         kcb = get_kprobe_ctlblk();
 290
 291         p = get_kprobe(addr);
 292         if (!p) {
 293                 unsigned int instr;
 294
 295                 if (get_kernel_nofault(instr, addr))
 296                         goto no_kprobe;
 297
 298                 if (instr != BREAKPOINT_INSTRUCTION) {
 299                         /*
 300                          * PowerPC has multiple variants of the "trap"
 301                          * instruction. If the current instruction is a
 302                          * trap variant, it could belong to someone else
 303                          */
 304                         if (is_trap(instr))
 305                                 goto no_kprobe;
 306                         /*
 307                          * The breakpoint instruction was removed right
 308                          * after we hit it.  Another cpu has removed
 309                          * either a probepoint or a debugger breakpoint
 310                          * at this address.  In either case, no further
 311                          * handling of this interrupt is appropriate.
 312                          */
 313                         ret = 1;
 314                 }
 315                 /* Not one of ours: let kernel handle it */
 316                 goto no_kprobe;
 317         }
 318
 319         /* Check we're not actually recursing */
 320         if (kprobe_running()) {
 321                 kprobe_opcode_t insn = *p->ainsn.insn;
 322                 if (kcb->kprobe_status == KPROBE_HIT_SS && is_trap(insn)) {
 323                         /* Turn off 'trace' bits */
 324                         regs_set_return_msr(regs,
 325                                 (regs->msr & ~MSR_SINGLESTEP) |
 326                                 kcb->kprobe_saved_msr);
 327                         goto no_kprobe;
 328                 }
 329
 330                 /*
 331                  * We have reentered the kprobe_handler(), since another probe
 332                  * was hit while within the handler. We here save the original
 333                  * kprobes variables and just single step on the instruction of
 334                  * the new probe without calling any user handlers.
 335                  */
 336                 save_previous_kprobe(kcb);
 337                 set_current_kprobe(p, regs, kcb);
 338                 kprobes_inc_nmissed_count(p);
 339                 kcb->kprobe_status = KPROBE_REENTER;
 340                 if (p->ainsn.boostable >= 0) {
 341                         ret = try_to_emulate(p, regs);
 342
 343                         if (ret > 0) {
 344                                 restore_previous_kprobe(kcb);
 345                                 preempt_enable();
 346                                 return 1;
 347                         }
 348                 }
 349                 prepare_singlestep(p, regs);
 350                 return 1;
 351         }
 352
 353         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 354         set_current_kprobe(p, regs, kcb);
 355         if (p->pre_handler && p->pre_handler(p, regs)) {
 356                 /* handler changed execution path, so skip ss setup */
 357                 reset_current_kprobe();
 358                 preempt_enable();
 359                 return 1;
 360         }
 361
 362         if (p->ainsn.boostable >= 0) {
 363                 ret = try_to_emulate(p, regs);
 364
 365                 if (ret > 0) {
 366                         if (p->post_handler)
 367                                 p->post_handler(p, regs, 0);
 368
 369                         kcb->kprobe_status = KPROBE_HIT_SSDONE;
 370                         reset_current_kprobe();
 371                         preempt_enable();
 372                         return 1;
 373                 }
 374         }
 375         prepare_singlestep(p, regs);
 376         kcb->kprobe_status = KPROBE_HIT_SS;
 377         return 1;
 378
 379 no_kprobe:
 380         preempt_enable();
 381         return ret;
 382 }
 383 NOKPROBE_SYMBOL(kprobe_handler);
 384
 385 /*
 386  * Called after single-stepping.  p->addr is the address of the
 387  * instruction whose first byte has been replaced by the "breakpoint"
 388  * instruction.  To avoid the SMP problems that can occur when we
 389  * temporarily put back the original opcode to single-step, we
 390  * single-stepped a copy of the instruction.  The address of this
 391  * copy is p->ainsn.insn.
 392  */
 393 int kprobe_post_handler(struct pt_regs *regs)
 394 {
 395         int len;
 396         struct kprobe *cur = kprobe_running();
 397         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 398
 399         if (!cur || user_mode(regs))
 400                 return 0;
 401
 402         len = ppc_inst_len(ppc_inst_read(cur->ainsn.insn));
 403         /* make sure we got here for instruction we have a kprobe on */
 404         if (((unsigned long)cur->ainsn.insn + len) != regs->nip)
 405                 return 0;
 406
 407         if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
 408                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 409                 cur->post_handler(cur, regs, 0);
 410         }
 411
 412         /* Adjust nip to after the single-stepped instruction */
 413         regs_set_return_ip(regs, (unsigned long)cur->addr + len);
 414         regs_set_return_msr(regs, regs->msr | kcb->kprobe_saved_msr);
 415
 416         /*Restore back the original saved kprobes variables and continue. */
 417         if (kcb->kprobe_status == KPROBE_REENTER) {
 418                 restore_previous_kprobe(kcb);
 419                 goto out;
 420         }
 421         reset_current_kprobe();
 422 out:
 423         preempt_enable();
 424
 425         /*
 426          * if somebody else is singlestepping across a probe point, msr
 427          * will have DE/SE set, in which case, continue the remaining processing
 428          * of do_debug, as if this is not a probe hit.
 429          */
 430         if (regs->msr & MSR_SINGLESTEP)
 431                 return 0;
 432
 433         return 1;
 434 }
 435 NOKPROBE_SYMBOL(kprobe_post_handler);
 436
 437 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 438 {
 439         struct kprobe *cur = kprobe_running();
 440         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 441         const struct exception_table_entry *entry;
 442
 443         switch(kcb->kprobe_status) {
 444         case KPROBE_HIT_SS:
 445         case KPROBE_REENTER:
 446                 /*
 447                  * We are here because the instruction being single
 448                  * stepped caused a page fault. We reset the current
 449                  * kprobe and the nip points back to the probe address
 450                  * and allow the page fault handler to continue as a
 451                  * normal page fault.
 452                  */
 453                 regs_set_return_ip(regs, (unsigned long)cur->addr);
 454                 /* Turn off 'trace' bits */
 455                 regs_set_return_msr(regs,
 456                         (regs->msr & ~MSR_SINGLESTEP) |
 457                         kcb->kprobe_saved_msr);
 458                 if (kcb->kprobe_status == KPROBE_REENTER)
 459                         restore_previous_kprobe(kcb);
 460                 else
 461                         reset_current_kprobe();
 462                 preempt_enable();
 463                 break;
 464         case KPROBE_HIT_ACTIVE:
 465         case KPROBE_HIT_SSDONE:
 466                 /*
 467                  * In case the user-specified fault handler returned
 468                  * zero, try to fix up.
 469                  */
 470                 if ((entry = search_exception_tables(regs->nip)) != NULL) {
 471                         regs_set_return_ip(regs, extable_fixup(entry));
 472                         return 1;
 473                 }
 474
 475                 /*
 476                  * fixup_exception() could not handle it,
 477                  * Let do_page_fault() fix it.
 478                  */
 479                 break;
 480         default:
 481                 break;
 482         }
 483         return 0;
 484 }
 485 NOKPROBE_SYMBOL(kprobe_fault_handler);
 486
 487 int arch_trampoline_kprobe(struct kprobe *p)
 488 {
 489         if (p->addr == (kprobe_opcode_t *)&arch_rethook_trampoline)
 490                 return 1;
 491
 492         return 0;
 493 }
 494 NOKPROBE_SYMBOL(arch_trampoline_kprobe);