arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <[email protected]>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/moduleloader.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/preempt.h>
  16 #include <linux/stop_machine.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <asm/set_memory.h>
  25 #include <asm/sections.h>
  26 #include <asm/dis.h>
  27 #include "entry.h"
  28
  29 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  30 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  31
  32 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  33
  34 DEFINE_INSN_CACHE_OPS(s390_insn);
  35
  36 static int insn_page_in_use;
  37
  38 void *alloc_insn_page(void)
  39 {
  40         void *page;
  41
  42         page = module_alloc(PAGE_SIZE);
  43         if (!page)
  44                 return NULL;
  45         __set_memory((unsigned long) page, 1, SET_MEMORY_RO | SET_MEMORY_X);
  46         return page;
  47 }
  48
  49 static void *alloc_s390_insn_page(void)
  50 {
  51         if (xchg(&insn_page_in_use, 1) == 1)
  52                 return NULL;
  53         return &kprobes_insn_page;
  54 }
  55
  56 static void free_s390_insn_page(void *page)
  57 {
  58         xchg(&insn_page_in_use, 0);
  59 }
  60
  61 struct kprobe_insn_cache kprobe_s390_insn_slots = {
  62         .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
  63         .alloc = alloc_s390_insn_page,
  64         .free = free_s390_insn_page,
  65         .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
  66         .insn_size = MAX_INSN_SIZE,
  67 };
  68
  69 static void copy_instruction(struct kprobe *p)
  70 {
  71         kprobe_opcode_t insn[MAX_INSN_SIZE];
  72         s64 disp, new_disp;
  73         u64 addr, new_addr;
  74         unsigned int len;
  75
  76         len = insn_length(*p->addr >> 8);
  77         memcpy(&insn, p->addr, len);
  78         p->opcode = insn[0];
  79         if (probe_is_insn_relative_long(&insn[0])) {
  80                 /*
  81                  * For pc-relative instructions in RIL-b or RIL-c format patch
  82                  * the RI2 displacement field. We have already made sure that
  83                  * the insn slot for the patched instruction is within the same
  84                  * 2GB area as the original instruction (either kernel image or
  85                  * module area). Therefore the new displacement will always fit.
  86                  */
  87                 disp = *(s32 *)&insn[1];
  88                 addr = (u64)(unsigned long)p->addr;
  89                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  90                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  91                 *(s32 *)&insn[1] = new_disp;
  92         }
  93         s390_kernel_write(p->ainsn.insn, &insn, len);
  94 }
  95 NOKPROBE_SYMBOL(copy_instruction);
  96
  97 static int s390_get_insn_slot(struct kprobe *p)
  98 {
  99         /*
 100          * Get an insn slot that is within the same 2GB area like the original
 101          * instruction. That way instructions with a 32bit signed displacement
 102          * field can be patched and executed within the insn slot.
 103          */
 104         p->ainsn.insn = NULL;
 105         if (is_kernel((unsigned long)p->addr))
 106                 p->ainsn.insn = get_s390_insn_slot();
 107         else if (is_module_addr(p->addr))
 108                 p->ainsn.insn = get_insn_slot();
 109         return p->ainsn.insn ? 0 : -ENOMEM;
 110 }
 111 NOKPROBE_SYMBOL(s390_get_insn_slot);
 112
 113 static void s390_free_insn_slot(struct kprobe *p)
 114 {
 115         if (!p->ainsn.insn)
 116                 return;
 117         if (is_kernel((unsigned long)p->addr))
 118                 free_s390_insn_slot(p->ainsn.insn, 0);
 119         else
 120                 free_insn_slot(p->ainsn.insn, 0);
 121         p->ainsn.insn = NULL;
 122 }
 123 NOKPROBE_SYMBOL(s390_free_insn_slot);
 124
 125 /* Check if paddr is at an instruction boundary */
 126 static bool can_probe(unsigned long paddr)
 127 {
 128         unsigned long addr, offset = 0;
 129         kprobe_opcode_t insn;
 130         struct kprobe *kp;
 131
 132         if (paddr & 0x01)
 133                 return false;
 134
 135         if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
 136                 return false;
 137
 138         /* Decode instructions */
 139         addr = paddr - offset;
 140         while (addr < paddr) {
 141                 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
 142                         return false;
 143
 144                 if (insn >> 8 == 0) {
 145                         if (insn != BREAKPOINT_INSTRUCTION) {
 146                                 /*
 147                                  * Note that QEMU inserts opcode 0x0000 to implement
 148                                  * software breakpoints for guests. Since the size of
 149                                  * the original instruction is unknown, stop following
 150                                  * instructions and prevent setting a kprobe.
 151                                  */
 152                                 return false;
 153                         }
 154                         /*
 155                          * Check if the instruction has been modified by another
 156                          * kprobe, in which case the original instruction is
 157                          * decoded.
 158                          */
 159                         kp = get_kprobe((void *)addr);
 160                         if (!kp) {
 161                                 /* not a kprobe */
 162                                 return false;
 163                         }
 164                         insn = kp->opcode;
 165                 }
 166                 addr += insn_length(insn >> 8);
 167         }
 168         return addr == paddr;
 169 }
 170
 171 int arch_prepare_kprobe(struct kprobe *p)
 172 {
 173         if (!can_probe((unsigned long)p->addr))
 174                 return -EINVAL;
 175         /* Make sure the probe isn't going on a difficult instruction */
 176         if (probe_is_prohibited_opcode(p->addr))
 177                 return -EINVAL;
 178         if (s390_get_insn_slot(p))
 179                 return -ENOMEM;
 180         copy_instruction(p);
 181         return 0;
 182 }
 183 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 184
 185 struct swap_insn_args {
 186         struct kprobe *p;
 187         unsigned int arm_kprobe : 1;
 188 };
 189
 190 static int swap_instruction(void *data)
 191 {
 192         struct swap_insn_args *args = data;
 193         struct kprobe *p = args->p;
 194         u16 opc;
 195
 196         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 197         s390_kernel_write(p->addr, &opc, sizeof(opc));
 198         return 0;
 199 }
 200 NOKPROBE_SYMBOL(swap_instruction);
 201
 202 void arch_arm_kprobe(struct kprobe *p)
 203 {
 204         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 205
 206         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 207 }
 208 NOKPROBE_SYMBOL(arch_arm_kprobe);
 209
 210 void arch_disarm_kprobe(struct kprobe *p)
 211 {
 212         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 213
 214         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 215 }
 216 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 217
 218 void arch_remove_kprobe(struct kprobe *p)
 219 {
 220         s390_free_insn_slot(p);
 221 }
 222 NOKPROBE_SYMBOL(arch_remove_kprobe);
 223
 224 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 225                               struct pt_regs *regs,
 226                               unsigned long ip)
 227 {
 228         struct per_regs per_kprobe;
 229
 230         /* Set up the PER control registers %cr9-%cr11 */
 231         per_kprobe.control = PER_EVENT_IFETCH;
 232         per_kprobe.start = ip;
 233         per_kprobe.end = ip;
 234
 235         /* Save control regs and psw mask */
 236         __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
 237         kcb->kprobe_saved_imask = regs->psw.mask &
 238                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 239
 240         /* Set PER control regs, turns on single step for the given address */
 241         __ctl_load(per_kprobe, 9, 11);
 242         regs->psw.mask |= PSW_MASK_PER;
 243         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 244         regs->psw.addr = ip;
 245 }
 246 NOKPROBE_SYMBOL(enable_singlestep);
 247
 248 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 249                                struct pt_regs *regs,
 250                                unsigned long ip)
 251 {
 252         /* Restore control regs and psw mask, set new psw address */
 253         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
 254         regs->psw.mask &= ~PSW_MASK_PER;
 255         regs->psw.mask |= kcb->kprobe_saved_imask;
 256         regs->psw.addr = ip;
 257 }
 258 NOKPROBE_SYMBOL(disable_singlestep);
 259
 260 /*
 261  * Activate a kprobe by storing its pointer to current_kprobe. The
 262  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 263  * two kprobes can be active, see KPROBE_REENTER.
 264  */
 265 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 266 {
 267         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 268         kcb->prev_kprobe.status = kcb->kprobe_status;
 269         __this_cpu_write(current_kprobe, p);
 270 }
 271 NOKPROBE_SYMBOL(push_kprobe);
 272
 273 /*
 274  * Deactivate a kprobe by backing up to the previous state. If the
 275  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 276  * for any other state prev_kprobe.kp will be NULL.
 277  */
 278 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 279 {
 280         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 281         kcb->kprobe_status = kcb->prev_kprobe.status;
 282 }
 283 NOKPROBE_SYMBOL(pop_kprobe);
 284
 285 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
 286 {
 287         ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
 288         ri->fp = NULL;
 289
 290         /* Replace the return addr with trampoline addr */
 291         regs->gprs[14] = (unsigned long) &__kretprobe_trampoline;
 292 }
 293 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
 294
 295 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 296 {
 297         switch (kcb->kprobe_status) {
 298         case KPROBE_HIT_SSDONE:
 299         case KPROBE_HIT_ACTIVE:
 300                 kprobes_inc_nmissed_count(p);
 301                 break;
 302         case KPROBE_HIT_SS:
 303         case KPROBE_REENTER:
 304         default:
 305                 /*
 306                  * A kprobe on the code path to single step an instruction
 307                  * is a BUG. The code path resides in the .kprobes.text
 308                  * section and is executed with interrupts disabled.
 309                  */
 310                 pr_err("Failed to recover from reentered kprobes.\n");
 311                 dump_kprobe(p);
 312                 BUG();
 313         }
 314 }
 315 NOKPROBE_SYMBOL(kprobe_reenter_check);
 316
 317 static int kprobe_handler(struct pt_regs *regs)
 318 {
 319         struct kprobe_ctlblk *kcb;
 320         struct kprobe *p;
 321
 322         /*
 323          * We want to disable preemption for the entire duration of kprobe
 324          * processing. That includes the calls to the pre/post handlers
 325          * and single stepping the kprobe instruction.
 326          */
 327         preempt_disable();
 328         kcb = get_kprobe_ctlblk();
 329         p = get_kprobe((void *)(regs->psw.addr - 2));
 330
 331         if (p) {
 332                 if (kprobe_running()) {
 333                         /*
 334                          * We have hit a kprobe while another is still
 335                          * active. This can happen in the pre and post
 336                          * handler. Single step the instruction of the
 337                          * new probe but do not call any handler function
 338                          * of this secondary kprobe.
 339                          * push_kprobe and pop_kprobe saves and restores
 340                          * the currently active kprobe.
 341                          */
 342                         kprobe_reenter_check(kcb, p);
 343                         push_kprobe(kcb, p);
 344                         kcb->kprobe_status = KPROBE_REENTER;
 345                 } else {
 346                         /*
 347                          * If we have no pre-handler or it returned 0, we
 348                          * continue with single stepping. If we have a
 349                          * pre-handler and it returned non-zero, it prepped
 350                          * for changing execution path, so get out doing
 351                          * nothing more here.
 352                          */
 353                         push_kprobe(kcb, p);
 354                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 355                         if (p->pre_handler && p->pre_handler(p, regs)) {
 356                                 pop_kprobe(kcb);
 357                                 preempt_enable_no_resched();
 358                                 return 1;
 359                         }
 360                         kcb->kprobe_status = KPROBE_HIT_SS;
 361                 }
 362                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 363                 return 1;
 364         } /* else:
 365            * No kprobe at this address and no active kprobe. The trap has
 366            * not been caused by a kprobe breakpoint. The race of breakpoint
 367            * vs. kprobe remove does not exist because on s390 as we use
 368            * stop_machine to arm/disarm the breakpoints.
 369            */
 370         preempt_enable_no_resched();
 371         return 0;
 372 }
 373 NOKPROBE_SYMBOL(kprobe_handler);
 374
 375 /*
 376  * Function return probe trampoline:
 377  *      - init_kprobes() establishes a probepoint here
 378  *      - When the probed function returns, this probe
 379  *              causes the handlers to fire
 380  */
 381 static void __used kretprobe_trampoline_holder(void)
 382 {
 383         asm volatile(".global __kretprobe_trampoline\n"
 384                      "__kretprobe_trampoline: bcr 0,0\n");
 385 }
 386
 387 /*
 388  * Called when the probe at kretprobe trampoline is hit
 389  */
 390 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 391 {
 392         regs->psw.addr = __kretprobe_trampoline_handler(regs, NULL);
 393         /*
 394          * By returning a non-zero value, we are telling
 395          * kprobe_handler() that we don't want the post_handler
 396          * to run (and have re-enabled preemption)
 397          */
 398         return 1;
 399 }
 400 NOKPROBE_SYMBOL(trampoline_probe_handler);
 401
 402 /*
 403  * Called after single-stepping.  p->addr is the address of the
 404  * instruction whose first byte has been replaced by the "breakpoint"
 405  * instruction.  To avoid the SMP problems that can occur when we
 406  * temporarily put back the original opcode to single-step, we
 407  * single-stepped a copy of the instruction.  The address of this
 408  * copy is p->ainsn.insn.
 409  */
 410 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 411 {
 412         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 413         unsigned long ip = regs->psw.addr;
 414         int fixup = probe_get_fixup_type(p->ainsn.insn);
 415
 416         if (fixup & FIXUP_PSW_NORMAL)
 417                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 418
 419         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 420                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 421                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 422                         ip = (unsigned long) p->addr + ilen;
 423         }
 424
 425         if (fixup & FIXUP_RETURN_REGISTER) {
 426                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 427                 regs->gprs[reg] += (unsigned long) p->addr -
 428                                    (unsigned long) p->ainsn.insn;
 429         }
 430
 431         disable_singlestep(kcb, regs, ip);
 432 }
 433 NOKPROBE_SYMBOL(resume_execution);
 434
 435 static int post_kprobe_handler(struct pt_regs *regs)
 436 {
 437         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 438         struct kprobe *p = kprobe_running();
 439
 440         if (!p)
 441                 return 0;
 442
 443         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 444                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 445                 p->post_handler(p, regs, 0);
 446         }
 447
 448         resume_execution(p, regs);
 449         pop_kprobe(kcb);
 450         preempt_enable_no_resched();
 451
 452         /*
 453          * if somebody else is singlestepping across a probe point, psw mask
 454          * will have PER set, in which case, continue the remaining processing
 455          * of do_single_step, as if this is not a probe hit.
 456          */
 457         if (regs->psw.mask & PSW_MASK_PER)
 458                 return 0;
 459
 460         return 1;
 461 }
 462 NOKPROBE_SYMBOL(post_kprobe_handler);
 463
 464 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 465 {
 466         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 467         struct kprobe *p = kprobe_running();
 468         const struct exception_table_entry *entry;
 469
 470         switch(kcb->kprobe_status) {
 471         case KPROBE_HIT_SS:
 472         case KPROBE_REENTER:
 473                 /*
 474                  * We are here because the instruction being single
 475                  * stepped caused a page fault. We reset the current
 476                  * kprobe and the nip points back to the probe address
 477                  * and allow the page fault handler to continue as a
 478                  * normal page fault.
 479                  */
 480                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 481                 pop_kprobe(kcb);
 482                 preempt_enable_no_resched();
 483                 break;
 484         case KPROBE_HIT_ACTIVE:
 485         case KPROBE_HIT_SSDONE:
 486                 /*
 487                  * In case the user-specified fault handler returned
 488                  * zero, try to fix up.
 489                  */
 490                 entry = s390_search_extables(regs->psw.addr);
 491                 if (entry && ex_handle(entry, regs))
 492                         return 1;
 493
 494                 /*
 495                  * fixup_exception() could not handle it,
 496                  * Let do_page_fault() fix it.
 497                  */
 498                 break;
 499         default:
 500                 break;
 501         }
 502         return 0;
 503 }
 504 NOKPROBE_SYMBOL(kprobe_trap_handler);
 505
 506 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 507 {
 508         int ret;
 509
 510         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 511                 local_irq_disable();
 512         ret = kprobe_trap_handler(regs, trapnr);
 513         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 514                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 515         return ret;
 516 }
 517 NOKPROBE_SYMBOL(kprobe_fault_handler);
 518
 519 /*
 520  * Wrapper routine to for handling exceptions.
 521  */
 522 int kprobe_exceptions_notify(struct notifier_block *self,
 523                              unsigned long val, void *data)
 524 {
 525         struct die_args *args = (struct die_args *) data;
 526         struct pt_regs *regs = args->regs;
 527         int ret = NOTIFY_DONE;
 528
 529         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 530                 local_irq_disable();
 531
 532         switch (val) {
 533         case DIE_BPT:
 534                 if (kprobe_handler(regs))
 535                         ret = NOTIFY_STOP;
 536                 break;
 537         case DIE_SSTEP:
 538                 if (post_kprobe_handler(regs))
 539                         ret = NOTIFY_STOP;
 540                 break;
 541         case DIE_TRAP:
 542                 if (!preemptible() && kprobe_running() &&
 543                     kprobe_trap_handler(regs, args->trapnr))
 544                         ret = NOTIFY_STOP;
 545                 break;
 546         default:
 547                 break;
 548         }
 549
 550         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 551                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 552
 553         return ret;
 554 }
 555 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 556
 557 static struct kprobe trampoline = {
 558         .addr = (kprobe_opcode_t *) &__kretprobe_trampoline,
 559         .pre_handler = trampoline_probe_handler
 560 };
 561
 562 int __init arch_init_kprobes(void)
 563 {
 564         return register_kprobe(&trampoline);
 565 }
 566
 567 int arch_trampoline_kprobe(struct kprobe *p)
 568 {
 569         return p->addr == (kprobe_opcode_t *) &__kretprobe_trampoline;
 570 }
 571 NOKPROBE_SYMBOL(arch_trampoline_kprobe);