arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <[email protected]>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/moduleloader.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/preempt.h>
  16 #include <linux/stop_machine.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <asm/set_memory.h>
  25 #include <asm/sections.h>
  26 #include <asm/dis.h>
  27 #include "kprobes.h"
  28 #include "entry.h"
  29
  30 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  31 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  32
  33 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  34
  35 static int insn_page_in_use;
  36
  37 void *alloc_insn_page(void)
  38 {
  39         void *page;
  40
  41         page = module_alloc(PAGE_SIZE);
  42         if (!page)
  43                 return NULL;
  44         set_memory_rox((unsigned long)page, 1);
  45         return page;
  46 }
  47
  48 static void *alloc_s390_insn_page(void)
  49 {
  50         if (xchg(&insn_page_in_use, 1) == 1)
  51                 return NULL;
  52         return &kprobes_insn_page;
  53 }
  54
  55 static void free_s390_insn_page(void *page)
  56 {
  57         xchg(&insn_page_in_use, 0);
  58 }
  59
  60 struct kprobe_insn_cache kprobe_s390_insn_slots = {
  61         .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
  62         .alloc = alloc_s390_insn_page,
  63         .free = free_s390_insn_page,
  64         .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
  65         .insn_size = MAX_INSN_SIZE,
  66 };
  67
  68 static void copy_instruction(struct kprobe *p)
  69 {
  70         kprobe_opcode_t insn[MAX_INSN_SIZE];
  71         s64 disp, new_disp;
  72         u64 addr, new_addr;
  73         unsigned int len;
  74
  75         len = insn_length(*p->addr >> 8);
  76         memcpy(&insn, p->addr, len);
  77         p->opcode = insn[0];
  78         if (probe_is_insn_relative_long(&insn[0])) {
  79                 /*
  80                  * For pc-relative instructions in RIL-b or RIL-c format patch
  81                  * the RI2 displacement field. We have already made sure that
  82                  * the insn slot for the patched instruction is within the same
  83                  * 2GB area as the original instruction (either kernel image or
  84                  * module area). Therefore the new displacement will always fit.
  85                  */
  86                 disp = *(s32 *)&insn[1];
  87                 addr = (u64)(unsigned long)p->addr;
  88                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  89                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  90                 *(s32 *)&insn[1] = new_disp;
  91         }
  92         s390_kernel_write(p->ainsn.insn, &insn, len);
  93 }
  94 NOKPROBE_SYMBOL(copy_instruction);
  95
  96 static int s390_get_insn_slot(struct kprobe *p)
  97 {
  98         /*
  99          * Get an insn slot that is within the same 2GB area like the original
 100          * instruction. That way instructions with a 32bit signed displacement
 101          * field can be patched and executed within the insn slot.
 102          */
 103         p->ainsn.insn = NULL;
 104         if (is_kernel((unsigned long)p->addr))
 105                 p->ainsn.insn = get_s390_insn_slot();
 106         else if (is_module_addr(p->addr))
 107                 p->ainsn.insn = get_insn_slot();
 108         return p->ainsn.insn ? 0 : -ENOMEM;
 109 }
 110 NOKPROBE_SYMBOL(s390_get_insn_slot);
 111
 112 static void s390_free_insn_slot(struct kprobe *p)
 113 {
 114         if (!p->ainsn.insn)
 115                 return;
 116         if (is_kernel((unsigned long)p->addr))
 117                 free_s390_insn_slot(p->ainsn.insn, 0);
 118         else
 119                 free_insn_slot(p->ainsn.insn, 0);
 120         p->ainsn.insn = NULL;
 121 }
 122 NOKPROBE_SYMBOL(s390_free_insn_slot);
 123
 124 /* Check if paddr is at an instruction boundary */
 125 static bool can_probe(unsigned long paddr)
 126 {
 127         unsigned long addr, offset = 0;
 128         kprobe_opcode_t insn;
 129         struct kprobe *kp;
 130
 131         if (paddr & 0x01)
 132                 return false;
 133
 134         if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
 135                 return false;
 136
 137         /* Decode instructions */
 138         addr = paddr - offset;
 139         while (addr < paddr) {
 140                 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
 141                         return false;
 142
 143                 if (insn >> 8 == 0) {
 144                         if (insn != BREAKPOINT_INSTRUCTION) {
 145                                 /*
 146                                  * Note that QEMU inserts opcode 0x0000 to implement
 147                                  * software breakpoints for guests. Since the size of
 148                                  * the original instruction is unknown, stop following
 149                                  * instructions and prevent setting a kprobe.
 150                                  */
 151                                 return false;
 152                         }
 153                         /*
 154                          * Check if the instruction has been modified by another
 155                          * kprobe, in which case the original instruction is
 156                          * decoded.
 157                          */
 158                         kp = get_kprobe((void *)addr);
 159                         if (!kp) {
 160                                 /* not a kprobe */
 161                                 return false;
 162                         }
 163                         insn = kp->opcode;
 164                 }
 165                 addr += insn_length(insn >> 8);
 166         }
 167         return addr == paddr;
 168 }
 169
 170 int arch_prepare_kprobe(struct kprobe *p)
 171 {
 172         if (!can_probe((unsigned long)p->addr))
 173                 return -EINVAL;
 174         /* Make sure the probe isn't going on a difficult instruction */
 175         if (probe_is_prohibited_opcode(p->addr))
 176                 return -EINVAL;
 177         if (s390_get_insn_slot(p))
 178                 return -ENOMEM;
 179         copy_instruction(p);
 180         return 0;
 181 }
 182 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 183
 184 struct swap_insn_args {
 185         struct kprobe *p;
 186         unsigned int arm_kprobe : 1;
 187 };
 188
 189 static int swap_instruction(void *data)
 190 {
 191         struct swap_insn_args *args = data;
 192         struct kprobe *p = args->p;
 193         u16 opc;
 194
 195         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 196         s390_kernel_write(p->addr, &opc, sizeof(opc));
 197         return 0;
 198 }
 199 NOKPROBE_SYMBOL(swap_instruction);
 200
 201 void arch_arm_kprobe(struct kprobe *p)
 202 {
 203         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 204
 205         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 206 }
 207 NOKPROBE_SYMBOL(arch_arm_kprobe);
 208
 209 void arch_disarm_kprobe(struct kprobe *p)
 210 {
 211         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 212
 213         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 214 }
 215 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 216
 217 void arch_remove_kprobe(struct kprobe *p)
 218 {
 219         s390_free_insn_slot(p);
 220 }
 221 NOKPROBE_SYMBOL(arch_remove_kprobe);
 222
 223 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 224                               struct pt_regs *regs,
 225                               unsigned long ip)
 226 {
 227         struct per_regs per_kprobe;
 228
 229         /* Set up the PER control registers %cr9-%cr11 */
 230         per_kprobe.control = PER_EVENT_IFETCH;
 231         per_kprobe.start = ip;
 232         per_kprobe.end = ip;
 233
 234         /* Save control regs and psw mask */
 235         __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
 236         kcb->kprobe_saved_imask = regs->psw.mask &
 237                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 238
 239         /* Set PER control regs, turns on single step for the given address */
 240         __ctl_load(per_kprobe, 9, 11);
 241         regs->psw.mask |= PSW_MASK_PER;
 242         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 243         regs->psw.addr = ip;
 244 }
 245 NOKPROBE_SYMBOL(enable_singlestep);
 246
 247 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 248                                struct pt_regs *regs,
 249                                unsigned long ip)
 250 {
 251         /* Restore control regs and psw mask, set new psw address */
 252         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
 253         regs->psw.mask &= ~PSW_MASK_PER;
 254         regs->psw.mask |= kcb->kprobe_saved_imask;
 255         regs->psw.addr = ip;
 256 }
 257 NOKPROBE_SYMBOL(disable_singlestep);
 258
 259 /*
 260  * Activate a kprobe by storing its pointer to current_kprobe. The
 261  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 262  * two kprobes can be active, see KPROBE_REENTER.
 263  */
 264 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 265 {
 266         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 267         kcb->prev_kprobe.status = kcb->kprobe_status;
 268         __this_cpu_write(current_kprobe, p);
 269 }
 270 NOKPROBE_SYMBOL(push_kprobe);
 271
 272 /*
 273  * Deactivate a kprobe by backing up to the previous state. If the
 274  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 275  * for any other state prev_kprobe.kp will be NULL.
 276  */
 277 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 278 {
 279         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 280         kcb->kprobe_status = kcb->prev_kprobe.status;
 281         kcb->prev_kprobe.kp = NULL;
 282 }
 283 NOKPROBE_SYMBOL(pop_kprobe);
 284
 285 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 286 {
 287         switch (kcb->kprobe_status) {
 288         case KPROBE_HIT_SSDONE:
 289         case KPROBE_HIT_ACTIVE:
 290                 kprobes_inc_nmissed_count(p);
 291                 break;
 292         case KPROBE_HIT_SS:
 293         case KPROBE_REENTER:
 294         default:
 295                 /*
 296                  * A kprobe on the code path to single step an instruction
 297                  * is a BUG. The code path resides in the .kprobes.text
 298                  * section and is executed with interrupts disabled.
 299                  */
 300                 pr_err("Failed to recover from reentered kprobes.\n");
 301                 dump_kprobe(p);
 302                 BUG();
 303         }
 304 }
 305 NOKPROBE_SYMBOL(kprobe_reenter_check);
 306
 307 static int kprobe_handler(struct pt_regs *regs)
 308 {
 309         struct kprobe_ctlblk *kcb;
 310         struct kprobe *p;
 311
 312         /*
 313          * We want to disable preemption for the entire duration of kprobe
 314          * processing. That includes the calls to the pre/post handlers
 315          * and single stepping the kprobe instruction.
 316          */
 317         preempt_disable();
 318         kcb = get_kprobe_ctlblk();
 319         p = get_kprobe((void *)(regs->psw.addr - 2));
 320
 321         if (p) {
 322                 if (kprobe_running()) {
 323                         /*
 324                          * We have hit a kprobe while another is still
 325                          * active. This can happen in the pre and post
 326                          * handler. Single step the instruction of the
 327                          * new probe but do not call any handler function
 328                          * of this secondary kprobe.
 329                          * push_kprobe and pop_kprobe saves and restores
 330                          * the currently active kprobe.
 331                          */
 332                         kprobe_reenter_check(kcb, p);
 333                         push_kprobe(kcb, p);
 334                         kcb->kprobe_status = KPROBE_REENTER;
 335                 } else {
 336                         /*
 337                          * If we have no pre-handler or it returned 0, we
 338                          * continue with single stepping. If we have a
 339                          * pre-handler and it returned non-zero, it prepped
 340                          * for changing execution path, so get out doing
 341                          * nothing more here.
 342                          */
 343                         push_kprobe(kcb, p);
 344                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 345                         if (p->pre_handler && p->pre_handler(p, regs)) {
 346                                 pop_kprobe(kcb);
 347                                 preempt_enable_no_resched();
 348                                 return 1;
 349                         }
 350                         kcb->kprobe_status = KPROBE_HIT_SS;
 351                 }
 352                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 353                 return 1;
 354         } /* else:
 355            * No kprobe at this address and no active kprobe. The trap has
 356            * not been caused by a kprobe breakpoint. The race of breakpoint
 357            * vs. kprobe remove does not exist because on s390 as we use
 358            * stop_machine to arm/disarm the breakpoints.
 359            */
 360         preempt_enable_no_resched();
 361         return 0;
 362 }
 363 NOKPROBE_SYMBOL(kprobe_handler);
 364
 365 /*
 366  * Called after single-stepping.  p->addr is the address of the
 367  * instruction whose first byte has been replaced by the "breakpoint"
 368  * instruction.  To avoid the SMP problems that can occur when we
 369  * temporarily put back the original opcode to single-step, we
 370  * single-stepped a copy of the instruction.  The address of this
 371  * copy is p->ainsn.insn.
 372  */
 373 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 374 {
 375         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 376         unsigned long ip = regs->psw.addr;
 377         int fixup = probe_get_fixup_type(p->ainsn.insn);
 378
 379         if (fixup & FIXUP_PSW_NORMAL)
 380                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 381
 382         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 383                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 384                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 385                         ip = (unsigned long) p->addr + ilen;
 386         }
 387
 388         if (fixup & FIXUP_RETURN_REGISTER) {
 389                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 390                 regs->gprs[reg] += (unsigned long) p->addr -
 391                                    (unsigned long) p->ainsn.insn;
 392         }
 393
 394         disable_singlestep(kcb, regs, ip);
 395 }
 396 NOKPROBE_SYMBOL(resume_execution);
 397
 398 static int post_kprobe_handler(struct pt_regs *regs)
 399 {
 400         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 401         struct kprobe *p = kprobe_running();
 402
 403         if (!p)
 404                 return 0;
 405
 406         resume_execution(p, regs);
 407         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 408                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 409                 p->post_handler(p, regs, 0);
 410         }
 411         pop_kprobe(kcb);
 412         preempt_enable_no_resched();
 413
 414         /*
 415          * if somebody else is singlestepping across a probe point, psw mask
 416          * will have PER set, in which case, continue the remaining processing
 417          * of do_single_step, as if this is not a probe hit.
 418          */
 419         if (regs->psw.mask & PSW_MASK_PER)
 420                 return 0;
 421
 422         return 1;
 423 }
 424 NOKPROBE_SYMBOL(post_kprobe_handler);
 425
 426 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 427 {
 428         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 429         struct kprobe *p = kprobe_running();
 430
 431         switch(kcb->kprobe_status) {
 432         case KPROBE_HIT_SS:
 433         case KPROBE_REENTER:
 434                 /*
 435                  * We are here because the instruction being single
 436                  * stepped caused a page fault. We reset the current
 437                  * kprobe and the nip points back to the probe address
 438                  * and allow the page fault handler to continue as a
 439                  * normal page fault.
 440                  */
 441                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 442                 pop_kprobe(kcb);
 443                 preempt_enable_no_resched();
 444                 break;
 445         case KPROBE_HIT_ACTIVE:
 446         case KPROBE_HIT_SSDONE:
 447                 /*
 448                  * In case the user-specified fault handler returned
 449                  * zero, try to fix up.
 450                  */
 451                 if (fixup_exception(regs))
 452                         return 1;
 453                 /*
 454                  * fixup_exception() could not handle it,
 455                  * Let do_page_fault() fix it.
 456                  */
 457                 break;
 458         default:
 459                 break;
 460         }
 461         return 0;
 462 }
 463 NOKPROBE_SYMBOL(kprobe_trap_handler);
 464
 465 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 466 {
 467         int ret;
 468
 469         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 470                 local_irq_disable();
 471         ret = kprobe_trap_handler(regs, trapnr);
 472         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 473                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 474         return ret;
 475 }
 476 NOKPROBE_SYMBOL(kprobe_fault_handler);
 477
 478 /*
 479  * Wrapper routine to for handling exceptions.
 480  */
 481 int kprobe_exceptions_notify(struct notifier_block *self,
 482                              unsigned long val, void *data)
 483 {
 484         struct die_args *args = (struct die_args *) data;
 485         struct pt_regs *regs = args->regs;
 486         int ret = NOTIFY_DONE;
 487
 488         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 489                 local_irq_disable();
 490
 491         switch (val) {
 492         case DIE_BPT:
 493                 if (kprobe_handler(regs))
 494                         ret = NOTIFY_STOP;
 495                 break;
 496         case DIE_SSTEP:
 497                 if (post_kprobe_handler(regs))
 498                         ret = NOTIFY_STOP;
 499                 break;
 500         case DIE_TRAP:
 501                 if (!preemptible() && kprobe_running() &&
 502                     kprobe_trap_handler(regs, args->trapnr))
 503                         ret = NOTIFY_STOP;
 504                 break;
 505         default:
 506                 break;
 507         }
 508
 509         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 510                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 511
 512         return ret;
 513 }
 514 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 515
 516 int __init arch_init_kprobes(void)
 517 {
 518         return 0;
 519 }
 520
 521 int arch_trampoline_kprobe(struct kprobe *p)
 522 {
 523         return 0;
 524 }
 525 NOKPROBE_SYMBOL(arch_trampoline_kprobe);