arch/s390/kernel/kprobes.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Kernel Probes (KProbes)
   4  *
   5  * Copyright IBM Corp. 2002, 2006
   6  *
   7  * s390 port, used ppc64 as template. Mike Grundy <[email protected]>
   8  */
   9
  10 #define pr_fmt(fmt) "kprobes: " fmt
  11
  12 #include <linux/moduleloader.h>
  13 #include <linux/kprobes.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/preempt.h>
  16 #include <linux/stop_machine.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/uaccess.h>
  19 #include <linux/extable.h>
  20 #include <linux/module.h>
  21 #include <linux/slab.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/ftrace.h>
  24 #include <asm/set_memory.h>
  25 #include <asm/sections.h>
  26 #include <asm/dis.h>
  27 #include "kprobes.h"
  28 #include "entry.h"
  29
  30 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
  31 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  32
  33 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
  34
  35 static int insn_page_in_use;
  36
  37 void *alloc_insn_page(void)
  38 {
  39         void *page;
  40
  41         page = module_alloc(PAGE_SIZE);
  42         if (!page)
  43                 return NULL;
  44         set_memory_rox((unsigned long)page, 1);
  45         return page;
  46 }
  47
  48 static void *alloc_s390_insn_page(void)
  49 {
  50         if (xchg(&insn_page_in_use, 1) == 1)
  51                 return NULL;
  52         return &kprobes_insn_page;
  53 }
  54
  55 static void free_s390_insn_page(void *page)
  56 {
  57         xchg(&insn_page_in_use, 0);
  58 }
  59
  60 struct kprobe_insn_cache kprobe_s390_insn_slots = {
  61         .mutex = __MUTEX_INITIALIZER(kprobe_s390_insn_slots.mutex),
  62         .alloc = alloc_s390_insn_page,
  63         .free = free_s390_insn_page,
  64         .pages = LIST_HEAD_INIT(kprobe_s390_insn_slots.pages),
  65         .insn_size = MAX_INSN_SIZE,
  66 };
  67
  68 static void copy_instruction(struct kprobe *p)
  69 {
  70         kprobe_opcode_t insn[MAX_INSN_SIZE];
  71         s64 disp, new_disp;
  72         u64 addr, new_addr;
  73         unsigned int len;
  74
  75         len = insn_length(*p->addr >> 8);
  76         memcpy(&insn, p->addr, len);
  77         p->opcode = insn[0];
  78         if (probe_is_insn_relative_long(&insn[0])) {
  79                 /*
  80                  * For pc-relative instructions in RIL-b or RIL-c format patch
  81                  * the RI2 displacement field. We have already made sure that
  82                  * the insn slot for the patched instruction is within the same
  83                  * 2GB area as the original instruction (either kernel image or
  84                  * module area). Therefore the new displacement will always fit.
  85                  */
  86                 disp = *(s32 *)&insn[1];
  87                 addr = (u64)(unsigned long)p->addr;
  88                 new_addr = (u64)(unsigned long)p->ainsn.insn;
  89                 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
  90                 *(s32 *)&insn[1] = new_disp;
  91         }
  92         s390_kernel_write(p->ainsn.insn, &insn, len);
  93 }
  94 NOKPROBE_SYMBOL(copy_instruction);
  95
  96 static int s390_get_insn_slot(struct kprobe *p)
  97 {
  98         /*
  99          * Get an insn slot that is within the same 2GB area like the original
 100          * instruction. That way instructions with a 32bit signed displacement
 101          * field can be patched and executed within the insn slot.
 102          */
 103         p->ainsn.insn = NULL;
 104         if (is_kernel((unsigned long)p->addr))
 105                 p->ainsn.insn = get_s390_insn_slot();
 106         else if (is_module_addr(p->addr))
 107                 p->ainsn.insn = get_insn_slot();
 108         return p->ainsn.insn ? 0 : -ENOMEM;
 109 }
 110 NOKPROBE_SYMBOL(s390_get_insn_slot);
 111
 112 static void s390_free_insn_slot(struct kprobe *p)
 113 {
 114         if (!p->ainsn.insn)
 115                 return;
 116         if (is_kernel((unsigned long)p->addr))
 117                 free_s390_insn_slot(p->ainsn.insn, 0);
 118         else
 119                 free_insn_slot(p->ainsn.insn, 0);
 120         p->ainsn.insn = NULL;
 121 }
 122 NOKPROBE_SYMBOL(s390_free_insn_slot);
 123
 124 /* Check if paddr is at an instruction boundary */
 125 static bool can_probe(unsigned long paddr)
 126 {
 127         unsigned long addr, offset = 0;
 128         kprobe_opcode_t insn;
 129         struct kprobe *kp;
 130
 131         if (paddr & 0x01)
 132                 return false;
 133
 134         if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
 135                 return false;
 136
 137         /* Decode instructions */
 138         addr = paddr - offset;
 139         while (addr < paddr) {
 140                 if (copy_from_kernel_nofault(&insn, (void *)addr, sizeof(insn)))
 141                         return false;
 142
 143                 if (insn >> 8 == 0) {
 144                         if (insn != BREAKPOINT_INSTRUCTION) {
 145                                 /*
 146                                  * Note that QEMU inserts opcode 0x0000 to implement
 147                                  * software breakpoints for guests. Since the size of
 148                                  * the original instruction is unknown, stop following
 149                                  * instructions and prevent setting a kprobe.
 150                                  */
 151                                 return false;
 152                         }
 153                         /*
 154                          * Check if the instruction has been modified by another
 155                          * kprobe, in which case the original instruction is
 156                          * decoded.
 157                          */
 158                         kp = get_kprobe((void *)addr);
 159                         if (!kp) {
 160                                 /* not a kprobe */
 161                                 return false;
 162                         }
 163                         insn = kp->opcode;
 164                 }
 165                 addr += insn_length(insn >> 8);
 166         }
 167         return addr == paddr;
 168 }
 169
 170 int arch_prepare_kprobe(struct kprobe *p)
 171 {
 172         if (!can_probe((unsigned long)p->addr))
 173                 return -EINVAL;
 174         /* Make sure the probe isn't going on a difficult instruction */
 175         if (probe_is_prohibited_opcode(p->addr))
 176                 return -EINVAL;
 177         if (s390_get_insn_slot(p))
 178                 return -ENOMEM;
 179         copy_instruction(p);
 180         return 0;
 181 }
 182 NOKPROBE_SYMBOL(arch_prepare_kprobe);
 183
 184 struct swap_insn_args {
 185         struct kprobe *p;
 186         unsigned int arm_kprobe : 1;
 187 };
 188
 189 static int swap_instruction(void *data)
 190 {
 191         struct swap_insn_args *args = data;
 192         struct kprobe *p = args->p;
 193         u16 opc;
 194
 195         opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
 196         s390_kernel_write(p->addr, &opc, sizeof(opc));
 197         return 0;
 198 }
 199 NOKPROBE_SYMBOL(swap_instruction);
 200
 201 void arch_arm_kprobe(struct kprobe *p)
 202 {
 203         struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 204
 205         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 206 }
 207 NOKPROBE_SYMBOL(arch_arm_kprobe);
 208
 209 void arch_disarm_kprobe(struct kprobe *p)
 210 {
 211         struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 212
 213         stop_machine_cpuslocked(swap_instruction, &args, NULL);
 214 }
 215 NOKPROBE_SYMBOL(arch_disarm_kprobe);
 216
 217 void arch_remove_kprobe(struct kprobe *p)
 218 {
 219         s390_free_insn_slot(p);
 220 }
 221 NOKPROBE_SYMBOL(arch_remove_kprobe);
 222
 223 static void enable_singlestep(struct kprobe_ctlblk *kcb,
 224                               struct pt_regs *regs,
 225                               unsigned long ip)
 226 {
 227         union {
 228                 struct ctlreg regs[3];
 229                 struct {
 230                         struct ctlreg control;
 231                         struct ctlreg start;
 232                         struct ctlreg end;
 233                 };
 234         } per_kprobe;
 235
 236         /* Set up the PER control registers %cr9-%cr11 */
 237         per_kprobe.control.val = PER_EVENT_IFETCH;
 238         per_kprobe.start.val = ip;
 239         per_kprobe.end.val = ip;
 240
 241         /* Save control regs and psw mask */
 242         __local_ctl_store(9, 11, kcb->kprobe_saved_ctl);
 243         kcb->kprobe_saved_imask = regs->psw.mask &
 244                 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
 245
 246         /* Set PER control regs, turns on single step for the given address */
 247         __local_ctl_load(9, 11, per_kprobe.regs);
 248         regs->psw.mask |= PSW_MASK_PER;
 249         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
 250         regs->psw.addr = ip;
 251 }
 252 NOKPROBE_SYMBOL(enable_singlestep);
 253
 254 static void disable_singlestep(struct kprobe_ctlblk *kcb,
 255                                struct pt_regs *regs,
 256                                unsigned long ip)
 257 {
 258         /* Restore control regs and psw mask, set new psw address */
 259         __local_ctl_load(9, 11, kcb->kprobe_saved_ctl);
 260         regs->psw.mask &= ~PSW_MASK_PER;
 261         regs->psw.mask |= kcb->kprobe_saved_imask;
 262         regs->psw.addr = ip;
 263 }
 264 NOKPROBE_SYMBOL(disable_singlestep);
 265
 266 /*
 267  * Activate a kprobe by storing its pointer to current_kprobe. The
 268  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
 269  * two kprobes can be active, see KPROBE_REENTER.
 270  */
 271 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 272 {
 273         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
 274         kcb->prev_kprobe.status = kcb->kprobe_status;
 275         __this_cpu_write(current_kprobe, p);
 276 }
 277 NOKPROBE_SYMBOL(push_kprobe);
 278
 279 /*
 280  * Deactivate a kprobe by backing up to the previous state. If the
 281  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
 282  * for any other state prev_kprobe.kp will be NULL.
 283  */
 284 static void pop_kprobe(struct kprobe_ctlblk *kcb)
 285 {
 286         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
 287         kcb->kprobe_status = kcb->prev_kprobe.status;
 288         kcb->prev_kprobe.kp = NULL;
 289 }
 290 NOKPROBE_SYMBOL(pop_kprobe);
 291
 292 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 293 {
 294         switch (kcb->kprobe_status) {
 295         case KPROBE_HIT_SSDONE:
 296         case KPROBE_HIT_ACTIVE:
 297                 kprobes_inc_nmissed_count(p);
 298                 break;
 299         case KPROBE_HIT_SS:
 300         case KPROBE_REENTER:
 301         default:
 302                 /*
 303                  * A kprobe on the code path to single step an instruction
 304                  * is a BUG. The code path resides in the .kprobes.text
 305                  * section and is executed with interrupts disabled.
 306                  */
 307                 pr_err("Failed to recover from reentered kprobes.\n");
 308                 dump_kprobe(p);
 309                 BUG();
 310         }
 311 }
 312 NOKPROBE_SYMBOL(kprobe_reenter_check);
 313
 314 static int kprobe_handler(struct pt_regs *regs)
 315 {
 316         struct kprobe_ctlblk *kcb;
 317         struct kprobe *p;
 318
 319         /*
 320          * We want to disable preemption for the entire duration of kprobe
 321          * processing. That includes the calls to the pre/post handlers
 322          * and single stepping the kprobe instruction.
 323          */
 324         preempt_disable();
 325         kcb = get_kprobe_ctlblk();
 326         p = get_kprobe((void *)(regs->psw.addr - 2));
 327
 328         if (p) {
 329                 if (kprobe_running()) {
 330                         /*
 331                          * We have hit a kprobe while another is still
 332                          * active. This can happen in the pre and post
 333                          * handler. Single step the instruction of the
 334                          * new probe but do not call any handler function
 335                          * of this secondary kprobe.
 336                          * push_kprobe and pop_kprobe saves and restores
 337                          * the currently active kprobe.
 338                          */
 339                         kprobe_reenter_check(kcb, p);
 340                         push_kprobe(kcb, p);
 341                         kcb->kprobe_status = KPROBE_REENTER;
 342                 } else {
 343                         /*
 344                          * If we have no pre-handler or it returned 0, we
 345                          * continue with single stepping. If we have a
 346                          * pre-handler and it returned non-zero, it prepped
 347                          * for changing execution path, so get out doing
 348                          * nothing more here.
 349                          */
 350                         push_kprobe(kcb, p);
 351                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 352                         if (p->pre_handler && p->pre_handler(p, regs)) {
 353                                 pop_kprobe(kcb);
 354                                 preempt_enable_no_resched();
 355                                 return 1;
 356                         }
 357                         kcb->kprobe_status = KPROBE_HIT_SS;
 358                 }
 359                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
 360                 return 1;
 361         } /* else:
 362            * No kprobe at this address and no active kprobe. The trap has
 363            * not been caused by a kprobe breakpoint. The race of breakpoint
 364            * vs. kprobe remove does not exist because on s390 as we use
 365            * stop_machine to arm/disarm the breakpoints.
 366            */
 367         preempt_enable_no_resched();
 368         return 0;
 369 }
 370 NOKPROBE_SYMBOL(kprobe_handler);
 371
 372 /*
 373  * Called after single-stepping.  p->addr is the address of the
 374  * instruction whose first byte has been replaced by the "breakpoint"
 375  * instruction.  To avoid the SMP problems that can occur when we
 376  * temporarily put back the original opcode to single-step, we
 377  * single-stepped a copy of the instruction.  The address of this
 378  * copy is p->ainsn.insn.
 379  */
 380 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 381 {
 382         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 383         unsigned long ip = regs->psw.addr;
 384         int fixup = probe_get_fixup_type(p->ainsn.insn);
 385
 386         if (fixup & FIXUP_PSW_NORMAL)
 387                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 388
 389         if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
 390                 int ilen = insn_length(p->ainsn.insn[0] >> 8);
 391                 if (ip - (unsigned long) p->ainsn.insn == ilen)
 392                         ip = (unsigned long) p->addr + ilen;
 393         }
 394
 395         if (fixup & FIXUP_RETURN_REGISTER) {
 396                 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
 397                 regs->gprs[reg] += (unsigned long) p->addr -
 398                                    (unsigned long) p->ainsn.insn;
 399         }
 400
 401         disable_singlestep(kcb, regs, ip);
 402 }
 403 NOKPROBE_SYMBOL(resume_execution);
 404
 405 static int post_kprobe_handler(struct pt_regs *regs)
 406 {
 407         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 408         struct kprobe *p = kprobe_running();
 409
 410         if (!p)
 411                 return 0;
 412
 413         resume_execution(p, regs);
 414         if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
 415                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
 416                 p->post_handler(p, regs, 0);
 417         }
 418         pop_kprobe(kcb);
 419         preempt_enable_no_resched();
 420
 421         /*
 422          * if somebody else is singlestepping across a probe point, psw mask
 423          * will have PER set, in which case, continue the remaining processing
 424          * of do_single_step, as if this is not a probe hit.
 425          */
 426         if (regs->psw.mask & PSW_MASK_PER)
 427                 return 0;
 428
 429         return 1;
 430 }
 431 NOKPROBE_SYMBOL(post_kprobe_handler);
 432
 433 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 434 {
 435         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 436         struct kprobe *p = kprobe_running();
 437
 438         switch(kcb->kprobe_status) {
 439         case KPROBE_HIT_SS:
 440         case KPROBE_REENTER:
 441                 /*
 442                  * We are here because the instruction being single
 443                  * stepped caused a page fault. We reset the current
 444                  * kprobe and the nip points back to the probe address
 445                  * and allow the page fault handler to continue as a
 446                  * normal page fault.
 447                  */
 448                 disable_singlestep(kcb, regs, (unsigned long) p->addr);
 449                 pop_kprobe(kcb);
 450                 preempt_enable_no_resched();
 451                 break;
 452         case KPROBE_HIT_ACTIVE:
 453         case KPROBE_HIT_SSDONE:
 454                 /*
 455                  * In case the user-specified fault handler returned
 456                  * zero, try to fix up.
 457                  */
 458                 if (fixup_exception(regs))
 459                         return 1;
 460                 /*
 461                  * fixup_exception() could not handle it,
 462                  * Let do_page_fault() fix it.
 463                  */
 464                 break;
 465         default:
 466                 break;
 467         }
 468         return 0;
 469 }
 470 NOKPROBE_SYMBOL(kprobe_trap_handler);
 471
 472 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 473 {
 474         int ret;
 475
 476         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 477                 local_irq_disable();
 478         ret = kprobe_trap_handler(regs, trapnr);
 479         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 480                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 481         return ret;
 482 }
 483 NOKPROBE_SYMBOL(kprobe_fault_handler);
 484
 485 /*
 486  * Wrapper routine to for handling exceptions.
 487  */
 488 int kprobe_exceptions_notify(struct notifier_block *self,
 489                              unsigned long val, void *data)
 490 {
 491         struct die_args *args = (struct die_args *) data;
 492         struct pt_regs *regs = args->regs;
 493         int ret = NOTIFY_DONE;
 494
 495         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 496                 local_irq_disable();
 497
 498         switch (val) {
 499         case DIE_BPT:
 500                 if (kprobe_handler(regs))
 501                         ret = NOTIFY_STOP;
 502                 break;
 503         case DIE_SSTEP:
 504                 if (post_kprobe_handler(regs))
 505                         ret = NOTIFY_STOP;
 506                 break;
 507         case DIE_TRAP:
 508                 if (!preemptible() && kprobe_running() &&
 509                     kprobe_trap_handler(regs, args->trapnr))
 510                         ret = NOTIFY_STOP;
 511                 break;
 512         default:
 513                 break;
 514         }
 515
 516         if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
 517                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
 518
 519         return ret;
 520 }
 521 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 522
 523 int __init arch_init_kprobes(void)
 524 {
 525         return 0;
 526 }
 527
 528 int arch_trampoline_kprobe(struct kprobe *p)
 529 {
 530         return 0;
 531 }
 532 NOKPROBE_SYMBOL(arch_trampoline_kprobe);