uprobe/arch/x86/swap-asm/swap_uprobes.c

   1 /**
   2  * uprobe/arch/asm-x86/swap_uprobes.c
   3  * @author Alexey Gerenkov <a.gerenkov@samsung.com> User-Space Probes initial
   4  * implementation; Support x86/ARM/MIPS for both user and kernel spaces.
   5  * @author Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for
   6  * separating core and arch parts
   7  *
   8  * @section LICENSE
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  23  *
  24  * @section COPYRIGHT
  25  *
  26  * Copyright (C) Samsung Electronics, 2006-2010
  27  *
  28  * @section DESCRIPTION
  29  *
  30  * Arch-dependent uprobe interface implementation for x86.
  31  */
  32
  33
  34 #include <linux/kdebug.h>
  35
  36 #include <kprobe/swap_slots.h>
  37 #include <uprobe/swap_uprobes.h>
  38
  39 #include "swap_uprobes.h"
  40
  41
  42 /**
  43  * @struct uprobe_ctlblk
  44  * @brief Uprobe control block
  45  */
  46 struct uprobe_ctlblk {
  47         unsigned long flags;            /**< Flags */
  48         struct kprobe *p;               /**< Pointer to the uprobe's kprobe */
  49 };
  50
  51 static unsigned long trampoline_addr(struct uprobe *up)
  52 {
  53         return (unsigned long)(up->kp.ainsn.insn +
  54                                UPROBES_TRAMP_RET_BREAK_IDX);
  55 }
  56
  57 unsigned long arch_tramp_by_ri(struct uretprobe_instance *ri)
  58 {
  59         return trampoline_addr(&ri->rp->up);
  60 }
  61
  62 static struct uprobe_ctlblk *current_ucb(void)
  63 {
  64         /* FIXME hardcoded offset */
  65         return (struct uprobe_ctlblk *)(end_of_stack(current) + 20);
  66 }
  67
  68 static struct kprobe *get_current_probe(void)
  69 {
  70         return current_ucb()->p;
  71 }
  72
  73 static void set_current_probe(struct kprobe *p)
  74 {
  75         current_ucb()->p = p;
  76 }
  77
  78 static void save_current_flags(struct pt_regs *regs)
  79 {
  80         current_ucb()->flags = regs->flags;
  81 }
  82
  83 static void restore_current_flags(struct pt_regs *regs, unsigned long flags)
  84 {
  85         regs->flags &= ~IF_MASK;
  86         regs->flags |= flags & IF_MASK;
  87 }
  88
  89 /**
  90  * @brief Prepares uprobe for x86.
  91  *
  92  * @param up Pointer to the uprobe.
  93  * @return 0 on success,\n
  94  * -1 on error.
  95  */
  96 int arch_prepare_uprobe(struct uprobe *up)
  97 {
  98         struct kprobe *p = up2kp(up);
  99         struct task_struct *task = up->task;
 100         u8 *tramp = up->atramp.tramp;
 101         enum { call_relative_opcode = 0xe8 };
 102
 103         if (!read_proc_vm_atomic(task, (unsigned long)p->addr,
 104                                  tramp, MAX_INSN_SIZE)) {
 105                 printk(KERN_ERR "failed to read memory %p!\n", p->addr);
 106                 return -EINVAL;
 107         }
 108         /* TODO: this is a workaround */
 109         if (tramp[0] == call_relative_opcode) {
 110                 printk(KERN_INFO "cannot install probe: 1st instruction is call\n");
 111                 return -EINVAL;
 112         }
 113
 114         tramp[UPROBES_TRAMP_RET_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
 115
 116         /* TODO: remove dual info */
 117         p->opcode = tramp[0];
 118
 119         p->ainsn.boostable = swap_can_boost(tramp) ? 0 : -1;
 120
 121         p->ainsn.insn = swap_slot_alloc(up->sm);
 122         if (p->ainsn.insn == NULL) {
 123                 printk(KERN_ERR "trampoline out of memory\n");
 124                 return -ENOMEM;
 125         }
 126
 127         if (!write_proc_vm_atomic(task, (unsigned long)p->ainsn.insn,
 128                                   tramp, sizeof(up->atramp.tramp))) {
 129                 swap_slot_free(up->sm, p->ainsn.insn);
 130                 printk(KERN_INFO "failed to write memory %p!\n", tramp);
 131                 return -EINVAL;
 132         }
 133
 134         /* for uretprobe */
 135         add_uprobe_table(p);
 136
 137         return 0;
 138 }
 139
 140 /**
 141  * @brief Jump pre-handler.
 142  *
 143  * @param p Pointer to the uprobe's kprobe.
 144  * @param regs Pointer to CPU register data.
 145  * @return 0.
 146  */
 147 int setjmp_upre_handler(struct kprobe *p, struct pt_regs *regs)
 148 {
 149         struct uprobe *up = container_of(p, struct uprobe, kp);
 150         struct ujprobe *jp = container_of(up, struct ujprobe, up);
 151         kprobe_pre_entry_handler_t pre_entry =
 152                 (kprobe_pre_entry_handler_t)jp->pre_entry;
 153         entry_point_t entry = (entry_point_t)jp->entry;
 154         unsigned long args[6];
 155
 156         /* FIXME some user space apps crash if we clean interrupt bit */
 157         /* regs->EREG(flags) &= ~IF_MASK; */
 158 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
 159         trace_hardirqs_off();
 160 #endif
 161
 162         /* read first 6 args from stack */
 163         if (!read_proc_vm_atomic(current, regs->EREG(sp) + 4,
 164                                  args, sizeof(args)))
 165                 printk(KERN_WARNING
 166                        "failed to read user space func arguments %lx!\n",
 167                        regs->sp + 4);
 168
 169         if (pre_entry)
 170                 p->ss_addr[smp_processor_id()] = (kprobe_opcode_t *)
 171                                                  pre_entry(jp->priv_arg, regs);
 172
 173         if (entry)
 174                 entry(args[0], args[1], args[2], args[3], args[4], args[5]);
 175         else
 176                 arch_ujprobe_return();
 177
 178         return 0;
 179 }
 180
 181 /**
 182  * @brief Prepares uretprobe for x86.
 183  *
 184  * @param ri Pointer to the uretprobe instance.
 185  * @param regs Pointer to CPU register data.
 186  * @return Void.
 187  */
 188 int arch_prepare_uretprobe(struct uretprobe_instance *ri, struct pt_regs *regs)
 189 {
 190         /* Replace the return addr with trampoline addr */
 191         unsigned long ra = trampoline_addr(&ri->rp->up);
 192         ri->sp = (kprobe_opcode_t *)regs->sp;
 193
 194         if (!read_proc_vm_atomic(current, regs->EREG(sp), &(ri->ret_addr),
 195                                  sizeof(ri->ret_addr))) {
 196                 printk(KERN_ERR "failed to read user space func ra %lx addr=%p!\n",
 197                                 regs->EREG(sp), ri->rp->up.kp.addr);
 198                 return -EINVAL;
 199         }
 200
 201         if (!write_proc_vm_atomic(current, regs->EREG(sp), &ra, sizeof(ra))) {
 202                 printk(KERN_ERR "failed to write user space func ra %lx!\n",
 203                        regs->EREG(sp));
 204                 return -EINVAL;
 205         }
 206
 207         return 0;
 208 }
 209
 210 /**
 211  * @brief Disarms uretprobe on x86 arch.
 212  *
 213  * @param ri Pointer to the uretprobe instance.
 214  * @param task Pointer to the task for which the probe.
 215  * @return 0 on success,\n
 216  * negative error code on error.
 217  */
 218 int arch_disarm_urp_inst(struct uretprobe_instance *ri,
 219                          struct task_struct *task, unsigned long tr)
 220 {
 221         int len;
 222         unsigned long ret_addr;
 223         unsigned long sp = (unsigned long)ri->sp;
 224         unsigned long tramp_addr;
 225
 226         if (tr == 0)
 227                 tramp_addr = arch_tramp_by_ri(ri);
 228         else
 229                 tramp_addr = tr; /* ri - invalid */
 230
 231         len = read_proc_vm_atomic(task, sp, &ret_addr, sizeof(ret_addr));
 232         if (len != sizeof(ret_addr)) {
 233                 printk(KERN_INFO "---> %s (%d/%d): failed to read stack from %08lx\n",
 234                        task->comm, task->tgid, task->pid, sp);
 235                 return -EFAULT;
 236         }
 237
 238         if (tramp_addr == ret_addr) {
 239                 len = write_proc_vm_atomic(task, sp, &ri->ret_addr,
 240                                            sizeof(ri->ret_addr));
 241                 if (len != sizeof(ri->ret_addr)) {
 242                         printk(KERN_INFO "---> %s (%d/%d): failed to write "
 243                                "orig_ret_addr to %08lx",
 244                                task->comm, task->tgid, task->pid, sp);
 245                         return -EFAULT;
 246                 }
 247         } else {
 248                 printk(KERN_INFO "---> %s (%d/%d): trampoline NOT found at sp = %08lx\n",
 249                        task->comm, task->tgid, task->pid, sp);
 250                 return -ENOENT;
 251         }
 252
 253         return 0;
 254 }
 255
 256 /**
 257  * @brief Gets trampoline address.
 258  *
 259  * @param p Pointer to the uprobe's kprobe.
 260  * @param regs Pointer to CPU register data.
 261  * @return Trampoline address.
 262  */
 263 unsigned long arch_get_trampoline_addr(struct kprobe *p, struct pt_regs *regs)
 264 {
 265         return trampoline_addr(kp2up(p));
 266 }
 267
 268 /**
 269  * @brief Restores return address.
 270  *
 271  * @param orig_ret_addr Original return address.
 272  * @param regs Pointer to CPU register data.
 273  * @return Void.
 274  */
 275 void arch_set_orig_ret_addr(unsigned long orig_ret_addr, struct pt_regs *regs)
 276 {
 277         regs->EREG(ip) = orig_ret_addr;
 278 }
 279
 280 /**
 281  * @brief Removes uprobe.
 282  *
 283  * @param up Pointer to the target uprobe.
 284  * @return Void.
 285  */
 286 void arch_remove_uprobe(struct uprobe *up)
 287 {
 288         struct kprobe *p = up2kp(up);
 289
 290         swap_slot_free(up->sm, p->ainsn.insn);
 291 }
 292
 293 int arch_arm_uprobe(struct uprobe *p)
 294 {
 295         int ret;
 296         kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
 297         unsigned long vaddr = (unsigned long)p->kp.addr;
 298
 299         ret = write_proc_vm_atomic(p->task, vaddr, &insn, sizeof(insn));
 300         if (!ret) {
 301                 pr_err("arch_arm_uprobe: failed to write memory tgid=%u vaddr=%08lx\n",
 302                        p->task->tgid, vaddr);
 303
 304                 return -EACCES;
 305         }
 306
 307         return 0;
 308 }
 309
 310 void arch_disarm_uprobe(struct kprobe *p, struct task_struct *task)
 311 {
 312         int ret;
 313         unsigned long vaddr = (unsigned long)p->addr;
 314
 315         ret = write_proc_vm_atomic(task, vaddr, &p->opcode, sizeof(p->opcode));
 316         if (!ret) {
 317                 pr_err("arch_disarm_uprobe: failed to write memory tgid=%u, vaddr=%08lx\n",
 318                        task->tgid, vaddr);
 319         }
 320 }
 321
 322 static void set_user_jmp_op(void *from, void *to)
 323 {
 324         struct __arch_jmp_op {
 325                 char op;
 326                 long raddr;
 327         } __packed jop;
 328
 329         jop.raddr = (long)(to) - ((long)(from) + 5);
 330         jop.op = RELATIVEJUMP_INSTRUCTION;
 331
 332         if (!write_proc_vm_atomic(current, (unsigned long)from, &jop,
 333                                   sizeof(jop)))
 334                 printk(KERN_WARNING
 335                        "failed to write jump opcode to user space %p\n", from);
 336 }
 337
 338 static void resume_execution(struct kprobe *p,
 339                              struct pt_regs *regs,
 340                              unsigned long flags)
 341 {
 342         unsigned long *tos, tos_dword = 0;
 343         unsigned long copy_eip = (unsigned long)p->ainsn.insn;
 344         unsigned long orig_eip = (unsigned long)p->addr;
 345         kprobe_opcode_t insns[2];
 346
 347         regs->EREG(flags) &= ~TF_MASK;
 348
 349         tos = (unsigned long *)&tos_dword;
 350         if (!read_proc_vm_atomic(current, regs->EREG(sp), &tos_dword,
 351                                  sizeof(tos_dword))) {
 352                 printk(KERN_WARNING
 353                        "failed to read dword from top of the user space stack %lx!\n",
 354                        regs->sp);
 355                 return;
 356         }
 357
 358         if (!read_proc_vm_atomic(current, (unsigned long)p->ainsn.insn, insns,
 359                                  2 * sizeof(kprobe_opcode_t))) {
 360                 printk(KERN_WARNING
 361                        "failed to read first 2 opcodes of instruction copy from user space %p!\n",
 362                        p->ainsn.insn);
 363                 return;
 364         }
 365
 366         switch (insns[0]) {
 367         case 0x9c: /* pushfl */
 368                 *tos &= ~(TF_MASK | IF_MASK);
 369                 *tos |= flags & (TF_MASK | IF_MASK);
 370                 break;
 371         case 0xc2: /* iret/ret/lret */
 372         case 0xc3:
 373         case 0xca:
 374         case 0xcb:
 375         case 0xcf:
 376         case 0xea: /* jmp absolute -- eip is correct */
 377                 /* eip is already adjusted, no more changes required */
 378                 p->ainsn.boostable = 1;
 379                 goto no_change;
 380         case 0xe8: /* call relative - Fix return addr */
 381                 *tos = orig_eip + (*tos - copy_eip);
 382                 break;
 383         case 0x9a: /* call absolute -- same as call absolute, indirect */
 384                 *tos = orig_eip + (*tos - copy_eip);
 385
 386                 if (!write_proc_vm_atomic(current,
 387                                           regs->EREG(sp),
 388                                           &tos_dword,
 389                                           sizeof(tos_dword))) {
 390                         printk(KERN_WARNING
 391                                "failed to write dword to top of the user space stack %lx!\n",
 392                                regs->sp);
 393                         return;
 394                 }
 395
 396                 goto no_change;
 397         case 0xff:
 398                 if ((insns[1] & 0x30) == 0x10) {
 399                         /*
 400                          * call absolute, indirect
 401                          * Fix return addr; eip is correct.
 402                          * But this is not boostable
 403                          */
 404                         *tos = orig_eip + (*tos - copy_eip);
 405
 406                         if (!write_proc_vm_atomic(current, regs->EREG(sp),
 407                                                   &tos_dword,
 408                                                   sizeof(tos_dword))) {
 409                                 printk(KERN_WARNING
 410                                        "failed to write dword to top of the user space stack %lx!\n",
 411                                        regs->EREG(sp));
 412                                 return;
 413                         }
 414
 415                         goto no_change;
 416                 } else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute
 417                                                            * indirect */
 418                            ((insns[1] & 0x31) == 0x21)) {
 419                         /* jmp far, absolute indirect */
 420                         /* eip is correct. And this is boostable */
 421                         p->ainsn.boostable = 1;
 422                         goto no_change;
 423                 }
 424         case 0xf3:
 425                 if (insns[1] == 0xc3)
 426                         /* repz ret special handling: no more changes */
 427                         goto no_change;
 428                 break;
 429         default:
 430                 break;
 431         }
 432
 433         if (!write_proc_vm_atomic(current, regs->EREG(sp), &tos_dword,
 434                                   sizeof(tos_dword))) {
 435                 printk(KERN_WARNING
 436                        "failed to write dword to top of the user space stack %lx!\n",
 437                        regs->EREG(sp));
 438                 return;
 439         }
 440
 441         if (p->ainsn.boostable == 0) {
 442                 if ((regs->EREG(ip) > copy_eip) && (regs->EREG(ip) - copy_eip) +
 443                     5 < MAX_INSN_SIZE) {
 444                         /*
 445                          * These instructions can be executed directly if it
 446                          * jumps back to correct address.
 447                          */
 448                         set_user_jmp_op((void *) regs->EREG(ip),
 449                                         (void *)orig_eip +
 450                                         (regs->EREG(ip) - copy_eip));
 451                         p->ainsn.boostable = 1;
 452                 } else {
 453                         p->ainsn.boostable = -1;
 454                 }
 455         }
 456
 457         regs->EREG(ip) = orig_eip + (regs->EREG(ip) - copy_eip);
 458
 459 no_change:
 460         return;
 461 }
 462
 463 static bool prepare_ss_addr(struct kprobe *p, struct pt_regs *regs)
 464 {
 465         unsigned long *ss_addr = (long *)&p->ss_addr[smp_processor_id()];
 466
 467         if (*ss_addr) {
 468                 regs->ip = *ss_addr;
 469                 *ss_addr = 0;
 470                 return true;
 471         } else {
 472                 regs->ip = (unsigned long)p->ainsn.insn;
 473                 return false;
 474         }
 475 }
 476
 477 static void prepare_ss(struct pt_regs *regs)
 478 {
 479         /* set single step mode */
 480         regs->flags |= TF_MASK;
 481         regs->flags &= ~IF_MASK;
 482 }
 483
 484 static int uprobe_handler(struct pt_regs *regs)
 485 {
 486         struct kprobe *p;
 487         kprobe_opcode_t *addr;
 488         struct task_struct *task = current;
 489         pid_t tgid = task->tgid;
 490
 491         save_current_flags(regs);
 492
 493         addr = (kprobe_opcode_t *)(regs->EREG(ip) - sizeof(kprobe_opcode_t));
 494         p = get_ukprobe(addr, tgid);
 495
 496         if (p == NULL) {
 497                 void *tramp_addr = (void *)addr - UPROBES_TRAMP_RET_BREAK_IDX;
 498
 499                 p = get_ukprobe_by_insn_slot(tramp_addr, tgid, regs);
 500                 if (p == NULL) {
 501                         printk(KERN_INFO "no_uprobe\n");
 502                         return 0;
 503                 }
 504
 505                 trampoline_uprobe_handler(p, regs);
 506                 return 1;
 507         } else {
 508                 if (!p->pre_handler || !p->pre_handler(p, regs)) {
 509                         if (p->ainsn.boostable == 1 && !p->post_handler) {
 510                                 prepare_ss_addr(p, regs);
 511                                 return 1;
 512                         }
 513
 514                         if (prepare_ss_addr(p, regs) == false) {
 515                                 set_current_probe(p);
 516                                 prepare_ss(regs);
 517                         }
 518                 }
 519         }
 520
 521         return 1;
 522 }
 523
 524 static int post_uprobe_handler(struct pt_regs *regs)
 525 {
 526         struct kprobe *p = get_current_probe();
 527         unsigned long flags = current_ucb()->flags;
 528
 529         if (p == NULL) {
 530                 printk("task[%u %u %s] current uprobe is not found\n",
 531                        current->tgid, current->pid, current->comm);
 532                 return 0;
 533         }
 534
 535         resume_execution(p, regs, flags);
 536         restore_current_flags(regs, flags);
 537
 538         /* clean stack */
 539         current_ucb()->p = 0;
 540         current_ucb()->flags = 0;
 541
 542         return 1;
 543 }
 544
 545 static int uprobe_exceptions_notify(struct notifier_block *self,
 546                                     unsigned long val, void *data)
 547 {
 548         struct die_args *args = (struct die_args *)data;
 549         int ret = NOTIFY_DONE;
 550
 551         if (args->regs == NULL || !user_mode_vm(args->regs))
 552                 return ret;
 553
 554         switch (val) {
 555 #ifdef CONFIG_KPROBES
 556         case DIE_INT3:
 557 #else
 558         case DIE_TRAP:
 559 #endif
 560                 if (uprobe_handler(args->regs))
 561                         ret = NOTIFY_STOP;
 562                 break;
 563         case DIE_DEBUG:
 564                 if (post_uprobe_handler(args->regs))
 565                         ret = NOTIFY_STOP;
 566                 break;
 567         default:
 568                 break;
 569         }
 570
 571         return ret;
 572 }
 573
 574 static struct notifier_block uprobe_exceptions_nb = {
 575         .notifier_call = uprobe_exceptions_notify,
 576         .priority = INT_MAX
 577 };
 578
 579 /**
 580  * @brief Registers notify.
 581  *
 582  * @return register_die_notifier result.
 583  */
 584 int swap_arch_init_uprobes(void)
 585 {
 586         return register_die_notifier(&uprobe_exceptions_nb);
 587 }
 588
 589 /**
 590  * @brief Unregisters notify.
 591  *
 592  * @return Void.
 593  */
 594 void swap_arch_exit_uprobes(void)
 595 {
 596         unregister_die_notifier(&uprobe_exceptions_nb);
 597 }
 598