uprobe/arch/x86/swap-asm/swap_uprobes.c

   1 /**
   2  * uprobe/arch/asm-x86/swap_uprobes.c
   3  * @author Alexey Gerenkov <a.gerenkov@samsung.com> User-Space Probes initial
   4  * implementation; Support x86/ARM/MIPS for both user and kernel spaces.
   5  * @author Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for
   6  * separating core and arch parts
   7  *
   8  * @section LICENSE
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  23  *
  24  * @section COPYRIGHT
  25  *
  26  * Copyright (C) Samsung Electronics, 2006-2010
  27  *
  28  * @section DESCRIPTION
  29  *
  30  * Arch-dependent uprobe interface implementation for x86.
  31  */
  32
  33
  34 #include <linux/kdebug.h>
  35
  36 #include <kprobe/swap_slots.h>
  37 #include <uprobe/swap_uprobes.h>
  38
  39 #include "swap_uprobes.h"
  40
  41
  42 /**
  43  * @struct uprobe_ctlblk
  44  * @brief Uprobe control block
  45  */
  46 struct uprobe_ctlblk {
  47         unsigned long flags;            /**< Flags */
  48         struct uprobe *p;               /**< Pointer to the uprobe */
  49 };
  50
  51 static unsigned long trampoline_addr(struct uprobe *up)
  52 {
  53         return (unsigned long)(up->ainsn.insn +
  54                                UPROBES_TRAMP_RET_BREAK_IDX);
  55 }
  56
  57 unsigned long arch_tramp_by_ri(struct uretprobe_instance *ri)
  58 {
  59         return trampoline_addr(&ri->rp->up);
  60 }
  61
  62 static struct uprobe_ctlblk *current_ucb(void)
  63 {
  64         /* FIXME hardcoded offset */
  65         return (struct uprobe_ctlblk *)(end_of_stack(current) + 20);
  66 }
  67
  68 static struct uprobe *get_current_probe(void)
  69 {
  70         return current_ucb()->p;
  71 }
  72
  73 static void set_current_probe(struct uprobe *p)
  74 {
  75         current_ucb()->p = p;
  76 }
  77
  78 static void save_current_flags(struct pt_regs *regs)
  79 {
  80         current_ucb()->flags = regs->flags;
  81 }
  82
  83 static void restore_current_flags(struct pt_regs *regs, unsigned long flags)
  84 {
  85         regs->flags &= ~IF_MASK;
  86         regs->flags |= flags & IF_MASK;
  87 }
  88
  89 /**
  90  * @brief Prepares uprobe for x86.
  91  *
  92  * @param up Pointer to the uprobe.
  93  * @return 0 on success,\n
  94  * -1 on error.
  95  */
  96 int arch_prepare_uprobe(struct uprobe *p)
  97 {
  98         struct task_struct *task = p->task;
  99         u8 *tramp = p->atramp.tramp;
 100         enum { call_relative_opcode = 0xe8 };
 101
 102         if (!read_proc_vm_atomic(task, (unsigned long)p->addr,
 103                                  tramp, MAX_INSN_SIZE)) {
 104                 printk(KERN_ERR "failed to read memory %p!\n", p->addr);
 105                 return -EINVAL;
 106         }
 107         /* TODO: this is a workaround */
 108         if (tramp[0] == call_relative_opcode) {
 109                 printk(KERN_INFO "cannot install probe: 1st instruction is call\n");
 110                 return -EINVAL;
 111         }
 112
 113         tramp[UPROBES_TRAMP_RET_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
 114
 115         /* TODO: remove dual info */
 116         p->opcode = tramp[0];
 117
 118         p->ainsn.boostable = swap_can_boost(tramp) ? 0 : -1;
 119
 120         p->ainsn.insn = swap_slot_alloc(p->sm);
 121         if (p->ainsn.insn == NULL) {
 122                 printk(KERN_ERR "trampoline out of memory\n");
 123                 return -ENOMEM;
 124         }
 125
 126         if (!write_proc_vm_atomic(task, (unsigned long)p->ainsn.insn,
 127                                   tramp, sizeof(p->atramp.tramp))) {
 128                 swap_slot_free(p->sm, p->ainsn.insn);
 129                 printk(KERN_INFO "failed to write memory %p!\n", tramp);
 130                 return -EINVAL;
 131         }
 132
 133         /* for uretprobe */
 134         add_uprobe_table(p);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * @brief Jump pre-handler.
 141  *
 142  * @param p Pointer to the uprobe.
 143  * @param regs Pointer to CPU register data.
 144  * @return 0.
 145  */
 146 int setjmp_upre_handler(struct uprobe *p, struct pt_regs *regs)
 147 {
 148         struct ujprobe *jp = container_of(p, struct ujprobe, up);
 149         uprobe_pre_entry_handler_t pre_entry =
 150                 (uprobe_pre_entry_handler_t)jp->pre_entry;
 151         entry_point_t entry = (entry_point_t)jp->entry;
 152         unsigned long args[6];
 153
 154         /* FIXME some user space apps crash if we clean interrupt bit */
 155         /* regs->EREG(flags) &= ~IF_MASK; */
 156 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
 157         trace_hardirqs_off();
 158 #endif
 159
 160         /* read first 6 args from stack */
 161         if (!read_proc_vm_atomic(current, regs->EREG(sp) + 4,
 162                                  args, sizeof(args)))
 163                 printk(KERN_WARNING
 164                        "failed to read user space func arguments %lx!\n",
 165                        regs->sp + 4);
 166
 167         if (pre_entry)
 168                 p->ss_addr[smp_processor_id()] = (uprobe_opcode_t *)
 169                                                  pre_entry(jp->priv_arg, regs);
 170
 171         if (entry)
 172                 entry(args[0], args[1], args[2], args[3], args[4], args[5]);
 173         else
 174                 arch_ujprobe_return();
 175
 176         return 0;
 177 }
 178
 179 /**
 180  * @brief Prepares uretprobe for x86.
 181  *
 182  * @param ri Pointer to the uretprobe instance.
 183  * @param regs Pointer to CPU register data.
 184  * @return Void.
 185  */
 186 int arch_prepare_uretprobe(struct uretprobe_instance *ri, struct pt_regs *regs)
 187 {
 188         /* Replace the return addr with trampoline addr */
 189         unsigned long ra = trampoline_addr(&ri->rp->up);
 190         unsigned long ret_addr;
 191         ri->sp = (kprobe_opcode_t *)regs->sp;
 192
 193         if (get_user(ret_addr, (unsigned long *)regs->sp)) {
 194                 pr_err("failed to read user space func ra %lx addr=%p!\n",
 195                        regs->sp, ri->rp->up.addr);
 196                 return -EINVAL;
 197         }
 198
 199         if (put_user(ra, (unsigned long *)regs->sp)) {
 200                 pr_err("failed to write user space func ra %lx!\n", regs->sp);
 201                 return -EINVAL;
 202         }
 203
 204         ri->ret_addr = (uprobe_opcode_t *)ret_addr;
 205
 206         return 0;
 207 }
 208
 209 static bool get_long(struct task_struct *task,
 210                      unsigned long vaddr, unsigned long *val)
 211 {
 212         return sizeof(*val) != read_proc_vm_atomic(task, vaddr,
 213                                                    val, sizeof(*val));
 214 }
 215
 216 static bool put_long(struct task_struct *task,
 217                      unsigned long vaddr, unsigned long *val)
 218 {
 219         return sizeof(*val) != write_proc_vm_atomic(task, vaddr,
 220                                                     val, sizeof(*val));
 221 }
 222
 223 /**
 224  * @brief Disarms uretprobe on x86 arch.
 225  *
 226  * @param ri Pointer to the uretprobe instance.
 227  * @param task Pointer to the task for which the probe.
 228  * @return 0 on success,\n
 229  * negative error code on error.
 230  */
 231 int arch_disarm_urp_inst(struct uretprobe_instance *ri,
 232                          struct task_struct *task, unsigned long tr)
 233 {
 234         unsigned long ret_addr;
 235         unsigned long sp = (unsigned long)ri->sp;
 236         unsigned long tramp_addr;
 237
 238         if (tr == 0)
 239                 tramp_addr = arch_tramp_by_ri(ri);
 240         else
 241                 tramp_addr = tr; /* ri - invalid */
 242
 243         if (get_long(task, sp, &ret_addr)) {
 244                 printk(KERN_INFO "---> %s (%d/%d): failed to read stack from %08lx\n",
 245                        task->comm, task->tgid, task->pid, sp);
 246                 return -EFAULT;
 247         }
 248
 249         if (tramp_addr == ret_addr) {
 250                 if (put_long(task, sp, (unsigned long *)&ri->ret_addr)) {
 251                         printk(KERN_INFO "---> %s (%d/%d): failed to write "
 252                                "orig_ret_addr to %08lx",
 253                                task->comm, task->tgid, task->pid, sp);
 254                         return -EFAULT;
 255                 }
 256         } else {
 257                 printk(KERN_INFO "---> %s (%d/%d): trampoline NOT found at sp = %08lx\n",
 258                        task->comm, task->tgid, task->pid, sp);
 259                 return -ENOENT;
 260         }
 261
 262         return 0;
 263 }
 264
 265 /**
 266  * @brief Gets trampoline address.
 267  *
 268  * @param p Pointer to the uprobe.
 269  * @param regs Pointer to CPU register data.
 270  * @return Trampoline address.
 271  */
 272 unsigned long arch_get_trampoline_addr(struct uprobe *p, struct pt_regs *regs)
 273 {
 274         return trampoline_addr(p);
 275 }
 276
 277 /**
 278  * @brief Restores return address.
 279  *
 280  * @param orig_ret_addr Original return address.
 281  * @param regs Pointer to CPU register data.
 282  * @return Void.
 283  */
 284 void arch_set_orig_ret_addr(unsigned long orig_ret_addr, struct pt_regs *regs)
 285 {
 286         regs->EREG(ip) = orig_ret_addr;
 287 }
 288
 289 /**
 290  * @brief Removes uprobe.
 291  *
 292  * @param up Pointer to the target uprobe.
 293  * @return Void.
 294  */
 295 void arch_remove_uprobe(struct uprobe *p)
 296 {
 297         swap_slot_free(p->sm, p->ainsn.insn);
 298 }
 299
 300 int arch_arm_uprobe(struct uprobe *p)
 301 {
 302         int ret;
 303         uprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
 304         unsigned long vaddr = (unsigned long)p->addr;
 305
 306         ret = write_proc_vm_atomic(p->task, vaddr, &insn, sizeof(insn));
 307         if (!ret) {
 308                 pr_err("arch_arm_uprobe: failed to write memory tgid=%u vaddr=%08lx\n",
 309                        p->task->tgid, vaddr);
 310
 311                 return -EACCES;
 312         }
 313
 314         return 0;
 315 }
 316
 317 void arch_disarm_uprobe(struct uprobe *p, struct task_struct *task)
 318 {
 319         int ret;
 320         unsigned long vaddr = (unsigned long)p->addr;
 321
 322         ret = write_proc_vm_atomic(task, vaddr, &p->opcode, sizeof(p->opcode));
 323         if (!ret) {
 324                 pr_err("arch_disarm_uprobe: failed to write memory tgid=%u, vaddr=%08lx\n",
 325                        task->tgid, vaddr);
 326         }
 327 }
 328
 329 static void set_user_jmp_op(void *from, void *to)
 330 {
 331         struct __arch_jmp_op {
 332                 char op;
 333                 long raddr;
 334         } __packed jop;
 335
 336         jop.raddr = (long)(to) - ((long)(from) + 5);
 337         jop.op = RELATIVEJUMP_INSTRUCTION;
 338
 339         if (put_user(jop.op, (char *)from) ||
 340             put_user(jop.raddr, (long *)(from + 1)))
 341                 pr_err("failed to write jump opcode to user space %p\n", from);
 342 }
 343
 344 static void resume_execution(struct uprobe *p,
 345                              struct pt_regs *regs,
 346                              unsigned long flags)
 347 {
 348         unsigned long *tos, tos_dword = 0;
 349         unsigned long copy_eip = (unsigned long)p->ainsn.insn;
 350         unsigned long orig_eip = (unsigned long)p->addr;
 351         uprobe_opcode_t insns[2];
 352
 353         regs->EREG(flags) &= ~TF_MASK;
 354
 355         tos = (unsigned long *)&tos_dword;
 356         if (get_user(tos_dword, (unsigned long *)regs->sp)) {
 357                 pr_err("failed to read from user space sp=%lx!\n", regs->sp);
 358                 return;
 359         }
 360
 361         if (get_user(*(unsigned short *)insns, (unsigned short *)p->ainsn.insn)) {
 362                 pr_err("failed to read first 2 opcodes %p!\n", p->ainsn.insn);
 363                 return;
 364         }
 365
 366         switch (insns[0]) {
 367         case 0x9c: /* pushfl */
 368                 *tos &= ~(TF_MASK | IF_MASK);
 369                 *tos |= flags & (TF_MASK | IF_MASK);
 370                 break;
 371         case 0xc2: /* iret/ret/lret */
 372         case 0xc3:
 373         case 0xca:
 374         case 0xcb:
 375         case 0xcf:
 376         case 0xea: /* jmp absolute -- eip is correct */
 377                 /* eip is already adjusted, no more changes required */
 378                 p->ainsn.boostable = 1;
 379                 goto no_change;
 380         case 0xe8: /* call relative - Fix return addr */
 381                 *tos = orig_eip + (*tos - copy_eip);
 382                 break;
 383         case 0x9a: /* call absolute -- same as call absolute, indirect */
 384                 *tos = orig_eip + (*tos - copy_eip);
 385
 386                 if (put_user(tos_dword, (unsigned long *)regs->sp)) {
 387                         pr_err("failed to write dword to sp=%lx\n", regs->sp);
 388                         return;
 389                 }
 390
 391                 goto no_change;
 392         case 0xff:
 393                 if ((insns[1] & 0x30) == 0x10) {
 394                         /*
 395                          * call absolute, indirect
 396                          * Fix return addr; eip is correct.
 397                          * But this is not boostable
 398                          */
 399                         *tos = orig_eip + (*tos - copy_eip);
 400
 401                         if (put_user(tos_dword, (unsigned long *)regs->sp)) {
 402                                 pr_err("failed to write dword to sp=%lx\n", regs->sp);
 403                                 return;
 404                         }
 405
 406                         goto no_change;
 407                 } else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute
 408                                                            * indirect */
 409                            ((insns[1] & 0x31) == 0x21)) {
 410                         /* jmp far, absolute indirect */
 411                         /* eip is correct. And this is boostable */
 412                         p->ainsn.boostable = 1;
 413                         goto no_change;
 414                 }
 415         case 0xf3:
 416                 if (insns[1] == 0xc3)
 417                         /* repz ret special handling: no more changes */
 418                         goto no_change;
 419                 break;
 420         default:
 421                 break;
 422         }
 423
 424         if (put_user(tos_dword, (unsigned long *)regs->sp)) {
 425                 pr_err("failed to write dword to sp=%lx\n", regs->sp);
 426                 return;
 427         }
 428
 429         if (p->ainsn.boostable == 0) {
 430                 if ((regs->EREG(ip) > copy_eip) && (regs->EREG(ip) - copy_eip) +
 431                     5 < MAX_INSN_SIZE) {
 432                         /*
 433                          * These instructions can be executed directly if it
 434                          * jumps back to correct address.
 435                          */
 436                         set_user_jmp_op((void *) regs->EREG(ip),
 437                                         (void *)orig_eip +
 438                                         (regs->EREG(ip) - copy_eip));
 439                         p->ainsn.boostable = 1;
 440                 } else {
 441                         p->ainsn.boostable = -1;
 442                 }
 443         }
 444
 445         regs->EREG(ip) = orig_eip + (regs->EREG(ip) - copy_eip);
 446
 447 no_change:
 448         return;
 449 }
 450
 451 static bool prepare_ss_addr(struct uprobe *p, struct pt_regs *regs)
 452 {
 453         unsigned long *ss_addr = (long *)&p->ss_addr[smp_processor_id()];
 454
 455         if (*ss_addr) {
 456                 regs->ip = *ss_addr;
 457                 *ss_addr = 0;
 458                 return true;
 459         } else {
 460                 regs->ip = (unsigned long)p->ainsn.insn;
 461                 return false;
 462         }
 463 }
 464
 465 static void prepare_ss(struct pt_regs *regs)
 466 {
 467         /* set single step mode */
 468         regs->flags |= TF_MASK;
 469         regs->flags &= ~IF_MASK;
 470 }
 471
 472 static int uprobe_handler(struct pt_regs *regs)
 473 {
 474         struct uprobe *p;
 475         uprobe_opcode_t *addr;
 476         struct task_struct *task = current;
 477         pid_t tgid = task->tgid;
 478
 479         save_current_flags(regs);
 480
 481         addr = (uprobe_opcode_t *)(regs->EREG(ip) - sizeof(uprobe_opcode_t));
 482         p = get_uprobe(addr, tgid);
 483
 484         if (p == NULL) {
 485                 void *tramp_addr = (void *)addr - UPROBES_TRAMP_RET_BREAK_IDX;
 486
 487                 p = get_uprobe_by_insn_slot(tramp_addr, tgid, regs);
 488                 if (p == NULL) {
 489                         printk(KERN_INFO "no_uprobe\n");
 490                         return 0;
 491                 }
 492
 493                 trampoline_uprobe_handler(p, regs);
 494                 return 1;
 495         } else {
 496                 if (!p->pre_handler || !p->pre_handler(p, regs)) {
 497                         if (p->ainsn.boostable == 1 && !p->post_handler) {
 498                                 prepare_ss_addr(p, regs);
 499                                 return 1;
 500                         }
 501
 502                         if (prepare_ss_addr(p, regs) == false) {
 503                                 set_current_probe(p);
 504                                 prepare_ss(regs);
 505                         }
 506                 }
 507         }
 508
 509         return 1;
 510 }
 511
 512 static int post_uprobe_handler(struct pt_regs *regs)
 513 {
 514         struct uprobe *p = get_current_probe();
 515         unsigned long flags = current_ucb()->flags;
 516
 517         if (p == NULL) {
 518                 printk("task[%u %u %s] current uprobe is not found\n",
 519                        current->tgid, current->pid, current->comm);
 520                 return 0;
 521         }
 522
 523         resume_execution(p, regs, flags);
 524         restore_current_flags(regs, flags);
 525
 526         /* clean stack */
 527         current_ucb()->p = 0;
 528         current_ucb()->flags = 0;
 529
 530         return 1;
 531 }
 532
 533 static int uprobe_exceptions_notify(struct notifier_block *self,
 534                                     unsigned long val, void *data)
 535 {
 536         struct die_args *args = (struct die_args *)data;
 537         int ret = NOTIFY_DONE;
 538
 539         if (args->regs == NULL || !user_mode_vm(args->regs))
 540                 return ret;
 541
 542         switch (val) {
 543 #ifdef CONFIG_KPROBES
 544         case DIE_INT3:
 545 #else
 546         case DIE_TRAP:
 547 #endif
 548                 if (uprobe_handler(args->regs))
 549                         ret = NOTIFY_STOP;
 550                 break;
 551         case DIE_DEBUG:
 552                 if (post_uprobe_handler(args->regs))
 553                         ret = NOTIFY_STOP;
 554                 break;
 555         default:
 556                 break;
 557         }
 558
 559         return ret;
 560 }
 561
 562 static struct notifier_block uprobe_exceptions_nb = {
 563         .notifier_call = uprobe_exceptions_notify,
 564         .priority = INT_MAX
 565 };
 566
 567 /**
 568  * @brief Registers notify.
 569  *
 570  * @return register_die_notifier result.
 571  */
 572 int swap_arch_init_uprobes(void)
 573 {
 574         return register_die_notifier(&uprobe_exceptions_nb);
 575 }
 576
 577 /**
 578  * @brief Unregisters notify.
 579  *
 580  * @return Void.
 581  */
 582 void swap_arch_exit_uprobes(void)
 583 {
 584         unregister_die_notifier(&uprobe_exceptions_nb);
 585 }
 586