kprobe/dbi_kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  */
  21
  22 /*
  23  *  Dynamic Binary Instrumentation Module based on KProbes
  24  *  modules/kprobe/dbi_kprobes.h
  25  *
  26  * This program is free software; you can redistribute it and/or modify
  27  * it under the terms of the GNU General Public License as published by
  28  * the Free Software Foundation; either version 2 of the License, or
  29  * (at your option) any later version.
  30  *
  31  * This program is distributed in the hope that it will be useful,
  32  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  33  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  34  * GNU General Public License for more details.
  35  *
  36  * You should have received a copy of the GNU General Public License
  37  * along with this program; if not, write to the Free Software
  38  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  39  *
  40  * Copyright (C) Samsung Electronics, 2006-2010
  41  *
  42  * 2006-2007    Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
  43  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  44  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  45  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  46  *
  47  */
  48
  49 #include "dbi_kprobes.h"
  50 #include "arch/asm/dbi_kprobes.h"
  51
  52 #include "dbi_kdebug.h"
  53 #include "dbi_kprobes_deps.h"
  54 #include "dbi_insn_slots.h"
  55 #include <ksyms.h>
  56
  57 #include <linux/version.h>
  58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
  59 #include <linux/config.h>
  60 #endif
  61
  62 #include <linux/hash.h>
  63 #include <linux/module.h>
  64 #include <linux/mm.h>
  65 #include <linux/pagemap.h>
  66
  67 struct slot_manager sm;
  68
  69 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  70 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  71
  72 DEFINE_SPINLOCK(kretprobe_lock);        /* Protects kretprobe_inst_table */
  73 EXPORT_SYMBOL_GPL(kretprobe_lock);
  74 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  75
  76 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  77 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  78
  79 atomic_t kprobe_count;
  80 EXPORT_SYMBOL_GPL(kprobe_count);
  81
  82 static void *sm_alloc(struct slot_manager *sm)
  83 {
  84         return kmalloc(PAGE_SIZE, GFP_ATOMIC);
  85 }
  86
  87 static void sm_free(struct slot_manager *sm, void *ptr)
  88 {
  89         kfree(ptr);
  90 }
  91
  92 static void init_sm()
  93 {
  94         sm.slot_size = KPROBES_TRAMP_LEN;
  95         sm.alloc = sm_alloc;
  96         sm.free = sm_free;
  97         INIT_HLIST_NODE(&sm.page_list);
  98 }
  99
 100 static void exit_sm()
 101 {
 102         /* FIXME: free */
 103 }
 104
 105 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
 106 {
 107         if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
 108                 struct task_struct *task;
 109                 if (ri == NULL) {
 110                         panic("kretprobe BUG!: ri = NULL\n");
 111                 }
 112
 113                 task = ri->task;
 114
 115                 if (task == NULL) {
 116                         panic("kretprobe BUG!: task = NULL\n");
 117                 }
 118
 119                 if (ri->rp == NULL) {
 120                         panic("kretprobe BUG!: ri->rp = NULL\n");
 121                 }
 122
 123                 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
 124                       ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
 125         }
 126 }
 127
 128 /* We have preemption disabled.. so it is safe to use __ versions */
 129 static inline void set_kprobe_instance(struct kprobe *kp)
 130 {
 131         __get_cpu_var(kprobe_instance) = kp;
 132 }
 133
 134 static inline void reset_kprobe_instance(void)
 135 {
 136         __get_cpu_var(kprobe_instance) = NULL;
 137 }
 138
 139 /* kprobe_running() will just return the current_kprobe on this CPU */
 140 struct kprobe *kprobe_running(void)
 141 {
 142         return __get_cpu_var(current_kprobe);
 143 }
 144
 145 void reset_current_kprobe(void)
 146 {
 147         __get_cpu_var(current_kprobe) = NULL;
 148 }
 149
 150 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
 151 {
 152         return &__get_cpu_var(kprobe_ctlblk);
 153 }
 154
 155 /*
 156  * This routine is called either:
 157  *      - under the kprobe_mutex - during kprobe_[un]register()
 158  *                              OR
 159  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 160  */
 161 struct kprobe *get_kprobe(void *addr)
 162 {
 163         struct hlist_head *head;
 164         struct hlist_node *node;
 165         struct kprobe *p;
 166
 167         head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
 168         swap_hlist_for_each_entry_rcu(p, node, head, hlist) {
 169                 if (p->addr == addr) {
 170                         return p;
 171                 }
 172         }
 173
 174         return NULL;
 175 }
 176
 177 /*
 178  * Aggregate handlers for multiple kprobes support - these handlers
 179  * take care of invoking the individual kprobe handlers on p->list
 180  */
 181 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 182 {
 183         struct kprobe *kp;
 184         int ret;
 185
 186         list_for_each_entry_rcu(kp, &p->list, list) {
 187                 if (kp->pre_handler) {
 188                         set_kprobe_instance(kp);
 189                         ret = kp->pre_handler(kp, regs);
 190                         if (ret)
 191                                 return ret;
 192                 }
 193                 reset_kprobe_instance();
 194         }
 195
 196         return 0;
 197 }
 198
 199 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
 200 {
 201         struct kprobe *kp;
 202
 203         list_for_each_entry_rcu(kp, &p->list, list) {
 204                 if (kp->post_handler) {
 205                         set_kprobe_instance(kp);
 206                         kp->post_handler(kp, regs, flags);
 207                         reset_kprobe_instance();
 208                 }
 209         }
 210 }
 211
 212 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
 213 {
 214         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 215
 216         /*
 217          * if we faulted "during" the execution of a user specified
 218          * probe handler, invoke just that probe's fault handler
 219          */
 220         if (cur && cur->fault_handler) {
 221                 if (cur->fault_handler(cur, regs, trapnr))
 222                         return 1;
 223         }
 224
 225         return 0;
 226 }
 227
 228 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 229 {
 230         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 231         int ret = 0;
 232         DBPRINTF ("cur = 0x%p\n", cur);
 233         if (cur)
 234                 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
 235
 236         if (cur && cur->break_handler) {
 237                 if (cur->break_handler(cur, regs))
 238                         ret = 1;
 239         }
 240         reset_kprobe_instance();
 241
 242         return ret;
 243 }
 244
 245 /* Walks the list and increments nmissed count for multiprobe case */
 246 void kprobes_inc_nmissed_count(struct kprobe *p)
 247 {
 248         struct kprobe *kp;
 249         if (p->pre_handler != aggr_pre_handler) {
 250                 p->nmissed++;
 251         } else {
 252                 list_for_each_entry_rcu(kp, &p->list, list) {
 253                         ++kp->nmissed;
 254                 }
 255         }
 256 }
 257
 258 /* Called with kretprobe_lock held */
 259 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
 260 {
 261         struct hlist_node *node;
 262         struct kretprobe_instance *ri;
 263
 264         swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 265                 return ri;
 266         }
 267
 268         if (!alloc_nodes_kretprobe(rp)) {
 269                 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 270                         return ri;
 271                 }
 272         }
 273
 274         return NULL;
 275 }
 276 EXPORT_SYMBOL_GPL(get_free_rp_inst);
 277
 278 /* Called with kretprobe_lock held */
 279 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
 280 {
 281         struct hlist_node *node;
 282         struct kretprobe_instance *ri;
 283
 284         swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 285                 return ri;
 286         }
 287
 288         return NULL;
 289 }
 290
 291 /* Called with kretprobe_lock held */
 292 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
 293 {
 294         struct hlist_node *node;
 295         struct kretprobe_instance *ri;
 296
 297         swap_hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
 298                 return ri;
 299         }
 300
 301         return NULL;
 302 }
 303 EXPORT_SYMBOL_GPL(get_used_rp_inst);
 304
 305 /* Called with kretprobe_lock held */
 306 void add_rp_inst (struct kretprobe_instance *ri)
 307 {
 308         /*
 309          * Remove rp inst off the free list -
 310          * Add it back when probed function returns
 311          */
 312         hlist_del(&ri->uflist);
 313
 314         /* Add rp inst onto table */
 315         INIT_HLIST_NODE(&ri->hlist);
 316
 317         hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
 318
 319         /* Also add this rp inst to the used list. */
 320         INIT_HLIST_NODE(&ri->uflist);
 321         hlist_add_head(&ri->uflist, &ri->rp->used_instances);
 322 }
 323 EXPORT_SYMBOL_GPL(add_rp_inst);
 324
 325 /* Called with kretprobe_lock held */
 326 void recycle_rp_inst(struct kretprobe_instance *ri)
 327 {
 328         if (ri->rp) {
 329                 hlist_del(&ri->hlist);
 330                 /* remove rp inst off the used list */
 331                 hlist_del(&ri->uflist);
 332                 /* put rp inst back onto the free list */
 333                 INIT_HLIST_NODE(&ri->uflist);
 334                 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
 335         }
 336 }
 337 EXPORT_SYMBOL_GPL(recycle_rp_inst);
 338
 339 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
 340 {
 341         return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
 342 }
 343 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
 344
 345 void free_rp_inst(struct kretprobe *rp)
 346 {
 347         struct kretprobe_instance *ri;
 348         while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
 349                 hlist_del(&ri->uflist);
 350                 kfree(ri);
 351         }
 352 }
 353 EXPORT_SYMBOL_GPL(free_rp_inst);
 354
 355 /*
 356  * Keep all fields in the kprobe consistent
 357  */
 358 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 359 {
 360         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 361         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 362         p->ss_addr = old_p->ss_addr;
 363 #ifdef CONFIG_ARM
 364         p->safe_arm = old_p->safe_arm;
 365         p->safe_thumb = old_p->safe_thumb;
 366 #endif
 367 }
 368
 369 /*
 370  * Add the new probe to old_p->list. Fail if this is the
 371  * second jprobe at the address - two jprobes can't coexist
 372  */
 373 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
 374 {
 375         if (p->break_handler) {
 376                 if (old_p->break_handler) {
 377                         return -EEXIST;
 378                 }
 379
 380                 list_add_tail_rcu(&p->list, &old_p->list);
 381                 old_p->break_handler = aggr_break_handler;
 382         } else {
 383                 list_add_rcu(&p->list, &old_p->list);
 384         }
 385
 386         if (p->post_handler && !old_p->post_handler) {
 387                 old_p->post_handler = aggr_post_handler;
 388         }
 389
 390         return 0;
 391 }
 392
 393 /**
 394  * hlist_replace_rcu - replace old entry by new one
 395  * @old : the element to be replaced
 396  * @new : the new element to insert
 397  *
 398  * The @old entry will be replaced with the @new entry atomically.
 399  */
 400 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
 401 {
 402         struct hlist_node *next = old->next;
 403
 404         new->next = next;
 405         new->pprev = old->pprev;
 406         smp_wmb();
 407         if (next)
 408                 new->next->pprev = &new->next;
 409         if (new->pprev)
 410                 *new->pprev = new;
 411         old->pprev = LIST_POISON2;
 412 }
 413
 414 /*
 415  * Fill in the required fields of the "manager kprobe". Replace the
 416  * earlier kprobe in the hlist with the manager kprobe
 417  */
 418 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 419 {
 420         copy_kprobe(p, ap);
 421         ap->addr = p->addr;
 422         ap->pre_handler = aggr_pre_handler;
 423         ap->fault_handler = aggr_fault_handler;
 424         if (p->post_handler)
 425                 ap->post_handler = aggr_post_handler;
 426         if (p->break_handler)
 427                 ap->break_handler = aggr_break_handler;
 428
 429         INIT_LIST_HEAD(&ap->list);
 430         list_add_rcu(&p->list, &ap->list);
 431
 432         dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
 433 }
 434
 435 /*
 436  * This is the second or subsequent kprobe at the address - handle
 437  * the intricacies
 438  */
 439 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
 440 {
 441         int ret = 0;
 442         struct kprobe *ap;
 443         DBPRINTF ("start\n");
 444
 445         DBPRINTF ("p = %p old_p = %p \n", p, old_p);
 446         if (old_p->pre_handler == aggr_pre_handler) {
 447                 DBPRINTF ("aggr_pre_handler \n");
 448
 449                 copy_kprobe(old_p, p);
 450                 ret = add_new_kprobe(old_p, p);
 451         } else {
 452                 DBPRINTF ("kzalloc\n");
 453 #ifdef kzalloc
 454                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 455 #else
 456                 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
 457                 if (ap)
 458                         memset(ap, 0, sizeof(struct kprobe));
 459 #endif
 460                 if (!ap)
 461                         return -ENOMEM;
 462                 add_aggr_kprobe(ap, old_p);
 463                 copy_kprobe(ap, p);
 464                 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
 465                 ret = add_new_kprobe(ap, p);
 466         }
 467
 468         return ret;
 469 }
 470 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
 471
 472 static void remove_kprobe(struct kprobe *p)
 473 {
 474         /* TODO: check boostable for x86 and MIPS */
 475         free_insn_slot(&sm, p->ainsn.insn);
 476 }
 477
 478 int dbi_register_kprobe(struct kprobe *p)
 479 {
 480         struct kprobe *old_p;
 481         int ret = 0;
 482         /*
 483          * If we have a symbol_name argument look it up,
 484          * and add it to the address.  That way the addr
 485          * field can either be global or relative to a symbol.
 486          */
 487         if (p->symbol_name) {
 488                 if (p->addr)
 489                         return -EINVAL;
 490                 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
 491         }
 492
 493         if (!p->addr)
 494                 return -EINVAL;
 495         DBPRINTF ("p->addr = 0x%p\n", p->addr);
 496         p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
 497         DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
 498
 499 #ifdef KPROBES_PROFILE
 500         p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
 501         p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
 502         p->count = 0;
 503 #endif
 504         p->mod_refcounted = 0;
 505         p->nmissed = 0;
 506
 507         old_p = get_kprobe(p->addr);
 508         if (old_p) {
 509                 ret = register_aggr_kprobe(old_p, p);
 510                 if (!ret)
 511                         atomic_inc(&kprobe_count);
 512                 goto out;
 513         }
 514
 515         if ((ret = arch_prepare_kprobe(p, &sm)) != 0)
 516                 goto out;
 517
 518         DBPRINTF ("before out ret = 0x%x\n", ret);
 519         INIT_HLIST_NODE(&p->hlist);
 520         hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
 521         arch_arm_kprobe(p);
 522
 523 out:
 524         DBPRINTF ("out ret = 0x%x\n", ret);
 525         return ret;
 526 }
 527
 528 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
 529 {
 530         struct kprobe *old_p, *list_p;
 531         int cleanup_p;
 532
 533         old_p = get_kprobe(p->addr);
 534         DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
 535         if (unlikely (!old_p))
 536                 return;
 537
 538         if (p != old_p) {
 539                 list_for_each_entry_rcu(list_p, &old_p->list, list)
 540                         if (list_p == p)
 541                                 /* kprobe p is a valid probe */
 542                                 goto valid_p;
 543                 return;
 544         }
 545
 546 valid_p:
 547         DBPRINTF ("dbi_unregister_kprobe valid_p");
 548         if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
 549             (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
 550                 /* Only probe on the hash list */
 551                 arch_disarm_kprobe(p);
 552                 hlist_del_rcu(&old_p->hlist);
 553                 cleanup_p = 1;
 554         } else {
 555                 list_del_rcu(&p->list);
 556                 cleanup_p = 0;
 557         }
 558         DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
 559
 560         if (cleanup_p) {
 561                 if (p != old_p) {
 562                         list_del_rcu(&p->list);
 563                         kfree(old_p);
 564                 }
 565
 566                 if (!in_atomic()) {
 567                         synchronize_sched();
 568                 }
 569
 570                 remove_kprobe(p);
 571         } else {
 572                 if (p->break_handler)
 573                         old_p->break_handler = NULL;
 574                 if (p->post_handler) {
 575                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
 576                                 if (list_p->post_handler) {
 577                                         cleanup_p = 2;
 578                                         break;
 579                                 }
 580                         }
 581
 582                         if (cleanup_p == 0)
 583                                 old_p->post_handler = NULL;
 584                 }
 585         }
 586 }
 587
 588 int dbi_register_jprobe(struct jprobe *jp)
 589 {
 590         /* Todo: Verify probepoint is a function entry point */
 591         jp->kp.pre_handler = setjmp_pre_handler;
 592         jp->kp.break_handler = longjmp_break_handler;
 593
 594         return dbi_register_kprobe(&jp->kp);
 595 }
 596
 597 void dbi_unregister_jprobe(struct jprobe *jp)
 598 {
 599         dbi_unregister_kprobe(&jp->kp, NULL);
 600 }
 601
 602 /*
 603  * This kprobe pre_handler is registered with every kretprobe. When probe
 604  * hits it will set up the return probe.
 605  */
 606 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 607 {
 608         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
 609         struct kretprobe_instance *ri;
 610         unsigned long flags = 0;
 611
 612         /* TODO: consider to only swap the RA after the last pre_handler fired */
 613         spin_lock_irqsave(&kretprobe_lock, flags);
 614
 615         /* TODO: test - remove retprobe after func entry but before its exit */
 616         if ((ri = get_free_rp_inst(rp)) != NULL) {
 617                 ri->rp = rp;
 618                 ri->task = current;
 619
 620                 if (rp->entry_handler) {
 621                         rp->entry_handler(ri, regs, ri->rp->priv_arg);
 622                 }
 623
 624                 arch_prepare_kretprobe(ri, regs);
 625
 626                 add_rp_inst(ri);
 627         } else {
 628                 ++rp->nmissed;
 629         }
 630
 631         spin_unlock_irqrestore(&kretprobe_lock, flags);
 632
 633         return 0;
 634 }
 635
 636 int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 637 {
 638         struct kretprobe_instance *ri = NULL;
 639         struct hlist_head *head;
 640         struct hlist_node *node, *tmp;
 641         unsigned long flags, orig_ret_address = 0;
 642         unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
 643
 644         struct kretprobe *crp = NULL;
 645         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 646
 647         spin_lock_irqsave(&kretprobe_lock, flags);
 648
 649         /*
 650          * We are using different hash keys (current and mm) for finding kernel
 651          * space and user space probes.  Kernel space probes can change mm field in
 652          * task_struct.  User space probes can be shared between threads of one
 653          * process so they have different current but same mm.
 654          */
 655         head = kretprobe_inst_table_head(current);
 656
 657 #ifdef CONFIG_X86
 658         regs->XREG(cs) = __KERNEL_CS | get_kernel_rpl();
 659         regs->EREG(ip) = trampoline_address;
 660         regs->ORIG_EAX_REG = 0xffffffff;
 661 #endif
 662
 663         /*
 664          * It is possible to have multiple instances associated with a given
 665          * task either because an multiple functions in the call path
 666          * have a return probe installed on them, and/or more then one
 667          * return probe was registered for a target function.
 668          *
 669          * We can handle this because:
 670          *     - instances are always inserted at the head of the list
 671          *     - when multiple return probes are registered for the same
 672          *       function, the first instance's ret_addr will point to the
 673          *       real return address, and all the rest will point to
 674          *       kretprobe_trampoline
 675          */
 676         swap_hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 677                 if (ri->task != current)
 678                         /* another task is sharing our hash bucket */
 679                         continue;
 680                 if (ri->rp && ri->rp->handler) {
 681                         __get_cpu_var(current_kprobe) = &ri->rp->kp;
 682                         get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
 683                         ri->rp->handler(ri, regs, ri->rp->priv_arg);
 684                         __get_cpu_var(current_kprobe) = NULL;
 685                 }
 686
 687                 orig_ret_address = (unsigned long)ri->ret_addr;
 688                 recycle_rp_inst(ri);
 689                 if (orig_ret_address != trampoline_address)
 690                         /*
 691                          * This is the real return address. Any other
 692                          * instances associated with this task are for
 693                          * other calls deeper on the call stack
 694                          */
 695                         break;
 696         }
 697         kretprobe_assert(ri, orig_ret_address, trampoline_address);
 698
 699 #ifdef CONFIG_ARM
 700         regs->ARM_lr = orig_ret_address;
 701         regs->ARM_pc = orig_ret_address;
 702 #endif
 703
 704         if (kcb->kprobe_status == KPROBE_REENTER) {
 705                 restore_previous_kprobe(kcb);
 706         } else {
 707                 reset_current_kprobe();
 708         }
 709
 710         spin_unlock_irqrestore(&kretprobe_lock, flags);
 711
 712         /*
 713          * By returning a non-zero value, we are telling
 714          * kprobe_handler() that we don't want the post_handler
 715          * to run (and have re-enabled preemption)
 716          */
 717
 718 #if defined (CONFIG_X86)
 719         return (int)orig_ret_address;
 720 #elif defined (CONFIG_ARM)
 721         return 1;
 722 #endif
 723 }
 724
 725 struct kretprobe *sched_rp;
 726
 727 #define SCHED_RP_NR 200
 728 #define COMMON_RP_NR 10
 729
 730 int alloc_nodes_kretprobe(struct kretprobe *rp)
 731 {
 732         int alloc_nodes;
 733         struct kretprobe_instance *inst;
 734         int i;
 735
 736         DBPRINTF("Alloc aditional mem for retprobes");
 737
 738         if ((unsigned long)rp->kp.addr == sched_addr) {
 739                 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 740                 alloc_nodes = SCHED_RP_NR;
 741         } else {
 742 #if 1//def CONFIG_PREEMPT
 743                 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
 744 #else
 745                 rp->maxacpptive += NR_CPUS;
 746 #endif
 747                 alloc_nodes = COMMON_RP_NR;
 748         }
 749
 750         for (i = 0; i < alloc_nodes; i++) {
 751                 inst = kmalloc(sizeof(inst) + rp->data_size, GFP_ATOMIC);
 752                 if (inst == NULL) {
 753                         free_rp_inst(rp);
 754                         return -ENOMEM;
 755                 }
 756                 INIT_HLIST_NODE(&inst->uflist);
 757                 hlist_add_head(&inst->uflist, &rp->free_instances);
 758         }
 759
 760         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 761         return 0;
 762 }
 763
 764 int dbi_register_kretprobe(struct kretprobe *rp)
 765 {
 766         int ret = 0;
 767         struct kretprobe_instance *inst;
 768         int i;
 769         DBPRINTF ("START");
 770
 771         rp->kp.pre_handler = pre_handler_kretprobe;
 772         rp->kp.post_handler = NULL;
 773         rp->kp.fault_handler = NULL;
 774         rp->kp.break_handler = NULL;
 775
 776         /* Pre-allocate memory for max kretprobe instances */
 777         if ((unsigned long)rp->kp.addr == sched_addr) {
 778                 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 779                 rp->kp.pre_handler = NULL; //not needed for __switch_to
 780         } else if ((unsigned long)rp->kp.addr == exit_addr) {
 781                 rp->kp.pre_handler = NULL; //not needed for do_exit
 782                 rp->maxactive = 0;
 783         } else if ((unsigned long)rp->kp.addr == do_group_exit_addr) {
 784                 rp->kp.pre_handler = NULL;
 785                 rp->maxactive = 0;
 786         } else if ((unsigned long)rp->kp.addr == sys_exit_group_addr) {
 787                 rp->kp.pre_handler = NULL;
 788                 rp->maxactive = 0;
 789         } else if (rp->maxactive <= 0) {
 790 #if 1//def CONFIG_PREEMPT
 791                 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
 792 #else
 793                 rp->maxactive = NR_CPUS;
 794 #endif
 795         }
 796         INIT_HLIST_HEAD(&rp->used_instances);
 797         INIT_HLIST_HEAD(&rp->free_instances);
 798         for (i = 0; i < rp->maxactive; i++) {
 799                 inst = kmalloc(sizeof(*inst) + rp->data_size, GFP_KERNEL);
 800                 if (inst == NULL) {
 801                         free_rp_inst(rp);
 802                         return -ENOMEM;
 803                 }
 804                 INIT_HLIST_NODE(&inst->uflist);
 805                 hlist_add_head(&inst->uflist, &rp->free_instances);
 806         }
 807
 808         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 809         rp->nmissed = 0;
 810         /* Establish function entry probe point */
 811         if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
 812                 free_rp_inst(rp);
 813
 814         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 815         if ((unsigned long)rp->kp.addr == sched_addr) {
 816                 sched_rp = rp;
 817         }
 818
 819         return ret;
 820 }
 821
 822 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
 823
 824 void dbi_unregister_kretprobe(struct kretprobe *rp)
 825 {
 826         unsigned long flags;
 827         struct kretprobe_instance *ri;
 828
 829         dbi_unregister_kprobe(&rp->kp, NULL);
 830
 831         /* No race here */
 832         spin_lock_irqsave(&kretprobe_lock, flags);
 833
 834         if ((unsigned long)rp->kp.addr == sched_addr)
 835                 sched_rp = NULL;
 836
 837         while ((ri = get_used_rp_inst(rp)) != NULL) {
 838                 if (dbi_disarm_krp_inst(ri) == 0)
 839                         recycle_rp_inst(ri);
 840                 else
 841                         panic("%s (%d/%d): cannot disarm krp instance (%08lx)",
 842                                         ri->task->comm, ri->task->tgid, ri->task->pid,
 843                                         (unsigned long)rp->kp.addr);
 844         }
 845
 846         spin_unlock_irqrestore(&kretprobe_lock, flags);
 847         free_rp_inst(rp);
 848 }
 849
 850 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
 851 {
 852         struct kprobe *old_p;
 853         struct kretprobe *clone = NULL;
 854         int ret;
 855
 856         clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
 857         if (!clone) {
 858                 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
 859                 return NULL;
 860         }
 861         memcpy(clone, rp, sizeof(struct kretprobe));
 862         clone->kp.pre_handler = pre_handler_kretprobe;
 863         clone->kp.post_handler = NULL;
 864         clone->kp.fault_handler = NULL;
 865         clone->kp.break_handler = NULL;
 866         old_p = get_kprobe(rp->kp.addr);
 867         if (old_p) {
 868                 ret = register_aggr_kprobe(old_p, &clone->kp);
 869                 if (ret) {
 870                         kfree(clone);
 871                         return NULL;
 872                 }
 873                 atomic_inc(&kprobe_count);
 874         }
 875
 876         return clone;
 877 }
 878 EXPORT_SYMBOL_GPL(clone_kretprobe);
 879
 880 static void inline set_task_trampoline(unsigned long *patch_addr,
 881                                        struct kretprobe_instance *ri,
 882                                        unsigned long tramp_addr)
 883 {
 884         unsigned long pc = *patch_addr;
 885         if (pc == tramp_addr)
 886                 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
 887                       task_cpu(ri->task), ri->task->comm, ri->task->tgid,
 888                       ri->task->pid, pc, task_cpu(current), current->comm,
 889                       current->tgid, current->pid);
 890         ri->ret_addr = (kprobe_opcode_t *)pc;
 891         *patch_addr = tramp_addr;
 892 }
 893
 894 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
 895 {
 896         arch_set_task_pc(p, (unsigned long)ri->ret_addr);
 897 }
 898
 899 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
 900 {
 901         unsigned long *tramp = &kretprobe_trampoline;
 902         unsigned long *sp = ri->sp;
 903         unsigned long *found = NULL;
 904         int retval = -ENOENT;
 905
 906         if (!sp) {
 907                 unsigned long pc = arch_get_task_pc(ri->task);
 908
 909                 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
 910                                 task_cpu(ri->task),
 911                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 912                                 pc, ri->ret_addr, tramp,
 913                                 ri->rp ? ri->rp->kp.addr: NULL);
 914
 915                 /* __switch_to retprobe handling */
 916                 if (pc == tramp) {
 917                         rm_task_trampoline(ri->task, ri);
 918                         return 0;
 919                 }
 920
 921                 return -EINVAL;
 922         }
 923
 924         while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
 925                 if (*sp == tramp) {
 926                         found = sp;
 927                         break;
 928                 }
 929                 sp--;
 930         }
 931
 932         if (found) {
 933                 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
 934                                 task_cpu(ri->task),
 935                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 936                                 tramp, found, ri->sp, found - ri->sp,
 937                                 ri->rp ? ri->rp->kp.addr: NULL);
 938                 *found = ri->ret_addr;
 939                 retval = 0;
 940         } else {
 941                 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
 942                                 task_cpu(ri->task),
 943                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 944                                 tramp,
 945                                 ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
 946         }
 947
 948         return retval;
 949 }
 950
 951 int patch_suspended_task(struct kretprobe *rp,
 952                          struct task_struct *task,
 953                          struct pt_regs *regs)
 954 {
 955         struct kretprobe_instance *ri;
 956         unsigned long flags;
 957         kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
 958         unsigned long *patch_addr;
 959
 960         spin_lock_irqsave(&kretprobe_lock, flags);
 961
 962         ri = get_free_rp_inst(rp);
 963         if (!ri)
 964                 return -ENOMEM;
 965
 966         ri->rp = rp;
 967         ri->task = task;
 968         ri->sp = NULL;
 969         patch_addr = arch_get_patch_addr(task, regs);
 970         set_task_trampoline(patch_addr, ri, (unsigned long)tramp);
 971         add_rp_inst(ri);
 972
 973         spin_unlock_irqrestore(&kretprobe_lock, flags);
 974         return 0;
 975 }
 976
 977 static int init_module_deps(void)
 978 {
 979         int ret;
 980
 981         sched_addr = swap_ksyms("__switch_to");
 982         fork_addr = swap_ksyms("do_fork");
 983         exit_addr = swap_ksyms("do_exit");
 984
 985         if (sched_addr == 0 || fork_addr == 0 || exit_addr == 0) {
 986                 return -ESRCH;
 987         }
 988
 989         ret = init_module_dependencies();
 990         if (ret) {
 991                 return ret;
 992         }
 993
 994         return arch_init_module_deps();
 995 }
 996
 997 static int __init init_kprobes(void)
 998 {
 999         int i, err = 0;
1000
1001         init_sm();
1002
1003         /* FIXME allocate the probe table, currently defined statically */
1004         /* initialize all list heads */
1005         for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
1006                 INIT_HLIST_HEAD(&kprobe_table[i]);
1007                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1008         }
1009         atomic_set(&kprobe_count, 0);
1010
1011         err = init_module_deps();
1012         if (err) {
1013                 return err;
1014         }
1015
1016         err = arch_init_kprobes();
1017
1018         DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
1019
1020         return err;
1021 }
1022
1023 static void __exit exit_kprobes(void)
1024 {
1025         arch_exit_kprobes();
1026         exit_sm();
1027 }
1028
1029 module_init(init_kprobes);
1030 module_exit(exit_kprobes);
1031
1032 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
1033 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
1034 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
1035 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
1036 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
1037 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
1038 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
1039
1040 MODULE_LICENSE("Dual BSD/GPL");