kprobe/dbi_kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  */
  21
  22 /*
  23  *  Dynamic Binary Instrumentation Module based on KProbes
  24  *  modules/kprobe/dbi_kprobes.h
  25  *
  26  * This program is free software; you can redistribute it and/or modify
  27  * it under the terms of the GNU General Public License as published by
  28  * the Free Software Foundation; either version 2 of the License, or
  29  * (at your option) any later version.
  30  *
  31  * This program is distributed in the hope that it will be useful,
  32  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  33  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  34  * GNU General Public License for more details.
  35  *
  36  * You should have received a copy of the GNU General Public License
  37  * along with this program; if not, write to the Free Software
  38  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  39  *
  40  * Copyright (C) Samsung Electronics, 2006-2010
  41  *
  42  * 2006-2007    Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
  43  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  44  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  45  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  46  *
  47  */
  48
  49 #include "dbi_kprobes.h"
  50 #include "arch/asm/dbi_kprobes.h"
  51
  52 #include "dbi_kdebug.h"
  53 #include "dbi_kprobes_deps.h"
  54 #include "dbi_insn_slots.h"
  55 #include <ksyms.h>
  56
  57 #include <linux/version.h>
  58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
  59 #include <linux/config.h>
  60 #endif
  61
  62 #include <linux/hash.h>
  63 #include <linux/module.h>
  64 #include <linux/mm.h>
  65 #include <linux/pagemap.h>
  66
  67 unsigned long sched_addr;
  68 static unsigned long exit_addr;
  69 static unsigned long do_group_exit_addr;
  70 static unsigned long sys_exit_group_addr;
  71 static unsigned long sys_exit_addr;
  72
  73 struct slot_manager sm;
  74
  75 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  76 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  77
  78 DEFINE_SPINLOCK(kretprobe_lock);        /* Protects kretprobe_inst_table */
  79 EXPORT_SYMBOL_GPL(kretprobe_lock);
  80 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  81
  82 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  83 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  84
  85 atomic_t kprobe_count;
  86 EXPORT_SYMBOL_GPL(kprobe_count);
  87
  88 static void *sm_alloc(struct slot_manager *sm)
  89 {
  90         return kmalloc(PAGE_SIZE, GFP_ATOMIC);
  91 }
  92
  93 static void sm_free(struct slot_manager *sm, void *ptr)
  94 {
  95         kfree(ptr);
  96 }
  97
  98 static void init_sm()
  99 {
 100         sm.slot_size = KPROBES_TRAMP_LEN;
 101         sm.alloc = sm_alloc;
 102         sm.free = sm_free;
 103         INIT_HLIST_NODE(&sm.page_list);
 104 }
 105
 106 static void exit_sm()
 107 {
 108         /* FIXME: free */
 109 }
 110
 111 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
 112 {
 113         if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
 114                 struct task_struct *task;
 115                 if (ri == NULL) {
 116                         panic("kretprobe BUG!: ri = NULL\n");
 117                 }
 118
 119                 task = ri->task;
 120
 121                 if (task == NULL) {
 122                         panic("kretprobe BUG!: task = NULL\n");
 123                 }
 124
 125                 if (ri->rp == NULL) {
 126                         panic("kretprobe BUG!: ri->rp = NULL\n");
 127                 }
 128
 129                 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
 130                       ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
 131         }
 132 }
 133
 134 /* We have preemption disabled.. so it is safe to use __ versions */
 135 static inline void set_kprobe_instance(struct kprobe *kp)
 136 {
 137         __get_cpu_var(kprobe_instance) = kp;
 138 }
 139
 140 static inline void reset_kprobe_instance(void)
 141 {
 142         __get_cpu_var(kprobe_instance) = NULL;
 143 }
 144
 145 /* kprobe_running() will just return the current_kprobe on this CPU */
 146 struct kprobe *kprobe_running(void)
 147 {
 148         return __get_cpu_var(current_kprobe);
 149 }
 150
 151 void reset_current_kprobe(void)
 152 {
 153         __get_cpu_var(current_kprobe) = NULL;
 154 }
 155
 156 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
 157 {
 158         return &__get_cpu_var(kprobe_ctlblk);
 159 }
 160
 161 /*
 162  * This routine is called either:
 163  *      - under the kprobe_mutex - during kprobe_[un]register()
 164  *                              OR
 165  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 166  */
 167 struct kprobe *get_kprobe(void *addr)
 168 {
 169         struct hlist_head *head;
 170         struct hlist_node *node;
 171         struct kprobe *p;
 172
 173         head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
 174         swap_hlist_for_each_entry_rcu(p, node, head, hlist) {
 175                 if (p->addr == addr) {
 176                         return p;
 177                 }
 178         }
 179
 180         return NULL;
 181 }
 182
 183 /*
 184  * Aggregate handlers for multiple kprobes support - these handlers
 185  * take care of invoking the individual kprobe handlers on p->list
 186  */
 187 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 188 {
 189         struct kprobe *kp;
 190         int ret;
 191
 192         list_for_each_entry_rcu(kp, &p->list, list) {
 193                 if (kp->pre_handler) {
 194                         set_kprobe_instance(kp);
 195                         ret = kp->pre_handler(kp, regs);
 196                         if (ret)
 197                                 return ret;
 198                 }
 199                 reset_kprobe_instance();
 200         }
 201
 202         return 0;
 203 }
 204
 205 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
 206 {
 207         struct kprobe *kp;
 208
 209         list_for_each_entry_rcu(kp, &p->list, list) {
 210                 if (kp->post_handler) {
 211                         set_kprobe_instance(kp);
 212                         kp->post_handler(kp, regs, flags);
 213                         reset_kprobe_instance();
 214                 }
 215         }
 216 }
 217
 218 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
 219 {
 220         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 221
 222         /*
 223          * if we faulted "during" the execution of a user specified
 224          * probe handler, invoke just that probe's fault handler
 225          */
 226         if (cur && cur->fault_handler) {
 227                 if (cur->fault_handler(cur, regs, trapnr))
 228                         return 1;
 229         }
 230
 231         return 0;
 232 }
 233
 234 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 235 {
 236         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 237         int ret = 0;
 238         DBPRINTF ("cur = 0x%p\n", cur);
 239         if (cur)
 240                 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
 241
 242         if (cur && cur->break_handler) {
 243                 if (cur->break_handler(cur, regs))
 244                         ret = 1;
 245         }
 246         reset_kprobe_instance();
 247
 248         return ret;
 249 }
 250
 251 /* Walks the list and increments nmissed count for multiprobe case */
 252 void kprobes_inc_nmissed_count(struct kprobe *p)
 253 {
 254         struct kprobe *kp;
 255         if (p->pre_handler != aggr_pre_handler) {
 256                 p->nmissed++;
 257         } else {
 258                 list_for_each_entry_rcu(kp, &p->list, list) {
 259                         ++kp->nmissed;
 260                 }
 261         }
 262 }
 263
 264 /* Called with kretprobe_lock held */
 265 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
 266 {
 267         struct hlist_node *node;
 268         struct kretprobe_instance *ri;
 269
 270         swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 271                 return ri;
 272         }
 273
 274         if (!alloc_nodes_kretprobe(rp)) {
 275                 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 276                         return ri;
 277                 }
 278         }
 279
 280         return NULL;
 281 }
 282 EXPORT_SYMBOL_GPL(get_free_rp_inst);
 283
 284 /* Called with kretprobe_lock held */
 285 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
 286 {
 287         struct hlist_node *node;
 288         struct kretprobe_instance *ri;
 289
 290         swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 291                 return ri;
 292         }
 293
 294         return NULL;
 295 }
 296
 297 /* Called with kretprobe_lock held */
 298 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
 299 {
 300         struct hlist_node *node;
 301         struct kretprobe_instance *ri;
 302
 303         swap_hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
 304                 return ri;
 305         }
 306
 307         return NULL;
 308 }
 309 EXPORT_SYMBOL_GPL(get_used_rp_inst);
 310
 311 /* Called with kretprobe_lock held */
 312 void add_rp_inst (struct kretprobe_instance *ri)
 313 {
 314         /*
 315          * Remove rp inst off the free list -
 316          * Add it back when probed function returns
 317          */
 318         hlist_del(&ri->uflist);
 319
 320         /* Add rp inst onto table */
 321         INIT_HLIST_NODE(&ri->hlist);
 322
 323         hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
 324
 325         /* Also add this rp inst to the used list. */
 326         INIT_HLIST_NODE(&ri->uflist);
 327         hlist_add_head(&ri->uflist, &ri->rp->used_instances);
 328 }
 329 EXPORT_SYMBOL_GPL(add_rp_inst);
 330
 331 /* Called with kretprobe_lock held */
 332 void recycle_rp_inst(struct kretprobe_instance *ri)
 333 {
 334         if (ri->rp) {
 335                 hlist_del(&ri->hlist);
 336                 /* remove rp inst off the used list */
 337                 hlist_del(&ri->uflist);
 338                 /* put rp inst back onto the free list */
 339                 INIT_HLIST_NODE(&ri->uflist);
 340                 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
 341         }
 342 }
 343 EXPORT_SYMBOL_GPL(recycle_rp_inst);
 344
 345 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
 346 {
 347         return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
 348 }
 349 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
 350
 351 void free_rp_inst(struct kretprobe *rp)
 352 {
 353         struct kretprobe_instance *ri;
 354         while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
 355                 hlist_del(&ri->uflist);
 356                 kfree(ri);
 357         }
 358 }
 359 EXPORT_SYMBOL_GPL(free_rp_inst);
 360
 361 /*
 362  * Keep all fields in the kprobe consistent
 363  */
 364 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 365 {
 366         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 367         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 368         p->ss_addr = old_p->ss_addr;
 369 #ifdef CONFIG_ARM
 370         p->safe_arm = old_p->safe_arm;
 371         p->safe_thumb = old_p->safe_thumb;
 372 #endif
 373 }
 374
 375 /*
 376  * Add the new probe to old_p->list. Fail if this is the
 377  * second jprobe at the address - two jprobes can't coexist
 378  */
 379 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
 380 {
 381         if (p->break_handler) {
 382                 if (old_p->break_handler) {
 383                         return -EEXIST;
 384                 }
 385
 386                 list_add_tail_rcu(&p->list, &old_p->list);
 387                 old_p->break_handler = aggr_break_handler;
 388         } else {
 389                 list_add_rcu(&p->list, &old_p->list);
 390         }
 391
 392         if (p->post_handler && !old_p->post_handler) {
 393                 old_p->post_handler = aggr_post_handler;
 394         }
 395
 396         return 0;
 397 }
 398
 399 /**
 400  * hlist_replace_rcu - replace old entry by new one
 401  * @old : the element to be replaced
 402  * @new : the new element to insert
 403  *
 404  * The @old entry will be replaced with the @new entry atomically.
 405  */
 406 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
 407 {
 408         struct hlist_node *next = old->next;
 409
 410         new->next = next;
 411         new->pprev = old->pprev;
 412         smp_wmb();
 413         if (next)
 414                 new->next->pprev = &new->next;
 415         if (new->pprev)
 416                 *new->pprev = new;
 417         old->pprev = LIST_POISON2;
 418 }
 419
 420 /*
 421  * Fill in the required fields of the "manager kprobe". Replace the
 422  * earlier kprobe in the hlist with the manager kprobe
 423  */
 424 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 425 {
 426         copy_kprobe(p, ap);
 427         ap->addr = p->addr;
 428         ap->pre_handler = aggr_pre_handler;
 429         ap->fault_handler = aggr_fault_handler;
 430         if (p->post_handler)
 431                 ap->post_handler = aggr_post_handler;
 432         if (p->break_handler)
 433                 ap->break_handler = aggr_break_handler;
 434
 435         INIT_LIST_HEAD(&ap->list);
 436         list_add_rcu(&p->list, &ap->list);
 437
 438         dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
 439 }
 440
 441 /*
 442  * This is the second or subsequent kprobe at the address - handle
 443  * the intricacies
 444  */
 445 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
 446 {
 447         int ret = 0;
 448         struct kprobe *ap;
 449         DBPRINTF ("start\n");
 450
 451         DBPRINTF ("p = %p old_p = %p \n", p, old_p);
 452         if (old_p->pre_handler == aggr_pre_handler) {
 453                 DBPRINTF ("aggr_pre_handler \n");
 454
 455                 copy_kprobe(old_p, p);
 456                 ret = add_new_kprobe(old_p, p);
 457         } else {
 458                 DBPRINTF ("kzalloc\n");
 459 #ifdef kzalloc
 460                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 461 #else
 462                 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
 463                 if (ap)
 464                         memset(ap, 0, sizeof(struct kprobe));
 465 #endif
 466                 if (!ap)
 467                         return -ENOMEM;
 468                 add_aggr_kprobe(ap, old_p);
 469                 copy_kprobe(ap, p);
 470                 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
 471                 ret = add_new_kprobe(ap, p);
 472         }
 473
 474         return ret;
 475 }
 476 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
 477
 478 static void remove_kprobe(struct kprobe *p)
 479 {
 480         /* TODO: check boostable for x86 and MIPS */
 481         free_insn_slot(&sm, p->ainsn.insn);
 482 }
 483
 484 int dbi_register_kprobe(struct kprobe *p)
 485 {
 486         struct kprobe *old_p;
 487         int ret = 0;
 488         /*
 489          * If we have a symbol_name argument look it up,
 490          * and add it to the address.  That way the addr
 491          * field can either be global or relative to a symbol.
 492          */
 493         if (p->symbol_name) {
 494                 if (p->addr)
 495                         return -EINVAL;
 496                 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
 497         }
 498
 499         if (!p->addr)
 500                 return -EINVAL;
 501         DBPRINTF ("p->addr = 0x%p\n", p->addr);
 502         p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
 503         DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
 504
 505 #ifdef KPROBES_PROFILE
 506         p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
 507         p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
 508         p->count = 0;
 509 #endif
 510         p->mod_refcounted = 0;
 511         p->nmissed = 0;
 512
 513         old_p = get_kprobe(p->addr);
 514         if (old_p) {
 515                 ret = register_aggr_kprobe(old_p, p);
 516                 if (!ret)
 517                         atomic_inc(&kprobe_count);
 518                 goto out;
 519         }
 520
 521         if ((ret = arch_prepare_kprobe(p, &sm)) != 0)
 522                 goto out;
 523
 524         DBPRINTF ("before out ret = 0x%x\n", ret);
 525         INIT_HLIST_NODE(&p->hlist);
 526         hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
 527         arch_arm_kprobe(p);
 528
 529 out:
 530         DBPRINTF ("out ret = 0x%x\n", ret);
 531         return ret;
 532 }
 533
 534 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
 535 {
 536         struct kprobe *old_p, *list_p;
 537         int cleanup_p;
 538
 539         old_p = get_kprobe(p->addr);
 540         DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
 541         if (unlikely (!old_p))
 542                 return;
 543
 544         if (p != old_p) {
 545                 list_for_each_entry_rcu(list_p, &old_p->list, list)
 546                         if (list_p == p)
 547                                 /* kprobe p is a valid probe */
 548                                 goto valid_p;
 549                 return;
 550         }
 551
 552 valid_p:
 553         DBPRINTF ("dbi_unregister_kprobe valid_p");
 554         if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
 555             (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
 556                 /* Only probe on the hash list */
 557                 arch_disarm_kprobe(p);
 558                 hlist_del_rcu(&old_p->hlist);
 559                 cleanup_p = 1;
 560         } else {
 561                 list_del_rcu(&p->list);
 562                 cleanup_p = 0;
 563         }
 564         DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
 565
 566         if (cleanup_p) {
 567                 if (p != old_p) {
 568                         list_del_rcu(&p->list);
 569                         kfree(old_p);
 570                 }
 571
 572                 if (!in_atomic()) {
 573                         synchronize_sched();
 574                 }
 575
 576                 remove_kprobe(p);
 577         } else {
 578                 if (p->break_handler)
 579                         old_p->break_handler = NULL;
 580                 if (p->post_handler) {
 581                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
 582                                 if (list_p->post_handler) {
 583                                         cleanup_p = 2;
 584                                         break;
 585                                 }
 586                         }
 587
 588                         if (cleanup_p == 0)
 589                                 old_p->post_handler = NULL;
 590                 }
 591         }
 592 }
 593
 594 int dbi_register_jprobe(struct jprobe *jp)
 595 {
 596         /* Todo: Verify probepoint is a function entry point */
 597         jp->kp.pre_handler = setjmp_pre_handler;
 598         jp->kp.break_handler = longjmp_break_handler;
 599
 600         return dbi_register_kprobe(&jp->kp);
 601 }
 602
 603 void dbi_unregister_jprobe(struct jprobe *jp)
 604 {
 605         dbi_unregister_kprobe(&jp->kp, NULL);
 606 }
 607
 608 /*
 609  * This kprobe pre_handler is registered with every kretprobe. When probe
 610  * hits it will set up the return probe.
 611  */
 612 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 613 {
 614         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
 615         struct kretprobe_instance *ri;
 616         unsigned long flags = 0;
 617
 618         /* TODO: consider to only swap the RA after the last pre_handler fired */
 619         spin_lock_irqsave(&kretprobe_lock, flags);
 620
 621         /* TODO: test - remove retprobe after func entry but before its exit */
 622         if ((ri = get_free_rp_inst(rp)) != NULL) {
 623                 ri->rp = rp;
 624                 ri->task = current;
 625
 626                 if (rp->entry_handler) {
 627                         rp->entry_handler(ri, regs, ri->rp->priv_arg);
 628                 }
 629
 630                 arch_prepare_kretprobe(ri, regs);
 631
 632                 add_rp_inst(ri);
 633         } else {
 634                 ++rp->nmissed;
 635         }
 636
 637         spin_unlock_irqrestore(&kretprobe_lock, flags);
 638
 639         return 0;
 640 }
 641
 642 int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 643 {
 644         struct kretprobe_instance *ri = NULL;
 645         struct hlist_head *head;
 646         struct hlist_node *node, *tmp;
 647         unsigned long flags, orig_ret_address = 0;
 648         unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
 649
 650         struct kretprobe *crp = NULL;
 651         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
 652
 653         spin_lock_irqsave(&kretprobe_lock, flags);
 654
 655         /*
 656          * We are using different hash keys (current and mm) for finding kernel
 657          * space and user space probes.  Kernel space probes can change mm field in
 658          * task_struct.  User space probes can be shared between threads of one
 659          * process so they have different current but same mm.
 660          */
 661         head = kretprobe_inst_table_head(current);
 662
 663 #ifdef CONFIG_X86
 664         regs->XREG(cs) = __KERNEL_CS | get_kernel_rpl();
 665         regs->EREG(ip) = trampoline_address;
 666         regs->ORIG_EAX_REG = 0xffffffff;
 667 #endif
 668
 669         /*
 670          * It is possible to have multiple instances associated with a given
 671          * task either because an multiple functions in the call path
 672          * have a return probe installed on them, and/or more then one
 673          * return probe was registered for a target function.
 674          *
 675          * We can handle this because:
 676          *     - instances are always inserted at the head of the list
 677          *     - when multiple return probes are registered for the same
 678          *       function, the first instance's ret_addr will point to the
 679          *       real return address, and all the rest will point to
 680          *       kretprobe_trampoline
 681          */
 682         swap_hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 683                 if (ri->task != current)
 684                         /* another task is sharing our hash bucket */
 685                         continue;
 686                 if (ri->rp && ri->rp->handler) {
 687                         __get_cpu_var(current_kprobe) = &ri->rp->kp;
 688                         get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
 689                         ri->rp->handler(ri, regs, ri->rp->priv_arg);
 690                         __get_cpu_var(current_kprobe) = NULL;
 691                 }
 692
 693                 orig_ret_address = (unsigned long)ri->ret_addr;
 694                 recycle_rp_inst(ri);
 695                 if (orig_ret_address != trampoline_address)
 696                         /*
 697                          * This is the real return address. Any other
 698                          * instances associated with this task are for
 699                          * other calls deeper on the call stack
 700                          */
 701                         break;
 702         }
 703         kretprobe_assert(ri, orig_ret_address, trampoline_address);
 704
 705 #ifdef CONFIG_ARM
 706         regs->ARM_lr = orig_ret_address;
 707         regs->ARM_pc = orig_ret_address;
 708 #endif
 709
 710         if (kcb->kprobe_status == KPROBE_REENTER) {
 711                 restore_previous_kprobe(kcb);
 712         } else {
 713                 reset_current_kprobe();
 714         }
 715
 716         spin_unlock_irqrestore(&kretprobe_lock, flags);
 717
 718         /*
 719          * By returning a non-zero value, we are telling
 720          * kprobe_handler() that we don't want the post_handler
 721          * to run (and have re-enabled preemption)
 722          */
 723
 724 #if defined (CONFIG_X86)
 725         return (int)orig_ret_address;
 726 #elif defined (CONFIG_ARM)
 727         return 1;
 728 #endif
 729 }
 730
 731 struct kretprobe *sched_rp;
 732
 733 #define SCHED_RP_NR 200
 734 #define COMMON_RP_NR 10
 735
 736 int alloc_nodes_kretprobe(struct kretprobe *rp)
 737 {
 738         int alloc_nodes;
 739         struct kretprobe_instance *inst;
 740         int i;
 741
 742         DBPRINTF("Alloc aditional mem for retprobes");
 743
 744         if ((unsigned long)rp->kp.addr == sched_addr) {
 745                 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 746                 alloc_nodes = SCHED_RP_NR;
 747         } else {
 748 #if 1//def CONFIG_PREEMPT
 749                 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
 750 #else
 751                 rp->maxacpptive += NR_CPUS;
 752 #endif
 753                 alloc_nodes = COMMON_RP_NR;
 754         }
 755
 756         for (i = 0; i < alloc_nodes; i++) {
 757                 inst = kmalloc(sizeof(inst) + rp->data_size, GFP_ATOMIC);
 758                 if (inst == NULL) {
 759                         free_rp_inst(rp);
 760                         return -ENOMEM;
 761                 }
 762                 INIT_HLIST_NODE(&inst->uflist);
 763                 hlist_add_head(&inst->uflist, &rp->free_instances);
 764         }
 765
 766         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 767         return 0;
 768 }
 769
 770 int dbi_register_kretprobe(struct kretprobe *rp)
 771 {
 772         int ret = 0;
 773         struct kretprobe_instance *inst;
 774         int i;
 775         DBPRINTF ("START");
 776
 777         rp->kp.pre_handler = pre_handler_kretprobe;
 778         rp->kp.post_handler = NULL;
 779         rp->kp.fault_handler = NULL;
 780         rp->kp.break_handler = NULL;
 781
 782         /* Pre-allocate memory for max kretprobe instances */
 783         if ((unsigned long)rp->kp.addr == sched_addr) {
 784                 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 785                 rp->kp.pre_handler = NULL; //not needed for __switch_to
 786         } else if ((unsigned long)rp->kp.addr == exit_addr) {
 787                 rp->kp.pre_handler = NULL; //not needed for do_exit
 788                 rp->maxactive = 0;
 789         } else if ((unsigned long)rp->kp.addr == do_group_exit_addr) {
 790                 rp->kp.pre_handler = NULL;
 791                 rp->maxactive = 0;
 792         } else if ((unsigned long)rp->kp.addr == sys_exit_group_addr) {
 793                 rp->kp.pre_handler = NULL;
 794                 rp->maxactive = 0;
 795         } else if ((unsigned long)rp->kp.addr == sys_exit_addr) {
 796                 rp->kp.pre_handler = NULL;
 797                 rp->maxactive = 0;
 798         } else if (rp->maxactive <= 0) {
 799 #if 1//def CONFIG_PREEMPT
 800                 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
 801 #else
 802                 rp->maxactive = NR_CPUS;
 803 #endif
 804         }
 805         INIT_HLIST_HEAD(&rp->used_instances);
 806         INIT_HLIST_HEAD(&rp->free_instances);
 807         for (i = 0; i < rp->maxactive; i++) {
 808                 inst = kmalloc(sizeof(*inst) + rp->data_size, GFP_KERNEL);
 809                 if (inst == NULL) {
 810                         free_rp_inst(rp);
 811                         return -ENOMEM;
 812                 }
 813                 INIT_HLIST_NODE(&inst->uflist);
 814                 hlist_add_head(&inst->uflist, &rp->free_instances);
 815         }
 816
 817         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 818         rp->nmissed = 0;
 819         /* Establish function entry probe point */
 820         if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
 821                 free_rp_inst(rp);
 822
 823         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 824         if ((unsigned long)rp->kp.addr == sched_addr) {
 825                 sched_rp = rp;
 826         }
 827
 828         return ret;
 829 }
 830
 831 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
 832
 833 void dbi_unregister_kretprobe(struct kretprobe *rp)
 834 {
 835         unsigned long flags;
 836         struct kretprobe_instance *ri;
 837
 838         dbi_unregister_kprobe(&rp->kp, NULL);
 839
 840         /* No race here */
 841         spin_lock_irqsave(&kretprobe_lock, flags);
 842
 843         if ((unsigned long)rp->kp.addr == sched_addr)
 844                 sched_rp = NULL;
 845
 846         while ((ri = get_used_rp_inst(rp)) != NULL) {
 847                 if (dbi_disarm_krp_inst(ri) == 0)
 848                         recycle_rp_inst(ri);
 849                 else
 850                         panic("%s (%d/%d): cannot disarm krp instance (%08lx)",
 851                                         ri->task->comm, ri->task->tgid, ri->task->pid,
 852                                         (unsigned long)rp->kp.addr);
 853         }
 854
 855         spin_unlock_irqrestore(&kretprobe_lock, flags);
 856         free_rp_inst(rp);
 857 }
 858
 859 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
 860 {
 861         struct kprobe *old_p;
 862         struct kretprobe *clone = NULL;
 863         int ret;
 864
 865         clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
 866         if (!clone) {
 867                 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
 868                 return NULL;
 869         }
 870         memcpy(clone, rp, sizeof(struct kretprobe));
 871         clone->kp.pre_handler = pre_handler_kretprobe;
 872         clone->kp.post_handler = NULL;
 873         clone->kp.fault_handler = NULL;
 874         clone->kp.break_handler = NULL;
 875         old_p = get_kprobe(rp->kp.addr);
 876         if (old_p) {
 877                 ret = register_aggr_kprobe(old_p, &clone->kp);
 878                 if (ret) {
 879                         kfree(clone);
 880                         return NULL;
 881                 }
 882                 atomic_inc(&kprobe_count);
 883         }
 884
 885         return clone;
 886 }
 887 EXPORT_SYMBOL_GPL(clone_kretprobe);
 888
 889 static void inline set_task_trampoline(unsigned long *patch_addr,
 890                                        struct kretprobe_instance *ri,
 891                                        unsigned long tramp_addr)
 892 {
 893         unsigned long pc = *patch_addr;
 894         if (pc == tramp_addr)
 895                 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
 896                       task_cpu(ri->task), ri->task->comm, ri->task->tgid,
 897                       ri->task->pid, pc, task_cpu(current), current->comm,
 898                       current->tgid, current->pid);
 899         ri->ret_addr = (kprobe_opcode_t *)pc;
 900         *patch_addr = tramp_addr;
 901 }
 902
 903 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
 904 {
 905         arch_set_task_pc(p, (unsigned long)ri->ret_addr);
 906 }
 907
 908 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
 909 {
 910         unsigned long *tramp = &kretprobe_trampoline;
 911         unsigned long *sp = ri->sp;
 912         unsigned long *found = NULL;
 913         int retval = -ENOENT;
 914
 915         if (!sp) {
 916                 unsigned long pc = arch_get_task_pc(ri->task);
 917
 918                 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
 919                                 task_cpu(ri->task),
 920                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 921                                 pc, ri->ret_addr, tramp,
 922                                 ri->rp ? ri->rp->kp.addr: NULL);
 923
 924                 /* __switch_to retprobe handling */
 925                 if (pc == tramp) {
 926                         rm_task_trampoline(ri->task, ri);
 927                         return 0;
 928                 }
 929
 930                 return -EINVAL;
 931         }
 932
 933         while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
 934                 if (*sp == tramp) {
 935                         found = sp;
 936                         break;
 937                 }
 938                 sp--;
 939         }
 940
 941         if (found) {
 942                 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
 943                                 task_cpu(ri->task),
 944                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 945                                 tramp, found, ri->sp, found - ri->sp,
 946                                 ri->rp ? ri->rp->kp.addr: NULL);
 947                 *found = ri->ret_addr;
 948                 retval = 0;
 949         } else {
 950                 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
 951                                 task_cpu(ri->task),
 952                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 953                                 tramp,
 954                                 ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
 955         }
 956
 957         return retval;
 958 }
 959
 960 int patch_suspended_task(struct kretprobe *rp,
 961                          struct task_struct *task,
 962                          struct pt_regs *regs)
 963 {
 964         struct kretprobe_instance *ri;
 965         unsigned long flags;
 966         kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
 967         unsigned long *patch_addr;
 968
 969         spin_lock_irqsave(&kretprobe_lock, flags);
 970
 971         ri = get_free_rp_inst(rp);
 972         if (!ri)
 973                 return -ENOMEM;
 974
 975         ri->rp = rp;
 976         ri->task = task;
 977         ri->sp = NULL;
 978         patch_addr = arch_get_patch_addr(task, regs);
 979         set_task_trampoline(patch_addr, ri, (unsigned long)tramp);
 980         add_rp_inst(ri);
 981
 982         spin_unlock_irqrestore(&kretprobe_lock, flags);
 983         return 0;
 984 }
 985
 986 static int init_module_deps(void)
 987 {
 988         int ret;
 989
 990         sched_addr = swap_ksyms("__switch_to");
 991         exit_addr = swap_ksyms("do_exit");
 992         sys_exit_group_addr = swap_ksyms("sys_exit_group");
 993         do_group_exit_addr = swap_ksyms("do_group_exit");
 994         sys_exit_addr = swap_ksyms("sys_exit");
 995
 996         if (sched_addr == 0 ||
 997             exit_addr == 0 ||
 998             sys_exit_group_addr == 0 ||
 999             do_group_exit_addr == 0 ||
1000             sys_exit_addr == 0) {
1001                 return -ESRCH;
1002         }
1003
1004         ret = init_module_dependencies();
1005         if (ret) {
1006                 return ret;
1007         }
1008
1009         return arch_init_module_deps();
1010 }
1011
1012 static int __init init_kprobes(void)
1013 {
1014         int i, err = 0;
1015
1016         init_sm();
1017
1018         /* FIXME allocate the probe table, currently defined statically */
1019         /* initialize all list heads */
1020         for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
1021                 INIT_HLIST_HEAD(&kprobe_table[i]);
1022                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1023         }
1024         atomic_set(&kprobe_count, 0);
1025
1026         err = init_module_deps();
1027         if (err) {
1028                 return err;
1029         }
1030
1031         err = arch_init_kprobes();
1032
1033         DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
1034
1035         return err;
1036 }
1037
1038 static void __exit exit_kprobes(void)
1039 {
1040         arch_exit_kprobes();
1041         exit_sm();
1042 }
1043
1044 module_init(init_kprobes);
1045 module_exit(exit_kprobes);
1046
1047 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
1048 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
1049 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
1050 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
1051 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
1052 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
1053 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
1054
1055 MODULE_LICENSE("Dual BSD/GPL");