kprobe/dbi_kprobes.c

   1 /*
   2  *  Kernel Probes (KProbes)
   3  *  kernel/kprobes.c
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) IBM Corporation, 2002, 2004
  20  */
  21
  22 /*
  23  *  Dynamic Binary Instrumentation Module based on KProbes
  24  *  modules/kprobe/dbi_kprobes.h
  25  *
  26  * This program is free software; you can redistribute it and/or modify
  27  * it under the terms of the GNU General Public License as published by
  28  * the Free Software Foundation; either version 2 of the License, or
  29  * (at your option) any later version.
  30  *
  31  * This program is distributed in the hope that it will be useful,
  32  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  33  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  34  * GNU General Public License for more details.
  35  *
  36  * You should have received a copy of the GNU General Public License
  37  * along with this program; if not, write to the Free Software
  38  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  39  *
  40  * Copyright (C) Samsung Electronics, 2006-2010
  41  *
  42  * 2006-2007    Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
  43  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  44  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  45  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  46  *
  47  */
  48
  49 #include "dbi_kprobes.h"
  50 #include "arch/asm/dbi_kprobes.h"
  51
  52 #include "dbi_kdebug.h"
  53 #include "dbi_kprobes_deps.h"
  54 #include "dbi_insn_slots.h"
  55 #include <ksyms.h>
  56
  57 #include <linux/version.h>
  58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
  59 #include <linux/config.h>
  60 #endif
  61
  62 #include <linux/hash.h>
  63 #include <linux/module.h>
  64 #include <linux/mm.h>
  65 #include <linux/pagemap.h>
  66
  67 struct hlist_head kprobe_insn_pages;
  68
  69 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
  70 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
  71
  72 DEFINE_SPINLOCK(kretprobe_lock);        /* Protects kretprobe_inst_table */
  73 EXPORT_SYMBOL_GPL(kretprobe_lock);
  74 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
  75
  76 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
  77 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
  78
  79 atomic_t kprobe_count;
  80 EXPORT_SYMBOL_GPL(kprobe_count);
  81
  82 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
  83 {
  84         if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
  85                 struct task_struct *task;
  86                 if (ri == NULL) {
  87                         panic("kretprobe BUG!: ri = NULL\n");
  88                 }
  89
  90                 task = ri->task;
  91
  92                 if (task == NULL) {
  93                         panic("kretprobe BUG!: task = NULL\n");
  94                 }
  95
  96                 if (ri->rp == NULL) {
  97                         panic("kretprobe BUG!: ri->rp = NULL\n");
  98                 }
  99
 100                 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
 101                       ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
 102         }
 103 }
 104
 105 /* We have preemption disabled.. so it is safe to use __ versions */
 106 static inline void set_kprobe_instance(struct kprobe *kp)
 107 {
 108         __get_cpu_var(kprobe_instance) = kp;
 109 }
 110
 111 static inline void reset_kprobe_instance(void)
 112 {
 113         __get_cpu_var(kprobe_instance) = NULL;
 114 }
 115
 116 /* kprobe_running() will just return the current_kprobe on this CPU */
 117 struct kprobe *kprobe_running(void)
 118 {
 119         return __get_cpu_var(current_kprobe);
 120 }
 121
 122 void reset_current_kprobe(void)
 123 {
 124         __get_cpu_var(current_kprobe) = NULL;
 125 }
 126
 127 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
 128 {
 129         return &__get_cpu_var(kprobe_ctlblk);
 130 }
 131
 132 /*
 133  * This routine is called either:
 134  *      - under the kprobe_mutex - during kprobe_[un]register()
 135  *                              OR
 136  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
 137  */
 138 struct kprobe *get_kprobe(void *addr)
 139 {
 140         struct hlist_head *head;
 141         struct hlist_node *node;
 142         struct kprobe *p;
 143
 144         head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
 145         hlist_for_each_entry_rcu(p, node, head, hlist) {
 146                 if (p->addr == addr) {
 147                         return p;
 148                 }
 149         }
 150
 151         return NULL;
 152 }
 153
 154 /*
 155  * Aggregate handlers for multiple kprobes support - these handlers
 156  * take care of invoking the individual kprobe handlers on p->list
 157  */
 158 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
 159 {
 160         struct kprobe *kp;
 161         int ret;
 162
 163         list_for_each_entry_rcu(kp, &p->list, list) {
 164                 if (kp->pre_handler) {
 165                         set_kprobe_instance(kp);
 166                         ret = kp->pre_handler(kp, regs);
 167                         if (ret)
 168                                 return ret;
 169                 }
 170                 reset_kprobe_instance();
 171         }
 172
 173         return 0;
 174 }
 175
 176 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
 177 {
 178         struct kprobe *kp;
 179
 180         list_for_each_entry_rcu(kp, &p->list, list) {
 181                 if (kp->post_handler) {
 182                         set_kprobe_instance(kp);
 183                         kp->post_handler(kp, regs, flags);
 184                         reset_kprobe_instance();
 185                 }
 186         }
 187 }
 188
 189 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
 190 {
 191         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 192
 193         /*
 194          * if we faulted "during" the execution of a user specified
 195          * probe handler, invoke just that probe's fault handler
 196          */
 197         if (cur && cur->fault_handler) {
 198                 if (cur->fault_handler(cur, regs, trapnr))
 199                         return 1;
 200         }
 201
 202         return 0;
 203 }
 204
 205 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
 206 {
 207         struct kprobe *cur = __get_cpu_var(kprobe_instance);
 208         int ret = 0;
 209         DBPRINTF ("cur = 0x%p\n", cur);
 210         if (cur)
 211                 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
 212
 213         if (cur && cur->break_handler) {
 214                 if (cur->break_handler(cur, regs))
 215                         ret = 1;
 216         }
 217         reset_kprobe_instance();
 218
 219         return ret;
 220 }
 221
 222 /* Walks the list and increments nmissed count for multiprobe case */
 223 void kprobes_inc_nmissed_count(struct kprobe *p)
 224 {
 225         struct kprobe *kp;
 226         if (p->pre_handler != aggr_pre_handler) {
 227                 p->nmissed++;
 228         } else {
 229                 list_for_each_entry_rcu(kp, &p->list, list) {
 230                         ++kp->nmissed;
 231                 }
 232         }
 233 }
 234
 235 /* Called with kretprobe_lock held */
 236 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
 237 {
 238         struct hlist_node *node;
 239         struct kretprobe_instance *ri;
 240
 241         hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 242                 return ri;
 243         }
 244
 245         if (!alloc_nodes_kretprobe(rp)) {
 246                 hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 247                         return ri;
 248                 }
 249         }
 250
 251         return NULL;
 252 }
 253 EXPORT_SYMBOL_GPL(get_free_rp_inst);
 254
 255 /* Called with kretprobe_lock held */
 256 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
 257 {
 258         struct hlist_node *node;
 259         struct kretprobe_instance *ri;
 260
 261         hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 262                 return ri;
 263         }
 264
 265         return NULL;
 266 }
 267
 268 /* Called with kretprobe_lock held */
 269 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
 270 {
 271         struct hlist_node *node;
 272         struct kretprobe_instance *ri;
 273
 274         hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
 275                 return ri;
 276         }
 277
 278         return NULL;
 279 }
 280 EXPORT_SYMBOL_GPL(get_used_rp_inst);
 281
 282 /* Called with kretprobe_lock held */
 283 void add_rp_inst (struct kretprobe_instance *ri)
 284 {
 285         /*
 286          * Remove rp inst off the free list -
 287          * Add it back when probed function returns
 288          */
 289         hlist_del(&ri->uflist);
 290
 291         /* Add rp inst onto table */
 292         INIT_HLIST_NODE(&ri->hlist);
 293
 294         hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
 295
 296         /* Also add this rp inst to the used list. */
 297         INIT_HLIST_NODE(&ri->uflist);
 298         hlist_add_head(&ri->uflist, &ri->rp->used_instances);
 299 }
 300 EXPORT_SYMBOL_GPL(add_rp_inst);
 301
 302 /* Called with kretprobe_lock held */
 303 void recycle_rp_inst(struct kretprobe_instance *ri)
 304 {
 305         if (ri->rp) {
 306                 hlist_del(&ri->hlist);
 307                 /* remove rp inst off the used list */
 308                 hlist_del(&ri->uflist);
 309                 /* put rp inst back onto the free list */
 310                 INIT_HLIST_NODE(&ri->uflist);
 311                 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
 312         }
 313 }
 314 EXPORT_SYMBOL_GPL(recycle_rp_inst);
 315
 316 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
 317 {
 318         return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
 319 }
 320 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
 321
 322 void free_rp_inst(struct kretprobe *rp)
 323 {
 324         struct kretprobe_instance *ri;
 325         while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
 326                 hlist_del(&ri->uflist);
 327                 kfree(ri);
 328         }
 329 }
 330 EXPORT_SYMBOL_GPL(free_rp_inst);
 331
 332 /*
 333  * Keep all fields in the kprobe consistent
 334  */
 335 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 336 {
 337         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 338         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 339         p->ss_addr = old_p->ss_addr;
 340 #ifdef CONFIG_ARM
 341         p->safe_arm = old_p->safe_arm;
 342         p->safe_thumb = old_p->safe_thumb;
 343 #endif
 344 }
 345
 346 /*
 347  * Add the new probe to old_p->list. Fail if this is the
 348  * second jprobe at the address - two jprobes can't coexist
 349  */
 350 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
 351 {
 352         if (p->break_handler) {
 353                 if (old_p->break_handler) {
 354                         return -EEXIST;
 355                 }
 356
 357                 list_add_tail_rcu(&p->list, &old_p->list);
 358                 old_p->break_handler = aggr_break_handler;
 359         } else {
 360                 list_add_rcu(&p->list, &old_p->list);
 361         }
 362
 363         if (p->post_handler && !old_p->post_handler) {
 364                 old_p->post_handler = aggr_post_handler;
 365         }
 366
 367         return 0;
 368 }
 369
 370 /**
 371  * hlist_replace_rcu - replace old entry by new one
 372  * @old : the element to be replaced
 373  * @new : the new element to insert
 374  *
 375  * The @old entry will be replaced with the @new entry atomically.
 376  */
 377 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
 378 {
 379         struct hlist_node *next = old->next;
 380
 381         new->next = next;
 382         new->pprev = old->pprev;
 383         smp_wmb();
 384         if (next)
 385                 new->next->pprev = &new->next;
 386         if (new->pprev)
 387                 *new->pprev = new;
 388         old->pprev = LIST_POISON2;
 389 }
 390
 391 /*
 392  * Fill in the required fields of the "manager kprobe". Replace the
 393  * earlier kprobe in the hlist with the manager kprobe
 394  */
 395 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 396 {
 397         copy_kprobe(p, ap);
 398         ap->addr = p->addr;
 399         ap->pre_handler = aggr_pre_handler;
 400         ap->fault_handler = aggr_fault_handler;
 401         if (p->post_handler)
 402                 ap->post_handler = aggr_post_handler;
 403         if (p->break_handler)
 404                 ap->break_handler = aggr_break_handler;
 405
 406         INIT_LIST_HEAD(&ap->list);
 407         list_add_rcu(&p->list, &ap->list);
 408
 409         dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
 410 }
 411
 412 /*
 413  * This is the second or subsequent kprobe at the address - handle
 414  * the intricacies
 415  */
 416 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
 417 {
 418         int ret = 0;
 419         struct kprobe *ap;
 420         DBPRINTF ("start\n");
 421
 422         DBPRINTF ("p = %p old_p = %p \n", p, old_p);
 423         if (old_p->pre_handler == aggr_pre_handler) {
 424                 DBPRINTF ("aggr_pre_handler \n");
 425
 426                 copy_kprobe(old_p, p);
 427                 ret = add_new_kprobe(old_p, p);
 428         } else {
 429                 DBPRINTF ("kzalloc\n");
 430 #ifdef kzalloc
 431                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
 432 #else
 433                 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
 434                 if (ap)
 435                         memset(ap, 0, sizeof(struct kprobe));
 436 #endif
 437                 if (!ap)
 438                         return -ENOMEM;
 439                 add_aggr_kprobe(ap, old_p);
 440                 copy_kprobe(ap, p);
 441                 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
 442                 ret = add_new_kprobe(ap, p);
 443         }
 444
 445         return ret;
 446 }
 447 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
 448
 449 static void remove_kprobe(struct kprobe *p)
 450 {
 451         /* TODO: check boostable for x86 and MIPS */
 452         free_insn_slot(&kprobe_insn_pages, NULL, p->ainsn.insn);
 453 }
 454
 455 int dbi_register_kprobe(struct kprobe *p)
 456 {
 457         struct kprobe *old_p;
 458         int ret = 0;
 459         /*
 460          * If we have a symbol_name argument look it up,
 461          * and add it to the address.  That way the addr
 462          * field can either be global or relative to a symbol.
 463          */
 464         if (p->symbol_name) {
 465                 if (p->addr)
 466                         return -EINVAL;
 467                 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
 468         }
 469
 470         if (!p->addr)
 471                 return -EINVAL;
 472         DBPRINTF ("p->addr = 0x%p\n", p->addr);
 473         p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
 474         DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
 475
 476 #ifdef KPROBES_PROFILE
 477         p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
 478         p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
 479         p->count = 0;
 480 #endif
 481         p->mod_refcounted = 0;
 482         p->nmissed = 0;
 483
 484         old_p = get_kprobe(p->addr);
 485         if (old_p) {
 486                 ret = register_aggr_kprobe(old_p, p);
 487                 if (!ret)
 488                         atomic_inc(&kprobe_count);
 489                 goto out;
 490         }
 491
 492         if ((ret = arch_prepare_kprobe(p, &kprobe_insn_pages)) != 0)
 493                 goto out;
 494
 495         DBPRINTF ("before out ret = 0x%x\n", ret);
 496         INIT_HLIST_NODE(&p->hlist);
 497         hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
 498         arch_arm_kprobe(p);
 499
 500 out:
 501         DBPRINTF ("out ret = 0x%x\n", ret);
 502         return ret;
 503 }
 504
 505 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
 506 {
 507         struct kprobe *old_p, *list_p;
 508         int cleanup_p;
 509
 510         old_p = get_kprobe(p->addr);
 511         DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
 512         if (unlikely (!old_p))
 513                 return;
 514
 515         if (p != old_p) {
 516                 list_for_each_entry_rcu(list_p, &old_p->list, list)
 517                         if (list_p == p)
 518                                 /* kprobe p is a valid probe */
 519                                 goto valid_p;
 520                 return;
 521         }
 522
 523 valid_p:
 524         DBPRINTF ("dbi_unregister_kprobe valid_p");
 525         if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
 526             (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
 527                 /* Only probe on the hash list */
 528                 arch_disarm_kprobe(p);
 529                 hlist_del_rcu(&old_p->hlist);
 530                 cleanup_p = 1;
 531         } else {
 532                 list_del_rcu(&p->list);
 533                 cleanup_p = 0;
 534         }
 535         DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
 536
 537         if (cleanup_p) {
 538                 if (p != old_p) {
 539                         list_del_rcu(&p->list);
 540                         kfree(old_p);
 541                 }
 542
 543                 if (!in_atomic()) {
 544                         synchronize_sched();
 545                 }
 546
 547                 remove_kprobe(p);
 548         } else {
 549                 if (p->break_handler)
 550                         old_p->break_handler = NULL;
 551                 if (p->post_handler) {
 552                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
 553                                 if (list_p->post_handler) {
 554                                         cleanup_p = 2;
 555                                         break;
 556                                 }
 557                         }
 558
 559                         if (cleanup_p == 0)
 560                                 old_p->post_handler = NULL;
 561                 }
 562         }
 563 }
 564
 565 int dbi_register_jprobe(struct jprobe *jp)
 566 {
 567         /* Todo: Verify probepoint is a function entry point */
 568         jp->kp.pre_handler = setjmp_pre_handler;
 569         jp->kp.break_handler = longjmp_break_handler;
 570
 571         return dbi_register_kprobe(&jp->kp);
 572 }
 573
 574 void dbi_unregister_jprobe(struct jprobe *jp)
 575 {
 576         dbi_unregister_kprobe(&jp->kp, NULL);
 577 }
 578
 579 /*
 580  * This kprobe pre_handler is registered with every kretprobe. When probe
 581  * hits it will set up the return probe.
 582  */
 583 int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
 584 {
 585         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
 586         unsigned long flags = 0;
 587         DBPRINTF ("START\n");
 588
 589         /*TODO: consider to only swap the RA after the last pre_handler fired */
 590         spin_lock_irqsave(&kretprobe_lock, flags);
 591         if (!rp->disarm)
 592                 arch_prepare_kretprobe(rp, regs);
 593         spin_unlock_irqrestore(&kretprobe_lock, flags);
 594         DBPRINTF ("END\n");
 595         return 0;
 596 }
 597 EXPORT_SYMBOL_GPL(pre_handler_kretprobe);
 598
 599 struct kretprobe *sched_rp;
 600
 601 #define SCHED_RP_NR 200
 602 #define COMMON_RP_NR 10
 603
 604 int alloc_nodes_kretprobe(struct kretprobe *rp)
 605 {
 606         int alloc_nodes;
 607         struct kretprobe_instance *inst;
 608         int i;
 609
 610         DBPRINTF("Alloc aditional mem for retprobes");
 611
 612         if ((unsigned long)rp->kp.addr == sched_addr) {
 613                 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 614                 alloc_nodes = SCHED_RP_NR;
 615         } else {
 616 #if 1//def CONFIG_PREEMPT
 617                 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
 618 #else
 619                 rp->maxacpptive += NR_CPUS;
 620 #endif
 621                 alloc_nodes = COMMON_RP_NR;
 622         }
 623
 624         for (i = 0; i < alloc_nodes; i++) {
 625                 inst = kmalloc(sizeof(struct kretprobe_instance), GFP_ATOMIC);
 626                 if (inst == NULL) {
 627                         free_rp_inst(rp);
 628                         return -ENOMEM;
 629                 }
 630                 INIT_HLIST_NODE(&inst->uflist);
 631                 hlist_add_head(&inst->uflist, &rp->free_instances);
 632         }
 633
 634         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 635         return 0;
 636 }
 637
 638 int dbi_register_kretprobe(struct kretprobe *rp)
 639 {
 640         int ret = 0;
 641         struct kretprobe_instance *inst;
 642         int i;
 643         DBPRINTF ("START");
 644
 645         rp->kp.pre_handler = pre_handler_kretprobe;
 646         rp->kp.post_handler = NULL;
 647         rp->kp.fault_handler = NULL;
 648         rp->kp.break_handler = NULL;
 649
 650         rp->disarm = 0;
 651
 652         /* Pre-allocate memory for max kretprobe instances */
 653         if ((unsigned long)rp->kp.addr == sched_addr) {
 654                 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
 655                 rp->kp.pre_handler = NULL; //not needed for __switch_to
 656         } else if ((unsigned long)rp->kp.addr == exit_addr) {
 657                 rp->kp.pre_handler = NULL; //not needed for do_exit
 658                 rp->maxactive = 0;
 659         } else if (rp->maxactive <= 0) {
 660 #if 1//def CONFIG_PREEMPT
 661                 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
 662 #else
 663                 rp->maxactive = NR_CPUS;
 664 #endif
 665         }
 666         INIT_HLIST_HEAD(&rp->used_instances);
 667         INIT_HLIST_HEAD(&rp->free_instances);
 668         for (i = 0; i < rp->maxactive; i++) {
 669                 inst = kmalloc(sizeof(struct kretprobe_instance), GFP_KERNEL);
 670                 if (inst == NULL) {
 671                         free_rp_inst(rp);
 672                         return -ENOMEM;
 673                 }
 674                 INIT_HLIST_NODE(&inst->uflist);
 675                 hlist_add_head(&inst->uflist, &rp->free_instances);
 676         }
 677
 678         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 679         rp->nmissed = 0;
 680         /* Establish function entry probe point */
 681         if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
 682                 free_rp_inst(rp);
 683
 684         DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
 685         if ((unsigned long)rp->kp.addr == sched_addr) {
 686                 sched_rp = rp;
 687         }
 688
 689         return ret;
 690 }
 691
 692 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
 693
 694 void dbi_unregister_kretprobe(struct kretprobe *rp)
 695 {
 696         unsigned long flags;
 697         struct kretprobe_instance *ri;
 698
 699         dbi_unregister_kprobe(&rp->kp, NULL);
 700
 701         /* No race here */
 702         spin_lock_irqsave(&kretprobe_lock, flags);
 703
 704         if ((unsigned long)rp->kp.addr == sched_addr)
 705                 sched_rp = NULL;
 706
 707         while ((ri = get_used_rp_inst(rp)) != NULL) {
 708                 if (dbi_disarm_krp_inst(ri) == 0)
 709                         recycle_rp_inst(ri);
 710                 else
 711                         panic("%s (%d/%d): cannot disarm krp instance (%08lx)",
 712                                         ri->task->comm, ri->task->tgid, ri->task->pid,
 713                                         (unsigned long)rp->kp.addr);
 714         }
 715
 716         spin_unlock_irqrestore(&kretprobe_lock, flags);
 717         free_rp_inst(rp);
 718 }
 719
 720 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
 721 {
 722         struct kprobe *old_p;
 723         struct kretprobe *clone = NULL;
 724         int ret;
 725
 726         clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
 727         if (!clone) {
 728                 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
 729                 return NULL;
 730         }
 731         memcpy(clone, rp, sizeof(struct kretprobe));
 732         clone->kp.pre_handler = pre_handler_kretprobe;
 733         clone->kp.post_handler = NULL;
 734         clone->kp.fault_handler = NULL;
 735         clone->kp.break_handler = NULL;
 736         old_p = get_kprobe(rp->kp.addr);
 737         if (old_p) {
 738                 ret = register_aggr_kprobe(old_p, &clone->kp);
 739                 if (ret) {
 740                         kfree(clone);
 741                         return NULL;
 742                 }
 743                 atomic_inc(&kprobe_count);
 744         }
 745
 746         return clone;
 747 }
 748 EXPORT_SYMBOL_GPL(clone_kretprobe);
 749
 750 static void inline set_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri, unsigned long tramp_addr)
 751 {
 752         unsigned long pc = arch_get_task_pc(p);
 753         if (pc == tramp_addr)
 754                 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
 755                                 task_cpu(p), p->comm, p->tgid, p->pid, pc,
 756                                 task_cpu(current), current->comm, current->tgid, current->pid);
 757         ri->ret_addr = (kprobe_opcode_t *)pc;
 758         arch_set_task_pc(p, tramp_addr);
 759 }
 760
 761 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
 762 {
 763         arch_set_task_pc(p, (unsigned long)ri->ret_addr);
 764 }
 765
 766 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
 767 {
 768         kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
 769         kprobe_opcode_t *sp = ri->sp;
 770         kprobe_opcode_t *found = NULL;
 771         int retval = -ENOENT;
 772
 773         if (!sp) {
 774                 unsigned long pc = arch_get_task_pc(ri->task);
 775
 776                 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
 777                                 task_cpu(ri->task),
 778                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 779                                 pc, (unsigned long)ri->ret_addr,
 780                                 (unsigned long)tramp,
 781                                 (unsigned long)(ri->rp ? ri->rp->kp.addr: NULL));
 782
 783                 /* __switch_to retprobe handling */
 784                 if (pc == (unsigned long)tramp) {
 785                         rm_task_trampoline(ri->task, ri);
 786                         return 0;
 787                 }
 788
 789                 return -EINVAL;
 790         }
 791
 792         while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
 793                 if ((unsigned long)*sp == (unsigned long)tramp) {
 794                         found = sp;
 795                         break;
 796                 }
 797                 sp--;
 798         }
 799
 800         if (found) {
 801                 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
 802                                 task_cpu(ri->task),
 803                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 804                                 (unsigned long)tramp,
 805                                 (unsigned long)found, (unsigned long)ri->sp, found - ri->sp,
 806                                 ri->rp ? ri->rp->kp.addr: NULL);
 807                 *found = (unsigned long)ri->ret_addr;
 808                 retval = 0;
 809         } else {
 810                 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
 811                                 task_cpu(ri->task),
 812                                 ri->task->comm, ri->task->tgid, ri->task->pid,
 813                                 (unsigned long)tramp,
 814                                 (unsigned long)ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
 815         }
 816
 817         return retval;
 818 }
 819
 820 int patch_suspended_task(struct kretprobe *rp, struct task_struct *task)
 821 {
 822         struct kretprobe_instance *ri;
 823         unsigned long flags;
 824         kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
 825
 826         spin_lock_irqsave(&kretprobe_lock, flags);
 827
 828         ri = get_free_rp_inst(rp);
 829         if (!ri)
 830                 return -ENOMEM;
 831
 832         ri->rp = rp;
 833         ri->task = task;
 834         ri->sp = NULL;
 835         set_task_trampoline(task, ri, (unsigned long)tramp);
 836         add_rp_inst(ri);
 837
 838         spin_unlock_irqrestore(&kretprobe_lock, flags);
 839         return 0;
 840 }
 841
 842 static int init_module_deps(void)
 843 {
 844         int ret;
 845
 846         sched_addr = swap_ksyms("__switch_to");
 847         fork_addr = swap_ksyms("do_fork");
 848         exit_addr = swap_ksyms("do_exit");
 849
 850         if (sched_addr == 0 || fork_addr == 0 || exit_addr == 0) {
 851                 return -ESRCH;
 852         }
 853
 854         ret = init_module_dependencies();
 855         if (ret) {
 856                 return ret;
 857         }
 858
 859         return arch_init_module_deps();
 860 }
 861
 862 static int __init init_kprobes(void)
 863 {
 864         int i, err = 0;
 865
 866         /* FIXME allocate the probe table, currently defined statically */
 867         /* initialize all list heads */
 868         for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
 869                 INIT_HLIST_HEAD(&kprobe_table[i]);
 870                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
 871         }
 872         atomic_set(&kprobe_count, 0);
 873
 874         err = init_module_deps();
 875         if (err) {
 876                 return err;
 877         }
 878
 879         err = arch_init_kprobes();
 880
 881         DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
 882
 883         return err;
 884 }
 885
 886 static void __exit exit_kprobes(void)
 887 {
 888         arch_exit_kprobes();
 889 }
 890
 891 module_init(init_kprobes);
 892 module_exit(exit_kprobes);
 893
 894 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
 895 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
 896 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
 897 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
 898 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
 899 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
 900 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
 901
 902 MODULE_LICENSE("Dual BSD/GPL");