uprobe/swap_uprobes.c

   1 /*
   2  *  Dynamic Binary Instrumentation Module based on KProbes
   3  *  modules/kprobe/dbi_uprobes.h
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) Samsung Electronics, 2006-2010
  20  *
  21  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  22  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  23  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  24  *
  25  */
  26
  27
  28 #include "swap_uprobes.h"
  29 #include "dbi_kdebug.h"
  30
  31 #include <asm/swap_uprobes.h>
  32
  33 #include <linux/hash.h>
  34 #include <linux/mempolicy.h>
  35 #include <linux/module.h>
  36 #include <dbi_insn_slots.h>
  37
  38 enum {
  39         UPROBE_HASH_BITS  = 10,
  40         UPROBE_TABLE_SIZE = (1 << UPROBE_HASH_BITS)
  41 };
  42
  43 struct hlist_head uprobe_insn_slot_table[UPROBE_TABLE_SIZE];
  44 struct hlist_head uprobe_table[UPROBE_TABLE_SIZE];
  45 struct hlist_head uprobe_insn_pages;
  46
  47 DEFINE_SPINLOCK(uretprobe_lock);        /* Protects uretprobe_inst_table */
  48 static struct hlist_head uretprobe_inst_table[UPROBE_TABLE_SIZE];
  49
  50 #define DEBUG_PRINT_HASH_TABLE 0
  51
  52 #if DEBUG_PRINT_HASH_TABLE
  53 void print_kprobe_hash_table(void)
  54 {
  55         int i;
  56         struct hlist_head *head;
  57         struct hlist_node *node;
  58         struct kprobe *p;
  59
  60         // print uprobe table
  61         for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
  62                 head = &kprobe_table[i];
  63                 hlist_for_each_entry_rcu (p, node, head, is_hlist_arm) {
  64                         printk("####### find K tgid=%u, addr=%x\n",
  65                                         p->tgid, p->addr);
  66                 }
  67         }
  68 }
  69
  70 void print_kretprobe_hash_table(void)
  71 {
  72         int i;
  73         struct hlist_head *head;
  74         struct hlist_node *node;
  75         struct kprobe *p;
  76
  77         // print uprobe table
  78         for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
  79                 head = &kretprobe_inst_table[i];
  80                 hlist_for_each_entry_rcu (p, node, head, is_hlist_arm) {
  81                         printk("####### find KR tgid=%u, addr=%x\n",
  82                                         p->tgid, p->addr);
  83                 }
  84         }
  85 }
  86
  87 void print_uprobe_hash_table(void)
  88 {
  89         int i;
  90         struct hlist_head *head;
  91         struct hlist_node *node;
  92         struct kprobe *p;
  93
  94         // print uprobe table
  95         for (i = 0; i < UPROBE_TABLE_SIZE; ++i) {
  96                 head = &uprobe_insn_slot_table[i];
  97                 hlist_for_each_entry_rcu (p, node, head, is_hlist_arm) {
  98                         printk("####### find U tgid=%u, addr=%x\n",
  99                                         p->tgid, p->addr);
 100                 }
 101         }
 102 }
 103 #endif
 104
 105 /*
 106  * Keep all fields in the uprobe consistent
 107  */
 108 static inline void copy_uprobe(struct kprobe *old_p, struct kprobe *p)
 109 {
 110         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
 111         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
 112         p->tgid = old_p->tgid;
 113         p->ss_addr = old_p->ss_addr;
 114 #ifdef CONFIG_ARM
 115         p->safe_arm = old_p->safe_arm;
 116         p->safe_thumb = old_p->safe_thumb;
 117 #endif
 118 }
 119
 120 /*
 121  * Aggregate handlers for multiple uprobes support - these handlers
 122  * take care of invoking the individual uprobe handlers on p->list
 123  */
 124 static int aggr_pre_uhandler(struct kprobe *p, struct pt_regs *regs)
 125 {
 126         struct kprobe *kp;
 127         int ret;
 128
 129         list_for_each_entry_rcu(kp, &p->list, list) {
 130                 if (kp->pre_handler) {
 131                         ret = kp->pre_handler(kp, regs);
 132                         if (ret) {
 133                                 return ret;
 134                         }
 135                 }
 136         }
 137
 138         return 0;
 139 }
 140
 141 static void aggr_post_uhandler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
 142 {
 143         struct kprobe *kp;
 144
 145         list_for_each_entry_rcu(kp, &p->list, list) {
 146                 if (kp->post_handler) {
 147                         kp->post_handler(kp, regs, flags);
 148                 }
 149         }
 150 }
 151
 152 static int aggr_fault_uhandler(struct kprobe *p, struct pt_regs *regs, int trapnr)
 153 {
 154         return 0;
 155 }
 156
 157 static int aggr_break_uhandler(struct kprobe *p, struct pt_regs *regs)
 158 {
 159         return 0;
 160 }
 161
 162 /*
 163  * Add the new probe to old_p->list. Fail if this is the
 164  * second ujprobe at the address - two ujprobes can't coexist
 165  */
 166 static int add_new_uprobe(struct kprobe *old_p, struct kprobe *p)
 167 {
 168         if (p->break_handler) {
 169                 if (old_p->break_handler) {
 170                         return -EEXIST;
 171                 }
 172
 173                 list_add_tail_rcu(&p->list, &old_p->list);
 174                 old_p->break_handler = aggr_break_uhandler;
 175         } else {
 176                 list_add_rcu (&p->list, &old_p->list);
 177         }
 178
 179         if (p->post_handler && !old_p->post_handler) {
 180                 old_p->post_handler = aggr_post_uhandler;
 181         }
 182
 183         return 0;
 184 }
 185
 186 /*
 187  * Fill in the required fields of the "manager uprobe". Replace the
 188  * earlier uprobe in the hlist with the manager uprobe
 189  */
 190 static inline void add_aggr_uprobe(struct kprobe *ap, struct kprobe *p)
 191 {
 192         copy_uprobe(p, ap);
 193
 194         ap->addr = p->addr;
 195         ap->pre_handler = aggr_pre_uhandler;
 196         ap->fault_handler = aggr_fault_uhandler;
 197
 198         if (p->post_handler) {
 199                 ap->post_handler = aggr_post_uhandler;
 200         }
 201
 202         if (p->break_handler) {
 203                 ap->break_handler = aggr_break_uhandler;
 204         }
 205
 206         INIT_LIST_HEAD(&ap->list);
 207         list_add_rcu(&p->list, &ap->list);
 208
 209         hlist_replace_rcu(&p->hlist, &ap->hlist);
 210 }
 211
 212 /*
 213  * This is the second or subsequent uprobe at the address - handle
 214  * the intricacies
 215  */
 216 static int register_aggr_uprobe(struct kprobe *old_p, struct kprobe *p)
 217 {
 218         int ret = 0;
 219         struct kprobe *ap;
 220
 221         if (old_p->pre_handler == aggr_pre_uhandler) {
 222                 copy_uprobe(old_p, p);
 223                 ret = add_new_uprobe(old_p, p);
 224         } else {
 225                 struct uprobe *uap = kzalloc(sizeof(*uap), GFP_KERNEL);
 226                 if (!uap) {
 227                         return -ENOMEM;
 228                 }
 229
 230                 uap->task = kp2up(p)->task;
 231                 ap = up2kp(uap);
 232                 add_aggr_uprobe(ap, old_p);
 233                 copy_uprobe(ap, p);
 234                 ret = add_new_uprobe(ap, p);
 235         }
 236
 237         return ret;
 238 }
 239
 240 static void arm_uprobe(struct uprobe *p)
 241 {
 242         kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
 243         int ret = write_proc_vm_atomic(p->task, (unsigned long)p->kp.addr,
 244                                        &insn, sizeof(insn));
 245         if (!ret) {
 246                 panic("arm_uprobe: failed to write memory "
 247                       "tgid=%u addr=%p!\n", p->task->tgid, p->kp.addr);
 248         }
 249 }
 250
 251 void disarm_uprobe(struct uprobe *p)
 252 {
 253         int ret = write_proc_vm_atomic(p->task, (unsigned long)p->kp.addr,
 254                                        &p->kp.opcode, sizeof(p->kp.opcode));
 255         if (!ret) {
 256                 panic("disarm_uprobe: failed to write memory "
 257                       "tgid=%u, addr=%p!\n", p->task->tgid, p->kp.addr);
 258         }
 259 }
 260 EXPORT_SYMBOL_GPL(disarm_uprobe);
 261
 262 static void init_uprobes_insn_slots(void)
 263 {
 264         int i;
 265         for (i = 0; i < UPROBE_TABLE_SIZE; ++i) {
 266                 INIT_HLIST_HEAD(&uprobe_insn_slot_table[i]);
 267         }
 268 }
 269
 270 static void init_uprobe_table(void)
 271 {
 272         int i;
 273         for (i = 0; i < UPROBE_TABLE_SIZE; ++i) {
 274                 INIT_HLIST_HEAD(&uprobe_table[i]);
 275         }
 276 }
 277
 278 static void init_uretprobe_inst_table(void)
 279 {
 280         int i;
 281         for (i = 0; i < UPROBE_TABLE_SIZE; ++i) {
 282                 INIT_HLIST_HEAD (&uretprobe_inst_table[i]);
 283         }
 284 }
 285
 286 struct kprobe *get_ukprobe(void *addr, pid_t tgid)
 287 {
 288         struct hlist_head *head;
 289         struct hlist_node *node;
 290         struct kprobe *p;
 291
 292         head = &uprobe_table[hash_ptr(addr, UPROBE_HASH_BITS)];
 293         hlist_for_each_entry_rcu(p, node, head, hlist) {
 294                 if (p->addr == addr && kp2up(p)->task->tgid == tgid) {
 295                         return p;
 296                 }
 297         }
 298
 299         return NULL;
 300 }
 301
 302 static void add_uprobe_table(struct kprobe *p)
 303 {
 304 #ifdef CONFIG_ARM
 305         INIT_HLIST_NODE(&p->is_hlist_arm);
 306         hlist_add_head_rcu(&p->is_hlist_arm, &uprobe_insn_slot_table[hash_ptr(p->ainsn.insn_arm, UPROBE_HASH_BITS)]);
 307         INIT_HLIST_NODE(&p->is_hlist_thumb);
 308         hlist_add_head_rcu(&p->is_hlist_thumb, &uprobe_insn_slot_table[hash_ptr(p->ainsn.insn_thumb, UPROBE_HASH_BITS)]);
 309 #else /* CONFIG_ARM */
 310         INIT_HLIST_NODE(&p->is_hlist);
 311         hlist_add_head_rcu(&p->is_hlist, &uprobe_insn_slot_table[hash_ptr(p->ainsn.insn, UPROBE_HASH_BITS)]);
 312 #endif /* CONFIG_ARM */
 313 }
 314
 315 #ifdef CONFIG_ARM
 316 static struct kprobe *get_kprobe_by_insn_slot_arm(kprobe_opcode_t *addr, pid_t tgid)
 317 {
 318         struct hlist_head *head;
 319         struct hlist_node *node;
 320         struct kprobe *p, *ret = NULL;
 321
 322         /* TODO: test - two processes invokes instrumented function */
 323         head = &uprobe_insn_slot_table[hash_ptr(addr, UPROBE_HASH_BITS)];
 324         hlist_for_each_entry_rcu(p, node, head, is_hlist_arm) {
 325                 if (p->ainsn.insn == addr && tgid == p->tgid) {
 326                         ret = p;
 327                         break;
 328                 }
 329         }
 330
 331         return ret;
 332 }
 333
 334 static struct kprobe *get_kprobe_by_insn_slot_thumb(kprobe_opcode_t *addr, pid_t tgid)
 335 {
 336         struct hlist_head *head;
 337         struct hlist_node *node;
 338         struct kprobe *p, *ret = NULL;
 339
 340         /* TODO: test - two processes invokes instrumented function */
 341         head = &uprobe_insn_slot_table[hash_ptr(addr, UPROBE_HASH_BITS)];
 342         hlist_for_each_entry_rcu(p, node, head, is_hlist_thumb) {
 343                 if (p->ainsn.insn == addr && tgid == p->tgid) {
 344                         ret = p;
 345                         break;
 346                 }
 347         }
 348
 349         return ret;
 350 }
 351
 352 struct kprobe *get_kprobe_by_insn_slot(kprobe_opcode_t *addr, pid_t tgid, struct pt_regs *regs)
 353 {
 354         struct kprobe *p = NULL;
 355
 356         if (!thumb_mode(regs)) {
 357                 p = get_kprobe_by_insn_slot_arm(addr - UPROBES_TRAMP_RET_BREAK_IDX, tgid);
 358         } else {
 359                 p = get_kprobe_by_insn_slot_thumb((kprobe_opcode_t *)((unsigned long)addr - 0x1a), tgid);
 360         }
 361
 362         return p;
 363 }
 364 #else /* CONFIG_ARM */
 365 struct kprobe *get_kprobe_by_insn_slot(void *addr, int tgid, struct task_struct *ctask)
 366 {
 367         struct hlist_head *head;
 368         struct hlist_node *node;
 369         struct kprobe *p, *ret = NULL;
 370
 371         /* TODO: test - two processes invokes instrumented function */
 372         head = &uprobe_insn_slot_table[hash_ptr(addr, UPROBE_HASH_BITS)];
 373         hlist_for_each_entry_rcu(p, node, head, is_hlist) {
 374                 if (p->ainsn.insn == addr && tgid == p->tgid) {
 375                         ret = p;
 376                         break;
 377                 }
 378         }
 379
 380         return ret;
 381 }
 382 #endif /* CONFIG_ARM */
 383
 384
 385 static void remove_uprobe(struct uprobe *up)
 386 {
 387         struct kprobe *p = &up->kp;
 388         struct task_struct *task = up->task;
 389
 390         if (p->tgid == 0) {
 391                 panic("remove_uprobe for tgid == 0!!!");
 392         }
 393
 394 #ifdef CONFIG_ARM
 395         free_insn_slot(&uprobe_insn_pages, task, p->ainsn.insn_arm);
 396         free_insn_slot(&uprobe_insn_pages, task, p->ainsn.insn_thumb);
 397 #else /* CONFIG_ARM */
 398         free_insn_slot(&uprobe_insn_pages, task, p->ainsn.insn);
 399 #endif /* CONFIG_ARM */
 400 }
 401
 402 static struct hlist_head *uretprobe_inst_table_head(void *hash_key)
 403 {
 404         return &uretprobe_inst_table[hash_ptr (hash_key, UPROBE_HASH_BITS)];
 405 }
 406
 407 /* Called with uretprobe_lock held */
 408 static void add_urp_inst(struct uretprobe_instance *ri)
 409 {
 410         /*
 411          * Remove rp inst off the free list -
 412          * Add it back when probed function returns
 413          */
 414         hlist_del(&ri->uflist);
 415
 416         /* Add rp inst onto table */
 417         INIT_HLIST_NODE(&ri->hlist);
 418         hlist_add_head(&ri->hlist, uretprobe_inst_table_head(ri->task->mm));
 419
 420         /* Also add this rp inst to the used list. */
 421         INIT_HLIST_NODE(&ri->uflist);
 422         hlist_add_head(&ri->uflist, &ri->rp->used_instances);
 423 }
 424
 425 /* Called with uretprobe_lock held */
 426 static void recycle_urp_inst(struct uretprobe_instance *ri)
 427 {
 428         if (ri->rp) {
 429                 hlist_del(&ri->hlist);
 430                 /* remove rp inst off the used list */
 431                 hlist_del(&ri->uflist);
 432                 /* put rp inst back onto the free list */
 433                 INIT_HLIST_NODE(&ri->uflist);
 434                 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
 435         }
 436 }
 437
 438 /* Called with uretprobe_lock held */
 439 static struct uretprobe_instance *get_used_urp_inst(struct uretprobe *rp)
 440 {
 441         struct hlist_node *node;
 442         struct uretprobe_instance *ri;
 443
 444         hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
 445                 return ri;
 446         }
 447
 448         return NULL;
 449 }
 450
 451 /* Called with uretprobe_lock held */
 452 struct uretprobe_instance *get_free_urp_inst_no_alloc(struct uretprobe *rp)
 453 {
 454         struct hlist_node *node;
 455         struct uretprobe_instance *ri;
 456
 457         hlist_for_each_entry (ri, node, &rp->free_instances, uflist) {
 458                 return ri;
 459         }
 460
 461         return NULL;
 462 }
 463
 464 /* Called with uretprobe_lock held */
 465 static void free_urp_inst(struct uretprobe *rp)
 466 {
 467         struct uretprobe_instance *ri;
 468         while ((ri = get_free_urp_inst_no_alloc(rp)) != NULL) {
 469                 hlist_del(&ri->uflist);
 470                 kfree(ri);
 471         }
 472 }
 473
 474 #define COMMON_URP_NR 10
 475
 476 static int alloc_nodes_uretprobe(struct uretprobe *rp)
 477 {
 478         int alloc_nodes;
 479         struct uretprobe_instance *inst;
 480         int i;
 481
 482 #if 1//def CONFIG_PREEMPT
 483         rp->maxactive += max(COMMON_URP_NR, 2 * NR_CPUS);
 484 #else
 485         rp->maxacpptive += NR_CPUS;
 486 #endif
 487         alloc_nodes = COMMON_URP_NR;
 488
 489         for (i = 0; i < alloc_nodes; ++i) {
 490                 inst = kmalloc(sizeof(*inst), GFP_ATOMIC);
 491                 if (inst == NULL) {
 492                         free_urp_inst(rp);
 493                         return -ENOMEM;
 494                 }
 495                 INIT_HLIST_NODE(&inst->uflist);
 496                 hlist_add_head(&inst->uflist, &rp->free_instances);
 497         }
 498
 499         return 0;
 500 }
 501
 502 /* Called with uretprobe_lock held */
 503 static struct uretprobe_instance *get_free_urp_inst(struct uretprobe *rp)
 504 {
 505         struct hlist_node *node;
 506         struct uretprobe_instance *ri;
 507
 508         hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 509                 return ri;
 510         }
 511
 512         if (!alloc_nodes_uretprobe(rp)) {
 513                 hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
 514                         return ri;
 515                 }
 516         }
 517
 518         return NULL;
 519 }
 520 // ===================================================================
 521
 522 int dbi_register_uprobe(struct uprobe *up, int atomic)
 523 {
 524         int ret = 0;
 525         struct kprobe *p, *old_p;
 526
 527         p = &up->kp;
 528         if (!p->addr) {
 529                 return -EINVAL;
 530         }
 531
 532         DBPRINTF("p->addr = 0x%p p = 0x%p\n", p->addr, p);
 533
 534 // thumb address = address-1;
 535 #if defined(CONFIG_ARM)
 536         // TODO: must be corrected in 'bundle'
 537         if ((unsigned long) p->addr & 0x01) {
 538                 p->addr = (kprobe_opcode_t *)((unsigned long)p->addr & 0xfffffffe);
 539         }
 540 #endif
 541
 542         p->mod_refcounted = 0;
 543         p->nmissed = 0;
 544         INIT_LIST_HEAD(&p->list);
 545 #ifdef KPROBES_PROFILE
 546         p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
 547         p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
 548         p->count = 0;
 549 #endif
 550
 551         // get the first item
 552         old_p = get_ukprobe(p->addr, p->tgid);
 553         if (old_p) {
 554 #ifdef CONFIG_ARM
 555                 p->safe_arm = old_p->safe_arm;
 556                 p->safe_thumb = old_p->safe_thumb;
 557 #endif
 558                 ret = register_aggr_uprobe(old_p, p);
 559                 if (!ret) {
 560 //                      atomic_inc(&kprobe_count);
 561                         add_uprobe_table(p);
 562                 }
 563                 DBPRINTF("goto out\n", ret);
 564                 goto out;
 565         }
 566
 567         ret = arch_prepare_uprobe(up, atomic);
 568         if (ret) {
 569                 DBPRINTF("goto out\n", ret);
 570                 goto out;
 571         }
 572
 573         DBPRINTF ("before out ret = 0x%x\n", ret);
 574
 575         // TODO: add uprobe (must be in function)
 576         INIT_HLIST_NODE(&p->hlist);
 577         hlist_add_head_rcu(&p->hlist, &uprobe_table[hash_ptr(p->addr, UPROBE_HASH_BITS)]);
 578         add_uprobe_table(p);
 579         arm_uprobe(up);
 580
 581 out:
 582         DBPRINTF("out ret = 0x%x\n", ret);
 583         return ret;
 584 }
 585
 586 void dbi_unregister_uprobe(struct uprobe *up, int atomic)
 587 {
 588         struct kprobe *p, *old_p, *list_p;
 589         int cleanup_p;
 590
 591         p = &up->kp;
 592         old_p = get_ukprobe(p->addr, p->tgid);
 593         if (unlikely(!old_p)) {
 594                 return;
 595         }
 596
 597         if (p != old_p) {
 598                 list_for_each_entry_rcu(list_p, &old_p->list, list) {
 599                         if (list_p == p) {
 600                                 /* uprobe p is a valid probe */
 601                                 goto valid_p;
 602                         }
 603                 }
 604
 605                 return;
 606         }
 607
 608 valid_p:
 609         if ((old_p == p) || ((old_p->pre_handler == aggr_pre_uhandler) &&
 610             (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
 611                 /* Only probe on the hash list */
 612                 disarm_uprobe(up);
 613                 hlist_del_rcu(&old_p->hlist);
 614                 cleanup_p = 1;
 615         } else {
 616                 list_del_rcu(&p->list);
 617                 cleanup_p = 0;
 618         }
 619
 620         if (cleanup_p) {
 621                 if (p != old_p) {
 622                         list_del_rcu(&p->list);
 623                         kfree(old_p);
 624                 }
 625
 626                 if (!in_atomic()) {
 627                         synchronize_sched();
 628                 }
 629
 630                 remove_uprobe(up);
 631         } else {
 632                 if (p->break_handler) {
 633                         old_p->break_handler = NULL;
 634                 }
 635
 636                 if (p->post_handler) {
 637                         list_for_each_entry_rcu (list_p, &old_p->list, list) {
 638                                 if (list_p->post_handler) {
 639                                         cleanup_p = 2;
 640                                         break;
 641                                 }
 642                         }
 643
 644                         if (cleanup_p == 0) {
 645                                 old_p->post_handler = NULL;
 646                         }
 647                 }
 648         }
 649 }
 650
 651 int dbi_register_ujprobe(struct ujprobe *jp, int atomic)
 652 {
 653         int ret = 0;
 654
 655         /* Todo: Verify probepoint is a function entry point */
 656         jp->up.kp.pre_handler = setjmp_upre_handler;
 657         jp->up.kp.break_handler = longjmp_break_uhandler;
 658
 659         ret = dbi_register_uprobe(&jp->up, atomic);
 660
 661         return ret;
 662 }
 663
 664 void dbi_unregister_ujprobe(struct ujprobe *jp, int atomic)
 665 {
 666         dbi_unregister_uprobe(&jp->up, atomic);
 667         /*
 668          * Here is an attempt to unregister even those probes that have not been
 669          * installed (hence not added to the hlist).
 670          * So if we try to delete them from the hlist we will get NULL pointer
 671          * dereference error. That is why we check whether this node
 672          * really belongs to the hlist.
 673          */
 674 #ifdef CONFIG_ARM
 675         if (!(hlist_unhashed(&jp->up.kp.is_hlist_arm))) {
 676                 hlist_del_rcu(&jp->up.kp.is_hlist_arm);
 677         }
 678         if (!(hlist_unhashed(&jp->up.kp.is_hlist_thumb))) {
 679                 hlist_del_rcu(&jp->up.kp.is_hlist_thumb);
 680         }
 681 #else /* CONFIG_ARM */
 682         if (!(hlist_unhashed(&jp->up.kp.is_hlist))) {
 683                 hlist_del_rcu(&jp->up.kp.is_hlist);
 684         }
 685 #endif /* CONFIG_ARM */
 686 }
 687
 688 int trampoline_uprobe_handler(struct kprobe *p, struct pt_regs *regs)
 689 {
 690         struct uretprobe_instance *ri = NULL;
 691         struct hlist_head *head;
 692         struct hlist_node *node, *tmp;
 693         unsigned long flags, tramp_addr, orig_ret_addr = 0;
 694
 695         tramp_addr = arch_get_trampoline_addr(p, regs);
 696         spin_lock_irqsave(&uretprobe_lock, flags);
 697
 698         head = uretprobe_inst_table_head(current->mm);
 699
 700         /*
 701          * It is possible to have multiple instances associated with a given
 702          * task either because an multiple functions in the call path
 703          * have a return probe installed on them, and/or more then one
 704          * return probe was registered for a target function.
 705          *
 706          * We can handle this because:
 707          *     - instances are always inserted at the head of the list
 708          *     - when multiple return probes are registered for the same
 709          *       function, the first instance's ret_addr will point to the
 710          *       real return address, and all the rest will point to
 711          *       uretprobe_trampoline
 712          */
 713         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 714                 if (ri->task != current) {
 715                         /* another task is sharing our hash bucket */
 716                         continue;
 717                 }
 718
 719                 if (ri->rp && ri->rp->handler) {
 720                         ri->rp->handler(ri, regs, ri->rp->priv_arg);
 721                 }
 722
 723                 orig_ret_addr = (unsigned long)ri->ret_addr;
 724                 recycle_urp_inst(ri);
 725
 726                 if (orig_ret_addr != tramp_addr) {
 727                         /*
 728                          * This is the real return address. Any other
 729                          * instances associated with this task are for
 730                          * other calls deeper on the call stack
 731                          */
 732                         break;
 733                 }
 734         }
 735
 736         spin_unlock_irqrestore(&uretprobe_lock, flags);
 737         arch_set_orig_ret_addr(orig_ret_addr, regs);
 738
 739         return 1;
 740 }
 741
 742 static int pre_handler_uretprobe(struct kprobe *p, struct pt_regs *regs)
 743 {
 744         struct uprobe *up = container_of(p, struct uprobe, kp);
 745         struct uretprobe *rp = container_of(up, struct uretprobe, up);
 746         struct uretprobe_instance *ri;
 747         unsigned long flags;
 748
 749         /* TODO: consider to only swap the RA after the last pre_handler fired */
 750         spin_lock_irqsave(&uretprobe_lock, flags);
 751
 752         /* TODO: test - remove retprobe after func entry but before its exit */
 753         if ((ri = get_free_urp_inst(rp)) != NULL) {
 754                 ri->rp = rp;
 755                 ri->task = current;
 756
 757                 arch_prepare_uretprobe_hl(ri, regs);
 758
 759                 add_urp_inst(ri);
 760         } else {
 761                 ++rp->nmissed;
 762         }
 763
 764         spin_unlock_irqrestore(&uretprobe_lock, flags);
 765
 766         return 0;
 767 }
 768
 769 int dbi_register_uretprobe(struct uretprobe *rp, int atomic)
 770 {
 771         int i, ret = 0;
 772         struct uretprobe_instance *inst;
 773
 774         DBPRINTF ("START\n");
 775
 776         rp->up.kp.pre_handler = pre_handler_uretprobe;
 777         rp->up.kp.post_handler = NULL;
 778         rp->up.kp.fault_handler = NULL;
 779         rp->up.kp.break_handler = NULL;
 780
 781         /* Pre-allocate memory for max kretprobe instances */
 782         if (rp->maxactive <= 0) {
 783 #if 1//def CONFIG_PREEMPT
 784                 rp->maxactive = max(10, 2 * NR_CPUS);
 785 #else
 786                 rp->maxactive = NR_CPUS;
 787 #endif
 788         }
 789
 790         INIT_HLIST_HEAD(&rp->used_instances);
 791         INIT_HLIST_HEAD(&rp->free_instances);
 792
 793         for (i = 0; i < rp->maxactive; i++) {
 794                 inst = kmalloc(sizeof(*inst), GFP_KERNEL);
 795                 if (inst == NULL) {
 796                         free_urp_inst(rp);
 797                         ret = -ENOMEM;
 798                         goto out;
 799                 }
 800
 801                 INIT_HLIST_NODE(&inst->uflist);
 802                 hlist_add_head(&inst->uflist, &rp->free_instances);
 803         }
 804
 805         rp->nmissed = 0;
 806
 807         /* Establish function entry probe point */
 808         ret = dbi_register_uprobe(&rp->up, atomic);
 809         if (ret) {
 810                 free_urp_inst(rp);
 811                 goto out;
 812         }
 813
 814 out:
 815         return ret;
 816 }
 817
 818 int dbi_disarm_urp_inst(struct uretprobe_instance *ri, struct task_struct *rm_task)
 819 {
 820         struct task_struct *task = rm_task ? rm_task : ri->task;
 821         kprobe_opcode_t *tramp;
 822         kprobe_opcode_t *sp = (kprobe_opcode_t *)((long)ri->sp & ~1);
 823         kprobe_opcode_t *stack = sp - RETPROBE_STACK_DEPTH + 1;
 824         kprobe_opcode_t *found = NULL;
 825         kprobe_opcode_t *buf[RETPROBE_STACK_DEPTH];
 826         int i, retval;
 827
 828         /* Understand function mode */
 829         if ((long)ri->sp & 1) {
 830                 tramp = (kprobe_opcode_t *)
 831                         ((unsigned long)ri->rp->up.kp.ainsn.insn + 0x1b);
 832         } else {
 833                 tramp = (kprobe_opcode_t *)
 834                         (ri->rp->up.kp.ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
 835         }
 836
 837         retval = read_proc_vm_atomic(task, (unsigned long)stack, buf, sizeof(buf));
 838         if (retval != sizeof(buf)) {
 839                 printk("---> %s (%d/%d): failed to read stack from %08lx",
 840                         task->comm, task->tgid, task->pid, (unsigned long)stack);
 841                 retval = -EFAULT;
 842                 goto out;
 843         }
 844
 845         /* search the stack from the bottom */
 846         for (i = RETPROBE_STACK_DEPTH - 1; i >= 0; i--) {
 847                 if (buf[i] == tramp) {
 848                         found = stack + i;
 849                         break;
 850                 }
 851         }
 852
 853         if (found) {
 854                 printk("---> %s (%d/%d): trampoline found at %08lx (%08lx /%+d) - %p\n",
 855                                 task->comm, task->tgid, task->pid,
 856                                 (unsigned long)found, (unsigned long)sp,
 857                                 found - sp, ri->rp->up.kp.addr);
 858                 retval = write_proc_vm_atomic(task, (unsigned long)found, &ri->ret_addr,
 859                                 sizeof(ri->ret_addr));
 860                 if (retval != sizeof(ri->ret_addr)) {
 861                         printk("---> %s (%d/%d): failed to write value to %08lx",
 862                                 task->comm, task->tgid, task->pid, (unsigned long)found);
 863                         retval = -EFAULT;
 864                 } else {
 865                         retval = 0;
 866                 }
 867         } else {
 868                 struct pt_regs *uregs = task_pt_regs(ri->task);
 869                 unsigned long ra = dbi_get_ret_addr(uregs);
 870                 if (ra == (unsigned long)tramp) {
 871                         printk("---> %s (%d/%d): trampoline found at lr = %08lx - %p\n",
 872                                         task->comm, task->tgid, task->pid, ra, ri->rp->up.kp.addr);
 873                         dbi_set_ret_addr(uregs, (unsigned long)tramp);
 874                         retval = 0;
 875                 } else {
 876                         printk("---> %s (%d/%d): trampoline NOT found at sp = %08lx, lr = %08lx - %p\n",
 877                                         task->comm, task->tgid, task->pid,
 878                                         (unsigned long)sp, ra, ri->rp->up.kp.addr);
 879                         retval = -ENOENT;
 880                 }
 881         }
 882
 883 out:
 884         return retval;
 885 }
 886
 887 /* Called with uretprobe_lock held */
 888 int dbi_disarm_urp_inst_for_task(struct task_struct *parent, struct task_struct *task)
 889 {
 890         struct uretprobe_instance *ri;
 891         struct hlist_node *node, *tmp;
 892         struct hlist_head *head = uretprobe_inst_table_head(parent->mm);
 893
 894         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
 895                 if (parent == ri->task && ri->rp->up.kp.tgid) {
 896                         dbi_disarm_urp_inst(ri, task);
 897                 }
 898         }
 899
 900         return 0;
 901 }
 902 EXPORT_SYMBOL_GPL(dbi_disarm_urp_inst_for_task);
 903
 904 void dbi_unregister_uretprobe(struct uretprobe *rp, int atomic)
 905 {
 906         unsigned long flags;
 907         struct uretprobe_instance *ri;
 908
 909         spin_lock_irqsave (&uretprobe_lock, flags);
 910
 911         while ((ri = get_used_urp_inst(rp)) != NULL) {
 912                 if (dbi_disarm_urp_inst(ri, NULL) != 0)
 913                         /*panic*/printk("%s (%d/%d): cannot disarm urp instance (%08lx)\n",
 914                                         ri->task->comm, ri->task->tgid, ri->task->pid,
 915                                         (unsigned long)rp->up.kp.addr);
 916                 recycle_urp_inst(ri);
 917         }
 918
 919         if (hlist_empty(&rp->used_instances)) {
 920                 struct kprobe *p = &rp->up.kp;
 921 #ifdef CONFIG_ARM
 922                 if (!(hlist_unhashed(&p->is_hlist_arm))) {
 923                         hlist_del_rcu(&p->is_hlist_arm);
 924                 }
 925
 926                 if (!(hlist_unhashed(&p->is_hlist_thumb))) {
 927                         hlist_del_rcu(&p->is_hlist_thumb);
 928                 }
 929 #else /* CONFIG_ARM */
 930                 if (!(hlist_unhashed(&p->is_hlist))) {
 931                         hlist_del_rcu(&p->is_hlist);
 932                 }
 933 #endif /* CONFIG_ARM */
 934         }
 935
 936         while ((ri = get_used_urp_inst(rp)) != NULL) {
 937                 ri->rp = NULL;
 938                 hlist_del(&ri->uflist);
 939         }
 940
 941         spin_unlock_irqrestore(&uretprobe_lock, flags);
 942         free_urp_inst(rp);
 943
 944         dbi_unregister_uprobe(&rp->up, atomic);
 945 }
 946
 947 void dbi_unregister_all_uprobes(struct task_struct *task, int atomic)
 948 {
 949         struct hlist_head *head;
 950         struct hlist_node *node, *tnode;
 951         struct kprobe *p;
 952         int i;
 953
 954         for (i = 0; i < UPROBE_TABLE_SIZE; ++i) {
 955                 head = &uprobe_table[i];
 956                 hlist_for_each_entry_safe(p, node, tnode, head, hlist) {
 957                         if (p->tgid == task->tgid) {
 958                                 struct uprobe *up = container_of(p, struct uprobe, kp);
 959                                 printk("dbi_unregister_all_uprobes: delete uprobe at %p[%lx] for %s/%d\n",
 960                                                 p->addr, (unsigned long)p->opcode, task->comm, task->pid);
 961                                 dbi_unregister_uprobe(up, atomic);
 962                         }
 963                 }
 964         }
 965 }
 966
 967 void dbi_uprobe_return(void)
 968 {
 969         dbi_arch_uprobe_return();
 970 }
 971
 972 static int __init init_uprobes(void)
 973 {
 974         init_uprobe_table();
 975         init_uprobes_insn_slots();
 976         init_uretprobe_inst_table();
 977
 978         return swap_arch_init_uprobes();
 979 }
 980
 981 static void __exit exit_uprobes(void)
 982 {
 983         swap_arch_exit_uprobes();
 984 }
 985
 986 EXPORT_SYMBOL_GPL(dbi_uprobe_return);
 987 EXPORT_SYMBOL_GPL(dbi_register_ujprobe);
 988 EXPORT_SYMBOL_GPL(dbi_unregister_ujprobe);
 989 EXPORT_SYMBOL_GPL(dbi_register_uretprobe);
 990 EXPORT_SYMBOL_GPL(dbi_unregister_uretprobe);
 991 EXPORT_SYMBOL_GPL(dbi_unregister_all_uprobes);
 992
 993 module_init(init_uprobes);
 994 module_exit(exit_uprobes);
 995
 996 MODULE_LICENSE ("GPL");