2 * Kernel Probes (KProbes)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) IBM Corporation, 2002, 2004
23 * Dynamic Binary Instrumentation Module based on KProbes
24 * modules/kprobe/dbi_kprobes.h
26 * This program is free software; you can redistribute it and/or modify
27 * it under the terms of the GNU General Public License as published by
28 * the Free Software Foundation; either version 2 of the License, or
29 * (at your option) any later version.
31 * This program is distributed in the hope that it will be useful,
32 * but WITHOUT ANY WARRANTY; without even the implied warranty of
33 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
34 * GNU General Public License for more details.
36 * You should have received a copy of the GNU General Public License
37 * along with this program; if not, write to the Free Software
38 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
40 * Copyright (C) Samsung Electronics, 2006-2010
42 * 2006-2007 Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
43 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
44 * Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
45 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
49 #include "dbi_kprobes.h"
50 #include "arch/asm/dbi_kprobes.h"
52 #include "dbi_kdebug.h"
53 #include "dbi_kprobes_deps.h"
54 #include "dbi_insn_slots.h"
57 #include <linux/version.h>
58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
59 #include <linux/config.h>
62 #include <linux/hash.h>
63 #include <linux/module.h>
65 #include <linux/pagemap.h>
67 unsigned long sched_addr;
68 static unsigned long exit_addr;
69 static unsigned long do_group_exit_addr;
70 static unsigned long sys_exit_group_addr;
71 static unsigned long sys_exit_addr;
73 struct slot_manager sm;
75 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
76 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
78 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */
79 EXPORT_SYMBOL_GPL(kretprobe_lock);
80 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
82 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
83 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
85 atomic_t kprobe_count;
86 EXPORT_SYMBOL_GPL(kprobe_count);
88 static void *sm_alloc(struct slot_manager *sm)
90 return kmalloc(PAGE_SIZE, GFP_ATOMIC);
93 static void sm_free(struct slot_manager *sm, void *ptr)
100 sm.slot_size = KPROBES_TRAMP_LEN;
103 INIT_HLIST_NODE(&sm.page_list);
106 static void exit_sm()
111 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
113 if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
114 struct task_struct *task;
116 panic("kretprobe BUG!: ri = NULL\n");
122 panic("kretprobe BUG!: task = NULL\n");
125 if (ri->rp == NULL) {
126 panic("kretprobe BUG!: ri->rp = NULL\n");
129 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
130 ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
134 /* We have preemption disabled.. so it is safe to use __ versions */
135 static inline void set_kprobe_instance(struct kprobe *kp)
137 __get_cpu_var(kprobe_instance) = kp;
140 static inline void reset_kprobe_instance(void)
142 __get_cpu_var(kprobe_instance) = NULL;
145 /* kprobe_running() will just return the current_kprobe on this CPU */
146 struct kprobe *kprobe_running(void)
148 return __get_cpu_var(current_kprobe);
151 void reset_current_kprobe(void)
153 __get_cpu_var(current_kprobe) = NULL;
156 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
158 return &__get_cpu_var(kprobe_ctlblk);
162 * This routine is called either:
163 * - under the kprobe_mutex - during kprobe_[un]register()
165 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
167 struct kprobe *get_kprobe(void *addr)
169 struct hlist_head *head;
170 struct hlist_node *node;
173 head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
174 swap_hlist_for_each_entry_rcu(p, node, head, hlist) {
175 if (p->addr == addr) {
184 * Aggregate handlers for multiple kprobes support - these handlers
185 * take care of invoking the individual kprobe handlers on p->list
187 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
192 list_for_each_entry_rcu(kp, &p->list, list) {
193 if (kp->pre_handler) {
194 set_kprobe_instance(kp);
195 ret = kp->pre_handler(kp, regs);
199 reset_kprobe_instance();
205 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
209 list_for_each_entry_rcu(kp, &p->list, list) {
210 if (kp->post_handler) {
211 set_kprobe_instance(kp);
212 kp->post_handler(kp, regs, flags);
213 reset_kprobe_instance();
218 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
220 struct kprobe *cur = __get_cpu_var(kprobe_instance);
223 * if we faulted "during" the execution of a user specified
224 * probe handler, invoke just that probe's fault handler
226 if (cur && cur->fault_handler) {
227 if (cur->fault_handler(cur, regs, trapnr))
234 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
236 struct kprobe *cur = __get_cpu_var(kprobe_instance);
238 DBPRINTF ("cur = 0x%p\n", cur);
240 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
242 if (cur && cur->break_handler) {
243 if (cur->break_handler(cur, regs))
246 reset_kprobe_instance();
251 /* Walks the list and increments nmissed count for multiprobe case */
252 void kprobes_inc_nmissed_count(struct kprobe *p)
255 if (p->pre_handler != aggr_pre_handler) {
258 list_for_each_entry_rcu(kp, &p->list, list) {
264 /* Called with kretprobe_lock held */
265 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
267 struct hlist_node *node;
268 struct kretprobe_instance *ri;
270 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
274 if (!alloc_nodes_kretprobe(rp)) {
275 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
282 EXPORT_SYMBOL_GPL(get_free_rp_inst);
284 /* Called with kretprobe_lock held */
285 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
287 struct hlist_node *node;
288 struct kretprobe_instance *ri;
290 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
297 /* Called with kretprobe_lock held */
298 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
300 struct hlist_node *node;
301 struct kretprobe_instance *ri;
303 swap_hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
309 EXPORT_SYMBOL_GPL(get_used_rp_inst);
311 /* Called with kretprobe_lock held */
312 void add_rp_inst (struct kretprobe_instance *ri)
315 * Remove rp inst off the free list -
316 * Add it back when probed function returns
318 hlist_del(&ri->uflist);
320 /* Add rp inst onto table */
321 INIT_HLIST_NODE(&ri->hlist);
323 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
325 /* Also add this rp inst to the used list. */
326 INIT_HLIST_NODE(&ri->uflist);
327 hlist_add_head(&ri->uflist, &ri->rp->used_instances);
329 EXPORT_SYMBOL_GPL(add_rp_inst);
331 /* Called with kretprobe_lock held */
332 void recycle_rp_inst(struct kretprobe_instance *ri)
335 hlist_del(&ri->hlist);
336 /* remove rp inst off the used list */
337 hlist_del(&ri->uflist);
338 /* put rp inst back onto the free list */
339 INIT_HLIST_NODE(&ri->uflist);
340 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
343 EXPORT_SYMBOL_GPL(recycle_rp_inst);
345 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
347 return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
349 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
351 void free_rp_inst(struct kretprobe *rp)
353 struct kretprobe_instance *ri;
354 while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
355 hlist_del(&ri->uflist);
359 EXPORT_SYMBOL_GPL(free_rp_inst);
362 * Keep all fields in the kprobe consistent
364 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
366 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
367 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
368 p->ss_addr = old_p->ss_addr;
370 p->safe_arm = old_p->safe_arm;
371 p->safe_thumb = old_p->safe_thumb;
376 * Add the new probe to old_p->list. Fail if this is the
377 * second jprobe at the address - two jprobes can't coexist
379 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
381 if (p->break_handler) {
382 if (old_p->break_handler) {
386 list_add_tail_rcu(&p->list, &old_p->list);
387 old_p->break_handler = aggr_break_handler;
389 list_add_rcu(&p->list, &old_p->list);
392 if (p->post_handler && !old_p->post_handler) {
393 old_p->post_handler = aggr_post_handler;
400 * hlist_replace_rcu - replace old entry by new one
401 * @old : the element to be replaced
402 * @new : the new element to insert
404 * The @old entry will be replaced with the @new entry atomically.
406 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
408 struct hlist_node *next = old->next;
411 new->pprev = old->pprev;
414 new->next->pprev = &new->next;
417 old->pprev = LIST_POISON2;
421 * Fill in the required fields of the "manager kprobe". Replace the
422 * earlier kprobe in the hlist with the manager kprobe
424 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
428 ap->pre_handler = aggr_pre_handler;
429 ap->fault_handler = aggr_fault_handler;
431 ap->post_handler = aggr_post_handler;
432 if (p->break_handler)
433 ap->break_handler = aggr_break_handler;
435 INIT_LIST_HEAD(&ap->list);
436 list_add_rcu(&p->list, &ap->list);
438 dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
442 * This is the second or subsequent kprobe at the address - handle
445 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
449 DBPRINTF ("start\n");
451 DBPRINTF ("p = %p old_p = %p \n", p, old_p);
452 if (old_p->pre_handler == aggr_pre_handler) {
453 DBPRINTF ("aggr_pre_handler \n");
455 copy_kprobe(old_p, p);
456 ret = add_new_kprobe(old_p, p);
458 DBPRINTF ("kzalloc\n");
460 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
462 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
464 memset(ap, 0, sizeof(struct kprobe));
468 add_aggr_kprobe(ap, old_p);
470 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
471 ret = add_new_kprobe(ap, p);
476 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
478 static void remove_kprobe(struct kprobe *p)
480 /* TODO: check boostable for x86 and MIPS */
481 free_insn_slot(&sm, p->ainsn.insn);
484 int dbi_register_kprobe(struct kprobe *p)
486 struct kprobe *old_p;
489 * If we have a symbol_name argument look it up,
490 * and add it to the address. That way the addr
491 * field can either be global or relative to a symbol.
493 if (p->symbol_name) {
496 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
501 DBPRINTF ("p->addr = 0x%p\n", p->addr);
502 p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
503 DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
505 #ifdef KPROBES_PROFILE
506 p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
507 p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
510 p->mod_refcounted = 0;
513 old_p = get_kprobe(p->addr);
515 ret = register_aggr_kprobe(old_p, p);
517 atomic_inc(&kprobe_count);
521 if ((ret = arch_prepare_kprobe(p, &sm)) != 0)
524 DBPRINTF ("before out ret = 0x%x\n", ret);
525 INIT_HLIST_NODE(&p->hlist);
526 hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
530 DBPRINTF ("out ret = 0x%x\n", ret);
534 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
536 struct kprobe *old_p, *list_p;
539 old_p = get_kprobe(p->addr);
540 DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
541 if (unlikely (!old_p))
545 list_for_each_entry_rcu(list_p, &old_p->list, list)
547 /* kprobe p is a valid probe */
553 DBPRINTF ("dbi_unregister_kprobe valid_p");
554 if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
555 (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
556 /* Only probe on the hash list */
557 arch_disarm_kprobe(p);
558 hlist_del_rcu(&old_p->hlist);
561 list_del_rcu(&p->list);
564 DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
568 list_del_rcu(&p->list);
578 if (p->break_handler)
579 old_p->break_handler = NULL;
580 if (p->post_handler) {
581 list_for_each_entry_rcu(list_p, &old_p->list, list) {
582 if (list_p->post_handler) {
589 old_p->post_handler = NULL;
594 int dbi_register_jprobe(struct jprobe *jp)
596 /* Todo: Verify probepoint is a function entry point */
597 jp->kp.pre_handler = setjmp_pre_handler;
598 jp->kp.break_handler = longjmp_break_handler;
600 return dbi_register_kprobe(&jp->kp);
603 void dbi_unregister_jprobe(struct jprobe *jp)
605 dbi_unregister_kprobe(&jp->kp, NULL);
609 * This kprobe pre_handler is registered with every kretprobe. When probe
610 * hits it will set up the return probe.
612 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
614 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
615 struct kretprobe_instance *ri;
616 unsigned long flags = 0;
618 /* TODO: consider to only swap the RA after the last pre_handler fired */
619 spin_lock_irqsave(&kretprobe_lock, flags);
621 /* TODO: test - remove retprobe after func entry but before its exit */
622 if ((ri = get_free_rp_inst(rp)) != NULL) {
626 if (rp->entry_handler) {
627 rp->entry_handler(ri, regs, ri->rp->priv_arg);
630 arch_prepare_kretprobe(ri, regs);
637 spin_unlock_irqrestore(&kretprobe_lock, flags);
642 int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
644 struct kretprobe_instance *ri = NULL;
645 struct hlist_head *head;
646 struct hlist_node *node, *tmp;
647 unsigned long flags, orig_ret_address = 0;
648 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
650 struct kretprobe *crp = NULL;
651 struct kprobe_ctlblk *kcb;
654 kcb = get_kprobe_ctlblk();
656 spin_lock_irqsave(&kretprobe_lock, flags);
659 * We are using different hash keys (current and mm) for finding kernel
660 * space and user space probes. Kernel space probes can change mm field in
661 * task_struct. User space probes can be shared between threads of one
662 * process so they have different current but same mm.
664 head = kretprobe_inst_table_head(current);
667 regs->XREG(cs) = __KERNEL_CS | get_kernel_rpl();
668 regs->EREG(ip) = trampoline_address;
669 regs->ORIG_EAX_REG = 0xffffffff;
673 * It is possible to have multiple instances associated with a given
674 * task either because an multiple functions in the call path
675 * have a return probe installed on them, and/or more then one
676 * return probe was registered for a target function.
678 * We can handle this because:
679 * - instances are always inserted at the head of the list
680 * - when multiple return probes are registered for the same
681 * function, the first instance's ret_addr will point to the
682 * real return address, and all the rest will point to
683 * kretprobe_trampoline
685 swap_hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
686 if (ri->task != current)
687 /* another task is sharing our hash bucket */
689 if (ri->rp && ri->rp->handler) {
690 __get_cpu_var(current_kprobe) = &ri->rp->kp;
691 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
692 ri->rp->handler(ri, regs, ri->rp->priv_arg);
693 __get_cpu_var(current_kprobe) = NULL;
696 orig_ret_address = (unsigned long)ri->ret_addr;
698 if (orig_ret_address != trampoline_address)
700 * This is the real return address. Any other
701 * instances associated with this task are for
702 * other calls deeper on the call stack
706 kretprobe_assert(ri, orig_ret_address, trampoline_address);
708 if (kcb->kprobe_status == KPROBE_REENTER) {
709 restore_previous_kprobe(kcb);
711 reset_current_kprobe();
714 spin_unlock_irqrestore(&kretprobe_lock, flags);
715 preempt_enable_no_resched();
718 * By returning a non-zero value, we are telling
719 * kprobe_handler() that we don't want the post_handler
720 * to run (and have re-enabled preemption)
723 return (int)orig_ret_address;
726 struct kretprobe *sched_rp;
728 #define SCHED_RP_NR 200
729 #define COMMON_RP_NR 10
731 int alloc_nodes_kretprobe(struct kretprobe *rp)
734 struct kretprobe_instance *inst;
737 DBPRINTF("Alloc aditional mem for retprobes");
739 if ((unsigned long)rp->kp.addr == sched_addr) {
740 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
741 alloc_nodes = SCHED_RP_NR;
743 #if 1//def CONFIG_PREEMPT
744 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
746 rp->maxacpptive += NR_CPUS;
748 alloc_nodes = COMMON_RP_NR;
751 for (i = 0; i < alloc_nodes; i++) {
752 inst = kmalloc(sizeof(inst) + rp->data_size, GFP_ATOMIC);
757 INIT_HLIST_NODE(&inst->uflist);
758 hlist_add_head(&inst->uflist, &rp->free_instances);
761 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
765 int dbi_register_kretprobe(struct kretprobe *rp)
768 struct kretprobe_instance *inst;
772 rp->kp.pre_handler = pre_handler_kretprobe;
773 rp->kp.post_handler = NULL;
774 rp->kp.fault_handler = NULL;
775 rp->kp.break_handler = NULL;
777 /* Pre-allocate memory for max kretprobe instances */
778 if ((unsigned long)rp->kp.addr == sched_addr) {
779 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
780 rp->kp.pre_handler = NULL; //not needed for __switch_to
781 } else if ((unsigned long)rp->kp.addr == exit_addr) {
782 rp->kp.pre_handler = NULL; //not needed for do_exit
784 } else if ((unsigned long)rp->kp.addr == do_group_exit_addr) {
785 rp->kp.pre_handler = NULL;
787 } else if ((unsigned long)rp->kp.addr == sys_exit_group_addr) {
788 rp->kp.pre_handler = NULL;
790 } else if ((unsigned long)rp->kp.addr == sys_exit_addr) {
791 rp->kp.pre_handler = NULL;
793 } else if (rp->maxactive <= 0) {
794 #if 1//def CONFIG_PREEMPT
795 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
797 rp->maxactive = NR_CPUS;
800 INIT_HLIST_HEAD(&rp->used_instances);
801 INIT_HLIST_HEAD(&rp->free_instances);
802 for (i = 0; i < rp->maxactive; i++) {
803 inst = kmalloc(sizeof(*inst) + rp->data_size, GFP_KERNEL);
808 INIT_HLIST_NODE(&inst->uflist);
809 hlist_add_head(&inst->uflist, &rp->free_instances);
812 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
814 /* Establish function entry probe point */
815 if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
818 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
819 if ((unsigned long)rp->kp.addr == sched_addr) {
826 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
828 void dbi_unregister_kretprobe(struct kretprobe *rp)
831 struct kretprobe_instance *ri;
833 dbi_unregister_kprobe(&rp->kp, NULL);
836 spin_lock_irqsave(&kretprobe_lock, flags);
838 if ((unsigned long)rp->kp.addr == sched_addr)
841 while ((ri = get_used_rp_inst (rp)) != NULL) {
842 if (!dbi_disarm_krp_inst(ri)) {
843 printk("%s (%d/%d): cannot disarm krp instance (%08lx)\n",
844 ri->task->comm, ri->task->tgid, ri->task->pid,
845 (unsigned long)rp->kp.addr);
850 spin_unlock_irqrestore(&kretprobe_lock, flags);
854 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
856 struct kprobe *old_p;
857 struct kretprobe *clone = NULL;
860 clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
862 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
865 memcpy(clone, rp, sizeof(struct kretprobe));
866 clone->kp.pre_handler = pre_handler_kretprobe;
867 clone->kp.post_handler = NULL;
868 clone->kp.fault_handler = NULL;
869 clone->kp.break_handler = NULL;
870 old_p = get_kprobe(rp->kp.addr);
872 ret = register_aggr_kprobe(old_p, &clone->kp);
877 atomic_inc(&kprobe_count);
882 EXPORT_SYMBOL_GPL(clone_kretprobe);
884 static void inline set_task_trampoline(unsigned long *patch_addr,
885 struct kretprobe_instance *ri,
886 unsigned long tramp_addr)
888 unsigned long pc = *patch_addr;
889 if (pc == tramp_addr)
890 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
891 task_cpu(ri->task), ri->task->comm, ri->task->tgid,
892 ri->task->pid, pc, task_cpu(current), current->comm,
893 current->tgid, current->pid);
894 ri->ret_addr = (kprobe_opcode_t *)pc;
895 *patch_addr = tramp_addr;
898 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
900 arch_set_task_pc(p, (unsigned long)ri->ret_addr);
903 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
905 unsigned long *tramp = &kretprobe_trampoline;
906 unsigned long *sp = ri->sp;
907 unsigned long *found = NULL;
908 int retval = -ENOENT;
911 unsigned long pc = arch_get_task_pc(ri->task);
913 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
915 ri->task->comm, ri->task->tgid, ri->task->pid,
916 pc, ri->ret_addr, tramp,
917 ri->rp ? ri->rp->kp.addr: NULL);
919 /* __switch_to retprobe handling */
921 rm_task_trampoline(ri->task, ri);
928 while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
937 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
939 ri->task->comm, ri->task->tgid, ri->task->pid,
940 tramp, found, ri->sp, found - ri->sp,
941 ri->rp ? ri->rp->kp.addr: NULL);
942 *found = ri->ret_addr;
945 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
947 ri->task->comm, ri->task->tgid, ri->task->pid,
949 ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
955 int patch_suspended_task(struct kretprobe *rp,
956 struct task_struct *task,
957 struct pt_regs *regs)
959 struct kretprobe_instance *ri;
961 kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
962 unsigned long *patch_addr;
964 spin_lock_irqsave(&kretprobe_lock, flags);
966 ri = get_free_rp_inst(rp);
973 patch_addr = arch_get_patch_addr(task, regs);
974 set_task_trampoline(patch_addr, ri, (unsigned long)tramp);
977 spin_unlock_irqrestore(&kretprobe_lock, flags);
981 static int init_module_deps(void)
985 sched_addr = swap_ksyms("__switch_to");
986 exit_addr = swap_ksyms("do_exit");
987 sys_exit_group_addr = swap_ksyms("sys_exit_group");
988 do_group_exit_addr = swap_ksyms("do_group_exit");
989 sys_exit_addr = swap_ksyms("sys_exit");
991 if (sched_addr == 0 ||
993 sys_exit_group_addr == 0 ||
994 do_group_exit_addr == 0 ||
995 sys_exit_addr == 0) {
999 ret = init_module_dependencies();
1004 return arch_init_module_deps();
1007 static int __init init_kprobes(void)
1013 /* FIXME allocate the probe table, currently defined statically */
1014 /* initialize all list heads */
1015 for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
1016 INIT_HLIST_HEAD(&kprobe_table[i]);
1017 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1019 atomic_set(&kprobe_count, 0);
1021 err = init_module_deps();
1026 err = arch_init_kprobes();
1028 DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
1033 static void __exit exit_kprobes(void)
1035 arch_exit_kprobes();
1039 module_init(init_kprobes);
1040 module_exit(exit_kprobes);
1042 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
1043 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
1044 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
1045 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
1046 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
1047 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
1048 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
1050 MODULE_LICENSE("Dual BSD/GPL");