2 * Kernel Probes (KProbes)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) IBM Corporation, 2002, 2004
23 * Dynamic Binary Instrumentation Module based on KProbes
24 * modules/kprobe/dbi_kprobes.h
26 * This program is free software; you can redistribute it and/or modify
27 * it under the terms of the GNU General Public License as published by
28 * the Free Software Foundation; either version 2 of the License, or
29 * (at your option) any later version.
31 * This program is distributed in the hope that it will be useful,
32 * but WITHOUT ANY WARRANTY; without even the implied warranty of
33 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
34 * GNU General Public License for more details.
36 * You should have received a copy of the GNU General Public License
37 * along with this program; if not, write to the Free Software
38 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
40 * Copyright (C) Samsung Electronics, 2006-2010
42 * 2006-2007 Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
43 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
44 * Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
45 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
49 #include "dbi_kprobes.h"
50 #include "arch/asm/dbi_kprobes.h"
52 #include "dbi_kdebug.h"
53 #include "dbi_kprobes_deps.h"
54 #include "dbi_insn_slots.h"
57 #include <linux/version.h>
58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
59 #include <linux/config.h>
62 #include <linux/hash.h>
63 #include <linux/module.h>
65 #include <linux/pagemap.h>
67 unsigned long sched_addr;
68 static unsigned long exit_addr;
69 static unsigned long do_group_exit_addr;
70 static unsigned long sys_exit_group_addr;
71 static unsigned long sys_exit_addr;
73 struct slot_manager sm;
75 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
76 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
78 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */
79 EXPORT_SYMBOL_GPL(kretprobe_lock);
80 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
82 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
83 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
85 atomic_t kprobe_count;
86 EXPORT_SYMBOL_GPL(kprobe_count);
88 static void *sm_alloc(struct slot_manager *sm)
90 return kmalloc(PAGE_SIZE, GFP_ATOMIC);
93 static void sm_free(struct slot_manager *sm, void *ptr)
100 sm.slot_size = KPROBES_TRAMP_LEN;
103 INIT_HLIST_NODE(&sm.page_list);
106 static void exit_sm()
111 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
113 if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
114 struct task_struct *task;
116 panic("kretprobe BUG!: ri = NULL\n");
122 panic("kretprobe BUG!: task = NULL\n");
125 if (ri->rp == NULL) {
126 panic("kretprobe BUG!: ri->rp = NULL\n");
129 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
130 ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
134 /* We have preemption disabled.. so it is safe to use __ versions */
135 static inline void set_kprobe_instance(struct kprobe *kp)
137 __get_cpu_var(kprobe_instance) = kp;
140 static inline void reset_kprobe_instance(void)
142 __get_cpu_var(kprobe_instance) = NULL;
145 /* kprobe_running() will just return the current_kprobe on this CPU */
146 struct kprobe *kprobe_running(void)
148 return __get_cpu_var(current_kprobe);
151 void reset_current_kprobe(void)
153 __get_cpu_var(current_kprobe) = NULL;
156 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
158 return &__get_cpu_var(kprobe_ctlblk);
162 * This routine is called either:
163 * - under the kprobe_mutex - during kprobe_[un]register()
165 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
167 struct kprobe *get_kprobe(void *addr)
169 struct hlist_head *head;
170 struct hlist_node *node;
173 head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
174 swap_hlist_for_each_entry_rcu(p, node, head, hlist) {
175 if (p->addr == addr) {
184 * Aggregate handlers for multiple kprobes support - these handlers
185 * take care of invoking the individual kprobe handlers on p->list
187 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
192 list_for_each_entry_rcu(kp, &p->list, list) {
193 if (kp->pre_handler) {
194 set_kprobe_instance(kp);
195 ret = kp->pre_handler(kp, regs);
199 reset_kprobe_instance();
205 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
209 list_for_each_entry_rcu(kp, &p->list, list) {
210 if (kp->post_handler) {
211 set_kprobe_instance(kp);
212 kp->post_handler(kp, regs, flags);
213 reset_kprobe_instance();
218 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
220 struct kprobe *cur = __get_cpu_var(kprobe_instance);
223 * if we faulted "during" the execution of a user specified
224 * probe handler, invoke just that probe's fault handler
226 if (cur && cur->fault_handler) {
227 if (cur->fault_handler(cur, regs, trapnr))
234 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
236 struct kprobe *cur = __get_cpu_var(kprobe_instance);
238 DBPRINTF ("cur = 0x%p\n", cur);
240 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
242 if (cur && cur->break_handler) {
243 if (cur->break_handler(cur, regs))
246 reset_kprobe_instance();
251 /* Walks the list and increments nmissed count for multiprobe case */
252 void kprobes_inc_nmissed_count(struct kprobe *p)
255 if (p->pre_handler != aggr_pre_handler) {
258 list_for_each_entry_rcu(kp, &p->list, list) {
264 /* Called with kretprobe_lock held */
265 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
267 struct hlist_node *node;
268 struct kretprobe_instance *ri;
270 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
274 if (!alloc_nodes_kretprobe(rp)) {
275 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
282 EXPORT_SYMBOL_GPL(get_free_rp_inst);
284 /* Called with kretprobe_lock held */
285 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
287 struct hlist_node *node;
288 struct kretprobe_instance *ri;
290 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
297 /* Called with kretprobe_lock held */
298 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
300 struct hlist_node *node;
301 struct kretprobe_instance *ri;
303 swap_hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
309 EXPORT_SYMBOL_GPL(get_used_rp_inst);
311 /* Called with kretprobe_lock held */
312 void add_rp_inst (struct kretprobe_instance *ri)
315 * Remove rp inst off the free list -
316 * Add it back when probed function returns
318 hlist_del(&ri->uflist);
320 /* Add rp inst onto table */
321 INIT_HLIST_NODE(&ri->hlist);
323 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
325 /* Also add this rp inst to the used list. */
326 INIT_HLIST_NODE(&ri->uflist);
327 hlist_add_head(&ri->uflist, &ri->rp->used_instances);
329 EXPORT_SYMBOL_GPL(add_rp_inst);
331 /* Called with kretprobe_lock held */
332 void recycle_rp_inst(struct kretprobe_instance *ri)
335 hlist_del(&ri->hlist);
336 /* remove rp inst off the used list */
337 hlist_del(&ri->uflist);
338 /* put rp inst back onto the free list */
339 INIT_HLIST_NODE(&ri->uflist);
340 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
343 EXPORT_SYMBOL_GPL(recycle_rp_inst);
345 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
347 return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
349 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
351 void free_rp_inst(struct kretprobe *rp)
353 struct kretprobe_instance *ri;
354 while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
355 hlist_del(&ri->uflist);
359 EXPORT_SYMBOL_GPL(free_rp_inst);
362 * Keep all fields in the kprobe consistent
364 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
366 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
367 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
368 p->ss_addr = old_p->ss_addr;
370 p->safe_arm = old_p->safe_arm;
371 p->safe_thumb = old_p->safe_thumb;
376 * Add the new probe to old_p->list. Fail if this is the
377 * second jprobe at the address - two jprobes can't coexist
379 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
381 if (p->break_handler) {
382 if (old_p->break_handler) {
386 list_add_tail_rcu(&p->list, &old_p->list);
387 old_p->break_handler = aggr_break_handler;
389 list_add_rcu(&p->list, &old_p->list);
392 if (p->post_handler && !old_p->post_handler) {
393 old_p->post_handler = aggr_post_handler;
400 * hlist_replace_rcu - replace old entry by new one
401 * @old : the element to be replaced
402 * @new : the new element to insert
404 * The @old entry will be replaced with the @new entry atomically.
406 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
408 struct hlist_node *next = old->next;
411 new->pprev = old->pprev;
414 new->next->pprev = &new->next;
417 old->pprev = LIST_POISON2;
421 * Fill in the required fields of the "manager kprobe". Replace the
422 * earlier kprobe in the hlist with the manager kprobe
424 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
428 ap->pre_handler = aggr_pre_handler;
429 ap->fault_handler = aggr_fault_handler;
431 ap->post_handler = aggr_post_handler;
432 if (p->break_handler)
433 ap->break_handler = aggr_break_handler;
435 INIT_LIST_HEAD(&ap->list);
436 list_add_rcu(&p->list, &ap->list);
438 dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
442 * This is the second or subsequent kprobe at the address - handle
445 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
449 DBPRINTF ("start\n");
451 DBPRINTF ("p = %p old_p = %p \n", p, old_p);
452 if (old_p->pre_handler == aggr_pre_handler) {
453 DBPRINTF ("aggr_pre_handler \n");
455 copy_kprobe(old_p, p);
456 ret = add_new_kprobe(old_p, p);
458 DBPRINTF ("kzalloc\n");
460 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
462 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
464 memset(ap, 0, sizeof(struct kprobe));
468 add_aggr_kprobe(ap, old_p);
470 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
471 ret = add_new_kprobe(ap, p);
476 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
478 static void remove_kprobe(struct kprobe *p)
480 /* TODO: check boostable for x86 and MIPS */
481 free_insn_slot(&sm, p->ainsn.insn);
484 int dbi_register_kprobe(struct kprobe *p)
486 struct kprobe *old_p;
489 * If we have a symbol_name argument look it up,
490 * and add it to the address. That way the addr
491 * field can either be global or relative to a symbol.
493 if (p->symbol_name) {
496 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
501 DBPRINTF ("p->addr = 0x%p\n", p->addr);
502 p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
503 DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
505 #ifdef KPROBES_PROFILE
506 p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
507 p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
510 p->mod_refcounted = 0;
513 old_p = get_kprobe(p->addr);
515 ret = register_aggr_kprobe(old_p, p);
517 atomic_inc(&kprobe_count);
521 if ((ret = arch_prepare_kprobe(p, &sm)) != 0)
524 DBPRINTF ("before out ret = 0x%x\n", ret);
525 INIT_HLIST_NODE(&p->hlist);
526 hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
530 DBPRINTF ("out ret = 0x%x\n", ret);
534 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
536 struct kprobe *old_p, *list_p;
539 old_p = get_kprobe(p->addr);
540 DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
541 if (unlikely (!old_p))
545 list_for_each_entry_rcu(list_p, &old_p->list, list)
547 /* kprobe p is a valid probe */
553 DBPRINTF ("dbi_unregister_kprobe valid_p");
554 if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
555 (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
556 /* Only probe on the hash list */
557 arch_disarm_kprobe(p);
558 hlist_del_rcu(&old_p->hlist);
561 list_del_rcu(&p->list);
564 DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
568 list_del_rcu(&p->list);
578 if (p->break_handler)
579 old_p->break_handler = NULL;
580 if (p->post_handler) {
581 list_for_each_entry_rcu(list_p, &old_p->list, list) {
582 if (list_p->post_handler) {
589 old_p->post_handler = NULL;
594 int dbi_register_jprobe(struct jprobe *jp)
596 /* Todo: Verify probepoint is a function entry point */
597 jp->kp.pre_handler = setjmp_pre_handler;
598 jp->kp.break_handler = longjmp_break_handler;
600 return dbi_register_kprobe(&jp->kp);
603 void dbi_unregister_jprobe(struct jprobe *jp)
605 dbi_unregister_kprobe(&jp->kp, NULL);
609 * This kprobe pre_handler is registered with every kretprobe. When probe
610 * hits it will set up the return probe.
612 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
614 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
615 struct kretprobe_instance *ri;
616 unsigned long flags = 0;
618 /* TODO: consider to only swap the RA after the last pre_handler fired */
619 spin_lock_irqsave(&kretprobe_lock, flags);
621 /* TODO: test - remove retprobe after func entry but before its exit */
622 if ((ri = get_free_rp_inst(rp)) != NULL) {
626 if (rp->entry_handler) {
627 rp->entry_handler(ri, regs, ri->rp->priv_arg);
630 arch_prepare_kretprobe(ri, regs);
637 spin_unlock_irqrestore(&kretprobe_lock, flags);
642 int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
644 struct kretprobe_instance *ri = NULL;
645 struct hlist_head *head;
646 struct hlist_node *node, *tmp;
647 unsigned long flags, orig_ret_address = 0;
648 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
650 struct kretprobe *crp = NULL;
651 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
653 spin_lock_irqsave(&kretprobe_lock, flags);
656 * We are using different hash keys (current and mm) for finding kernel
657 * space and user space probes. Kernel space probes can change mm field in
658 * task_struct. User space probes can be shared between threads of one
659 * process so they have different current but same mm.
661 head = kretprobe_inst_table_head(current);
664 regs->XREG(cs) = __KERNEL_CS | get_kernel_rpl();
665 regs->EREG(ip) = trampoline_address;
666 regs->ORIG_EAX_REG = 0xffffffff;
670 * It is possible to have multiple instances associated with a given
671 * task either because an multiple functions in the call path
672 * have a return probe installed on them, and/or more then one
673 * return probe was registered for a target function.
675 * We can handle this because:
676 * - instances are always inserted at the head of the list
677 * - when multiple return probes are registered for the same
678 * function, the first instance's ret_addr will point to the
679 * real return address, and all the rest will point to
680 * kretprobe_trampoline
682 swap_hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
683 if (ri->task != current)
684 /* another task is sharing our hash bucket */
686 if (ri->rp && ri->rp->handler) {
687 __get_cpu_var(current_kprobe) = &ri->rp->kp;
688 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
689 ri->rp->handler(ri, regs, ri->rp->priv_arg);
690 __get_cpu_var(current_kprobe) = NULL;
693 orig_ret_address = (unsigned long)ri->ret_addr;
695 if (orig_ret_address != trampoline_address)
697 * This is the real return address. Any other
698 * instances associated with this task are for
699 * other calls deeper on the call stack
703 kretprobe_assert(ri, orig_ret_address, trampoline_address);
706 regs->ARM_lr = orig_ret_address;
707 regs->ARM_pc = orig_ret_address;
710 if (kcb->kprobe_status == KPROBE_REENTER) {
711 restore_previous_kprobe(kcb);
713 reset_current_kprobe();
716 spin_unlock_irqrestore(&kretprobe_lock, flags);
719 * By returning a non-zero value, we are telling
720 * kprobe_handler() that we don't want the post_handler
721 * to run (and have re-enabled preemption)
724 #if defined (CONFIG_X86)
725 return (int)orig_ret_address;
726 #elif defined (CONFIG_ARM)
731 struct kretprobe *sched_rp;
733 #define SCHED_RP_NR 200
734 #define COMMON_RP_NR 10
736 int alloc_nodes_kretprobe(struct kretprobe *rp)
739 struct kretprobe_instance *inst;
742 DBPRINTF("Alloc aditional mem for retprobes");
744 if ((unsigned long)rp->kp.addr == sched_addr) {
745 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
746 alloc_nodes = SCHED_RP_NR;
748 #if 1//def CONFIG_PREEMPT
749 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
751 rp->maxacpptive += NR_CPUS;
753 alloc_nodes = COMMON_RP_NR;
756 for (i = 0; i < alloc_nodes; i++) {
757 inst = kmalloc(sizeof(inst) + rp->data_size, GFP_ATOMIC);
762 INIT_HLIST_NODE(&inst->uflist);
763 hlist_add_head(&inst->uflist, &rp->free_instances);
766 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
770 int dbi_register_kretprobe(struct kretprobe *rp)
773 struct kretprobe_instance *inst;
777 rp->kp.pre_handler = pre_handler_kretprobe;
778 rp->kp.post_handler = NULL;
779 rp->kp.fault_handler = NULL;
780 rp->kp.break_handler = NULL;
782 /* Pre-allocate memory for max kretprobe instances */
783 if ((unsigned long)rp->kp.addr == sched_addr) {
784 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
785 rp->kp.pre_handler = NULL; //not needed for __switch_to
786 } else if ((unsigned long)rp->kp.addr == exit_addr) {
787 rp->kp.pre_handler = NULL; //not needed for do_exit
789 } else if ((unsigned long)rp->kp.addr == do_group_exit_addr) {
790 rp->kp.pre_handler = NULL;
792 } else if ((unsigned long)rp->kp.addr == sys_exit_group_addr) {
793 rp->kp.pre_handler = NULL;
795 } else if ((unsigned long)rp->kp.addr == sys_exit_addr) {
796 rp->kp.pre_handler = NULL;
798 } else if (rp->maxactive <= 0) {
799 #if 1//def CONFIG_PREEMPT
800 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
802 rp->maxactive = NR_CPUS;
805 INIT_HLIST_HEAD(&rp->used_instances);
806 INIT_HLIST_HEAD(&rp->free_instances);
807 for (i = 0; i < rp->maxactive; i++) {
808 inst = kmalloc(sizeof(*inst) + rp->data_size, GFP_KERNEL);
813 INIT_HLIST_NODE(&inst->uflist);
814 hlist_add_head(&inst->uflist, &rp->free_instances);
817 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
819 /* Establish function entry probe point */
820 if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
823 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
824 if ((unsigned long)rp->kp.addr == sched_addr) {
831 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
833 void dbi_unregister_kretprobe(struct kretprobe *rp)
836 struct kretprobe_instance *ri;
838 dbi_unregister_kprobe(&rp->kp, NULL);
841 spin_lock_irqsave(&kretprobe_lock, flags);
843 if ((unsigned long)rp->kp.addr == sched_addr)
846 while ((ri = get_used_rp_inst(rp)) != NULL) {
847 if (dbi_disarm_krp_inst(ri) == 0)
850 panic("%s (%d/%d): cannot disarm krp instance (%08lx)",
851 ri->task->comm, ri->task->tgid, ri->task->pid,
852 (unsigned long)rp->kp.addr);
855 spin_unlock_irqrestore(&kretprobe_lock, flags);
859 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
861 struct kprobe *old_p;
862 struct kretprobe *clone = NULL;
865 clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
867 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
870 memcpy(clone, rp, sizeof(struct kretprobe));
871 clone->kp.pre_handler = pre_handler_kretprobe;
872 clone->kp.post_handler = NULL;
873 clone->kp.fault_handler = NULL;
874 clone->kp.break_handler = NULL;
875 old_p = get_kprobe(rp->kp.addr);
877 ret = register_aggr_kprobe(old_p, &clone->kp);
882 atomic_inc(&kprobe_count);
887 EXPORT_SYMBOL_GPL(clone_kretprobe);
889 static void inline set_task_trampoline(unsigned long *patch_addr,
890 struct kretprobe_instance *ri,
891 unsigned long tramp_addr)
893 unsigned long pc = *patch_addr;
894 if (pc == tramp_addr)
895 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
896 task_cpu(ri->task), ri->task->comm, ri->task->tgid,
897 ri->task->pid, pc, task_cpu(current), current->comm,
898 current->tgid, current->pid);
899 ri->ret_addr = (kprobe_opcode_t *)pc;
900 *patch_addr = tramp_addr;
903 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
905 arch_set_task_pc(p, (unsigned long)ri->ret_addr);
908 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
910 unsigned long *tramp = &kretprobe_trampoline;
911 unsigned long *sp = ri->sp;
912 unsigned long *found = NULL;
913 int retval = -ENOENT;
916 unsigned long pc = arch_get_task_pc(ri->task);
918 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
920 ri->task->comm, ri->task->tgid, ri->task->pid,
921 pc, ri->ret_addr, tramp,
922 ri->rp ? ri->rp->kp.addr: NULL);
924 /* __switch_to retprobe handling */
926 rm_task_trampoline(ri->task, ri);
933 while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
942 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
944 ri->task->comm, ri->task->tgid, ri->task->pid,
945 tramp, found, ri->sp, found - ri->sp,
946 ri->rp ? ri->rp->kp.addr: NULL);
947 *found = ri->ret_addr;
950 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
952 ri->task->comm, ri->task->tgid, ri->task->pid,
954 ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
960 int patch_suspended_task(struct kretprobe *rp,
961 struct task_struct *task,
962 struct pt_regs *regs)
964 struct kretprobe_instance *ri;
966 kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
967 unsigned long *patch_addr;
969 spin_lock_irqsave(&kretprobe_lock, flags);
971 ri = get_free_rp_inst(rp);
978 patch_addr = arch_get_patch_addr(task, regs);
979 set_task_trampoline(patch_addr, ri, (unsigned long)tramp);
982 spin_unlock_irqrestore(&kretprobe_lock, flags);
986 static int init_module_deps(void)
990 sched_addr = swap_ksyms("__switch_to");
991 exit_addr = swap_ksyms("do_exit");
992 sys_exit_group_addr = swap_ksyms("sys_exit_group");
993 do_group_exit_addr = swap_ksyms("do_group_exit");
994 sys_exit_addr = swap_ksyms("sys_exit");
996 if (sched_addr == 0 ||
998 sys_exit_group_addr == 0 ||
999 do_group_exit_addr == 0 ||
1000 sys_exit_addr == 0) {
1004 ret = init_module_dependencies();
1009 return arch_init_module_deps();
1012 static int __init init_kprobes(void)
1018 /* FIXME allocate the probe table, currently defined statically */
1019 /* initialize all list heads */
1020 for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
1021 INIT_HLIST_HEAD(&kprobe_table[i]);
1022 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1024 atomic_set(&kprobe_count, 0);
1026 err = init_module_deps();
1031 err = arch_init_kprobes();
1033 DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
1038 static void __exit exit_kprobes(void)
1040 arch_exit_kprobes();
1044 module_init(init_kprobes);
1045 module_exit(exit_kprobes);
1047 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
1048 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
1049 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
1050 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
1051 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
1052 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
1053 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
1055 MODULE_LICENSE("Dual BSD/GPL");