2 * Kernel Probes (KProbes)
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) IBM Corporation, 2002, 2004
23 * Dynamic Binary Instrumentation Module based on KProbes
24 * modules/kprobe/dbi_kprobes.h
26 * This program is free software; you can redistribute it and/or modify
27 * it under the terms of the GNU General Public License as published by
28 * the Free Software Foundation; either version 2 of the License, or
29 * (at your option) any later version.
31 * This program is distributed in the hope that it will be useful,
32 * but WITHOUT ANY WARRANTY; without even the implied warranty of
33 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
34 * GNU General Public License for more details.
36 * You should have received a copy of the GNU General Public License
37 * along with this program; if not, write to the Free Software
38 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
40 * Copyright (C) Samsung Electronics, 2006-2010
42 * 2006-2007 Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM and MIPS
43 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
44 * Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
45 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
49 #include "dbi_kprobes.h"
50 #include "arch/asm/dbi_kprobes.h"
52 #include "dbi_kdebug.h"
53 #include "dbi_kprobes_deps.h"
54 #include "dbi_insn_slots.h"
57 #include <linux/version.h>
58 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,19)
59 #include <linux/config.h>
62 #include <linux/hash.h>
63 #include <linux/module.h>
65 #include <linux/pagemap.h>
67 struct slot_manager sm;
69 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
70 static DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
72 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */
73 EXPORT_SYMBOL_GPL(kretprobe_lock);
74 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
76 struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
77 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
79 atomic_t kprobe_count;
80 EXPORT_SYMBOL_GPL(kprobe_count);
82 static void *sm_alloc(struct slot_manager *sm)
84 return kmalloc(PAGE_SIZE, GFP_ATOMIC);
87 static void sm_free(struct slot_manager *sm, void *ptr)
94 sm.slot_size = KPROBES_TRAMP_LEN;
97 INIT_HLIST_NODE(&sm.page_list);
100 static void exit_sm()
105 void kretprobe_assert(struct kretprobe_instance *ri, unsigned long orig_ret_address, unsigned long trampoline_address)
107 if (!orig_ret_address || (orig_ret_address == trampoline_address)) {
108 struct task_struct *task;
110 panic("kretprobe BUG!: ri = NULL\n");
116 panic("kretprobe BUG!: task = NULL\n");
119 if (ri->rp == NULL) {
120 panic("kretprobe BUG!: ri->rp = NULL\n");
123 panic("kretprobe BUG!: Processing kretprobe %p @ %p (%d/%d - %s)\n",
124 ri->rp, ri->rp->kp.addr, ri->task->tgid, ri->task->pid, ri->task->comm);
128 /* We have preemption disabled.. so it is safe to use __ versions */
129 static inline void set_kprobe_instance(struct kprobe *kp)
131 __get_cpu_var(kprobe_instance) = kp;
134 static inline void reset_kprobe_instance(void)
136 __get_cpu_var(kprobe_instance) = NULL;
139 /* kprobe_running() will just return the current_kprobe on this CPU */
140 struct kprobe *kprobe_running(void)
142 return __get_cpu_var(current_kprobe);
145 void reset_current_kprobe(void)
147 __get_cpu_var(current_kprobe) = NULL;
150 struct kprobe_ctlblk *get_kprobe_ctlblk(void)
152 return &__get_cpu_var(kprobe_ctlblk);
156 * This routine is called either:
157 * - under the kprobe_mutex - during kprobe_[un]register()
159 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
161 struct kprobe *get_kprobe(void *addr)
163 struct hlist_head *head;
164 struct hlist_node *node;
167 head = &kprobe_table[hash_ptr (addr, KPROBE_HASH_BITS)];
168 swap_hlist_for_each_entry_rcu(p, node, head, hlist) {
169 if (p->addr == addr) {
178 * Aggregate handlers for multiple kprobes support - these handlers
179 * take care of invoking the individual kprobe handlers on p->list
181 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
186 list_for_each_entry_rcu(kp, &p->list, list) {
187 if (kp->pre_handler) {
188 set_kprobe_instance(kp);
189 ret = kp->pre_handler(kp, regs);
193 reset_kprobe_instance();
199 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
203 list_for_each_entry_rcu(kp, &p->list, list) {
204 if (kp->post_handler) {
205 set_kprobe_instance(kp);
206 kp->post_handler(kp, regs, flags);
207 reset_kprobe_instance();
212 static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, int trapnr)
214 struct kprobe *cur = __get_cpu_var(kprobe_instance);
217 * if we faulted "during" the execution of a user specified
218 * probe handler, invoke just that probe's fault handler
220 if (cur && cur->fault_handler) {
221 if (cur->fault_handler(cur, regs, trapnr))
228 static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
230 struct kprobe *cur = __get_cpu_var(kprobe_instance);
232 DBPRINTF ("cur = 0x%p\n", cur);
234 DBPRINTF ("cur = 0x%p cur->break_handler = 0x%p\n", cur, cur->break_handler);
236 if (cur && cur->break_handler) {
237 if (cur->break_handler(cur, regs))
240 reset_kprobe_instance();
245 /* Walks the list and increments nmissed count for multiprobe case */
246 void kprobes_inc_nmissed_count(struct kprobe *p)
249 if (p->pre_handler != aggr_pre_handler) {
252 list_for_each_entry_rcu(kp, &p->list, list) {
258 /* Called with kretprobe_lock held */
259 struct kretprobe_instance *get_free_rp_inst(struct kretprobe *rp)
261 struct hlist_node *node;
262 struct kretprobe_instance *ri;
264 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
268 if (!alloc_nodes_kretprobe(rp)) {
269 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
276 EXPORT_SYMBOL_GPL(get_free_rp_inst);
278 /* Called with kretprobe_lock held */
279 struct kretprobe_instance *get_free_rp_inst_no_alloc(struct kretprobe *rp)
281 struct hlist_node *node;
282 struct kretprobe_instance *ri;
284 swap_hlist_for_each_entry(ri, node, &rp->free_instances, uflist) {
291 /* Called with kretprobe_lock held */
292 struct kretprobe_instance *get_used_rp_inst(struct kretprobe *rp)
294 struct hlist_node *node;
295 struct kretprobe_instance *ri;
297 swap_hlist_for_each_entry(ri, node, &rp->used_instances, uflist) {
303 EXPORT_SYMBOL_GPL(get_used_rp_inst);
305 /* Called with kretprobe_lock held */
306 void add_rp_inst (struct kretprobe_instance *ri)
309 * Remove rp inst off the free list -
310 * Add it back when probed function returns
312 hlist_del(&ri->uflist);
314 /* Add rp inst onto table */
315 INIT_HLIST_NODE(&ri->hlist);
317 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]);
319 /* Also add this rp inst to the used list. */
320 INIT_HLIST_NODE(&ri->uflist);
321 hlist_add_head(&ri->uflist, &ri->rp->used_instances);
323 EXPORT_SYMBOL_GPL(add_rp_inst);
325 /* Called with kretprobe_lock held */
326 void recycle_rp_inst(struct kretprobe_instance *ri)
329 hlist_del(&ri->hlist);
330 /* remove rp inst off the used list */
331 hlist_del(&ri->uflist);
332 /* put rp inst back onto the free list */
333 INIT_HLIST_NODE(&ri->uflist);
334 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
337 EXPORT_SYMBOL_GPL(recycle_rp_inst);
339 struct hlist_head *kretprobe_inst_table_head(void *hash_key)
341 return &kretprobe_inst_table[hash_ptr(hash_key, KPROBE_HASH_BITS)];
343 EXPORT_SYMBOL_GPL(kretprobe_inst_table_head);
345 void free_rp_inst(struct kretprobe *rp)
347 struct kretprobe_instance *ri;
348 while ((ri = get_free_rp_inst_no_alloc(rp)) != NULL) {
349 hlist_del(&ri->uflist);
353 EXPORT_SYMBOL_GPL(free_rp_inst);
356 * Keep all fields in the kprobe consistent
358 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
360 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
361 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
362 p->ss_addr = old_p->ss_addr;
364 p->safe_arm = old_p->safe_arm;
365 p->safe_thumb = old_p->safe_thumb;
370 * Add the new probe to old_p->list. Fail if this is the
371 * second jprobe at the address - two jprobes can't coexist
373 static int add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
375 if (p->break_handler) {
376 if (old_p->break_handler) {
380 list_add_tail_rcu(&p->list, &old_p->list);
381 old_p->break_handler = aggr_break_handler;
383 list_add_rcu(&p->list, &old_p->list);
386 if (p->post_handler && !old_p->post_handler) {
387 old_p->post_handler = aggr_post_handler;
394 * hlist_replace_rcu - replace old entry by new one
395 * @old : the element to be replaced
396 * @new : the new element to insert
398 * The @old entry will be replaced with the @new entry atomically.
400 inline void dbi_hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new)
402 struct hlist_node *next = old->next;
405 new->pprev = old->pprev;
408 new->next->pprev = &new->next;
411 old->pprev = LIST_POISON2;
415 * Fill in the required fields of the "manager kprobe". Replace the
416 * earlier kprobe in the hlist with the manager kprobe
418 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
422 ap->pre_handler = aggr_pre_handler;
423 ap->fault_handler = aggr_fault_handler;
425 ap->post_handler = aggr_post_handler;
426 if (p->break_handler)
427 ap->break_handler = aggr_break_handler;
429 INIT_LIST_HEAD(&ap->list);
430 list_add_rcu(&p->list, &ap->list);
432 dbi_hlist_replace_rcu(&p->hlist, &ap->hlist);
436 * This is the second or subsequent kprobe at the address - handle
439 int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
443 DBPRINTF ("start\n");
445 DBPRINTF ("p = %p old_p = %p \n", p, old_p);
446 if (old_p->pre_handler == aggr_pre_handler) {
447 DBPRINTF ("aggr_pre_handler \n");
449 copy_kprobe(old_p, p);
450 ret = add_new_kprobe(old_p, p);
452 DBPRINTF ("kzalloc\n");
454 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
456 ap = kmalloc(sizeof(struct kprobe), GFP_KERNEL);
458 memset(ap, 0, sizeof(struct kprobe));
462 add_aggr_kprobe(ap, old_p);
464 DBPRINTF ("ap = %p p = %p old_p = %p \n", ap, p, old_p);
465 ret = add_new_kprobe(ap, p);
470 EXPORT_SYMBOL_GPL(register_aggr_kprobe);
472 static void remove_kprobe(struct kprobe *p)
474 /* TODO: check boostable for x86 and MIPS */
475 free_insn_slot(&sm, p->ainsn.insn);
478 int dbi_register_kprobe(struct kprobe *p)
480 struct kprobe *old_p;
483 * If we have a symbol_name argument look it up,
484 * and add it to the address. That way the addr
485 * field can either be global or relative to a symbol.
487 if (p->symbol_name) {
490 p->addr = (kprobe_opcode_t *)swap_ksyms(p->symbol_name);
495 DBPRINTF ("p->addr = 0x%p\n", p->addr);
496 p->addr = (kprobe_opcode_t *)(((char *)p->addr) + p->offset);
497 DBPRINTF ("p->addr = 0x%p p = 0x%p\n", p->addr, p);
499 #ifdef KPROBES_PROFILE
500 p->start_tm.tv_sec = p->start_tm.tv_usec = 0;
501 p->hnd_tm_sum.tv_sec = p->hnd_tm_sum.tv_usec = 0;
504 p->mod_refcounted = 0;
507 old_p = get_kprobe(p->addr);
509 ret = register_aggr_kprobe(old_p, p);
511 atomic_inc(&kprobe_count);
515 if ((ret = arch_prepare_kprobe(p, &sm)) != 0)
518 DBPRINTF ("before out ret = 0x%x\n", ret);
519 INIT_HLIST_NODE(&p->hlist);
520 hlist_add_head_rcu(&p->hlist, &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
524 DBPRINTF ("out ret = 0x%x\n", ret);
528 void dbi_unregister_kprobe(struct kprobe *p, struct task_struct *task)
530 struct kprobe *old_p, *list_p;
533 old_p = get_kprobe(p->addr);
534 DBPRINTF ("dbi_unregister_kprobe p=%p old_p=%p", p, old_p);
535 if (unlikely (!old_p))
539 list_for_each_entry_rcu(list_p, &old_p->list, list)
541 /* kprobe p is a valid probe */
547 DBPRINTF ("dbi_unregister_kprobe valid_p");
548 if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
549 (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
550 /* Only probe on the hash list */
551 arch_disarm_kprobe(p);
552 hlist_del_rcu(&old_p->hlist);
555 list_del_rcu(&p->list);
558 DBPRINTF ("dbi_unregister_kprobe cleanup_p=%d", cleanup_p);
562 list_del_rcu(&p->list);
572 if (p->break_handler)
573 old_p->break_handler = NULL;
574 if (p->post_handler) {
575 list_for_each_entry_rcu(list_p, &old_p->list, list) {
576 if (list_p->post_handler) {
583 old_p->post_handler = NULL;
588 int dbi_register_jprobe(struct jprobe *jp)
590 /* Todo: Verify probepoint is a function entry point */
591 jp->kp.pre_handler = setjmp_pre_handler;
592 jp->kp.break_handler = longjmp_break_handler;
594 return dbi_register_kprobe(&jp->kp);
597 void dbi_unregister_jprobe(struct jprobe *jp)
599 dbi_unregister_kprobe(&jp->kp, NULL);
603 * This kprobe pre_handler is registered with every kretprobe. When probe
604 * hits it will set up the return probe.
606 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
608 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
609 struct kretprobe_instance *ri;
610 unsigned long flags = 0;
612 /* TODO: consider to only swap the RA after the last pre_handler fired */
613 spin_lock_irqsave(&kretprobe_lock, flags);
615 /* TODO: test - remove retprobe after func entry but before its exit */
616 if ((ri = get_free_rp_inst(rp)) != NULL) {
620 if (rp->entry_handler) {
621 rp->entry_handler(ri, regs, ri->rp->priv_arg);
624 arch_prepare_kretprobe(ri, regs);
631 spin_unlock_irqrestore(&kretprobe_lock, flags);
636 int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
638 struct kretprobe_instance *ri = NULL;
639 struct hlist_head *head;
640 struct hlist_node *node, *tmp;
641 unsigned long flags, orig_ret_address = 0;
642 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
644 struct kretprobe *crp = NULL;
645 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
647 spin_lock_irqsave(&kretprobe_lock, flags);
650 * We are using different hash keys (current and mm) for finding kernel
651 * space and user space probes. Kernel space probes can change mm field in
652 * task_struct. User space probes can be shared between threads of one
653 * process so they have different current but same mm.
655 head = kretprobe_inst_table_head(current);
658 * It is possible to have multiple instances associated with a given
659 * task either because an multiple functions in the call path
660 * have a return probe installed on them, and/or more then one
661 * return probe was registered for a target function.
663 * We can handle this because:
664 * - instances are always inserted at the head of the list
665 * - when multiple return probes are registered for the same
666 * function, the first instance's ret_addr will point to the
667 * real return address, and all the rest will point to
668 * kretprobe_trampoline
670 swap_hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
671 if (ri->task != current)
672 /* another task is sharing our hash bucket */
674 if (ri->rp && ri->rp->handler) {
675 ri->rp->handler(ri, regs, ri->rp->priv_arg);
678 orig_ret_address = (unsigned long)ri->ret_addr;
680 if (orig_ret_address != trampoline_address)
682 * This is the real return address. Any other
683 * instances associated with this task are for
684 * other calls deeper on the call stack
688 kretprobe_assert(ri, orig_ret_address, trampoline_address);
690 dbi_set_ret_addr(regs, orig_ret_address);
691 dbi_set_instr_ptr(regs, orig_ret_address);
693 if (kcb->kprobe_status == KPROBE_REENTER) {
694 restore_previous_kprobe(kcb);
696 reset_current_kprobe();
699 spin_unlock_irqrestore(&kretprobe_lock, flags);
702 * By returning a non-zero value, we are telling
703 * kprobe_handler() that we don't want the post_handler
704 * to run (and have re-enabled preemption)
710 struct kretprobe *sched_rp;
712 #define SCHED_RP_NR 200
713 #define COMMON_RP_NR 10
715 int alloc_nodes_kretprobe(struct kretprobe *rp)
718 struct kretprobe_instance *inst;
721 DBPRINTF("Alloc aditional mem for retprobes");
723 if ((unsigned long)rp->kp.addr == sched_addr) {
724 rp->maxactive += SCHED_RP_NR;//max (100, 2 * NR_CPUS);
725 alloc_nodes = SCHED_RP_NR;
727 #if 1//def CONFIG_PREEMPT
728 rp->maxactive += max (COMMON_RP_NR, 2 * NR_CPUS);
730 rp->maxacpptive += NR_CPUS;
732 alloc_nodes = COMMON_RP_NR;
735 for (i = 0; i < alloc_nodes; i++) {
736 inst = kmalloc(sizeof(inst) + rp->data_size, GFP_ATOMIC);
741 INIT_HLIST_NODE(&inst->uflist);
742 hlist_add_head(&inst->uflist, &rp->free_instances);
745 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
749 int dbi_register_kretprobe(struct kretprobe *rp)
752 struct kretprobe_instance *inst;
756 rp->kp.pre_handler = pre_handler_kretprobe;
757 rp->kp.post_handler = NULL;
758 rp->kp.fault_handler = NULL;
759 rp->kp.break_handler = NULL;
761 /* Pre-allocate memory for max kretprobe instances */
762 if ((unsigned long)rp->kp.addr == sched_addr) {
763 rp->maxactive = SCHED_RP_NR;//max (100, 2 * NR_CPUS);
764 rp->kp.pre_handler = NULL; //not needed for __switch_to
765 } else if ((unsigned long)rp->kp.addr == exit_addr) {
766 rp->kp.pre_handler = NULL; //not needed for do_exit
768 } else if ((unsigned long)rp->kp.addr == do_group_exit_addr) {
769 rp->kp.pre_handler = NULL;
771 } else if ((unsigned long)rp->kp.addr == sys_exit_group_addr) {
772 rp->kp.pre_handler = NULL;
774 } else if (rp->maxactive <= 0) {
775 #if 1//def CONFIG_PREEMPT
776 rp->maxactive = max (COMMON_RP_NR, 2 * NR_CPUS);
778 rp->maxactive = NR_CPUS;
781 INIT_HLIST_HEAD(&rp->used_instances);
782 INIT_HLIST_HEAD(&rp->free_instances);
783 for (i = 0; i < rp->maxactive; i++) {
784 inst = kmalloc(sizeof(*inst) + rp->data_size, GFP_KERNEL);
789 INIT_HLIST_NODE(&inst->uflist);
790 hlist_add_head(&inst->uflist, &rp->free_instances);
793 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
795 /* Establish function entry probe point */
796 if ((ret = dbi_register_kprobe(&rp->kp)) != 0)
799 DBPRINTF ("addr=%p, *addr=[%lx %lx %lx]", rp->kp.addr, (unsigned long) (*(rp->kp.addr)), (unsigned long) (*(rp->kp.addr + 1)), (unsigned long) (*(rp->kp.addr + 2)));
800 if ((unsigned long)rp->kp.addr == sched_addr) {
807 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri);
809 void dbi_unregister_kretprobe(struct kretprobe *rp)
812 struct kretprobe_instance *ri;
814 dbi_unregister_kprobe(&rp->kp, NULL);
817 spin_lock_irqsave(&kretprobe_lock, flags);
819 if ((unsigned long)rp->kp.addr == sched_addr)
822 while ((ri = get_used_rp_inst(rp)) != NULL) {
823 if (dbi_disarm_krp_inst(ri) == 0)
826 panic("%s (%d/%d): cannot disarm krp instance (%08lx)",
827 ri->task->comm, ri->task->tgid, ri->task->pid,
828 (unsigned long)rp->kp.addr);
831 spin_unlock_irqrestore(&kretprobe_lock, flags);
835 struct kretprobe *clone_kretprobe(struct kretprobe *rp)
837 struct kprobe *old_p;
838 struct kretprobe *clone = NULL;
841 clone = kmalloc(sizeof(struct kretprobe), GFP_KERNEL);
843 DBPRINTF ("failed to alloc memory for clone probe %p!", rp->kp.addr);
846 memcpy(clone, rp, sizeof(struct kretprobe));
847 clone->kp.pre_handler = pre_handler_kretprobe;
848 clone->kp.post_handler = NULL;
849 clone->kp.fault_handler = NULL;
850 clone->kp.break_handler = NULL;
851 old_p = get_kprobe(rp->kp.addr);
853 ret = register_aggr_kprobe(old_p, &clone->kp);
858 atomic_inc(&kprobe_count);
863 EXPORT_SYMBOL_GPL(clone_kretprobe);
865 static void inline set_task_trampoline(unsigned long *patch_addr,
866 struct kretprobe_instance *ri,
867 unsigned long tramp_addr)
869 unsigned long pc = *patch_addr;
870 if (pc == tramp_addr)
871 panic("[%d] %s (%d/%d): pc = %08lx --- [%d] %s (%d/%d)\n",
872 task_cpu(ri->task), ri->task->comm, ri->task->tgid,
873 ri->task->pid, pc, task_cpu(current), current->comm,
874 current->tgid, current->pid);
875 ri->ret_addr = (kprobe_opcode_t *)pc;
876 *patch_addr = tramp_addr;
879 static void inline rm_task_trampoline(struct task_struct *p, struct kretprobe_instance *ri)
881 arch_set_task_pc(p, (unsigned long)ri->ret_addr);
884 static int dbi_disarm_krp_inst(struct kretprobe_instance *ri)
886 unsigned long *tramp = &kretprobe_trampoline;
887 unsigned long *sp = ri->sp;
888 unsigned long *found = NULL;
889 int retval = -ENOENT;
892 unsigned long pc = arch_get_task_pc(ri->task);
894 printk("---> [%d] %s (%d/%d): pc = %08lx, ra = %08lx, tramp= %08lx (%08lx)\n",
896 ri->task->comm, ri->task->tgid, ri->task->pid,
897 pc, ri->ret_addr, tramp,
898 ri->rp ? ri->rp->kp.addr: NULL);
900 /* __switch_to retprobe handling */
902 rm_task_trampoline(ri->task, ri);
909 while (sp > ri->sp - RETPROBE_STACK_DEPTH) {
918 printk("---> [%d] %s (%d/%d): tramp (%08lx) found at %08lx (%08lx /%+d) - %p\n",
920 ri->task->comm, ri->task->tgid, ri->task->pid,
921 tramp, found, ri->sp, found - ri->sp,
922 ri->rp ? ri->rp->kp.addr: NULL);
923 *found = ri->ret_addr;
926 printk("---> [%d] %s (%d/%d): tramp (%08lx) NOT found at sp = %08lx - %p\n",
928 ri->task->comm, ri->task->tgid, ri->task->pid,
930 ri->sp, ri->rp ? ri->rp->kp.addr: NULL);
936 int patch_suspended_task(struct kretprobe *rp,
937 struct task_struct *task,
938 struct pt_regs *regs)
940 struct kretprobe_instance *ri;
942 kprobe_opcode_t *tramp = (kprobe_opcode_t *)&kretprobe_trampoline;
943 unsigned long *patch_addr;
945 spin_lock_irqsave(&kretprobe_lock, flags);
947 ri = get_free_rp_inst(rp);
954 patch_addr = arch_get_patch_addr(task, regs);
955 set_task_trampoline(patch_addr, ri, (unsigned long)tramp);
958 spin_unlock_irqrestore(&kretprobe_lock, flags);
962 static int init_module_deps(void)
966 sched_addr = swap_ksyms("__switch_to");
967 fork_addr = swap_ksyms("do_fork");
968 exit_addr = swap_ksyms("do_exit");
970 if (sched_addr == 0 || fork_addr == 0 || exit_addr == 0) {
974 ret = init_module_dependencies();
979 return arch_init_module_deps();
982 static int __init init_kprobes(void)
988 /* FIXME allocate the probe table, currently defined statically */
989 /* initialize all list heads */
990 for (i = 0; i < KPROBE_TABLE_SIZE; ++i) {
991 INIT_HLIST_HEAD(&kprobe_table[i]);
992 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
994 atomic_set(&kprobe_count, 0);
996 err = init_module_deps();
1001 err = arch_init_kprobes();
1003 DBPRINTF ("init_kprobes: arch_init_kprobes - %d", err);
1008 static void __exit exit_kprobes(void)
1010 arch_exit_kprobes();
1014 module_init(init_kprobes);
1015 module_exit(exit_kprobes);
1017 EXPORT_SYMBOL_GPL(dbi_register_kprobe);
1018 EXPORT_SYMBOL_GPL(dbi_unregister_kprobe);
1019 EXPORT_SYMBOL_GPL(dbi_register_jprobe);
1020 EXPORT_SYMBOL_GPL(dbi_unregister_jprobe);
1021 EXPORT_SYMBOL_GPL(dbi_jprobe_return);
1022 EXPORT_SYMBOL_GPL(dbi_register_kretprobe);
1023 EXPORT_SYMBOL_GPL(dbi_unregister_kretprobe);
1025 MODULE_LICENSE("Dual BSD/GPL");