From eac8871cfc124bff85ff4c8c713218276939c820 Mon Sep 17 00:00:00 2001 From: Vyacheslav Cherkashin Date: Fri, 29 Apr 2016 15:52:13 +0300 Subject: [PATCH] [IMPROVE] call kprobe handlers at current context (for x86) Move kprobe handler from atomic context to current Change-Id: I8c3cc6bb964c6df1a2a83786f5bfca9c10b594a0 Signed-off-by: Vyacheslav Cherkashin --- kprobe/arch/x86/swap-asm/swap_kprobes.c | 87 ++++++++++++++++++++++++++++----- kprobe/swap_kprobes.c | 20 ++++++-- kprobe/swap_kprobes.h | 12 +++++ 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/kprobe/arch/x86/swap-asm/swap_kprobes.c b/kprobe/arch/x86/swap-asm/swap_kprobes.c index 29cd7e8..0f9cc49 100644 --- a/kprobe/arch/x86/swap-asm/swap_kprobes.c +++ b/kprobe/arch/x86/swap-asm/swap_kprobes.c @@ -35,7 +35,7 @@ #include "swap_kprobes.h" #include - +#include #include #include #include @@ -325,10 +325,25 @@ static int setup_singlestep(struct kprobe *p, struct pt_regs *regs, return 1; } + +static struct td_raw kp_tdraw; + +static struct pt_regs *current_regs(void) +{ + return (struct pt_regs *)swap_td_raw(&kp_tdraw, current); +} + + +void restore_int3(void); +__asm( + "restore_int3:\n" + "int3\n" +); + static int __kprobe_handler(struct pt_regs *regs) { struct kprobe *p = 0; - int ret = 0, reenter = 0; + int ret = 0; kprobe_opcode_t *addr = NULL; struct kprobe_ctlblk *kcb; @@ -360,7 +375,7 @@ static int __kprobe_handler(struct pt_regs *regs) prepare_singlestep(p, regs); kcb->kprobe_status = KPROBE_REENTER; - return 1; + goto out_get_kp_if_TF; } else { if (*addr != BREAKPOINT_INSTRUCTION) { /* The breakpoint instruction was removed by @@ -403,30 +418,62 @@ static int __kprobe_handler(struct pt_regs *regs) } set_current_kprobe(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; - if (!reenter) - kcb->kprobe_status = KPROBE_HIT_ACTIVE; + /* save regs to stack */ + *current_regs() = *regs; - if (p->pre_handler) { - ret = p->pre_handler(p, regs); - if (ret) - return ret; - } + regs->ip = (unsigned long)restore_int3; + get_kp(p); + + return 1; ss_probe: setup_singlestep(p, regs, kcb); +out_get_kp_if_TF: + if ((regs->flags & TF_MASK)) + get_kp(p); + return 1; no_kprobe: return ret; } +static int restore_handler(struct pt_regs *regs) +{ + struct kprobe *p = swap_kprobe_running(); + struct kprobe_ctlblk *kcb = swap_get_kprobe_ctlblk(); + + /* restore regs from stack */ + *regs = *current_regs(); + + if (p->pre_handler) { + int ret; + + ret = p->pre_handler(p, regs); + if (ret) { + put_kp(p); + return ret; + } + } + + setup_singlestep(p, regs, kcb); + if (!(regs->flags & TF_MASK)) + put_kp(p); + + return 1; +} + static int kprobe_handler(struct pt_regs *regs) { int ret; - ret = __kprobe_handler(regs); + if (regs->ip == (unsigned long)restore_int3 + 1) + ret = restore_handler(regs); + else + ret = __kprobe_handler(regs); return ret; } @@ -616,6 +663,9 @@ static int post_kprobe_handler(struct pt_regs *regs) if (!cur) return 0; + + put_kp(cur); + if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { kcb->kprobe_status = KPROBE_HIT_SSDONE; cur->post_handler(cur, regs, 0); @@ -1085,14 +1135,24 @@ int swap_arch_init_kprobes(void) { int ret; - ret = register_die_notifier(&kprobe_exceptions_nb); + ret = swap_td_raw_reg(&kp_tdraw, sizeof(struct pt_regs)); if (ret) return ret; + ret = register_die_notifier(&kprobe_exceptions_nb); + if (ret) + goto unreg_tdraw; + ret = kjump_init(); if (ret) - unregister_die_notifier(&kprobe_exceptions_nb); + goto unreg_die; + return 0; + +unreg_die: + unregister_die_notifier(&kprobe_exceptions_nb); +unreg_tdraw: + swap_td_raw_unreg(&kp_tdraw); return ret; } @@ -1105,4 +1165,5 @@ void swap_arch_exit_kprobes(void) { kjump_exit(); unregister_die_notifier(&kprobe_exceptions_nb); + swap_td_raw_unreg(&kp_tdraw); } diff --git a/kprobe/swap_kprobes.c b/kprobe/swap_kprobes.c index c90757e..2c3e934 100644 --- a/kprobe/swap_kprobes.c +++ b/kprobe/swap_kprobes.c @@ -544,6 +544,8 @@ static int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p) #endif if (!ap) return -ENOMEM; + + atomic_set(&ap->usage, 0); add_aggr_kprobe(ap, old_p); copy_kprobe(ap, p); DBPRINTF("ap = %p p = %p old_p = %p\n", ap, p, old_p); @@ -559,6 +561,12 @@ static void remove_kprobe(struct kprobe *p) swap_slot_free(&sm, p->ainsn.insn); } +static void wait_kp(struct kprobe *p) +{ + while (atomic_read(&p->usage)) + schedule(); +} + /** * @brief Registers kprobe. * @@ -588,6 +596,7 @@ int swap_register_kprobe(struct kprobe *p) p->nmissed = 0; INIT_LIST_HEAD(&p->list); + atomic_set(&p->usage, 0); old_p = swap_get_kprobe(p->addr); if (old_p) { @@ -616,14 +625,15 @@ static void swap_unregister_valid_kprobe(struct kprobe *p, struct kprobe *old_p) { struct kprobe *list_p; + BUG_ON(in_atomic()); + if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) && (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) { /* Only probe on the hash list */ swap_arch_disarm_kprobe(p); - /* FIXME: move sync out from atomic context */ - if (!in_atomic()) - synchronize_sched(); + synchronize_sched(); + wait_kp(old_p); hlist_del_rcu(&old_p->hlist); remove_kprobe(old_p); @@ -641,11 +651,13 @@ static void swap_unregister_valid_kprobe(struct kprobe *p, struct kprobe *old_p) if (p->post_handler) { list_for_each_entry_rcu(list_p, &old_p->list, list) if (list_p->post_handler) - return; + goto out; old_p->post_handler = NULL; } } + +out: /* Set NULL addr for reusability if symbol_name is used */ if (p->symbol_name) p->addr = NULL; diff --git a/kprobe/swap_kprobes.h b/kprobe/swap_kprobes.h index c69824c..cadb852 100644 --- a/kprobe/swap_kprobes.h +++ b/kprobe/swap_kprobes.h @@ -151,6 +151,7 @@ struct kprobe { * NULL if original function should be called. * Not supported for X86, not tested for MIPS. */ kprobe_opcode_t *ss_addr[NR_CPUS]; + atomic_t usage; }; /** @@ -252,6 +253,17 @@ extern void swap_kprobes_inc_nmissed_count(struct kprobe *p); #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) +static void inline get_kp(struct kprobe *p) +{ + atomic_inc(&p->usage); +} + +static void inline put_kp(struct kprobe *p) +{ + atomic_dec(&p->usage); +} + + /* Get the kprobe at this addr (if any) - called with preemption disabled */ struct kprobe *swap_get_kprobe(void *addr); -- 2.7.4