[IMPROVE] call kprobe handlers at current context (for x86)

Move kprobe handler from atomic context to current

Change-Id: I8c3cc6bb964c6df1a2a83786f5bfca9c10b594a0
Signed-off-by: Vyacheslav Cherkashin <v.cherkashin@samsung.com>

diff --git a/kprobe/arch/x86/swap-asm/swap_kprobes.c b/kprobe/arch/x86/swap-asm/swap_kprobes.c
index 29cd7e8..0f9cc49 100644 (file)
--- a/kprobe/arch/x86/swap-asm/swap_kprobes.c
+++ b/kprobe/arch/x86/swap-asm/swap_kprobes.c
@@ -35,7 +35,7 @@
 
 #include "swap_kprobes.h"
 #include <kprobe/swap_kprobes.h>
-
+#include <kprobe/swap_td_raw.h>
 #include <kprobe/swap_kdebug.h>
 #include <kprobe/swap_slots.h>
 #include <kprobe/swap_kprobes_deps.h>
@@ -325,10 +325,25 @@ static int setup_singlestep(struct kprobe *p, struct pt_regs *regs,
        return 1;
 }
 
+
+static struct td_raw kp_tdraw;
+
+static struct pt_regs *current_regs(void)
+{
+       return (struct pt_regs *)swap_td_raw(&kp_tdraw, current);
+}
+
+
+void restore_int3(void);
+__asm(
+       "restore_int3:\n"
+       "int3\n"
+);
+
 static int __kprobe_handler(struct pt_regs *regs)
 {
        struct kprobe *p = 0;
-       int ret = 0, reenter = 0;
+       int ret = 0;
        kprobe_opcode_t *addr = NULL;
        struct kprobe_ctlblk *kcb;
 
@@ -360,7 +375,7 @@ static int __kprobe_handler(struct pt_regs *regs)
                        prepare_singlestep(p, regs);
                        kcb->kprobe_status = KPROBE_REENTER;
 
-                       return 1;
+                       goto out_get_kp_if_TF;
                } else {
                        if (*addr != BREAKPOINT_INSTRUCTION) {
                                /* The breakpoint instruction was removed by
@@ -403,30 +418,62 @@ static int __kprobe_handler(struct pt_regs *regs)
        }
 
        set_current_kprobe(p, regs, kcb);
+       kcb->kprobe_status = KPROBE_HIT_ACTIVE;
 
-       if (!reenter)
-               kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+       /* save regs to stack */
+       *current_regs() = *regs;
 
-       if (p->pre_handler) {
-               ret = p->pre_handler(p, regs);
-               if (ret)
-                       return ret;
-       }
+       regs->ip = (unsigned long)restore_int3;
+       get_kp(p);
+
+       return 1;
 
 ss_probe:
        setup_singlestep(p, regs, kcb);
 
+out_get_kp_if_TF:
+       if ((regs->flags & TF_MASK))
+               get_kp(p);
+
        return 1;
 
 no_kprobe:
        return ret;
 }
 
+static int restore_handler(struct pt_regs *regs)
+{
+       struct kprobe *p = swap_kprobe_running();
+       struct kprobe_ctlblk *kcb = swap_get_kprobe_ctlblk();
+
+       /* restore regs from stack */
+       *regs = *current_regs();
+
+       if (p->pre_handler) {
+               int ret;
+
+               ret = p->pre_handler(p, regs);
+               if (ret) {
+                       put_kp(p);
+                       return ret;
+               }
+       }
+
+       setup_singlestep(p, regs, kcb);
+       if (!(regs->flags & TF_MASK))
+               put_kp(p);
+
+       return 1;
+}
+
 static int kprobe_handler(struct pt_regs *regs)
 {
        int ret;
 
-       ret = __kprobe_handler(regs);
+       if (regs->ip == (unsigned long)restore_int3 + 1)
+               ret = restore_handler(regs);
+       else
+               ret = __kprobe_handler(regs);
 
        return ret;
 }
@@ -616,6 +663,9 @@ static int post_kprobe_handler(struct pt_regs *regs)
 
        if (!cur)
                return 0;
+
+       put_kp(cur);
+
        if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
                kcb->kprobe_status = KPROBE_HIT_SSDONE;
                cur->post_handler(cur, regs, 0);
@@ -1085,14 +1135,24 @@ int swap_arch_init_kprobes(void)
 {
        int ret;
 
-       ret = register_die_notifier(&kprobe_exceptions_nb);
+       ret = swap_td_raw_reg(&kp_tdraw, sizeof(struct pt_regs));
        if (ret)
                return ret;
 
+       ret = register_die_notifier(&kprobe_exceptions_nb);
+       if (ret)
+               goto unreg_tdraw;
+
        ret = kjump_init();
        if (ret)
-               unregister_die_notifier(&kprobe_exceptions_nb);
+               goto unreg_die;
 
+       return 0;
+
+unreg_die:
+       unregister_die_notifier(&kprobe_exceptions_nb);
+unreg_tdraw:
+       swap_td_raw_unreg(&kp_tdraw);
        return ret;
 }
 
@@ -1105,4 +1165,5 @@ void swap_arch_exit_kprobes(void)
 {
        kjump_exit();
        unregister_die_notifier(&kprobe_exceptions_nb);
+       swap_td_raw_unreg(&kp_tdraw);
 }

diff --git a/kprobe/swap_kprobes.c b/kprobe/swap_kprobes.c
index c90757e..2c3e934 100644 (file)
--- a/kprobe/swap_kprobes.c
+++ b/kprobe/swap_kprobes.c
@@ -544,6 +544,8 @@ static int register_aggr_kprobe(struct kprobe *old_p, struct kprobe *p)
 #endif
                if (!ap)
                        return -ENOMEM;
+
+               atomic_set(&ap->usage, 0);
                add_aggr_kprobe(ap, old_p);
                copy_kprobe(ap, p);
                DBPRINTF("ap = %p p = %p old_p = %p\n", ap, p, old_p);
@@ -559,6 +561,12 @@ static void remove_kprobe(struct kprobe *p)
        swap_slot_free(&sm, p->ainsn.insn);
 }
 
+static void wait_kp(struct kprobe *p)
+{
+       while (atomic_read(&p->usage))
+               schedule();
+}
+
 /**
  * @brief Registers kprobe.
  *
@@ -588,6 +596,7 @@ int swap_register_kprobe(struct kprobe *p)
 
        p->nmissed = 0;
        INIT_LIST_HEAD(&p->list);
+       atomic_set(&p->usage, 0);
 
        old_p = swap_get_kprobe(p->addr);
        if (old_p) {
@@ -616,14 +625,15 @@ static void swap_unregister_valid_kprobe(struct kprobe *p, struct kprobe *old_p)
 {
        struct kprobe *list_p;
 
+       BUG_ON(in_atomic());
+
        if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
            (p->list.next == &old_p->list) && (p->list.prev == &old_p->list))) {
                /* Only probe on the hash list */
                swap_arch_disarm_kprobe(p);
 
-               /* FIXME: move sync out from atomic context */
-               if (!in_atomic())
-                       synchronize_sched();
+               synchronize_sched();
+               wait_kp(old_p);
 
                hlist_del_rcu(&old_p->hlist);
                remove_kprobe(old_p);
@@ -641,11 +651,13 @@ static void swap_unregister_valid_kprobe(struct kprobe *p, struct kprobe *old_p)
                if (p->post_handler) {
                        list_for_each_entry_rcu(list_p, &old_p->list, list)
                                if (list_p->post_handler)
-                                       return;
+                                       goto out;
 
                        old_p->post_handler = NULL;
                }
        }
+
+out:
        /* Set NULL addr for reusability if symbol_name is used */
        if (p->symbol_name)
                p->addr = NULL;

diff --git a/kprobe/swap_kprobes.h b/kprobe/swap_kprobes.h
index c69824c..cadb852 100644 (file)
--- a/kprobe/swap_kprobes.h
+++ b/kprobe/swap_kprobes.h
@@ -151,6 +151,7 @@ struct kprobe {
         * NULL if original function should be called.
         * Not supported for X86, not tested for MIPS. */
        kprobe_opcode_t                                 *ss_addr[NR_CPUS];
+       atomic_t usage;
 };
 
 /**
@@ -252,6 +253,17 @@ extern void swap_kprobes_inc_nmissed_count(struct kprobe *p);
 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
 
 
+static void inline get_kp(struct kprobe *p)
+{
+       atomic_inc(&p->usage);
+}
+
+static void inline put_kp(struct kprobe *p)
+{
+       atomic_dec(&p->usage);
+}
+
+
 /* Get the kprobe at this addr (if any) - called with preemption disabled */
 struct kprobe *swap_get_kprobe(void *addr);