[FIX] call kprobe from IRQ context (x86)

If kprobe is called from IRQ context, we call handler in current
context and use PER_CPU to store temporary values.

Otherwise, we jump to exec_trampoline to call handler in process
context and use stack (swap_ktd) to store temporary values.

Change-Id: If377dbb02a47e9ae03f44d426f74b7ce8a40bb6f
Signed-off-by: Vyacheslav Cherkashin <v.cherkashin@samsung.com>

diff --git a/kprobe/arch/x86/swap-asm/swap_kprobes.c b/kprobe/arch/x86/swap-asm/swap_kprobes.c
index 9e5d2ed..fc805d9 100644 (file)
--- a/kprobe/arch/x86/swap-asm/swap_kprobes.c
+++ b/kprobe/arch/x86/swap-asm/swap_kprobes.c
@@ -317,10 +317,14 @@ static int setup_singlestep(struct kp_core *p, struct pt_regs *regs,
 
 
 static struct td_raw kp_tdraw;
+static DEFINE_PER_CPU(struct pt_regs, per_cpu_regs);
 
 static struct pt_regs *current_regs(void)
 {
-       return (struct pt_regs *)swap_td_raw(&kp_tdraw, current);
+       if (able2resched())
+               return (struct pt_regs *)swap_td_raw(&kp_tdraw, current);
+
+       return &__get_cpu_var(per_cpu_regs);
 }
 
 
@@ -341,7 +345,7 @@ static int __used exec_handler(void)
        return p->handlers.pre(p, regs);
 }
 
-static int befor_exec_trampoline(struct pt_regs *regs)
+static int after_exec_trampoline(struct pt_regs *regs)
 {
        int ret = (int)regs->ax;
        struct kp_core *p = kp_core_running();
@@ -352,7 +356,7 @@ static int befor_exec_trampoline(struct pt_regs *regs)
 
        if (ret) {
                kp_core_put(p);
-               return ret;
+               return 1;
        }
 
        setup_singlestep(p, regs, kcb);
@@ -362,19 +366,16 @@ static int befor_exec_trampoline(struct pt_regs *regs)
        return 1;
 }
 
-static int __kprobe_handler(struct pt_regs *regs)
+#define KSTAT_NOT_FOUND                0x00
+#define KSTAT_FOUND            0x01
+#define KSTAT_PREPARE_KCB      0x02
+
+static unsigned long kprobe_pre_handler(struct kp_core *p,
+                                       struct pt_regs *regs,
+                                       struct kp_core_ctlblk *kcb)
 {
-       struct kp_core *p;
-       int ret = 0;
+       int ret = KSTAT_NOT_FOUND;
        unsigned long addr = regs->ip - 1;
-       struct kp_core_ctlblk *kcb;
-
-       kcb = kp_core_ctlblk();
-
-       rcu_read_lock();
-       p = kp_core_by_addr(addr);
-       kp_core_get(p);
-       rcu_read_unlock();
 
        /* Check we're not actually recursing */
        if (kp_core_running()) {
@@ -383,10 +384,9 @@ static int __kprobe_handler(struct pt_regs *regs)
                            *p->ainsn.insn == BREAKPOINT_INSTRUCTION) {
                                regs->flags &= ~TF_MASK;
                                regs->flags |= kcb->kp_core_saved_eflags;
-                               goto no_kprobe;
+                               goto out;
                        }
 
-
                        /* We have reentered the kprobe_handler(), since
                         * another probe was hit while within the handler.
                         * We here save the original kprobes variables and
@@ -398,7 +398,8 @@ static int __kprobe_handler(struct pt_regs *regs)
                        prepare_singlestep(p, regs);
                        kcb->kp_core_status = KPROBE_REENTER;
 
-                       goto out_get_kp_if_TF;
+                       ret = KSTAT_FOUND;
+                       goto out;
                } else {
                        if (*(char *)addr != BREAKPOINT_INSTRUCTION) {
                                /* The breakpoint instruction was removed by
@@ -406,11 +407,12 @@ static int __kprobe_handler(struct pt_regs *regs)
                                 * handling of this interrupt is appropriate
                                 */
                                regs->EREG(ip) -= sizeof(kprobe_opcode_t);
-                               ret = 1;
-                               goto no_kprobe;
+
+                               ret = KSTAT_FOUND;
+                               goto out;
                        }
 
-                       goto no_kprobe;
+                       goto out;
                }
        }
 
@@ -426,29 +428,70 @@ static int __kprobe_handler(struct pt_regs *regs)
                         * the original instruction.
                         */
                        regs->EREG(ip) -= sizeof(kprobe_opcode_t);
-                       ret = 1;
+
+                       ret = KSTAT_FOUND;
                }
 
-               goto no_kprobe;
+               goto out;
        }
 
        set_current_kp_core(p, regs, kcb);
        kcb->kp_core_status = KPROBE_HIT_ACTIVE;
 
-       /* save regs to stack */
-       *current_regs() = *regs;
+       ret = KSTAT_PREPARE_KCB;
+out:
+       return ret;
+}
 
-       regs->ip = (unsigned long)exec_trampoline;
-       return 1;
+static int __kprobe_handler(struct pt_regs *regs)
+{
+       int ret;
+       struct kp_core *p;
+       struct kp_core_ctlblk *kcb;
+       unsigned long addr = regs->ip - 1;
 
-out_get_kp_if_TF:
-       if (!(regs->flags & TF_MASK))
-               kp_core_put(p);
+       kcb = kp_core_ctlblk();
 
-       return 1;
+       rcu_read_lock();
+       p = kp_core_by_addr(addr);
+       kp_core_get(p);
+       rcu_read_unlock();
 
-no_kprobe:
-       return ret;
+       if (able2resched()) {
+               ret = kprobe_pre_handler(p, regs, kcb);
+               if (ret == KSTAT_PREPARE_KCB) {
+                       /* save regs to stack */
+                       *current_regs() = *regs;
+
+                       regs->ip = (unsigned long)exec_trampoline;
+                       return 1;
+               }
+
+               if (!(regs->flags & TF_MASK))
+                       kp_core_put(p);
+       } else {
+               ret = kprobe_pre_handler(p, regs, kcb);
+               if (ret == KSTAT_PREPARE_KCB) {
+                       int rr = p->handlers.pre(p, regs);
+                       if (rr) {
+                               kp_core_put(p);
+                               return 1;
+                       }
+
+                       setup_singlestep(p, regs, kcb);
+               }
+
+               /*
+                * If TF is enabled then processing instruction
+                * takes place in two stages.
+                */
+               if (regs->flags & TF_MASK)
+                       preempt_disable();
+               else
+                       kp_core_put(p);
+       }
+
+       return !!ret;
 }
 
 static int kprobe_handler(struct pt_regs *regs)
@@ -456,7 +499,7 @@ static int kprobe_handler(struct pt_regs *regs)
        int ret;
 
        if (regs->ip == (unsigned long)exec_trampoline_int3 + 1)
-               ret = befor_exec_trampoline(regs);
+               ret = after_exec_trampoline(regs);
        else
                ret = __kprobe_handler(regs);
 
@@ -644,8 +687,6 @@ static int post_kprobe_handler(struct pt_regs *regs)
        if (!cur)
                return 0;
 
-       kp_core_put(cur);
-
        resume_execution(cur, regs, kcb);
        regs->flags |= kcb->kp_core_saved_eflags;
 #ifndef CONFIG_X86
@@ -657,7 +698,12 @@ static int post_kprobe_handler(struct pt_regs *regs)
                goto out;
        }
        kp_core_running_set(NULL);
+
 out:
+       kp_core_put(cur);
+       if (!able2resched())
+               swap_preempt_enable_no_resched();
+
        /*
         * if somebody else is singlestepping across a probe point, eflags
         * will have TF set, in which case, continue the remaining processing

diff --git a/kprobe/swap_kprobes.c b/kprobe/swap_kprobes.c
index bd26f8a..dfc48d9 100644 (file)
--- a/kprobe/swap_kprobes.c
+++ b/kprobe/swap_kprobes.c
@@ -177,11 +177,10 @@ static DEFINE_PER_CPU(struct kpc_data, per_cpu_kpc_data);
 
 static struct kpc_data *kp_core_data(void)
 {
-       if (in_interrupt()) {
-               return &__get_cpu_var(per_cpu_kpc_data);
-       } else {
+       if (able2resched())
                return (struct kpc_data *)swap_ktd(&ktd_cur, current);
-       }
+
+       return &__get_cpu_var(per_cpu_kpc_data);
 }
 
 static int kprobe_cur_reg(void)

diff --git a/kprobe/swap_kprobes.h b/kprobe/swap_kprobes.h
index c1609ec..0aaee09 100644 (file)
--- a/kprobe/swap_kprobes.h
+++ b/kprobe/swap_kprobes.h
@@ -290,5 +290,14 @@ void kp_core_running_set(struct kp_core *p);
 struct kp_core_ctlblk *kp_core_ctlblk(void);
 
 
+static inline int able2resched(void)
+{
+       if (in_interrupt())
+               return 0;
+
+       return 1;
+}
+
+
 #endif /* _SWAP_KPROBES_H */
 

diff --git a/kprobe/swap_kprobes_deps.h b/kprobe/swap_kprobes_deps.h
index 8b05b13..f34a54e 100644 (file)
--- a/kprobe/swap_kprobes_deps.h
+++ b/kprobe/swap_kprobes_deps.h
@@ -67,6 +67,26 @@
 #endif
 
 
+/*
+ * swap_preempt_enable_no_resched()
+ */
+#if (defined(MODULE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0)))
+
+#ifdef CONFIG_PREEMPT_COUNT
+#define swap_preempt_enable_no_resched() \
+do { \
+       barrier(); \
+       preempt_count_dec(); \
+} while (0)
+#else /* !CONFIG_PREEMPT_COUNT */
+#define swap_preempt_enable_no_resched() barrier()
+#endif /* CONFIG_PREEMPT_COUNT */
+
+#else /* !(defined(MODULE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) */
+#define swap_preempt_enable_no_resched() preempt_enable_no_resched()
+#endif /* !(defined(MODULE) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) */
+
+
 #if LINUX_VERSION_CODE > KERNEL_VERSION(3, 1, 0)
     #define task_job(task) (task->jobctl)
 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(3, 1, 0) */