From 90e1564690c281ad00759b621daa41fb232a44a4 Mon Sep 17 00:00:00 2001 From: Vyacheslav Cherkashin Date: Fri, 3 Jun 2016 18:57:22 +0300 Subject: [PATCH] [FIX] __switch_to for x86 In order not to miss calls __switch_to() we use special flags (which are stored in the stack). These flags we set when enter __switch_to and reset when return from __switch_to. If flags have been set we use PER_CPU (exclusive for __switch_to) to store temporary values. To identify __switch_to kretprobe we set SWITCH_TO_RP flag to next executing process. And reset this flag in trampoline_probe_handler(). Change-Id: Id86e40853755bf9acdde642df8a7db337440e1db Signed-off-by: Vyacheslav Cherkashin --- kprobe/arch/x86/swap-asm/swap_kprobes.c | 35 ++++++++++++++++------- kprobe/swap_kprobes.c | 49 +++++++++++++++++++++++++++------ kprobe/swap_kprobes.h | 18 ++++++++++-- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/kprobe/arch/x86/swap-asm/swap_kprobes.c b/kprobe/arch/x86/swap-asm/swap_kprobes.c index fc805d9..e31f865 100644 --- a/kprobe/arch/x86/swap-asm/swap_kprobes.c +++ b/kprobe/arch/x86/swap-asm/swap_kprobes.c @@ -317,14 +317,17 @@ static int setup_singlestep(struct kp_core *p, struct pt_regs *regs, static struct td_raw kp_tdraw; -static DEFINE_PER_CPU(struct pt_regs, per_cpu_regs); +static DEFINE_PER_CPU(struct pt_regs, per_cpu_regs_i); +static DEFINE_PER_CPU(struct pt_regs, per_cpu_regs_st); static struct pt_regs *current_regs(void) { - if (able2resched()) - return (struct pt_regs *)swap_td_raw(&kp_tdraw, current); + if (in_interrupt()) + return &__get_cpu_var(per_cpu_regs_i); + else if (switch_to_bits_get(current_kctx, SWITCH_TO_ALL)) + return &__get_cpu_var(per_cpu_regs_st); - return &__get_cpu_var(per_cpu_regs); + return (struct pt_regs *)swap_td_raw(&kp_tdraw, current); } @@ -449,6 +452,10 @@ static int __kprobe_handler(struct pt_regs *regs) struct kp_core *p; struct kp_core_ctlblk *kcb; unsigned long addr = regs->ip - 1; + struct kctx *ctx = current_kctx; + + if (addr == sched_addr) + switch_to_bits_set(ctx, SWITCH_TO_KP); kcb = kp_core_ctlblk(); @@ -457,7 +464,7 @@ static int __kprobe_handler(struct pt_regs *regs) kp_core_get(p); rcu_read_unlock(); - if (able2resched()) { + if (able2resched(ctx)) { ret = kprobe_pre_handler(p, regs, kcb); if (ret == KSTAT_PREPARE_KCB) { /* save regs to stack */ @@ -474,6 +481,7 @@ static int __kprobe_handler(struct pt_regs *regs) if (ret == KSTAT_PREPARE_KCB) { int rr = p->handlers.pre(p, regs); if (rr) { + switch_to_bits_reset(ctx, SWITCH_TO_KP); kp_core_put(p); return 1; } @@ -485,10 +493,12 @@ static int __kprobe_handler(struct pt_regs *regs) * If TF is enabled then processing instruction * takes place in two stages. */ - if (regs->flags & TF_MASK) + if (regs->flags & TF_MASK) { preempt_disable(); - else + } else { + switch_to_bits_reset(ctx, SWITCH_TO_KP); kp_core_put(p); + } } return !!ret; @@ -681,6 +691,7 @@ no_change: */ static int post_kprobe_handler(struct pt_regs *regs) { + struct kctx *ctx = current_kctx; struct kp_core *cur = kp_core_running(); struct kp_core_ctlblk *kcb = kp_core_ctlblk(); @@ -700,10 +711,12 @@ static int post_kprobe_handler(struct pt_regs *regs) kp_core_running_set(NULL); out: - kp_core_put(cur); - if (!able2resched()) + if (!able2resched(ctx)) swap_preempt_enable_no_resched(); + switch_to_bits_reset(ctx, SWITCH_TO_KP); + kp_core_put(cur); + /* * if somebody else is singlestepping across a probe point, eflags * will have TF set, in which case, continue the remaining processing @@ -849,8 +862,10 @@ void swap_arch_prepare_kretprobe(struct kretprobe_instance *ri, /* for __switch_to probe */ if ((unsigned long)ri->rp->kp.addr == sched_addr) { + struct task_struct *next = (struct task_struct *)swap_get_karg(regs, 1); ri->sp = NULL; - ri->task = (struct task_struct *)regs->dx; + ri->task = next; + switch_to_bits_set(kctx_by_task(next), SWITCH_TO_RP); } else { ri->sp = ptr_ret_addr; } diff --git a/kprobe/swap_kprobes.c b/kprobe/swap_kprobes.c index dfc48d9..0c51d14 100644 --- a/kprobe/swap_kprobes.c +++ b/kprobe/swap_kprobes.c @@ -153,34 +153,64 @@ struct kpc_data { struct kp_core_ctlblk ctlblk; }; +struct kctx { + struct kpc_data kpc; + unsigned long st_flags; +}; + static void ktd_cur_init(struct task_struct *task, void *data) { - struct kpc_data *d = (struct kpc_data *)data; + struct kctx *ctx = (struct kctx *)data; - memset(d, 0, sizeof(*d)); + memset(ctx, 0, sizeof(*ctx)); } static void ktd_cur_exit(struct task_struct *task, void *data) { - struct kpc_data *d = (struct kpc_data *)data; + struct kctx *ctx = (struct kctx *)data; - WARN(d->running, "running probe is not NULL"); + WARN(ctx->kpc.running, "running=%p\n", ctx->kpc.running); } struct ktask_data ktd_cur = { .init = ktd_cur_init, .exit = ktd_cur_exit, - .size = sizeof(struct kpc_data), + .size = sizeof(struct kctx), }; -static DEFINE_PER_CPU(struct kpc_data, per_cpu_kpc_data); +struct kctx *kctx_by_task(struct task_struct *task) +{ + return (struct kctx *)swap_ktd(&ktd_cur, task); +} + +void switch_to_bits_set(struct kctx *ctx, unsigned long mask) +{ + ctx->st_flags |= mask; +} + +void switch_to_bits_reset(struct kctx *ctx, unsigned long mask) +{ + ctx->st_flags &= ~mask; +} + +unsigned long switch_to_bits_get(struct kctx *ctx, unsigned long mask) +{ + return ctx->st_flags & mask; +} + +static DEFINE_PER_CPU(struct kpc_data, per_cpu_kpc_data_i); +static DEFINE_PER_CPU(struct kpc_data, per_cpu_kpc_data_st); static struct kpc_data *kp_core_data(void) { - if (able2resched()) - return (struct kpc_data *)swap_ktd(&ktd_cur, current); + struct kctx *ctx = current_kctx; + + if (in_interrupt()) + return &__get_cpu_var(per_cpu_kpc_data_i); + else if (switch_to_bits_get(ctx, SWITCH_TO_ALL)) + return &__get_cpu_var(per_cpu_kpc_data_st); - return &__get_cpu_var(per_cpu_kpc_data); + return &ctx->kpc; } static int kprobe_cur_reg(void) @@ -698,6 +728,7 @@ int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) else kp_core_running_set(NULL); + switch_to_bits_reset(current_kctx, SWITCH_TO_RP); spin_unlock_irqrestore(&kretprobe_lock, flags); /* diff --git a/kprobe/swap_kprobes.h b/kprobe/swap_kprobes.h index 0aaee09..df154a6 100644 --- a/kprobe/swap_kprobes.h +++ b/kprobe/swap_kprobes.h @@ -290,9 +290,23 @@ void kp_core_running_set(struct kp_core *p); struct kp_core_ctlblk *kp_core_ctlblk(void); -static inline int able2resched(void) +struct kctx; + +/* for __switch_to support */ +#define SWITCH_TO_KP 0b0001 +#define SWITCH_TO_RP 0b0010 +#define SWITCH_TO_ALL (SWITCH_TO_KP | SWITCH_TO_RP) + +#define current_kctx kctx_by_task(current) +struct kctx *kctx_by_task(struct task_struct *task); + +void switch_to_bits_set(struct kctx *ctx, unsigned long mask); +void switch_to_bits_reset(struct kctx *ctx, unsigned long mask); +unsigned long switch_to_bits_get(struct kctx *ctx, unsigned long mask); + +static inline int able2resched(struct kctx *ctx) { - if (in_interrupt()) + if (in_interrupt() || switch_to_bits_get(ctx, SWITCH_TO_ALL)) return 0; return 1; -- 2.7.4