* @brief Uprobe control block
*/
struct uprobe_ctlblk {
- unsigned long flags; /**< Flags */
- struct kprobe *p; /**< Pointer to the uprobe's kprobe */
+ unsigned long flags; /**< Flags */
+ struct kprobe *p; /**< Pointer to the uprobe's kprobe */
};
static unsigned long trampoline_addr(struct uprobe *up)
UPROBES_TRAMP_RET_BREAK_IDX);
}
-static DEFINE_PER_CPU(struct uprobe_ctlblk, ucb) = { 0, NULL };
+unsigned long arch_tramp_by_ri(struct uretprobe_instance *ri)
+{
+ return trampoline_addr(&ri->rp->up);
+}
-static struct kprobe *get_current_probe(void)
+static struct uprobe_ctlblk *current_ucb(void)
{
- return __get_cpu_var(ucb).p;
+ /* FIXME hardcoded offset */
+ return (struct uprobe_ctlblk *)(end_of_stack(current) + 20);
}
-static void set_current_probe(struct kprobe *p)
+static struct kprobe *get_current_probe(void)
{
- __get_cpu_var(ucb).p = p;
+ return current_ucb()->p;
}
-static void reset_current_probe(void)
+static void set_current_probe(struct kprobe *p)
{
- set_current_probe(NULL);
+ current_ucb()->p = p;
}
static void save_current_flags(struct pt_regs *regs)
{
- __get_cpu_var(ucb).flags = regs->EREG(flags);
+ current_ucb()->flags = regs->flags;
}
-static void restore_current_flags(struct pt_regs *regs)
+static void restore_current_flags(struct pt_regs *regs, unsigned long flags)
{
- regs->EREG(flags) &= ~IF_MASK;
- regs->EREG(flags) |= __get_cpu_var(ucb).flags & IF_MASK;
+ regs->flags &= ~IF_MASK;
+ regs->flags |= flags & IF_MASK;
}
/**
*/
int arch_prepare_uprobe(struct uprobe *up)
{
- int ret = 0;
struct kprobe *p = up2kp(up);
struct task_struct *task = up->task;
u8 *tramp = up->atramp.tramp;
enum { call_relative_opcode = 0xe8 };
if (!read_proc_vm_atomic(task, (unsigned long)p->addr,
- tramp, MAX_INSN_SIZE))
- panic("failed to read memory %p!\n", p->addr);
+ tramp, MAX_INSN_SIZE)) {
+ printk(KERN_ERR "failed to read memory %p!\n", p->addr);
+ return -EINVAL;
+ }
/* TODO: this is a workaround */
if (tramp[0] == call_relative_opcode) {
- printk("cannot install probe: 1st instruction is call\n");
- return -1;
+ printk(KERN_INFO "cannot install probe: 1st instruction is call\n");
+ return -EINVAL;
}
tramp[UPROBES_TRAMP_RET_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
p->ainsn.boostable = swap_can_boost(tramp) ? 0 : -1;
- return ret;
+ p->ainsn.insn = swap_slot_alloc(up->sm);
+ if (p->ainsn.insn == NULL) {
+ printk(KERN_ERR "trampoline out of memory\n");
+ return -ENOMEM;
+ }
+
+ if (!write_proc_vm_atomic(task, (unsigned long)p->ainsn.insn,
+ tramp, sizeof(up->atramp.tramp))) {
+ swap_slot_free(up->sm, p->ainsn.insn);
+ printk(KERN_INFO "failed to write memory %p!\n", tramp);
+ return -EINVAL;
+ }
+
+ /* for uretprobe */
+ add_uprobe_table(p);
+
+ return 0;
}
/**
{
struct uprobe *up = container_of(p, struct uprobe, kp);
struct ujprobe *jp = container_of(up, struct ujprobe, up);
- kprobe_pre_entry_handler_t pre_entry = (kprobe_pre_entry_handler_t)jp->pre_entry;
+ kprobe_pre_entry_handler_t pre_entry =
+ (kprobe_pre_entry_handler_t)jp->pre_entry;
entry_point_t entry = (entry_point_t)jp->entry;
unsigned long args[6];
/* FIXME some user space apps crash if we clean interrupt bit */
- //regs->EREG(flags) &= ~IF_MASK;
+ /* regs->EREG(flags) &= ~IF_MASK; */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
trace_hardirqs_off();
#endif
/* read first 6 args from stack */
- if (!read_proc_vm_atomic(current, regs->EREG(sp) + 4, args, sizeof(args)))
- panic("failed to read user space func arguments %lx!\n", regs->EREG(sp) + 4);
+ if (!read_proc_vm_atomic(current, regs->EREG(sp) + 4,
+ args, sizeof(args)))
+ printk(KERN_WARNING
+ "failed to read user space func arguments %lx!\n",
+ regs->sp + 4);
if (pre_entry)
p->ss_addr[smp_processor_id()] = (kprobe_opcode_t *)
* @param regs Pointer to CPU register data.
* @return Void.
*/
-void arch_prepare_uretprobe(struct uretprobe_instance *ri, struct pt_regs *regs)
+int arch_prepare_uretprobe(struct uretprobe_instance *ri, struct pt_regs *regs)
{
/* Replace the return addr with trampoline addr */
unsigned long ra = trampoline_addr(&ri->rp->up);
+ unsigned long ret_addr;
ri->sp = (kprobe_opcode_t *)regs->sp;
- if (!read_proc_vm_atomic(current, regs->EREG(sp), &(ri->ret_addr), sizeof(ri->ret_addr)))
- panic("failed to read user space func ra %lx!\n", regs->EREG(sp));
+ if (get_user(ret_addr, (unsigned long *)regs->sp)) {
+ pr_err("failed to read user space func ra %lx addr=%p!\n",
+ regs->sp, ri->rp->up.kp.addr);
+ return -EINVAL;
+ }
+
+ if (put_user(ra, (unsigned long *)regs->sp)) {
+ pr_err("failed to write user space func ra %lx!\n", regs->sp);
+ return -EINVAL;
+ }
+
+ ri->ret_addr = (kprobe_opcode_t *)ret_addr;
- if (!write_proc_vm_atomic(current, regs->EREG(sp), &ra, sizeof(ra)))
- panic("failed to write user space func ra %lx!\n", regs->EREG(sp));
+ return 0;
}
/**
* negative error code on error.
*/
int arch_disarm_urp_inst(struct uretprobe_instance *ri,
- struct task_struct *task)
+ struct task_struct *task, unsigned long tr)
{
- int len;
unsigned long ret_addr;
unsigned long sp = (unsigned long)ri->sp;
- unsigned long tramp_addr = trampoline_addr(&ri->rp->up);
- len = read_proc_vm_atomic(task, sp, &ret_addr, sizeof(ret_addr));
- if (len != sizeof(ret_addr)) {
- printk("---> %s (%d/%d): failed to read stack from %08lx\n",
+ unsigned long tramp_addr;
+
+ if (tr == 0)
+ tramp_addr = arch_tramp_by_ri(ri);
+ else
+ tramp_addr = tr; /* ri - invalid */
+
+ if (get_user(ret_addr, (unsigned long *)sp)) {
+ printk(KERN_INFO "---> %s (%d/%d): failed to read stack from %08lx\n",
task->comm, task->tgid, task->pid, sp);
return -EFAULT;
}
if (tramp_addr == ret_addr) {
- len = write_proc_vm_atomic(task, sp, &ri->ret_addr,
- sizeof(ri->ret_addr));
- if (len != sizeof(ri->ret_addr)) {
- printk("---> %s (%d/%d): failed to write "
+ if (put_user((unsigned long)ri->ret_addr, (unsigned long *)sp)) {
+ printk(KERN_INFO "---> %s (%d/%d): failed to write "
"orig_ret_addr to %08lx",
task->comm, task->tgid, task->pid, sp);
return -EFAULT;
}
} else {
- printk("---> %s (%d/%d): trampoline NOT found at sp = %08lx\n",
+ printk(KERN_INFO "---> %s (%d/%d): trampoline NOT found at sp = %08lx\n",
task->comm, task->tgid, task->pid, sp);
return -ENOENT;
}
swap_slot_free(up->sm, p->ainsn.insn);
}
+int arch_arm_uprobe(struct uprobe *p)
+{
+ int ret;
+ kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
+ unsigned long vaddr = (unsigned long)p->kp.addr;
+
+ ret = write_proc_vm_atomic(p->task, vaddr, &insn, sizeof(insn));
+ if (!ret) {
+ pr_err("arch_arm_uprobe: failed to write memory tgid=%u vaddr=%08lx\n",
+ p->task->tgid, vaddr);
+
+ return -EACCES;
+ }
+
+ return 0;
+}
+
+void arch_disarm_uprobe(struct kprobe *p, struct task_struct *task)
+{
+ int ret;
+ unsigned long vaddr = (unsigned long)p->addr;
+
+ ret = write_proc_vm_atomic(task, vaddr, &p->opcode, sizeof(p->opcode));
+ if (!ret) {
+ pr_err("arch_disarm_uprobe: failed to write memory tgid=%u, vaddr=%08lx\n",
+ task->tgid, vaddr);
+ }
+}
+
static void set_user_jmp_op(void *from, void *to)
{
- struct __arch_jmp_op
- {
+ struct __arch_jmp_op {
char op;
long raddr;
- } __attribute__ ((packed)) jop;
+ } __packed jop;
jop.raddr = (long)(to) - ((long)(from) + 5);
jop.op = RELATIVEJUMP_INSTRUCTION;
- if (!write_proc_vm_atomic(current, (unsigned long)from, &jop, sizeof(jop)))
- panic("failed to write jump opcode to user space %p!\n", from);
+ if (put_user(jop.op, (char *)from) ||
+ put_user(jop.raddr, (long *)(from + 1)))
+ pr_err("failed to write jump opcode to user space %p\n", from);
}
-static void resume_execution(struct kprobe *p, struct pt_regs *regs, unsigned long flags)
+static void resume_execution(struct kprobe *p,
+ struct pt_regs *regs,
+ unsigned long flags)
{
unsigned long *tos, tos_dword = 0;
unsigned long copy_eip = (unsigned long)p->ainsn.insn;
regs->EREG(flags) &= ~TF_MASK;
tos = (unsigned long *)&tos_dword;
- if (!read_proc_vm_atomic(current, regs->EREG(sp), &tos_dword, sizeof(tos_dword)))
- panic("failed to read dword from top of the user space stack %lx!\n", regs->EREG(sp));
+ if (get_user(tos_dword, (unsigned long *)regs->sp)) {
+ pr_err("failed to read from user space sp=%lx!\n", regs->sp);
+ return;
+ }
- if (!read_proc_vm_atomic(current, (unsigned long)p->ainsn.insn, insns, 2 * sizeof(kprobe_opcode_t)))
- panic("failed to read first 2 opcodes of instruction copy from user space %p!\n", p->ainsn.insn);
+ if (get_user(*(unsigned short *)insns, (unsigned short *)p->ainsn.insn)) {
+ pr_err("failed to read first 2 opcodes %p!\n", p->ainsn.insn);
+ return;
+ }
switch (insns[0]) {
- case 0x9c: /* pushfl */
- *tos &= ~(TF_MASK | IF_MASK);
- *tos |= flags & (TF_MASK | IF_MASK);
- break;
- case 0xc2: /* iret/ret/lret */
- case 0xc3:
- case 0xca:
- case 0xcb:
- case 0xcf:
- case 0xea: /* jmp absolute -- eip is correct */
- /* eip is already adjusted, no more changes required */
- p->ainsn.boostable = 1;
- goto no_change;
- case 0xe8: /* call relative - Fix return addr */
- *tos = orig_eip + (*tos - copy_eip);
- break;
- case 0x9a: /* call absolute -- same as call absolute, indirect */
+ case 0x9c: /* pushfl */
+ *tos &= ~(TF_MASK | IF_MASK);
+ *tos |= flags & (TF_MASK | IF_MASK);
+ break;
+ case 0xc2: /* iret/ret/lret */
+ case 0xc3:
+ case 0xca:
+ case 0xcb:
+ case 0xcf:
+ case 0xea: /* jmp absolute -- eip is correct */
+ /* eip is already adjusted, no more changes required */
+ p->ainsn.boostable = 1;
+ goto no_change;
+ case 0xe8: /* call relative - Fix return addr */
+ *tos = orig_eip + (*tos - copy_eip);
+ break;
+ case 0x9a: /* call absolute -- same as call absolute, indirect */
+ *tos = orig_eip + (*tos - copy_eip);
+
+ if (put_user(tos_dword, (unsigned long *)regs->sp)) {
+ pr_err("failed to write dword to sp=%lx\n", regs->sp);
+ return;
+ }
+
+ goto no_change;
+ case 0xff:
+ if ((insns[1] & 0x30) == 0x10) {
+ /*
+ * call absolute, indirect
+ * Fix return addr; eip is correct.
+ * But this is not boostable
+ */
*tos = orig_eip + (*tos - copy_eip);
- if (!write_proc_vm_atomic(current, regs->EREG (sp), &tos_dword, sizeof(tos_dword)))
- panic("failed to write dword to top of the user space stack %lx!\n", regs->EREG (sp));
+ if (put_user(tos_dword, (unsigned long *)regs->sp)) {
+ pr_err("failed to write dword to sp=%lx\n", regs->sp);
+ return;
+ }
goto no_change;
- case 0xff:
- if ((insns[1] & 0x30) == 0x10) {
- /*
- * call absolute, indirect
- * Fix return addr; eip is correct.
- * But this is not boostable
- */
- *tos = orig_eip + (*tos - copy_eip);
-
- if (!write_proc_vm_atomic(current, regs->EREG(sp), &tos_dword, sizeof(tos_dword)))
- panic("failed to write dword to top of the user space stack %lx!\n", regs->EREG(sp));
-
- goto no_change;
- } else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute indirect */
- ((insns[1] & 0x31) == 0x21)) {
- /* jmp far, absolute indirect */
- /* eip is correct. And this is boostable */
- p->ainsn.boostable = 1;
- goto no_change;
- }
- case 0xf3:
- if (insns[1] == 0xc3)
- /* repz ret special handling: no more changes */
- goto no_change;
- break;
- default:
- break;
+ } else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute
+ * indirect */
+ ((insns[1] & 0x31) == 0x21)) {
+ /* jmp far, absolute indirect */
+ /* eip is correct. And this is boostable */
+ p->ainsn.boostable = 1;
+ goto no_change;
+ }
+ case 0xf3:
+ if (insns[1] == 0xc3)
+ /* repz ret special handling: no more changes */
+ goto no_change;
+ break;
+ default:
+ break;
}
- if (!write_proc_vm_atomic(current, regs->EREG(sp), &tos_dword, sizeof(tos_dword)))
- panic("failed to write dword to top of the user space stack %lx!\n", regs->EREG(sp));
+ if (put_user(tos_dword, (unsigned long *)regs->sp)) {
+ pr_err("failed to write dword to sp=%lx\n", regs->sp);
+ return;
+ }
if (p->ainsn.boostable == 0) {
- if ((regs->EREG(ip) > copy_eip) && (regs->EREG(ip) - copy_eip) + 5 < MAX_INSN_SIZE) {
+ if ((regs->EREG(ip) > copy_eip) && (regs->EREG(ip) - copy_eip) +
+ 5 < MAX_INSN_SIZE) {
/*
* These instructions can be executed directly if it
* jumps back to correct address.
*/
- set_user_jmp_op((void *) regs->EREG(ip), (void *)orig_eip + (regs->EREG(ip) - copy_eip));
+ set_user_jmp_op((void *) regs->EREG(ip),
+ (void *)orig_eip +
+ (regs->EREG(ip) - copy_eip));
p->ainsn.boostable = 1;
} else {
p->ainsn.boostable = -1;
return;
}
-static int make_trampoline(struct uprobe *up)
+static bool prepare_ss_addr(struct kprobe *p, struct pt_regs *regs)
{
- struct kprobe *p = up2kp(up);
- struct task_struct *task = up->task;
- void *tramp;
+ unsigned long *ss_addr = (long *)&p->ss_addr[smp_processor_id()];
- tramp = swap_slot_alloc(up->sm);
- if (tramp == 0) {
- printk("trampoline out of memory\n");
- return -ENOMEM;
- }
-
- if (!write_proc_vm_atomic(task, (unsigned long)tramp,
- up->atramp.tramp,
- sizeof(up->atramp.tramp))) {
- swap_slot_free(up->sm, tramp);
- panic("failed to write memory %p!\n", tramp);
- return -EINVAL;
+ if (*ss_addr) {
+ regs->ip = *ss_addr;
+ *ss_addr = 0;
+ return true;
+ } else {
+ regs->ip = (unsigned long)p->ainsn.insn;
+ return false;
}
+}
- p->ainsn.insn = tramp;
-
- return 0;
+static void prepare_ss(struct pt_regs *regs)
+{
+ /* set single step mode */
+ regs->flags |= TF_MASK;
+ regs->flags &= ~IF_MASK;
}
static int uprobe_handler(struct pt_regs *regs)
p = get_ukprobe_by_insn_slot(tramp_addr, tgid, regs);
if (p == NULL) {
- printk("no_uprobe\n");
+ printk(KERN_INFO "no_uprobe\n");
return 0;
}
trampoline_uprobe_handler(p, regs);
return 1;
} else {
- if (p->ainsn.insn == NULL) {
- struct uprobe *up = kp2up(p);
-
- make_trampoline(up);
-
- /* for uretprobe */
- add_uprobe_table(p);
- }
-
if (!p->pre_handler || !p->pre_handler(p, regs)) {
-
if (p->ainsn.boostable == 1 && !p->post_handler) {
- if (p->ss_addr[smp_processor_id()]) {
- regs->EREG(ip) = (unsigned long)p->ss_addr[smp_processor_id()];
- p->ss_addr[smp_processor_id()] = NULL;
- } else {
- regs->EREG(ip) = (unsigned long)p->ainsn.insn;
- }
+ prepare_ss_addr(p, regs);
return 1;
}
- prepare_singlestep(p, regs);
+ if (prepare_ss_addr(p, regs) == false) {
+ set_current_probe(p);
+ prepare_ss(regs);
+ }
}
}
- set_current_probe(p);
-
return 1;
}
static int post_uprobe_handler(struct pt_regs *regs)
{
struct kprobe *p = get_current_probe();
- unsigned long flags = __get_cpu_var(ucb).flags;
+ unsigned long flags = current_ucb()->flags;
- if (p == NULL)
+ if (p == NULL) {
+ printk("task[%u %u %s] current uprobe is not found\n",
+ current->tgid, current->pid, current->comm);
return 0;
+ }
resume_execution(p, regs, flags);
- restore_current_flags(regs);
+ restore_current_flags(regs, flags);
- reset_current_probe();
+ /* clean stack */
+ current_ucb()->p = 0;
+ current_ucb()->flags = 0;
return 1;
}
-static int uprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data)
+static int uprobe_exceptions_notify(struct notifier_block *self,
+ unsigned long val, void *data)
{
struct die_args *args = (struct die_args *)data;
int ret = NOTIFY_DONE;
switch (val) {
#ifdef CONFIG_KPROBES
- case DIE_INT3:
+ case DIE_INT3:
#else
- case DIE_TRAP:
+ case DIE_TRAP:
#endif
- if (uprobe_handler(args->regs))
- ret = NOTIFY_STOP;
- break;
- case DIE_DEBUG:
- if (post_uprobe_handler(args->regs))
- ret = NOTIFY_STOP;
- break;
- default:
- break;
+ if (uprobe_handler(args->regs))
+ ret = NOTIFY_STOP;
+ break;
+ case DIE_DEBUG:
+ if (post_uprobe_handler(args->regs))
+ ret = NOTIFY_STOP;
+ break;
+ default:
+ break;
}
return ret;
projects / kernel / swap-modules.git / blobdiff