2 * uprobe/arch/asm-x86/swap_uprobes.c
3 * @author Alexey Gerenkov <a.gerenkov@samsung.com> User-Space Probes initial
4 * implementation; Support x86/ARM/MIPS for both user and kernel spaces.
5 * @author Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for
6 * separating core and arch parts
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 * Copyright (C) Samsung Electronics, 2006-2010
28 * @section DESCRIPTION
30 * Arch-dependent uprobe interface implementation for x86.
34 #include <linux/kdebug.h>
36 #include <kprobe/swap_slots.h>
37 #include <uprobe/swap_uprobes.h>
39 #include "swap_uprobes.h"
43 * @struct uprobe_ctlblk
44 * @brief Uprobe control block
46 struct uprobe_ctlblk {
47 unsigned long flags; /**< Flags */
48 struct kprobe *p; /**< Pointer to the uprobe's kprobe */
51 static unsigned long trampoline_addr(struct uprobe *up)
53 return (unsigned long)(up->kp.ainsn.insn +
54 UPROBES_TRAMP_RET_BREAK_IDX);
57 unsigned long arch_tramp_by_ri(struct uretprobe_instance *ri)
59 return trampoline_addr(&ri->rp->up);
62 static struct uprobe_ctlblk *current_ucb(void)
64 /* FIXME hardcoded offset */
65 return (struct uprobe_ctlblk *)(end_of_stack(current) + 20);
68 static struct kprobe *get_current_probe(void)
70 return current_ucb()->p;
73 static void set_current_probe(struct kprobe *p)
78 static void save_current_flags(struct pt_regs *regs)
80 current_ucb()->flags = regs->flags;
83 static void restore_current_flags(struct pt_regs *regs, unsigned long flags)
85 regs->flags &= ~IF_MASK;
86 regs->flags |= flags & IF_MASK;
90 * @brief Prepares uprobe for x86.
92 * @param up Pointer to the uprobe.
93 * @return 0 on success,\n
96 int arch_prepare_uprobe(struct uprobe *up)
98 struct kprobe *p = up2kp(up);
99 struct task_struct *task = up->task;
100 u8 *tramp = up->atramp.tramp;
101 enum { call_relative_opcode = 0xe8 };
103 if (!read_proc_vm_atomic(task, (unsigned long)p->addr,
104 tramp, MAX_INSN_SIZE)) {
105 printk(KERN_ERR "failed to read memory %p!\n", p->addr);
108 /* TODO: this is a workaround */
109 if (tramp[0] == call_relative_opcode) {
110 printk(KERN_INFO "cannot install probe: 1st instruction is call\n");
114 tramp[UPROBES_TRAMP_RET_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
116 /* TODO: remove dual info */
117 p->opcode = tramp[0];
119 p->ainsn.boostable = swap_can_boost(tramp) ? 0 : -1;
121 p->ainsn.insn = swap_slot_alloc(up->sm);
122 if (p->ainsn.insn == NULL) {
123 printk(KERN_ERR "trampoline out of memory\n");
127 if (!write_proc_vm_atomic(task, (unsigned long)p->ainsn.insn,
128 tramp, sizeof(up->atramp.tramp))) {
129 swap_slot_free(up->sm, p->ainsn.insn);
130 printk(KERN_INFO "failed to write memory %p!\n", tramp);
141 * @brief Jump pre-handler.
143 * @param p Pointer to the uprobe's kprobe.
144 * @param regs Pointer to CPU register data.
147 int setjmp_upre_handler(struct kprobe *p, struct pt_regs *regs)
149 struct uprobe *up = container_of(p, struct uprobe, kp);
150 struct ujprobe *jp = container_of(up, struct ujprobe, up);
151 kprobe_pre_entry_handler_t pre_entry =
152 (kprobe_pre_entry_handler_t)jp->pre_entry;
153 entry_point_t entry = (entry_point_t)jp->entry;
154 unsigned long args[6];
156 /* FIXME some user space apps crash if we clean interrupt bit */
157 /* regs->EREG(flags) &= ~IF_MASK; */
158 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
159 trace_hardirqs_off();
162 /* read first 6 args from stack */
163 if (!read_proc_vm_atomic(current, regs->EREG(sp) + 4,
166 "failed to read user space func arguments %lx!\n",
170 p->ss_addr[smp_processor_id()] = (kprobe_opcode_t *)
171 pre_entry(jp->priv_arg, regs);
174 entry(args[0], args[1], args[2], args[3], args[4], args[5]);
176 arch_ujprobe_return();
182 * @brief Prepares uretprobe for x86.
184 * @param ri Pointer to the uretprobe instance.
185 * @param regs Pointer to CPU register data.
188 int arch_prepare_uretprobe(struct uretprobe_instance *ri, struct pt_regs *regs)
190 /* Replace the return addr with trampoline addr */
191 unsigned long ra = trampoline_addr(&ri->rp->up);
192 unsigned long ret_addr;
193 ri->sp = (kprobe_opcode_t *)regs->sp;
195 if (get_user(ret_addr, (unsigned long *)regs->sp)) {
196 pr_err("failed to read user space func ra %lx addr=%p!\n",
197 regs->sp, ri->rp->up.kp.addr);
201 if (put_user(ra, (unsigned long *)regs->sp)) {
202 pr_err("failed to write user space func ra %lx!\n", regs->sp);
206 ri->ret_addr = (kprobe_opcode_t *)ret_addr;
211 static bool get_long(struct task_struct *task,
212 unsigned long vaddr, unsigned long *val)
214 return task->mm == current->mm ?
215 !!get_user(*val, (unsigned long *)vaddr) :
216 sizeof(*val) != read_proc_vm_atomic(task, vaddr,
220 static bool put_long(struct task_struct *task,
221 unsigned long vaddr, unsigned long *val)
223 return task->mm == current->mm ?
224 !!put_user(*val, (unsigned long *)vaddr) :
225 sizeof(*val) != write_proc_vm_atomic(task, vaddr,
230 * @brief Disarms uretprobe on x86 arch.
232 * @param ri Pointer to the uretprobe instance.
233 * @param task Pointer to the task for which the probe.
234 * @return 0 on success,\n
235 * negative error code on error.
237 int arch_disarm_urp_inst(struct uretprobe_instance *ri,
238 struct task_struct *task, unsigned long tr)
240 unsigned long ret_addr;
241 unsigned long sp = (unsigned long)ri->sp;
242 unsigned long tramp_addr;
245 tramp_addr = arch_tramp_by_ri(ri);
247 tramp_addr = tr; /* ri - invalid */
249 if (get_long(task, sp, &ret_addr)) {
250 printk(KERN_INFO "---> %s (%d/%d): failed to read stack from %08lx\n",
251 task->comm, task->tgid, task->pid, sp);
255 if (tramp_addr == ret_addr) {
256 if (put_long(task, sp, (unsigned long *)&ri->ret_addr)) {
257 printk(KERN_INFO "---> %s (%d/%d): failed to write "
258 "orig_ret_addr to %08lx",
259 task->comm, task->tgid, task->pid, sp);
263 printk(KERN_INFO "---> %s (%d/%d): trampoline NOT found at sp = %08lx\n",
264 task->comm, task->tgid, task->pid, sp);
272 * @brief Gets trampoline address.
274 * @param p Pointer to the uprobe's kprobe.
275 * @param regs Pointer to CPU register data.
276 * @return Trampoline address.
278 unsigned long arch_get_trampoline_addr(struct kprobe *p, struct pt_regs *regs)
280 return trampoline_addr(kp2up(p));
284 * @brief Restores return address.
286 * @param orig_ret_addr Original return address.
287 * @param regs Pointer to CPU register data.
290 void arch_set_orig_ret_addr(unsigned long orig_ret_addr, struct pt_regs *regs)
292 regs->EREG(ip) = orig_ret_addr;
296 * @brief Removes uprobe.
298 * @param up Pointer to the target uprobe.
301 void arch_remove_uprobe(struct uprobe *up)
303 struct kprobe *p = up2kp(up);
305 swap_slot_free(up->sm, p->ainsn.insn);
308 int arch_arm_uprobe(struct uprobe *p)
311 kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
312 unsigned long vaddr = (unsigned long)p->kp.addr;
314 ret = write_proc_vm_atomic(p->task, vaddr, &insn, sizeof(insn));
316 pr_err("arch_arm_uprobe: failed to write memory tgid=%u vaddr=%08lx\n",
317 p->task->tgid, vaddr);
325 void arch_disarm_uprobe(struct kprobe *p, struct task_struct *task)
328 unsigned long vaddr = (unsigned long)p->addr;
330 ret = write_proc_vm_atomic(task, vaddr, &p->opcode, sizeof(p->opcode));
332 pr_err("arch_disarm_uprobe: failed to write memory tgid=%u, vaddr=%08lx\n",
337 static void set_user_jmp_op(void *from, void *to)
339 struct __arch_jmp_op {
344 jop.raddr = (long)(to) - ((long)(from) + 5);
345 jop.op = RELATIVEJUMP_INSTRUCTION;
347 if (put_user(jop.op, (char *)from) ||
348 put_user(jop.raddr, (long *)(from + 1)))
349 pr_err("failed to write jump opcode to user space %p\n", from);
352 static void resume_execution(struct kprobe *p,
353 struct pt_regs *regs,
356 unsigned long *tos, tos_dword = 0;
357 unsigned long copy_eip = (unsigned long)p->ainsn.insn;
358 unsigned long orig_eip = (unsigned long)p->addr;
359 kprobe_opcode_t insns[2];
361 regs->EREG(flags) &= ~TF_MASK;
363 tos = (unsigned long *)&tos_dword;
364 if (get_user(tos_dword, (unsigned long *)regs->sp)) {
365 pr_err("failed to read from user space sp=%lx!\n", regs->sp);
369 if (get_user(*(unsigned short *)insns, (unsigned short *)p->ainsn.insn)) {
370 pr_err("failed to read first 2 opcodes %p!\n", p->ainsn.insn);
375 case 0x9c: /* pushfl */
376 *tos &= ~(TF_MASK | IF_MASK);
377 *tos |= flags & (TF_MASK | IF_MASK);
379 case 0xc2: /* iret/ret/lret */
384 case 0xea: /* jmp absolute -- eip is correct */
385 /* eip is already adjusted, no more changes required */
386 p->ainsn.boostable = 1;
388 case 0xe8: /* call relative - Fix return addr */
389 *tos = orig_eip + (*tos - copy_eip);
391 case 0x9a: /* call absolute -- same as call absolute, indirect */
392 *tos = orig_eip + (*tos - copy_eip);
394 if (put_user(tos_dword, (unsigned long *)regs->sp)) {
395 pr_err("failed to write dword to sp=%lx\n", regs->sp);
401 if ((insns[1] & 0x30) == 0x10) {
403 * call absolute, indirect
404 * Fix return addr; eip is correct.
405 * But this is not boostable
407 *tos = orig_eip + (*tos - copy_eip);
409 if (put_user(tos_dword, (unsigned long *)regs->sp)) {
410 pr_err("failed to write dword to sp=%lx\n", regs->sp);
415 } else if (((insns[1] & 0x31) == 0x20) || /* jmp near, absolute
417 ((insns[1] & 0x31) == 0x21)) {
418 /* jmp far, absolute indirect */
419 /* eip is correct. And this is boostable */
420 p->ainsn.boostable = 1;
424 if (insns[1] == 0xc3)
425 /* repz ret special handling: no more changes */
432 if (put_user(tos_dword, (unsigned long *)regs->sp)) {
433 pr_err("failed to write dword to sp=%lx\n", regs->sp);
437 if (p->ainsn.boostable == 0) {
438 if ((regs->EREG(ip) > copy_eip) && (regs->EREG(ip) - copy_eip) +
441 * These instructions can be executed directly if it
442 * jumps back to correct address.
444 set_user_jmp_op((void *) regs->EREG(ip),
446 (regs->EREG(ip) - copy_eip));
447 p->ainsn.boostable = 1;
449 p->ainsn.boostable = -1;
453 regs->EREG(ip) = orig_eip + (regs->EREG(ip) - copy_eip);
459 static bool prepare_ss_addr(struct kprobe *p, struct pt_regs *regs)
461 unsigned long *ss_addr = (long *)&p->ss_addr[smp_processor_id()];
468 regs->ip = (unsigned long)p->ainsn.insn;
473 static void prepare_ss(struct pt_regs *regs)
475 /* set single step mode */
476 regs->flags |= TF_MASK;
477 regs->flags &= ~IF_MASK;
480 static int uprobe_handler(struct pt_regs *regs)
483 kprobe_opcode_t *addr;
484 struct task_struct *task = current;
485 pid_t tgid = task->tgid;
487 save_current_flags(regs);
489 addr = (kprobe_opcode_t *)(regs->EREG(ip) - sizeof(kprobe_opcode_t));
490 p = get_ukprobe(addr, tgid);
493 void *tramp_addr = (void *)addr - UPROBES_TRAMP_RET_BREAK_IDX;
495 p = get_ukprobe_by_insn_slot(tramp_addr, tgid, regs);
497 printk(KERN_INFO "no_uprobe\n");
501 trampoline_uprobe_handler(p, regs);
504 if (!p->pre_handler || !p->pre_handler(p, regs)) {
505 if (p->ainsn.boostable == 1 && !p->post_handler) {
506 prepare_ss_addr(p, regs);
510 if (prepare_ss_addr(p, regs) == false) {
511 set_current_probe(p);
520 static int post_uprobe_handler(struct pt_regs *regs)
522 struct kprobe *p = get_current_probe();
523 unsigned long flags = current_ucb()->flags;
526 printk("task[%u %u %s] current uprobe is not found\n",
527 current->tgid, current->pid, current->comm);
531 resume_execution(p, regs, flags);
532 restore_current_flags(regs, flags);
535 current_ucb()->p = 0;
536 current_ucb()->flags = 0;
541 static int uprobe_exceptions_notify(struct notifier_block *self,
542 unsigned long val, void *data)
544 struct die_args *args = (struct die_args *)data;
545 int ret = NOTIFY_DONE;
547 if (args->regs == NULL || !user_mode_vm(args->regs))
551 #ifdef CONFIG_KPROBES
556 if (uprobe_handler(args->regs))
560 if (post_uprobe_handler(args->regs))
570 static struct notifier_block uprobe_exceptions_nb = {
571 .notifier_call = uprobe_exceptions_notify,
576 * @brief Registers notify.
578 * @return register_die_notifier result.
580 int swap_arch_init_uprobes(void)
582 return register_die_notifier(&uprobe_exceptions_nb);
586 * @brief Unregisters notify.
590 void swap_arch_exit_uprobes(void)
592 unregister_die_notifier(&uprobe_exceptions_nb);