2 * Dynamic Binary Instrumentation Module based on KProbes
3 * modules/kprobe/arch/asm-arm/dbi_kprobes.c
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) Samsung Electronics, 2006-2010
21 * 2006-2007 Ekaterina Gorelkina <e.gorelkina@samsung.com>: initial implementation for ARM/MIPS
22 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
23 * Probes initial implementation; Support x86.
24 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
29 #include "dbi_kprobes.h"
30 #include "../dbi_kprobes.h"
33 #include "../../dbi_kdebug.h"
34 #include "../../dbi_insn_slots.h"
35 #include "../../dbi_kprobes_deps.h"
36 #include "../../dbi_uprobes.h"
38 #include <asm/cacheflush.h>
40 unsigned int *arr_traps_original;
42 extern unsigned int *sched_addr;
43 extern unsigned int *fork_addr;
45 extern struct kprobe * per_cpu__current_kprobe;
46 extern spinlock_t kretprobe_lock;
47 extern struct kretprobe *sched_rp;
49 extern struct hlist_head kprobe_insn_pages;
50 extern struct hlist_head uprobe_insn_pages;
52 extern unsigned long (*kallsyms_search) (const char *name);
54 extern struct kprobe *kprobe_running (void);
55 extern struct kprobe_ctlblk *get_kprobe_ctlblk (void);
56 extern void reset_current_kprobe (void);
58 unsigned int arr_traps_template[] = { 0xe1a0c00d, // mov ip, sp
59 0xe92dd800, // stmdb sp!, {fp, ip, lr, pc}
60 0xe24cb004, // sub fp, ip, #4 ; 0x4
62 0xe3500000, // cmp r0, #0 ; 0x0
63 0xe89da800, // ldmia sp, {fp, sp, pc}
70 * Function return probe trampoline:
71 * - init_kprobes() establishes a probepoint here
72 * - When the probed function returns, this probe
73 * causes the handlers to fire
75 void kretprobe_trampoline_holder (void)
77 asm volatile (".global kretprobe_trampoline\n"
78 "kretprobe_trampoline:\n"
85 struct kprobe trampoline_p =
87 .addr = (kprobe_opcode_t *) & kretprobe_trampoline,
88 .pre_handler = trampoline_probe_handler
92 void gen_insn_execbuf_holder (void)
94 asm volatile (".global gen_insn_execbuf\n"
98 "nop\n" // original instruction
100 "ldr pc, [pc, #4]\n" //ssbreak
103 "nop\n"); //stored PC-4(next insn addr)
108 * 0. push Rx on stack
109 * 1. load address to Rx
110 * 2. do insn using Rx
111 * 3. pop Rx from stack
115 * 7. stored PC-4(next insn addr)
117 void pc_dep_insn_execbuf_holder (void)
119 asm volatile (".global pc_dep_insn_execbuf\n"
120 "pc_dep_insn_execbuf:\n"
121 "str r0, [sp, #-4]\n"
122 "ldr r0, [pc, #12]\n"
123 "nop\n" // instruction with replaced PC
124 "ldr r0, [sp, #-4]\n"
125 "ldr pc, [pc, #4]\n" //ssbreak
128 "nop\n");// stored PC-4 (next insn addr)
131 int prep_pc_dep_insn_execbuf (kprobe_opcode_t * insns, kprobe_opcode_t insn, int uregs)
139 for (i = 0; i < 13; i++, reg_mask <<= 1)
141 if (!(insn & reg_mask))
147 for (i = 0; i < 13; i++)
149 // DBPRINTF("prep_pc_dep_insn_execbuf: check R%d/%d, changing regs %x in %x",
150 // i, ARM_INSN_REG_RN(insn), uregs, insn);
151 if ((uregs & 0x1) && (ARM_INSN_REG_RN (insn) == i))
153 if ((uregs & 0x2) && (ARM_INSN_REG_RD (insn) == i))
155 if ((uregs & 0x4) && (ARM_INSN_REG_RS (insn) == i))
157 if ((uregs & 0x8) && (ARM_INSN_REG_RM (insn) == i))
164 DBPRINTF ("there are no free register %x in insn %lx!", uregs, insn);
167 DBPRINTF ("prep_pc_dep_insn_execbuf: using R%d, changing regs %x", i, uregs);
169 // set register to save
170 ARM_INSN_REG_SET_RD (insns[0], i);
171 // set register to load address to
172 ARM_INSN_REG_SET_RD (insns[1], i);
173 // set instruction to execute and patch it
176 ARM_INSN_REG_CLEAR_MR (insn, 15);
177 ARM_INSN_REG_SET_MR (insn, i);
181 if ((uregs & 0x1) && (ARM_INSN_REG_RN (insn) == 15))
182 ARM_INSN_REG_SET_RN (insn, i);
183 if ((uregs & 0x2) && (ARM_INSN_REG_RD (insn) == 15))
184 ARM_INSN_REG_SET_RD (insn, i);
185 if ((uregs & 0x4) && (ARM_INSN_REG_RS (insn) == 15))
186 ARM_INSN_REG_SET_RS (insn, i);
187 if ((uregs & 0x8) && (ARM_INSN_REG_RM (insn) == 15))
188 ARM_INSN_REG_SET_RM (insn, i);
190 insns[UPROBES_TRAMP_INSN_IDX] = insn;
191 // set register to restore
192 ARM_INSN_REG_SET_RD (insns[3], i);
197 int arch_check_insn (struct arch_specific_insn *ainsn)
200 // check instructions that can change PC by nature
201 if (ARM_INSN_MATCH (UNDEF, ainsn->insn[0]) ||
202 ARM_INSN_MATCH (AUNDEF, ainsn->insn[0]) ||
203 ARM_INSN_MATCH (SWI, ainsn->insn[0]) ||
204 ARM_INSN_MATCH (BREAK, ainsn->insn[0]) ||
205 ARM_INSN_MATCH (B, ainsn->insn[0]) ||
206 ARM_INSN_MATCH (BL, ainsn->insn[0]) ||
207 ARM_INSN_MATCH (BLX1, ainsn->insn[0]) ||
208 ARM_INSN_MATCH (BLX2, ainsn->insn[0]) ||
209 ARM_INSN_MATCH (BX, ainsn->insn[0]) ||
210 ARM_INSN_MATCH (BXJ, ainsn->insn[0]))
212 DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
215 #ifndef CONFIG_CPU_V7
216 // check instructions that can write result to PC
217 else if ((ARM_INSN_MATCH (DPIS, ainsn->insn[0]) ||
218 ARM_INSN_MATCH (DPRS, ainsn->insn[0]) ||
219 ARM_INSN_MATCH (DPI, ainsn->insn[0]) ||
220 ARM_INSN_MATCH (LIO, ainsn->insn[0]) ||
221 ARM_INSN_MATCH (LRO, ainsn->insn[0])) &&
222 (ARM_INSN_REG_RD (ainsn->insn[0]) == 15))
224 DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
227 #endif // CONFIG_CPU_V7
228 // check special instruction loads store multiple registers
229 else if ((ARM_INSN_MATCH (LM, ainsn->insn[0]) || ARM_INSN_MATCH (SM, ainsn->insn[0])) &&
230 // store pc or load to pc
231 (ARM_INSN_REG_MR (ainsn->insn[0], 15) ||
232 // store/load with pc update
233 ((ARM_INSN_REG_RN (ainsn->insn[0]) == 15) && (ainsn->insn[0] & 0x200000))))
235 DBPRINTF ("arch_check_insn: %lx\n", ainsn->insn[0]);
241 int arch_prepare_kretprobe (struct kretprobe *p)
243 DBPRINTF("Warrning: arch_prepare_kretprobe is not implemented\n");
247 int arch_prepare_kprobe (struct kprobe *p)
249 kprobe_opcode_t insns[KPROBES_TRAMP_LEN];
255 kprobe_opcode_t insn[MAX_INSN_SIZE];
256 struct arch_specific_insn ainsn;
257 /* insn: must be on special executable page on i386. */
258 p->ainsn.insn = get_insn_slot (NULL, 0);
261 memcpy (insn, p->addr, MAX_INSN_SIZE * sizeof (kprobe_opcode_t));
263 ret = arch_check_insn (&ainsn);
266 p->opcode = *p->addr;
268 p->ainsn.boostable = 1;
271 if (ARM_INSN_MATCH (DPIS, insn[0]) || ARM_INSN_MATCH (LRO, insn[0]) ||
272 ARM_INSN_MATCH (SRO, insn[0]))
276 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
277 (ARM_INSN_MATCH (SRO, insn[0]) && (ARM_INSN_REG_RD (insn[0]) == 15)))
280 DBPRINTF ("Unboostable insn %lx, DPIS/LRO/SRO\n", insn[0]);
285 else if (ARM_INSN_MATCH (DPI, insn[0]) || ARM_INSN_MATCH (LIO, insn[0]) ||
286 ARM_INSN_MATCH (SIO, insn[0]))
290 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_MATCH (SIO, insn[0]) &&
291 (ARM_INSN_REG_RD (insn[0]) == 15)))
295 DBPRINTF ("Unboostable insn %lx/%p/%d, DPI/LIO/SIO\n", insn[0], p, p->ainsn.boostable);
299 else if (ARM_INSN_MATCH (DPRS, insn[0]))
303 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
304 (ARM_INSN_REG_RS (insn[0]) == 15))
308 DBPRINTF ("Unboostable insn %lx, DPRS\n", insn[0]);
312 else if (ARM_INSN_MATCH (SM, insn[0]))
316 if (ARM_INSN_REG_MR (insn[0], 15))
319 DBPRINTF ("Unboostable insn %lx, SM\n", insn[0]);
323 // check instructions that can write result to SP andu uses PC
324 if (pc_dep && (ARM_INSN_REG_RD (ainsn.insn[0]) == 13))
328 //printk ("insn writes result to SP and uses PC: %lx/%d\n", ainsn.insn[0], count);
329 free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
335 memcpy (insns, pc_dep_insn_execbuf, sizeof (insns));
336 if (prep_pc_dep_insn_execbuf (insns, insn[0], uregs) != 0)
338 DBPRINTF ("failed to prepare exec buffer for insn %lx!", insn[0]);
339 free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
342 //insns[KPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
343 insns[6] = (kprobe_opcode_t) (p->addr + 2);
347 memcpy (insns, gen_insn_execbuf, sizeof (insns));
348 insns[KPROBES_TRAMP_INSN_IDX] = insn[0];
350 //insns[KPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
351 insns[7] = (kprobe_opcode_t) (p->addr + 1);
352 DBPRINTF ("arch_prepare_kprobe: insn %lx", insn[0]);
353 DBPRINTF ("arch_prepare_kprobe: to %p - %lx %lx %lx %lx %lx %lx %lx %lx %lx",
354 p->ainsn.insn, insns[0], insns[1], insns[2], insns[3], insns[4],
355 insns[5], insns[6], insns[7], insns[8]);
356 memcpy (p->ainsn.insn, insns, sizeof(insns));
361 free_insn_slot (&kprobe_insn_pages, NULL, p->ainsn.insn, 0);
368 static unsigned int arch_construct_brunch (unsigned int base, unsigned int addr, int link)
370 kprobe_opcode_t insn;
371 unsigned int bpi = (unsigned int) base - (unsigned int) addr - 8;
373 DBPRINTF ("base=%x addr=%x base-addr-8=%x\n", base, addr, bpi);
374 if (abs (insn & 0xffffff) > 0xffffff)
376 DBPRINTF ("ERROR: kprobe address out of range\n");
379 insn = insn & 0xffffff;
380 insn = insn | ((link != 0) ? 0xeb000000 : 0xea000000);
381 DBPRINTF ("insn=%lX\n", insn);
382 return (unsigned int) insn;
385 int arch_prepare_uprobe (struct kprobe *p, struct task_struct *task, int atomic)
388 kprobe_opcode_t insns[UPROBES_TRAMP_LEN];
392 if ((unsigned long) p->addr & 0x01)
394 DBPRINTF ("Attempt to register kprobe at an unaligned address");
400 kprobe_opcode_t insn[MAX_INSN_SIZE];
401 struct arch_specific_insn ainsn;
403 if (!read_proc_vm_atomic (task, (unsigned long) p->addr, &insn, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
404 panic ("failed to read memory %p!\n", p->addr);
406 ret = arch_check_insn (&ainsn);
410 p->ainsn.insn = get_insn_slot(task, atomic);
414 p->ainsn.boostable = 1;
417 if (ARM_INSN_MATCH (DPIS, insn[0]) || ARM_INSN_MATCH (LRO, insn[0]) ||
418 ARM_INSN_MATCH (SRO, insn[0]))
422 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
423 (ARM_INSN_MATCH (SRO, insn[0]) && (ARM_INSN_REG_RD (insn[0]) == 15)))
426 DBPRINTF ("Unboostable insn %lx, DPIS/LRO/SRO\n", insn[0]);
431 else if (ARM_INSN_MATCH (DPI, insn[0]) || ARM_INSN_MATCH (LIO, insn[0]) ||
432 ARM_INSN_MATCH (SIO, insn[0]))
436 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_MATCH (SIO, insn[0]) &&
437 (ARM_INSN_REG_RD (insn[0]) == 15)))
441 DBPRINTF ("Unboostable insn %lx/%p/%d, DPI/LIO/SIO\n", insn[0], p, p->ainsn.boostable);
445 else if (ARM_INSN_MATCH (DPRS, insn[0]))
449 if ((ARM_INSN_REG_RN (insn[0]) == 15) || (ARM_INSN_REG_RM (insn[0]) == 15) ||
450 (ARM_INSN_REG_RS (insn[0]) == 15))
454 DBPRINTF ("Unboostable insn %lx, DPRS\n", insn[0]);
458 else if (ARM_INSN_MATCH (SM, insn[0]))
462 if (ARM_INSN_REG_MR (insn[0], 15))
465 DBPRINTF ("Unboostable insn %lx, SM\n", insn[0]);
469 // check instructions that can write result to SP andu uses PC
470 if (pc_dep && (ARM_INSN_REG_RD (ainsn.insn[0]) == 13))
474 //printk ("insn writes result to SP and uses PC: %lx/%d\n", ainsn.insn[0], count);
475 free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
481 memcpy (insns, pc_dep_insn_execbuf, sizeof (insns));
482 if (prep_pc_dep_insn_execbuf (insns, insn[0], uregs) != 0)
484 DBPRINTF ("failed to prepare exec buffer for insn %lx!", insn[0]);
485 free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
488 //insns[UPROBES_TRAMP_SS_BREAK_IDX] = BREAKPOINT_INSTRUCTION;
489 insns[6] = (kprobe_opcode_t) (p->addr + 2);
493 memcpy (insns, gen_insn_execbuf, sizeof (insns));
494 insns[UPROBES_TRAMP_INSN_IDX] = insn[0];
496 insns[UPROBES_TRAMP_RET_BREAK_IDX] = UNDEF_INSTRUCTION;
497 insns[7] = (kprobe_opcode_t) (p->addr + 1);
498 DBPRINTF ("arch_prepare_uprobe: to %p - %lx %lx %lx %lx %lx %lx %lx %lx %lx",
499 p->ainsn.insn, insns[0], insns[1], insns[2], insns[3], insns[4],
500 insns[5], insns[6], insns[7], insns[8]);
503 if (!write_proc_vm_atomic (task, (unsigned long) p->ainsn.insn, insns, sizeof (insns)))
505 panic("failed to write memory %p!\n", p->ainsn.insn);
506 DBPRINTF ("failed to write insn slot to process memory: insn %p, addr %p, probe %p!", insn, p->ainsn.insn, p->addr);
507 /*printk ("failed to write insn slot to process memory: %p/%d insn %lx, addr %p, probe %p!\n",
508 task, task->pid, insn, p->ainsn.insn, p->addr);*/
509 free_insn_slot (&uprobe_insn_pages, task, p->ainsn.insn, 0);
519 int arch_prepare_uretprobe (struct kretprobe *p, struct task_struct *task)
521 DBPRINTF("Warrning: arch_prepare_uretprobe is not implemented\n");
525 void prepare_singlestep (struct kprobe *p, struct pt_regs *regs)
529 regs->uregs[15] = (unsigned long) p->ss_addr;
533 regs->uregs[15] = (unsigned long) p->ainsn.insn;
536 void save_previous_kprobe (struct kprobe_ctlblk *kcb, struct kprobe *cur_p)
538 if (kcb->prev_kprobe.kp != NULL)
540 DBPRINTF ("no space to save new probe[]: task = %d/%s", current->pid, current->comm);
543 kcb->prev_kprobe.kp = kprobe_running ();
544 kcb->prev_kprobe.status = kcb->kprobe_status;
547 void restore_previous_kprobe (struct kprobe_ctlblk *kcb)
549 __get_cpu_var (current_kprobe) = kcb->prev_kprobe.kp;
550 kcb->kprobe_status = kcb->prev_kprobe.status;
551 kcb->prev_kprobe.kp = NULL;
552 kcb->prev_kprobe.status = 0;
555 void set_current_kprobe (struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
557 __get_cpu_var (current_kprobe) = p;
558 DBPRINTF ("set_current_kprobe: p=%p addr=%p\n", p, p->addr);
562 int kprobe_handler (struct pt_regs *regs)
564 struct kprobe *p = 0;
565 int ret = 0, pid = 0, retprobe = 0, reenter = 0;
566 kprobe_opcode_t *addr = NULL, *ssaddr = 0;
567 struct kprobe_ctlblk *kcb;
569 /* We're in an interrupt, but this is clear and BUG()-safe. */
571 addr = (kprobe_opcode_t *) (regs->uregs[15] - 4);
572 DBPRINTF ("KPROBE: regs->uregs[15] = 0x%lx addr = 0x%p\n", regs->uregs[15], addr);
573 regs->uregs[15] -= 4;
574 //DBPRINTF("regs->uregs[14] = 0x%lx\n", regs->uregs[14]);
578 kcb = get_kprobe_ctlblk ();
580 if (user_mode (regs))
582 //DBPRINTF("exception[%lu] from user mode %s/%u addr %p (%lx).", nCount, current->comm, current->pid, addr, regs->uregs[14]);
586 /* Check we're not actually recursing */
587 if (kprobe_running ())
589 DBPRINTF ("lock???");
590 p = get_kprobe (addr, pid, current);
593 if(!pid && (addr == (kprobe_opcode_t *)kretprobe_trampoline)){
594 save_previous_kprobe (kcb, p);
595 kcb->kprobe_status = KPROBE_REENTER;
599 /* We have reentered the kprobe_handler(), since
600 * another probe was hit while within the handler.
601 * We here save the original kprobes variables and
602 * just single step on the instruction of the new probe
603 * without calling any user handlers.
605 if(!p->ainsn.boostable){
606 save_previous_kprobe (kcb, p);
607 set_current_kprobe (p, regs, kcb);
609 kprobes_inc_nmissed_count (p);
610 prepare_singlestep (p, regs);
611 if(!p->ainsn.boostable)
612 kcb->kprobe_status = KPROBE_REENTER;
613 preempt_enable_no_resched ();
619 if(pid) { //we can reenter probe upon uretprobe exception
620 DBPRINTF ("check for UNDEF_INSTRUCTION %p\n", addr);
621 // UNDEF_INSTRUCTION from user space
622 p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
624 save_previous_kprobe (kcb, p);
625 kcb->kprobe_status = KPROBE_REENTER;
628 DBPRINTF ("uretprobe %p\n", addr);
632 p = __get_cpu_var (current_kprobe);
633 DBPRINTF ("kprobe_running !!! p = 0x%p p->break_handler = 0x%p", p, p->break_handler);
634 /*if (p->break_handler && p->break_handler(p, regs)) {
635 DBPRINTF("kprobe_running !!! goto ss");
638 DBPRINTF ("unknown uprobe at %p cur at %p/%p\n", addr, p->addr, p->ainsn.insn);
640 ssaddr = p->ainsn.insn + UPROBES_TRAMP_SS_BREAK_IDX;
642 ssaddr = p->ainsn.insn + KPROBES_TRAMP_SS_BREAK_IDX;
645 regs->uregs[15] = (unsigned long) (p->addr + 1);
646 DBPRINTF ("finish step at %p cur at %p/%p, redirect to %lx\n", addr, p->addr, p->ainsn.insn, regs->uregs[15]);
647 if (kcb->kprobe_status == KPROBE_REENTER) {
648 restore_previous_kprobe (kcb);
651 reset_current_kprobe ();
654 DBPRINTF ("kprobe_running !!! goto no");
656 /* If it's not ours, can't be delete race, (we hold lock). */
657 DBPRINTF ("no_kprobe");
663 //if(einsn != UNDEF_INSTRUCTION) {
664 DBPRINTF ("get_kprobe %p-%d", addr, pid);
666 p = get_kprobe (addr, pid, current);
670 DBPRINTF ("search UNDEF_INSTRUCTION %p\n", addr);
671 // UNDEF_INSTRUCTION from user space
672 p = get_kprobe_by_insn_slot (addr-UPROBES_TRAMP_RET_BREAK_IDX, pid, current);
674 /* Not one of ours: let kernel handle it */
675 DBPRINTF ("no_kprobe");
676 //printk("no_kprobe2 ret = %d\n", ret);
680 DBPRINTF ("uretprobe %p\n", addr);
683 /* Not one of ours: let kernel handle it */
684 DBPRINTF ("no_kprobe");
685 //printk("no_kprobe2 ret = %d\n", ret);
690 set_current_kprobe (p, regs, kcb);
692 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
694 if (retprobe) //(einsn == UNDEF_INSTRUCTION)
695 ret = trampoline_probe_handler (p, regs);
696 else if (p->pre_handler)
698 ret = p->pre_handler (p, regs);
699 if(!p->ainsn.boostable)
700 kcb->kprobe_status = KPROBE_HIT_SS;
701 else if(p->pre_handler != trampoline_probe_handler)
702 reset_current_kprobe ();
707 DBPRINTF ("p->pre_handler 1");
708 /* handler has already set things up, so skip ss setup */
711 DBPRINTF ("p->pre_handler 0");
714 preempt_enable_no_resched ();
719 void patch_suspended_task_ret_addr(struct task_struct *p, struct kretprobe *rp)
721 struct kretprobe_instance *ri = NULL;
722 struct hlist_node *node, *tmp;
723 struct hlist_head *head;
727 spin_lock_irqsave (&kretprobe_lock, flags);
728 head = kretprobe_inst_table_head (p);
729 hlist_for_each_entry_safe (ri, node, tmp, head, hlist){
730 if ((ri->rp == rp) && (p == ri->task)){
735 spin_unlock_irqrestore (&kretprobe_lock, flags);
737 #ifndef task_thread_info
738 #define task_thread_info(task) (task)->thread_info
739 #endif // task_thread_info
743 if(thread_saved_pc(p) != (unsigned long)&kretprobe_trampoline){
744 ri->ret_addr = (kprobe_opcode_t *)thread_saved_pc(p);
745 task_thread_info(p)->cpu_context.pc = (unsigned long) &kretprobe_trampoline;
750 if ((ri = get_free_rp_inst(rp)) != NULL)
755 ri->ret_addr = (kprobe_opcode_t *)thread_saved_pc(p);
756 task_thread_info(p)->cpu_context.pc = (unsigned long) &kretprobe_trampoline;
758 // printk("change2 saved pc %p->%p for %d/%d/%p\n", ri->ret_addr, &kretprobe_trampoline, p->tgid, p->pid, p);
761 printk("no ri for %d\n", p->pid);
766 int setjmp_pre_handler (struct kprobe *p, struct pt_regs *regs)
768 struct jprobe *jp = container_of (p, struct jprobe, kp);
769 kprobe_pre_entry_handler_t pre_entry;
773 p = __get_cpu_var (current_kprobe);
776 DBPRINTF ("pjp = 0x%p jp->entry = 0x%p", jp, jp->entry);
777 entry = (entry_point_t) jp->entry;
778 pre_entry = (kprobe_pre_entry_handler_t) jp->pre_entry;
780 // DIE("entry NULL", regs)
781 DBPRINTF ("entry = 0x%p jp->entry = 0x%p", entry, jp->entry);
783 //call handler for all kernel probes and user space ones which belong to current tgid
784 if (!p->tgid || (p->tgid == current->tgid))
786 if(!p->tgid && ((unsigned int)p->addr == sched_addr) && sched_rp){
787 struct task_struct *p, *g;
790 if(current != &init_task)
791 patch_suspended_task_ret_addr(&init_task, sched_rp);
793 do_each_thread(g, p){
796 patch_suspended_task_ret_addr(p, sched_rp);
797 } while_each_thread(g, p);
801 p->ss_addr = (void *)pre_entry (jp->priv_arg, regs);
803 entry (regs->ARM_r0, regs->ARM_r1, regs->ARM_r2, regs->ARM_r3, regs->ARM_r4, regs->ARM_r5);
807 arch_uprobe_return ();
813 arch_uprobe_return ();
815 prepare_singlestep (p, regs);
820 void jprobe_return (void)
822 preempt_enable_no_resched();
825 void arch_uprobe_return (void)
827 preempt_enable_no_resched();
830 int longjmp_break_handler (struct kprobe *p, struct pt_regs *regs)
833 //kprobe_opcode_t insn = BREAKPOINT_INSTRUCTION;
834 kprobe_opcode_t insns[2];
838 insns[0] = BREAKPOINT_INSTRUCTION;
839 insns[1] = p->opcode;
840 //p->opcode = *p->addr;
841 if (read_proc_vm_atomic (current, (unsigned long) (p->addr), &(p->opcode), sizeof (p->opcode)) < sizeof (p->opcode))
843 printk ("ERROR[%lu]: failed to read vm of proc %s/%u addr %p.", nCount, current->comm, current->pid, p->addr);
846 //*p->addr = BREAKPOINT_INSTRUCTION;
847 //*(p->addr+1) = p->opcode;
848 if (write_proc_vm_atomic (current, (unsigned long) (p->addr), insns, sizeof (insns)) < sizeof (insns))
850 printk ("ERROR[%lu]: failed to write vm of proc %s/%u addr %p.", nCount, current->comm, current->pid, p->addr);
856 DBPRINTF ("p->opcode = 0x%lx *p->addr = 0x%lx p->addr = 0x%p\n", p->opcode, *p->addr, p->addr);
857 *(p->addr + 1) = p->opcode;
858 p->opcode = *p->addr;
859 *p->addr = BREAKPOINT_INSTRUCTION;
860 flush_icache_range ((unsigned int) p->addr, (unsigned int) (((unsigned int) p->addr) + (sizeof (kprobe_opcode_t) * 2)));
863 reset_current_kprobe ();
871 void arch_arm_kprobe (struct kprobe *p)
873 *p->addr = BREAKPOINT_INSTRUCTION;
874 flush_icache_range ((unsigned long) p->addr, (unsigned long) p->addr + sizeof (kprobe_opcode_t));
877 void arch_disarm_kprobe (struct kprobe *p)
879 *p->addr = p->opcode;
880 flush_icache_range ((unsigned long) p->addr, (unsigned long) p->addr + sizeof (kprobe_opcode_t));
884 int trampoline_probe_handler (struct kprobe *p, struct pt_regs *regs)
887 struct kretprobe_instance *ri = NULL;
888 struct hlist_head *head, empty_rp;
889 struct hlist_node *node, *tmp;
890 unsigned long flags, orig_ret_address = 0;
891 unsigned long trampoline_address = (unsigned long) &kretprobe_trampoline;
892 struct kretprobe *crp = NULL;
893 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk ();
898 // in case of user space retprobe trampoline is at the Nth instruction of US tramp
899 trampoline_address = (unsigned long)(p->ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
902 INIT_HLIST_HEAD (&empty_rp);
903 spin_lock_irqsave (&kretprobe_lock, flags);
904 head = kretprobe_inst_table_head (current);
907 * It is possible to have multiple instances associated with a given
908 * task either because an multiple functions in the call path
909 * have a return probe installed on them, and/or more then one
910 * return probe was registered for a target function.
912 * We can handle this because:
913 * - instances are always inserted at the head of the list
914 * - when multiple return probes are registered for the same
915 * function, the first instance's ret_addr will point to the
916 * real return address, and all the rest will point to
917 * kretprobe_trampoline
919 hlist_for_each_entry_safe (ri, node, tmp, head, hlist)
921 if (ri->task != current)
922 /* another task is sharing our hash bucket */
924 if (ri->rp && ri->rp->handler){
925 ri->rp->handler (ri, regs, ri->rp->priv_arg);
928 orig_ret_address = (unsigned long) ri->ret_addr;
929 recycle_rp_inst (ri, &empty_rp);
930 if (orig_ret_address != trampoline_address)
932 * This is the real return address. Any other
933 * instances associated with this task are for
934 * other calls deeper on the call stack
938 kretprobe_assert (ri, orig_ret_address, trampoline_address);
939 //BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
940 if (trampoline_address != (unsigned long) &kretprobe_trampoline){
941 if (ri->rp2) BUG_ON (ri->rp2->kp.tgid == 0);
942 if (ri->rp) BUG_ON (ri->rp->kp.tgid == 0);
943 else if (ri->rp2) BUG_ON (ri->rp2->kp.tgid == 0);
945 if ((ri->rp && ri->rp->kp.tgid) || (ri->rp2 && ri->rp2->kp.tgid))
946 BUG_ON (trampoline_address == (unsigned long) &kretprobe_trampoline);
948 regs->uregs[14] = orig_ret_address;
949 DBPRINTF ("regs->uregs[14] = 0x%lx\n", regs->uregs[14]);
950 DBPRINTF ("regs->uregs[15] = 0x%lx\n", regs->uregs[15]);
951 if (trampoline_address != (unsigned long) &kretprobe_trampoline)
952 regs->uregs[15] = orig_ret_address;
954 regs->uregs[15] += 4;
955 DBPRINTF ("regs->uregs[15] = 0x%lx\n", regs->uregs[15]);
957 if(p){ // ARM, MIPS, X86 user space
958 if (kcb->kprobe_status == KPROBE_REENTER)
959 restore_previous_kprobe (kcb);
961 reset_current_kprobe ();
963 //TODO: test - enter function, delete us retprobe, exit function
964 // for user space retprobes only - deferred deletion
965 if (trampoline_address != (unsigned long) &kretprobe_trampoline)
967 // if we are not at the end of the list and current retprobe should be disarmed
971 /*sprintf(die_msg, "deferred disarm p->addr = %p [%lx %lx %lx]\n",
972 crp->kp.addr, *kaddrs[0], *kaddrs[1], *kaddrs[2]);
973 DIE(die_msg, regs); */
974 // look for other instances for the same retprobe
975 hlist_for_each_entry_continue (ri, node, hlist)
977 if (ri->task != current)
978 continue; /* another task is sharing our hash bucket */
979 if (ri->rp2 == crp) //if instance belong to the same retprobe
983 { // if there are no more instances for this retprobe
985 DBPRINTF ("defered retprobe deletion p->addr = %p", crp->kp.addr);
986 unregister_uprobe (&crp->kp, current, 1);
993 spin_unlock_irqrestore (&kretprobe_lock, flags);
994 hlist_for_each_entry_safe (ri, node, tmp, &empty_rp, hlist)
996 hlist_del (&ri->hlist);
1000 preempt_enable_no_resched ();
1002 * By returning a non-zero value, we are telling
1003 * kprobe_handler() that we don't want the post_handler
1004 * to run (and have re-enabled preemption)
1009 void __arch_prepare_kretprobe (struct kretprobe *rp, struct pt_regs *regs)
1012 struct kretprobe_instance *ri;
1014 DBPRINTF ("start\n");
1015 //TODO: test - remove retprobe after func entry but before its exit
1016 if ((ri = get_free_rp_inst (rp)) != NULL)
1021 ri->ret_addr = (kprobe_opcode_t *) regs->uregs[14];
1023 regs->uregs[14] = (unsigned long) (rp->kp.ainsn.insn + UPROBES_TRAMP_RET_BREAK_IDX);
1024 else /* Replace the return addr with trampoline addr */
1025 regs->uregs[14] = (unsigned long) &kretprobe_trampoline;
1026 DBPRINTF ("ret addr set to %p->%lx\n", ri->ret_addr, regs->uregs[14]);
1030 DBPRINTF ("WARNING: missed retprobe %p\n", rp->kp.addr);
1036 int asm_init_module_dependencies()
1038 //No module dependencies
1042 int __init arch_init_kprobes (void)
1045 unsigned int do_bp_handler;
1046 unsigned int kprobe_handler_addr;
1048 unsigned int insns_num = 0;
1049 unsigned int code_size = 0;
1053 if (arch_init_module_dependencies())
1055 DBPRINTF ("Unable to init module dependencies\n");
1059 do_bp_handler = (unsigned int) kallsyms_search ("do_undefinstr");
1061 kprobe_handler_addr = (unsigned int) &kprobe_handler;
1062 insns_num = sizeof (arr_traps_template) / sizeof (arr_traps_template[0]);
1063 code_size = insns_num * sizeof (unsigned int);
1064 DBPRINTF ("insns_num = %d\n", insns_num);
1065 // Save original code
1066 arr_traps_original = kmalloc (code_size, GFP_KERNEL);
1067 if (!arr_traps_original)
1069 DBPRINTF ("Unable to allocate space for original code of <do_bp>!\n");
1072 memcpy (arr_traps_original, (void *) do_bp_handler, code_size);
1074 arr_traps_template[NOTIFIER_CALL_CHAIN_INDEX] = arch_construct_brunch ((unsigned int)kprobe_handler, do_bp_handler + NOTIFIER_CALL_CHAIN_INDEX * 4, 1);
1077 memcpy ((void *) do_bp_handler, arr_traps_template, code_size);
1078 flush_icache_range (do_bp_handler, do_bp_handler + code_size);
1079 if((ret = register_kprobe (&trampoline_p, 0)) != 0){
1080 //unregister_jprobe(&do_exit_p, 0);
1087 void __exit arch_exit_kprobes (void)
1089 unsigned int do_bp_handler;
1091 unsigned int insns_num = 0;
1092 unsigned int code_size = 0;
1094 // Get instruction address
1095 do_bp_handler = (unsigned int) kallsyms_search ("do_undefinstr");
1097 //unregister_jprobe(&do_exit_p, 0);
1099 // Replace back the original code
1101 insns_num = sizeof (arr_traps_template) / sizeof (arr_traps_template[0]);
1102 code_size = insns_num * sizeof (unsigned int);
1103 memcpy ((void *) do_bp_handler, arr_traps_original, code_size);
1104 flush_icache_range (do_bp_handler, do_bp_handler + code_size);
1105 kfree (arr_traps_original);
1106 arr_traps_original = NULL;
1110 EXPORT_SYMBOL_GPL (arch_uprobe_return);
1111 EXPORT_SYMBOL_GPL (arch_exit_kprobes);