2 * Dynamic Binary Instrumentation Module based on KProbes
3 * modules/kprobe/dbi_kprobes_deps.h
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) Samsung Electronics, 2006-2010
21 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
22 * Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
23 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
27 #include <linux/module.h>
28 #include <linux/sched.h>
30 #include <asm/pgtable.h>
32 #include "dbi_kprobes_deps.h"
33 #include "dbi_kdebug.h"
36 #include <linux/slab.h>
39 unsigned long sched_addr;
40 unsigned long fork_addr;
42 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
43 struct mm_struct* init_mm_ptr;
44 struct mm_struct init_mm;
48 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
49 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff);
50 DECLARE_MOD_DEP_WRAPPER(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff)
51 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len, prot, flags, pgoff)
52 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) */
54 /* copy_to_user_page */
55 #ifndef copy_to_user_page
56 DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len);
57 DECLARE_MOD_DEP_WRAPPER(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len)
58 IMP_MOD_DEP_WRAPPER(copy_to_user_page, vma, page, uaddr, dst, src, len)
59 #endif /* copy_to_user_page */
62 DECLARE_MOD_CB_DEP(kallsyms_search, unsigned long, const char *name);
63 DECLARE_MOD_FUNC_DEP(access_process_vm, int, struct task_struct * tsk, unsigned long addr, void *buf, int len, int write);
65 DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr);
67 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
68 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
69 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access);
72 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags);
73 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
75 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
76 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct mm_struct *mm);
77 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
78 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct task_struct *tsk);
79 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
81 #ifdef CONFIG_HUGETLB_PAGE
82 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, int write);
85 #ifdef __HAVE_ARCH_GATE_AREA
86 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
87 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct mm_struct *mm, unsigned long addr);
88 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
89 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct task_struct *task, unsigned long addr);
90 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
91 #endif /* __HAVE_ARCH_GATE_AREA */
93 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
94 DECLARE_MOD_FUNC_DEP(in_gate_area_no_mm, int, unsigned long addr);
95 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
96 DECLARE_MOD_FUNC_DEP(in_gate_area_no_task, int, unsigned long addr);
97 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
100 DECLARE_MOD_FUNC_DEP(follow_page, \
101 struct page *, struct vm_area_struct * vma, \
102 unsigned long address, unsigned int foll_flags);
103 DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
104 void, struct vm_area_struct *vma, struct page *page, \
105 unsigned long vmaddr);
106 DECLARE_MOD_FUNC_DEP(vm_normal_page, \
107 struct page *, struct vm_area_struct *vma, \
108 unsigned long addr, pte_t pte);
110 DECLARE_MOD_FUNC_DEP(flush_ptrace_access, \
111 void, struct vm_area_struct *vma, struct page *page, \
112 unsigned long uaddr, void *kaddr, unsigned long len, int write);
115 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
116 DECLARE_MOD_FUNC_DEP(put_task_struct, \
117 void, struct task_struct *tsk);
119 DECLARE_MOD_FUNC_DEP(put_task_struct, \
120 void, struct rcu_head * rhp);
123 DECLARE_MOD_DEP_WRAPPER(access_process_vm, int, struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
124 IMP_MOD_DEP_WRAPPER (access_process_vm, tsk, addr, buf, len, write)
126 DECLARE_MOD_DEP_WRAPPER (find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr)
127 IMP_MOD_DEP_WRAPPER (find_extend_vma, mm, addr)
129 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
130 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
131 DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
132 int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access)
133 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, write_access)
136 DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
137 int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags)
138 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, flags)
141 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
142 DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
143 struct vm_area_struct *, struct mm_struct *mm)
144 IMP_MOD_DEP_WRAPPER (get_gate_vma, mm)
145 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
146 DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
147 struct vm_area_struct *, struct task_struct *tsk)
148 IMP_MOD_DEP_WRAPPER (get_gate_vma, tsk)
149 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
151 #ifdef CONFIG_HUGETLB_PAGE
152 DECLARE_MOD_DEP_WRAPPER (follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, unsigned int write)
153 IMP_MOD_DEP_WRAPPER (follow_hugetlb_page, mm, vma, pages, vmas, position, length, i, write)
156 static inline int dbi_in_gate_area(struct task_struct *task, unsigned long addr)
158 #ifdef __HAVE_ARCH_GATE_AREA
159 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
160 struct mm_struct *mm = task->mm;
161 IMP_MOD_DEP_WRAPPER (in_gate_area, mm, addr)
162 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
163 IMP_MOD_DEP_WRAPPER (in_gate_area, task, addr)
164 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
165 #else /*__HAVE_ARCH_GATE_AREA */
166 return in_gate_area(task, addr);
167 #endif/*__HAVE_ARCH_GATE_AREA */
171 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
172 DECLARE_MOD_DEP_WRAPPER(in_gate_area_no_mm, int, unsigned long addr)
173 IMP_MOD_DEP_WRAPPER(in_gate_area_no_mm, addr)
174 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
175 DECLARE_MOD_DEP_WRAPPER(in_gate_area_no_task, int, unsigned long addr)
176 IMP_MOD_DEP_WRAPPER(in_gate_area_no_task, addr)
177 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
179 static inline int dbi_in_gate_area_no_xxx(unsigned long addr)
181 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
182 return in_gate_area_no_mm(addr);
183 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
184 return in_gate_area_no_task(addr);
185 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
189 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 11))
190 DECLARE_MOD_DEP_WRAPPER (follow_page, \
191 struct page *, struct vm_area_struct * vma, \
192 unsigned long address, unsigned int foll_flags)
193 IMP_MOD_DEP_WRAPPER (follow_page, vma, address, foll_flags)
195 DECLARE_MOD_DEP_WRAPPER (__flush_anon_page, \
196 void, struct vm_area_struct *vma, \
197 struct page *page, unsigned long vmaddr)
198 IMP_MOD_DEP_WRAPPER (__flush_anon_page, vma, page, vmaddr)
200 DECLARE_MOD_DEP_WRAPPER(vm_normal_page, \
201 struct page *, struct vm_area_struct *vma, \
202 unsigned long addr, pte_t pte)
203 IMP_MOD_DEP_WRAPPER (vm_normal_page, vma, addr, pte)
205 DECLARE_MOD_DEP_WRAPPER (flush_ptrace_access, \
206 void, struct vm_area_struct *vma, struct page *page, \
207 unsigned long uaddr, void *kaddr, unsigned long len, int write)
208 IMP_MOD_DEP_WRAPPER (flush_ptrace_access, vma, page, uaddr, kaddr, len, write)
211 int init_module_dependencies()
214 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
215 init_mm_ptr = (struct mm_struct*) kallsyms_search ("init_mm");
216 memcmp(init_mm_ptr, &init_mm, sizeof(struct mm_struct));
219 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
220 INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
223 #ifndef copy_to_user_page
224 INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
225 #endif /* copy_to_user_page */
227 INIT_MOD_DEP_VAR(flush_ptrace_access, flush_ptrace_access);
228 INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
229 INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
231 #ifdef CONFIG_HUGETLB_PAGE
232 INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
235 #ifdef __HAVE_ARCH_GATE_AREA
236 INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
239 #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))
240 INIT_MOD_DEP_VAR(in_gate_area_no_mm, in_gate_area_no_mm);
241 #else /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
242 INIT_MOD_DEP_VAR(in_gate_area_no_task, in_gate_area_no_task);
243 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
245 INIT_MOD_DEP_VAR(follow_page, follow_page);
247 INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
248 INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
249 INIT_MOD_DEP_VAR(access_process_vm, access_process_vm);
251 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
252 # if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
253 INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
255 INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
258 INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
261 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
262 INIT_MOD_DEP_VAR(do_mmap_pgoff, do_mmap_pgoff);
268 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) /* FIXME: must be < 32 */
269 #define GUP_FLAGS_WRITE 0x1
270 #define GUP_FLAGS_FORCE 0x2
271 #define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
272 #define GUP_FLAGS_IGNORE_SIGKILL 0x8
273 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) */
275 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
276 static inline int use_zero_page(struct vm_area_struct *vma)
279 * We don't want to optimize FOLL_ANON for make_pages_present()
280 * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
281 * we want to get the page from the page tables to make sure
282 * that we serialize and update with any other user of that
285 if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
288 * And if we have a fault routine, it's not an anonymous region.
290 return !vma->vm_ops || !vma->vm_ops->fault;
294 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
295 unsigned long zero_pfn __read_mostly;
298 static inline int is_zero_pfn(unsigned long pfn)
300 return pfn == zero_pfn;
304 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
306 return stack_guard_page_start(vma, addr) ||
307 stack_guard_page_end(vma, addr+PAGE_SIZE);
310 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
311 unsigned long start, int nr_pages, unsigned int gup_flags,
312 struct page **pages, struct vm_area_struct **vmas,
316 unsigned long vm_flags;
322 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
325 * Require read or write permissions.
326 * If FOLL_FORCE is set, we only require the "MAY" flags.
328 vm_flags = (gup_flags & FOLL_WRITE) ?
329 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
330 vm_flags &= (gup_flags & FOLL_FORCE) ?
331 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
335 struct vm_area_struct *vma;
337 vma = find_extend_vma(mm, start);
338 if (!vma && dbi_in_gate_area_no_xxx(start)) {
339 unsigned long pg = start & PAGE_MASK;
345 /* user gate pages are read-only */
346 if (gup_flags & FOLL_WRITE) {
347 return i ? : -EFAULT;
350 pgd = pgd_offset_k(pg);
352 pgd = pgd_offset_gate(mm, pg);
353 BUG_ON(pgd_none(*pgd));
354 pud = pud_offset(pgd, pg);
355 BUG_ON(pud_none(*pud));
356 pmd = pmd_offset(pud, pg);
357 if (pmd_none(*pmd)) {
358 return i ? : -EFAULT;
360 VM_BUG_ON(pmd_trans_huge(*pmd));
361 pte = pte_offset_map(pmd, pg);
362 if (pte_none(*pte)) {
364 return i ? : -EFAULT;
366 vma = get_gate_vma(mm);
370 page = vm_normal_page(vma, start, *pte);
372 if (!(gup_flags & FOLL_DUMP) &&
373 is_zero_pfn(pte_pfn(*pte)))
374 page = pte_page(*pte);
377 return i ? : -EFAULT;
388 (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
389 !(vm_flags & vma->vm_flags)) {
390 return i ? : -EFAULT;
393 if (is_vm_hugetlb_page(vma)) {
394 i = follow_hugetlb_page(mm, vma, pages, vmas,
395 &start, &nr_pages, i, gup_flags);
401 unsigned int foll_flags = gup_flags;
404 * If we have a pending SIGKILL, don't keep faulting
405 * pages and potentially allocating memory.
407 if (unlikely(fatal_signal_pending(current))) {
408 return i ? i : -ERESTARTSYS;
411 /* cond_resched(); */
412 while (!(page = follow_page(vma, start, foll_flags))) {
414 unsigned int fault_flags = 0;
416 /* For mlock, just skip the stack guard page. */
417 if (foll_flags & FOLL_MLOCK) {
418 if (stack_guard_page(vma, start))
421 if (foll_flags & FOLL_WRITE)
422 fault_flags |= FAULT_FLAG_WRITE;
424 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
425 if (foll_flags & FOLL_NOWAIT)
426 fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT);
428 ret = handle_mm_fault(mm, vma, start,
431 if (ret & VM_FAULT_ERROR) {
432 if (ret & VM_FAULT_OOM) {
433 return i ? i : -ENOMEM;
435 if (ret & (VM_FAULT_HWPOISON |
436 VM_FAULT_HWPOISON_LARGE)) {
440 else if (gup_flags & FOLL_HWPOISON) {
447 if (ret & VM_FAULT_SIGBUS) {
448 return i ? i : -EFAULT;
454 if (ret & VM_FAULT_MAJOR)
460 if (ret & VM_FAULT_RETRY) {
467 * The VM_FAULT_WRITE bit tells us that
468 * do_wp_page has broken COW when necessary,
469 * even if maybe_mkwrite decided not to set
470 * pte_write. We can thus safely do subsequent
471 * page lookups as if they were reads. But only
472 * do so when looping for pte_write is futile:
473 * in some cases userspace may also be wanting
474 * to write to the gotten user page, which a
475 * read fault here might prevent (a readonly
476 * page might get reCOWed by userspace write).
478 if ((ret & VM_FAULT_WRITE) &&
479 !(vma->vm_flags & VM_WRITE))
480 foll_flags &= ~FOLL_WRITE;
482 /* cond_resched(); */
485 return i ? i : PTR_ERR(page);
490 flush_anon_page(vma, page, start);
491 flush_dcache_page(page);
499 } while (nr_pages && start < vma->vm_end);
504 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
506 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
507 unsigned long start, int len, int flags,
508 struct page **pages, struct vm_area_struct **vmas)
511 unsigned int vm_flags = 0;
512 int write = !!(flags & GUP_FLAGS_WRITE);
513 int force = !!(flags & GUP_FLAGS_FORCE);
514 int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
515 int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL);
520 * Require read or write permissions.
521 * If 'force' is set, we only require the "MAY" flags.
523 vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
524 vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
528 struct vm_area_struct *vma;
529 unsigned int foll_flags;
531 vma = find_vma(mm, start);
532 if (!vma && dbi_in_gate_area(tsk, start)) {
533 unsigned long pg = start & PAGE_MASK;
534 struct vm_area_struct *gate_vma = get_gate_vma(tsk);
540 /* user gate pages are read-only */
541 if (!ignore && write)
542 return i ? : -EFAULT;
544 pgd = pgd_offset_k(pg);
546 pgd = pgd_offset_gate(mm, pg);
547 BUG_ON(pgd_none(*pgd));
548 pud = pud_offset(pgd, pg);
549 BUG_ON(pud_none(*pud));
550 pmd = pmd_offset(pud, pg);
552 return i ? : -EFAULT;
553 pte = pte_offset_map(pmd, pg);
554 if (pte_none(*pte)) {
556 return i ? : -EFAULT;
559 struct page *page = vm_normal_page(gate_vma, start, *pte);
574 (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
575 (!ignore && !(vm_flags & vma->vm_flags)))
576 return i ? : -EFAULT;
578 if (is_vm_hugetlb_page(vma)) {
579 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
580 i = follow_hugetlb_page(mm, vma, pages, vmas,
583 i = follow_hugetlb_page(mm, vma, pages, vmas,
584 &start, &len, i, write);
589 foll_flags = FOLL_TOUCH;
591 foll_flags |= FOLL_GET;
593 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
594 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,30)
595 if (!write && use_zero_page(vma))
596 foll_flags |= FOLL_ANON;
605 * If we have a pending SIGKILL, don't keep faulting
606 * pages and potentially allocating memory, unless
607 * current is handling munlock--e.g., on exit. In
608 * that case, we are not allocating memory. Rather,
609 * we're only unlocking already resident/mapped pages.
611 if (unlikely(!ignore_sigkill &&
612 fatal_signal_pending(current)))
613 return i ? i : -ERESTARTSYS;
617 foll_flags |= FOLL_WRITE;
622 DBPRINTF ("pages = %p vma = %p\n", pages, vma);
623 while (!(page = follow_page(vma, start, foll_flags))) {
625 ret = handle_mm_fault(mm, vma, start,
626 foll_flags & FOLL_WRITE);
628 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
629 if (ret & VM_FAULT_WRITE)
630 foll_flags &= ~FOLL_WRITE;
632 switch (ret & ~VM_FAULT_WRITE) {
639 case VM_FAULT_SIGBUS:
640 return i ? i : -EFAULT;
642 return i ? i : -ENOMEM;
648 if (ret & VM_FAULT_ERROR) {
649 if (ret & VM_FAULT_OOM)
650 return i ? i : -ENOMEM;
651 else if (ret & VM_FAULT_SIGBUS)
652 return i ? i : -EFAULT;
655 if (ret & VM_FAULT_MAJOR)
661 * The VM_FAULT_WRITE bit tells us that
662 * do_wp_page has broken COW when necessary,
663 * even if maybe_mkwrite decided not to set
664 * pte_write. We can thus safely do subsequent
665 * page lookups as if they were reads. But only
666 * do so when looping for pte_write is futile:
667 * in some cases userspace may also be wanting
668 * to write to the gotten user page, which a
669 * read fault here might prevent (a readonly
670 * page might get reCOWed by userspace write).
672 if ((ret & VM_FAULT_WRITE) &&
673 !(vma->vm_flags & VM_WRITE))
674 foll_flags &= ~FOLL_WRITE;
682 return i ? i : PTR_ERR(page);
686 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
687 flush_anon_page(page, start);
689 flush_anon_page(vma, page, start);
691 flush_dcache_page(page);
698 } while (len && start < vma->vm_end);
703 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
705 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
706 unsigned long start, int len, int write, int force,
707 struct page **pages, struct vm_area_struct **vmas)
709 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
710 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) /* FIXME: must be >= 32! */
711 int flags = FOLL_TOUCH;
719 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
723 flags |= GUP_FLAGS_WRITE;
725 flags |= GUP_FLAGS_FORCE;
726 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
728 return __get_user_pages_uprobe(tsk, mm,
730 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
732 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
734 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
736 return get_user_pages(tsk, mm, start, len, write, force, pages, vmas);
740 #define ACCESS_PROCESS_OPTIMIZATION 0
742 #if ACCESS_PROCESS_OPTIMIZATION
744 #define GET_STEP_X(LEN, STEP) (((LEN) >= (STEP)) ? (STEP) : (LEN) % (STEP))
745 #define GET_STEP_4(LEN) GET_STEP_X((LEN), 4)
747 static void read_data_current(unsigned long addr, void *buf, int len)
752 for (step = GET_STEP_4(len); len; len -= step) {
753 switch (GET_STEP_4(len)) {
755 get_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
761 get_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
766 get_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
776 static void write_data_current(unsigned long addr, void *buf, int len)
781 for (step = GET_STEP_4(len); len; len -= step) {
782 switch (GET_STEP_4(len)) {
784 put_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
790 put_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
795 put_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
805 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
807 struct mm_struct *mm;
808 struct vm_area_struct *vma;
815 #if ACCESS_PROCESS_OPTIMIZATION
816 if (write == 0 && tsk == current) {
817 read_data_current(addr, buf, len);
822 mm = tsk->mm; /* function 'get_task_mm' is to be called */
826 /* ignore errors, just check how much was successfully transferred */
828 int bytes, ret, offset;
830 struct page *page = NULL;
832 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
833 write, 1, &page, &vma);
837 * Check if this is a VM_IO | VM_PFNMAP VMA, which
838 * we can access using slightly different code.
840 #ifdef CONFIG_HAVE_IOREMAP_PROT
841 vma = find_vma(mm, addr);
844 if (vma->vm_ops && vma->vm_ops->access)
845 ret = vma->vm_ops->access(vma, addr, buf,
853 offset = addr & (PAGE_SIZE-1);
854 if (bytes > PAGE_SIZE-offset)
855 bytes = PAGE_SIZE-offset;
857 maddr = kmap_atomic(page);
860 copy_to_user_page(vma, page, addr,
861 maddr + offset, buf, bytes);
862 set_page_dirty_lock(page);
864 copy_from_user_page(vma, page, addr,
865 buf, maddr + offset, bytes);
868 kunmap_atomic(maddr);
869 page_cache_release(page);
876 return buf - old_buf;
879 int page_present (struct mm_struct *mm, unsigned long address)
887 pgd = pgd_offset(mm, address);
888 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
891 pud = pud_offset(pgd, address);
892 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
895 pmd = pmd_offset(pud, address);
896 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
899 ptep = pte_offset_map(pmd, address);
905 if (pte_present(pte)) {
907 if (pfn_valid(pfn)) {
917 EXPORT_SYMBOL_GPL (page_present);
918 EXPORT_SYMBOL_GPL (get_user_pages_uprobe);
919 EXPORT_SYMBOL_GPL (access_process_vm_atomic);