2 * Dynamic Binary Instrumentation Module based on KProbes
3 * modules/kprobe/dbi_kprobes_deps.h
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * Copyright (C) Samsung Electronics, 2006-2010
21 * 2008-2009 Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
22 * Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
23 * 2010 Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
27 #include <linux/module.h>
28 #include <linux/sched.h>
30 #include <asm/pgtable.h>
32 #include "dbi_kprobes_deps.h"
33 #include "dbi_kdebug.h"
36 #include <linux/slab.h>
39 unsigned long sched_addr;
40 unsigned long fork_addr;
42 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
43 struct mm_struct* init_mm_ptr;
44 struct mm_struct init_mm;
48 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
49 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff);
50 DECLARE_MOD_DEP_WRAPPER(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff)
51 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len, prot, flags, pgoff)
52 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) */
54 /* copy_to_user_page */
55 #ifndef copy_to_user_page
56 DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len);
57 DECLARE_MOD_DEP_WRAPPER(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len)
58 IMP_MOD_DEP_WRAPPER(copy_to_user_page, vma, page, uaddr, dst, src, len)
59 #endif /* copy_to_user_page */
62 DECLARE_MOD_FUNC_DEP(access_process_vm, int, struct task_struct * tsk, unsigned long addr, void *buf, int len, int write);
64 DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr);
66 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
67 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
68 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access);
71 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags);
72 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
74 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
75 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct mm_struct *mm);
76 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
77 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct task_struct *tsk);
78 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
80 #ifdef CONFIG_HUGETLB_PAGE
81 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, int write);
84 #ifdef __HAVE_ARCH_GATE_AREA
85 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
86 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct mm_struct *mm, unsigned long addr);
87 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
88 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct task_struct *task, unsigned long addr);
89 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
90 #endif /* __HAVE_ARCH_GATE_AREA */
92 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
93 DECLARE_MOD_FUNC_DEP(in_gate_area_no_mm, int, unsigned long addr);
94 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
95 DECLARE_MOD_FUNC_DEP(in_gate_area_no_task, int, unsigned long addr);
96 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
99 DECLARE_MOD_FUNC_DEP(follow_page, \
100 struct page *, struct vm_area_struct * vma, \
101 unsigned long address, unsigned int foll_flags);
102 DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
103 void, struct vm_area_struct *vma, struct page *page, \
104 unsigned long vmaddr);
105 DECLARE_MOD_FUNC_DEP(vm_normal_page, \
106 struct page *, struct vm_area_struct *vma, \
107 unsigned long addr, pte_t pte);
110 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
111 DECLARE_MOD_FUNC_DEP(put_task_struct, \
112 void, struct task_struct *tsk);
114 DECLARE_MOD_FUNC_DEP(put_task_struct, \
115 void, struct rcu_head * rhp);
118 DECLARE_MOD_DEP_WRAPPER(access_process_vm, int, struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
119 IMP_MOD_DEP_WRAPPER (access_process_vm, tsk, addr, buf, len, write)
121 DECLARE_MOD_DEP_WRAPPER (find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr)
122 IMP_MOD_DEP_WRAPPER (find_extend_vma, mm, addr)
124 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
125 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
126 DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
127 int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access)
128 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, write_access)
131 DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
132 int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags)
133 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, flags)
136 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
137 DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
138 struct vm_area_struct *, struct mm_struct *mm)
139 IMP_MOD_DEP_WRAPPER (get_gate_vma, mm)
140 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
141 DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
142 struct vm_area_struct *, struct task_struct *tsk)
143 IMP_MOD_DEP_WRAPPER (get_gate_vma, tsk)
144 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
146 #ifdef CONFIG_HUGETLB_PAGE
147 DECLARE_MOD_DEP_WRAPPER (follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, unsigned int write)
148 IMP_MOD_DEP_WRAPPER (follow_hugetlb_page, mm, vma, pages, vmas, position, length, i, write)
151 static inline int dbi_in_gate_area(struct task_struct *task, unsigned long addr)
153 #ifdef __HAVE_ARCH_GATE_AREA
154 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
155 struct mm_struct *mm = task->mm;
156 IMP_MOD_DEP_WRAPPER (in_gate_area, mm, addr)
157 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
158 IMP_MOD_DEP_WRAPPER (in_gate_area, task, addr)
159 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
160 #else /*__HAVE_ARCH_GATE_AREA */
161 return in_gate_area(task, addr);
162 #endif/*__HAVE_ARCH_GATE_AREA */
166 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
167 DECLARE_MOD_DEP_WRAPPER(in_gate_area_no_mm, int, unsigned long addr)
168 IMP_MOD_DEP_WRAPPER(in_gate_area_no_mm, addr)
169 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
170 DECLARE_MOD_DEP_WRAPPER(in_gate_area_no_task, int, unsigned long addr)
171 IMP_MOD_DEP_WRAPPER(in_gate_area_no_task, addr)
172 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
174 static inline int dbi_in_gate_area_no_xxx(unsigned long addr)
176 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
177 return in_gate_area_no_mm(addr);
178 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
179 return in_gate_area_no_task(addr);
180 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
184 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 11))
185 DECLARE_MOD_DEP_WRAPPER (follow_page, \
186 struct page *, struct vm_area_struct * vma, \
187 unsigned long address, unsigned int foll_flags)
188 IMP_MOD_DEP_WRAPPER (follow_page, vma, address, foll_flags)
190 DECLARE_MOD_DEP_WRAPPER (__flush_anon_page, \
191 void, struct vm_area_struct *vma, \
192 struct page *page, unsigned long vmaddr)
193 IMP_MOD_DEP_WRAPPER (__flush_anon_page, vma, page, vmaddr)
195 DECLARE_MOD_DEP_WRAPPER(vm_normal_page, \
196 struct page *, struct vm_area_struct *vma, \
197 unsigned long addr, pte_t pte)
198 IMP_MOD_DEP_WRAPPER (vm_normal_page, vma, addr, pte)
201 int init_module_dependencies()
204 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
205 init_mm_ptr = (struct mm_struct*)swap_ksyms("init_mm");
206 memcmp(init_mm_ptr, &init_mm, sizeof(struct mm_struct));
209 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
210 INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
213 #ifndef copy_to_user_page
214 INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
215 #endif /* copy_to_user_page */
217 INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
218 INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
220 #ifdef CONFIG_HUGETLB_PAGE
221 INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
224 #ifdef __HAVE_ARCH_GATE_AREA
225 INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
228 #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))
229 INIT_MOD_DEP_VAR(in_gate_area_no_mm, in_gate_area_no_mm);
230 #else /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
231 INIT_MOD_DEP_VAR(in_gate_area_no_task, in_gate_area_no_task);
232 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
234 INIT_MOD_DEP_VAR(follow_page, follow_page);
236 INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
237 INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
238 INIT_MOD_DEP_VAR(access_process_vm, access_process_vm);
240 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
241 # if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
242 INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
244 INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
247 INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
250 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0))
251 INIT_MOD_DEP_VAR(do_mmap_pgoff, do_mmap_pgoff);
257 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) /* FIXME: must be < 32 */
258 #define GUP_FLAGS_WRITE 0x1
259 #define GUP_FLAGS_FORCE 0x2
260 #define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
261 #define GUP_FLAGS_IGNORE_SIGKILL 0x8
262 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) */
264 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
265 static inline int use_zero_page(struct vm_area_struct *vma)
268 * We don't want to optimize FOLL_ANON for make_pages_present()
269 * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
270 * we want to get the page from the page tables to make sure
271 * that we serialize and update with any other user of that
274 if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
277 * And if we have a fault routine, it's not an anonymous region.
279 return !vma->vm_ops || !vma->vm_ops->fault;
283 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
284 unsigned long zero_pfn __read_mostly;
287 static inline int is_zero_pfn(unsigned long pfn)
289 return pfn == zero_pfn;
293 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
295 return stack_guard_page_start(vma, addr) ||
296 stack_guard_page_end(vma, addr+PAGE_SIZE);
299 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
300 unsigned long start, int nr_pages, unsigned int gup_flags,
301 struct page **pages, struct vm_area_struct **vmas,
305 unsigned long vm_flags;
311 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
314 * Require read or write permissions.
315 * If FOLL_FORCE is set, we only require the "MAY" flags.
317 vm_flags = (gup_flags & FOLL_WRITE) ?
318 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
319 vm_flags &= (gup_flags & FOLL_FORCE) ?
320 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
324 struct vm_area_struct *vma;
326 vma = find_extend_vma(mm, start);
327 if (!vma && dbi_in_gate_area_no_xxx(start)) {
328 unsigned long pg = start & PAGE_MASK;
334 /* user gate pages are read-only */
335 if (gup_flags & FOLL_WRITE) {
336 return i ? : -EFAULT;
339 pgd = pgd_offset_k(pg);
341 pgd = pgd_offset_gate(mm, pg);
342 BUG_ON(pgd_none(*pgd));
343 pud = pud_offset(pgd, pg);
344 BUG_ON(pud_none(*pud));
345 pmd = pmd_offset(pud, pg);
346 if (pmd_none(*pmd)) {
347 return i ? : -EFAULT;
349 VM_BUG_ON(pmd_trans_huge(*pmd));
350 pte = pte_offset_map(pmd, pg);
351 if (pte_none(*pte)) {
353 return i ? : -EFAULT;
355 vma = get_gate_vma(mm);
359 page = vm_normal_page(vma, start, *pte);
361 if (!(gup_flags & FOLL_DUMP) &&
362 is_zero_pfn(pte_pfn(*pte)))
363 page = pte_page(*pte);
366 return i ? : -EFAULT;
377 (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
378 !(vm_flags & vma->vm_flags)) {
379 return i ? : -EFAULT;
382 if (is_vm_hugetlb_page(vma)) {
383 i = follow_hugetlb_page(mm, vma, pages, vmas,
384 &start, &nr_pages, i, gup_flags);
390 unsigned int foll_flags = gup_flags;
393 * If we have a pending SIGKILL, don't keep faulting
394 * pages and potentially allocating memory.
396 if (unlikely(fatal_signal_pending(current))) {
397 return i ? i : -ERESTARTSYS;
400 /* cond_resched(); */
401 while (!(page = follow_page(vma, start, foll_flags))) {
403 unsigned int fault_flags = 0;
405 /* For mlock, just skip the stack guard page. */
406 if (foll_flags & FOLL_MLOCK) {
407 if (stack_guard_page(vma, start))
410 if (foll_flags & FOLL_WRITE)
411 fault_flags |= FAULT_FLAG_WRITE;
413 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
414 if (foll_flags & FOLL_NOWAIT)
415 fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT);
417 ret = handle_mm_fault(mm, vma, start,
420 if (ret & VM_FAULT_ERROR) {
421 if (ret & VM_FAULT_OOM) {
422 return i ? i : -ENOMEM;
424 if (ret & (VM_FAULT_HWPOISON |
425 VM_FAULT_HWPOISON_LARGE)) {
429 else if (gup_flags & FOLL_HWPOISON) {
436 if (ret & VM_FAULT_SIGBUS) {
437 return i ? i : -EFAULT;
443 if (ret & VM_FAULT_MAJOR)
449 if (ret & VM_FAULT_RETRY) {
456 * The VM_FAULT_WRITE bit tells us that
457 * do_wp_page has broken COW when necessary,
458 * even if maybe_mkwrite decided not to set
459 * pte_write. We can thus safely do subsequent
460 * page lookups as if they were reads. But only
461 * do so when looping for pte_write is futile:
462 * in some cases userspace may also be wanting
463 * to write to the gotten user page, which a
464 * read fault here might prevent (a readonly
465 * page might get reCOWed by userspace write).
467 if ((ret & VM_FAULT_WRITE) &&
468 !(vma->vm_flags & VM_WRITE))
469 foll_flags &= ~FOLL_WRITE;
471 /* cond_resched(); */
474 return i ? i : PTR_ERR(page);
479 flush_anon_page(vma, page, start);
480 flush_dcache_page(page);
488 } while (nr_pages && start < vma->vm_end);
493 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
495 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
496 unsigned long start, int len, int flags,
497 struct page **pages, struct vm_area_struct **vmas)
500 unsigned int vm_flags = 0;
501 int write = !!(flags & GUP_FLAGS_WRITE);
502 int force = !!(flags & GUP_FLAGS_FORCE);
503 int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
504 int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL);
509 * Require read or write permissions.
510 * If 'force' is set, we only require the "MAY" flags.
512 vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
513 vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
517 struct vm_area_struct *vma;
518 unsigned int foll_flags;
520 vma = find_vma(mm, start);
521 if (!vma && dbi_in_gate_area(tsk, start)) {
522 unsigned long pg = start & PAGE_MASK;
523 struct vm_area_struct *gate_vma = get_gate_vma(tsk);
529 /* user gate pages are read-only */
530 if (!ignore && write)
531 return i ? : -EFAULT;
533 pgd = pgd_offset_k(pg);
535 pgd = pgd_offset_gate(mm, pg);
536 BUG_ON(pgd_none(*pgd));
537 pud = pud_offset(pgd, pg);
538 BUG_ON(pud_none(*pud));
539 pmd = pmd_offset(pud, pg);
541 return i ? : -EFAULT;
542 pte = pte_offset_map(pmd, pg);
543 if (pte_none(*pte)) {
545 return i ? : -EFAULT;
548 struct page *page = vm_normal_page(gate_vma, start, *pte);
563 (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
564 (!ignore && !(vm_flags & vma->vm_flags)))
565 return i ? : -EFAULT;
567 if (is_vm_hugetlb_page(vma)) {
568 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
569 i = follow_hugetlb_page(mm, vma, pages, vmas,
572 i = follow_hugetlb_page(mm, vma, pages, vmas,
573 &start, &len, i, write);
578 foll_flags = FOLL_TOUCH;
580 foll_flags |= FOLL_GET;
582 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
583 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,30)
584 if (!write && use_zero_page(vma))
585 foll_flags |= FOLL_ANON;
594 * If we have a pending SIGKILL, don't keep faulting
595 * pages and potentially allocating memory, unless
596 * current is handling munlock--e.g., on exit. In
597 * that case, we are not allocating memory. Rather,
598 * we're only unlocking already resident/mapped pages.
600 if (unlikely(!ignore_sigkill &&
601 fatal_signal_pending(current)))
602 return i ? i : -ERESTARTSYS;
606 foll_flags |= FOLL_WRITE;
611 DBPRINTF ("pages = %p vma = %p\n", pages, vma);
612 while (!(page = follow_page(vma, start, foll_flags))) {
614 ret = handle_mm_fault(mm, vma, start,
615 foll_flags & FOLL_WRITE);
617 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
618 if (ret & VM_FAULT_WRITE)
619 foll_flags &= ~FOLL_WRITE;
621 switch (ret & ~VM_FAULT_WRITE) {
628 case VM_FAULT_SIGBUS:
629 return i ? i : -EFAULT;
631 return i ? i : -ENOMEM;
637 if (ret & VM_FAULT_ERROR) {
638 if (ret & VM_FAULT_OOM)
639 return i ? i : -ENOMEM;
640 else if (ret & VM_FAULT_SIGBUS)
641 return i ? i : -EFAULT;
644 if (ret & VM_FAULT_MAJOR)
650 * The VM_FAULT_WRITE bit tells us that
651 * do_wp_page has broken COW when necessary,
652 * even if maybe_mkwrite decided not to set
653 * pte_write. We can thus safely do subsequent
654 * page lookups as if they were reads. But only
655 * do so when looping for pte_write is futile:
656 * in some cases userspace may also be wanting
657 * to write to the gotten user page, which a
658 * read fault here might prevent (a readonly
659 * page might get reCOWed by userspace write).
661 if ((ret & VM_FAULT_WRITE) &&
662 !(vma->vm_flags & VM_WRITE))
663 foll_flags &= ~FOLL_WRITE;
671 return i ? i : PTR_ERR(page);
675 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
676 flush_anon_page(page, start);
678 flush_anon_page(vma, page, start);
680 flush_dcache_page(page);
687 } while (len && start < vma->vm_end);
692 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
694 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
695 unsigned long start, int len, int write, int force,
696 struct page **pages, struct vm_area_struct **vmas)
698 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
699 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) /* FIXME: must be >= 32! */
700 int flags = FOLL_TOUCH;
708 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
712 flags |= GUP_FLAGS_WRITE;
714 flags |= GUP_FLAGS_FORCE;
715 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
717 return __get_user_pages_uprobe(tsk, mm,
719 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
721 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
723 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
725 return get_user_pages(tsk, mm, start, len, write, force, pages, vmas);
729 #define ACCESS_PROCESS_OPTIMIZATION 0
731 #if ACCESS_PROCESS_OPTIMIZATION
733 #define GET_STEP_X(LEN, STEP) (((LEN) >= (STEP)) ? (STEP) : (LEN) % (STEP))
734 #define GET_STEP_4(LEN) GET_STEP_X((LEN), 4)
736 static void read_data_current(unsigned long addr, void *buf, int len)
741 for (step = GET_STEP_4(len); len; len -= step) {
742 switch (GET_STEP_4(len)) {
744 get_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
750 get_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
755 get_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
765 static void write_data_current(unsigned long addr, void *buf, int len)
770 for (step = GET_STEP_4(len); len; len -= step) {
771 switch (GET_STEP_4(len)) {
773 put_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
779 put_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
784 put_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
794 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
796 struct mm_struct *mm;
797 struct vm_area_struct *vma;
804 #if ACCESS_PROCESS_OPTIMIZATION
805 if (write == 0 && tsk == current) {
806 read_data_current(addr, buf, len);
811 mm = tsk->mm; /* function 'get_task_mm' is to be called */
815 /* ignore errors, just check how much was successfully transferred */
817 int bytes, ret, offset;
819 struct page *page = NULL;
821 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
822 write, 1, &page, &vma);
826 * Check if this is a VM_IO | VM_PFNMAP VMA, which
827 * we can access using slightly different code.
829 #ifdef CONFIG_HAVE_IOREMAP_PROT
830 vma = find_vma(mm, addr);
833 if (vma->vm_ops && vma->vm_ops->access)
834 ret = vma->vm_ops->access(vma, addr, buf,
842 offset = addr & (PAGE_SIZE-1);
843 if (bytes > PAGE_SIZE-offset)
844 bytes = PAGE_SIZE-offset;
846 maddr = kmap_atomic(page);
849 copy_to_user_page(vma, page, addr,
850 maddr + offset, buf, bytes);
851 set_page_dirty_lock(page);
853 copy_from_user_page(vma, page, addr,
854 buf, maddr + offset, bytes);
857 kunmap_atomic(maddr);
858 page_cache_release(page);
865 return buf - old_buf;
868 int page_present (struct mm_struct *mm, unsigned long address)
876 pgd = pgd_offset(mm, address);
877 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
880 pud = pud_offset(pgd, address);
881 if (pud_none(*pud) || unlikely(pud_bad(*pud)))
884 pmd = pmd_offset(pud, address);
885 if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
888 ptep = pte_offset_map(pmd, address);
894 if (pte_present(pte)) {
896 if (pfn_valid(pfn)) {
906 EXPORT_SYMBOL_GPL (page_present);
907 EXPORT_SYMBOL_GPL (get_user_pages_uprobe);
908 EXPORT_SYMBOL_GPL (access_process_vm_atomic);