kprobe/swap_kprobes_deps.c

   1 /*
   2  *  Dynamic Binary Instrumentation Module based on KProbes
   3  *  modules/kprobe/swap_kprobes_deps.h
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) Samsung Electronics, 2006-2010
  20  *
  21  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  22  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  23  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  24  *
  25  */
  26
  27 #include <linux/module.h>
  28 #include <linux/sched.h>
  29
  30 #include <asm/pgtable.h>
  31
  32 #include "swap_kprobes_deps.h"
  33 #include "swap_kdebug.h"
  34
  35
  36 #include <linux/slab.h>
  37 #include <linux/mm.h>
  38
  39 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
  40 /* kernel define 'pgd_offset_k' redefinition */
  41 #undef pgd_offset_k
  42 #define pgd_offset_k(addr)      pgd_offset(init_task.active_mm, addr)
  43 #endif
  44
  45 #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))
  46 #ifndef is_zero_pfn
  47
  48 static unsigned long swap_zero_pfn = 0;
  49
  50 #endif /* is_zero_pfn */
  51 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
  52
  53 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)
  54 static inline void *swap_kmap_atomic(struct page *page)
  55 {
  56         return kmap_atomic(page);
  57 }
  58 static inline void swap_kunmap_atomic(void *kvaddr)
  59 {
  60         kunmap_atomic(kvaddr);
  61 }
  62 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36) */
  63 static inline void *swap_kmap_atomic(struct page *page)
  64 {
  65         return kmap_atomic(page, KM_USER0);
  66 }
  67
  68 static inline void swap_kunmap_atomic(void *kvaddr)
  69 {
  70         kunmap_atomic(kvaddr, KM_USER0);
  71 }
  72 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36) */
  73
  74 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
  75 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff, unsigned long *populate);
  76 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap_pgoff,
  77                         unsigned long,
  78                         struct file *file, unsigned long addr,
  79                         unsigned long len, unsigned long prot,
  80                         unsigned long flags, unsigned long pgoff,
  81                         unsigned long *populate)
  82 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len, prot, flags, pgoff, populate)
  83 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
  84 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff);
  85 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap_pgoff,
  86                         unsigned long,
  87                         struct file *file, unsigned long addr,
  88                         unsigned long len, unsigned long prot,
  89                         unsigned long flags, unsigned long pgoff)
  90 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len, prot, flags, pgoff)
  91 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) */
  92
  93 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
  94 EXPORT_SYMBOL_GPL(swap_do_mmap_pgoff);
  95 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) */
  96
  97 /* copy_to_user_page */
  98 #ifndef copy_to_user_page
  99 static DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len);
 100 DECLARE_MOD_DEP_WRAPPER(swap_copy_to_user_page,
 101                         void,
 102                         struct vm_area_struct *vma, struct page *page,
 103                         unsigned long uaddr, void *dst, const void *src,
 104                         unsigned long len)
 105 IMP_MOD_DEP_WRAPPER(copy_to_user_page, vma, page, uaddr, dst, src, len)
 106 #else /* copy_to_user_page */
 107 #define swap_copy_to_user_page copy_to_user_page
 108 #endif /* copy_to_user_page */
 109
 110
 111 static DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr);
 112
 113 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
 114 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 115 static DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access);
 116 #endif
 117 #else
 118 static DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags);
 119 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
 120
 121 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 122 static DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct mm_struct *mm);
 123 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 124 static DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct task_struct *tsk);
 125 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 126
 127 #ifdef CONFIG_HUGETLB_PAGE
 128 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page, int, struct mm_struct *mm, \
 129                 struct vm_area_struct *vma, struct page **pages, \
 130                 struct vm_area_struct **vmas, unsigned long *position, int *length, \
 131                 int i, int write);
 132 #endif
 133
 134 #ifdef __HAVE_ARCH_GATE_AREA
 135 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 136 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct mm_struct *mm, unsigned long addr);
 137 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 138 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct task_struct *task, unsigned long addr);
 139 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 140 #endif /* __HAVE_ARCH_GATE_AREA */
 141
 142 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 143 static DECLARE_MOD_FUNC_DEP(in_gate_area_no_mm, int, unsigned long addr);
 144 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 145 static DECLARE_MOD_FUNC_DEP(in_gate_area_no_task, int, unsigned long addr);
 146 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 147
 148 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 149 static DECLARE_MOD_FUNC_DEP(follow_page_mask, \
 150                 struct page *, struct vm_area_struct * vma, \
 151                 unsigned long address, unsigned int foll_flags, \
 152                 unsigned int *page_mask);
 153 DECLARE_MOD_DEP_WRAPPER(swap_follow_page_mask,
 154                         struct page *,
 155                         struct vm_area_struct * vma, unsigned long address,
 156                         unsigned int foll_flags, unsigned int *page_mask)
 157 IMP_MOD_DEP_WRAPPER (follow_page_mask, vma, address, foll_flags, page_mask)
 158 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 159 static DECLARE_MOD_FUNC_DEP(follow_page, \
 160                 struct page *, struct vm_area_struct * vma, \
 161                 unsigned long address, unsigned int foll_flags);
 162 DECLARE_MOD_DEP_WRAPPER(swap_follow_page,
 163                         struct page *,
 164                         struct vm_area_struct * vma, unsigned long address,
 165                         unsigned int foll_flags)
 166 IMP_MOD_DEP_WRAPPER (follow_page, vma, address, foll_flags)
 167 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 168
 169 static DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
 170                 void, struct vm_area_struct *vma, struct page *page, \
 171                 unsigned long vmaddr);
 172 static DECLARE_MOD_FUNC_DEP(vm_normal_page, \
 173                 struct page *, struct vm_area_struct *vma, \
 174                 unsigned long addr, pte_t pte);
 175
 176
 177 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
 178 static DECLARE_MOD_FUNC_DEP(put_task_struct, \
 179                 void, struct task_struct *tsk);
 180 #else
 181 static DECLARE_MOD_FUNC_DEP(put_task_struct, \
 182                 void, struct rcu_head * rhp);
 183 #endif
 184
 185 DECLARE_MOD_DEP_WRAPPER(swap_find_extend_vma,
 186                         struct vm_area_struct *,
 187                         struct mm_struct * mm, unsigned long addr)
 188 IMP_MOD_DEP_WRAPPER (find_extend_vma, mm, addr)
 189
 190 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
 191 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 192 DECLARE_MOD_DEP_WRAPPER(swap_handle_mm_fault,
 193                         int,
 194                         struct mm_struct *mm, struct vm_area_struct *vma,
 195                         unsigned long address, int write_access)
 196 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, write_access)
 197 #endif
 198 #else
 199 DECLARE_MOD_DEP_WRAPPER(swap_handle_mm_fault,
 200                         int,
 201                         struct mm_struct *mm, struct vm_area_struct *vma,
 202                         unsigned long address, unsigned int flags)
 203 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, flags)
 204 #endif
 205
 206 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 207 DECLARE_MOD_DEP_WRAPPER(swap_get_gate_vma,
 208                         struct vm_area_struct *,
 209                         struct mm_struct *mm)
 210 IMP_MOD_DEP_WRAPPER (get_gate_vma, mm)
 211 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 212 DECLARE_MOD_DEP_WRAPPER(swap_get_gate_vma,
 213                         struct vm_area_struct *,
 214                         struct task_struct *tsk)
 215 IMP_MOD_DEP_WRAPPER (get_gate_vma, tsk)
 216 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 217
 218 #ifdef CONFIG_HUGETLB_PAGE
 219 DECLARE_MOD_DEP_WRAPPER(swap_follow_hugetlb_page,
 220                         int,
 221                         struct mm_struct *mm, struct vm_area_struct *vma,
 222                         struct page **pages, struct vm_area_struct **vmas,
 223                         unsigned long *position, int *length, int i,
 224                         unsigned int write)
 225         IMP_MOD_DEP_WRAPPER (follow_hugetlb_page, mm, vma, pages, vmas, position, length, i, write)
 226 #else /* CONFIG_HUGETLB_PAGE */
 227 #define swap_follow_hugetlb_page follow_hugetlb_page
 228 #endif /* CONFIG_HUGETLB_PAGE */
 229
 230 static inline int swap_in_gate_area(struct task_struct *task, unsigned long addr)
 231 {
 232 #ifdef __HAVE_ARCH_GATE_AREA
 233 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 234         struct mm_struct *mm = task->mm;
 235         IMP_MOD_DEP_WRAPPER (in_gate_area, mm, addr)
 236 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 237         IMP_MOD_DEP_WRAPPER (in_gate_area, task, addr)
 238 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 239 #else /*__HAVE_ARCH_GATE_AREA */
 240         return in_gate_area(task, addr);
 241 #endif/*__HAVE_ARCH_GATE_AREA */
 242 }
 243
 244
 245 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 246 DECLARE_MOD_DEP_WRAPPER(swap_in_gate_area_no_mm, int, unsigned long addr)
 247 IMP_MOD_DEP_WRAPPER(in_gate_area_no_mm, addr)
 248 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 249 DECLARE_MOD_DEP_WRAPPER(swap_in_gate_area_no_task, int, unsigned long addr)
 250 IMP_MOD_DEP_WRAPPER(in_gate_area_no_task, addr)
 251 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 252
 253 static inline int swap_in_gate_area_no_xxx(unsigned long addr)
 254 {
 255 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 256         return swap_in_gate_area_no_mm(addr);
 257 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 258         return swap_in_gate_area_no_task(addr);
 259 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 260 }
 261
 262 DECLARE_MOD_DEP_WRAPPER(swap__flush_anon_page,
 263                         void,
 264                         struct vm_area_struct *vma, struct page *page,
 265                         unsigned long vmaddr)
 266 IMP_MOD_DEP_WRAPPER (__flush_anon_page, vma, page, vmaddr)
 267
 268 static inline void swap_flush_anon_page(struct vm_area_struct *vma,
 269                                         struct page *page,
 270                                         unsigned long vmaddr)
 271 {
 272 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 273         if (PageAnon(page))
 274                 swap__flush_anon_page(vma, page, vmaddr);
 275 #else /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 276         flush_anon_page(vma, page, vmaddr);
 277 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 278 }
 279
 280 DECLARE_MOD_DEP_WRAPPER(swap_vm_normal_page,
 281                         struct page *,
 282                         struct vm_area_struct *vma, unsigned long addr,
 283                         pte_t pte)
 284 IMP_MOD_DEP_WRAPPER (vm_normal_page, vma, addr, pte)
 285
 286
 287
 288
 289 int init_module_dependencies(void)
 290 {
 291
 292 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 293         INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
 294 #endif
 295
 296 #ifndef copy_to_user_page
 297         INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
 298 #endif /* copy_to_user_page */
 299
 300         INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
 301         INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
 302
 303 #ifdef CONFIG_HUGETLB_PAGE
 304         INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
 305 #endif
 306
 307 #ifdef  __HAVE_ARCH_GATE_AREA
 308         INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
 309 #endif
 310
 311 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 312         INIT_MOD_DEP_VAR(follow_page_mask, follow_page_mask);
 313 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 314         INIT_MOD_DEP_VAR(follow_page, follow_page);
 315 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 316
 317 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 318
 319 #ifndef is_zero_pfn
 320         swap_zero_pfn = page_to_pfn(ZERO_PAGE(0));
 321 #endif /* is_zero_pfn */
 322
 323         INIT_MOD_DEP_VAR(in_gate_area_no_mm, in_gate_area_no_mm);
 324 #else /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))  */
 325         INIT_MOD_DEP_VAR(in_gate_area_no_task, in_gate_area_no_task);
 326 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))  */
 327
 328 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 329         INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
 330 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 331
 332         INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
 333
 334 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
 335 # if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
 336         INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
 337 # else
 338         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
 339 # endif
 340 #else /*2.6.16 */
 341         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
 342 #endif
 343 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
 344         INIT_MOD_DEP_VAR(do_mmap_pgoff, do_mmap_pgoff);
 345 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) */
 346
 347         return 0;
 348 }
 349
 350 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) /* FIXME: must be < 32 */
 351 #define GUP_FLAGS_WRITE                  0x1
 352 #define GUP_FLAGS_FORCE                  0x2
 353 #define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
 354 #define GUP_FLAGS_IGNORE_SIGKILL         0x8
 355 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) */
 356
 357 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 358 static inline int use_zero_page(struct vm_area_struct *vma)
 359 {
 360         /*
 361          * We don't want to optimize FOLL_ANON for make_pages_present()
 362          * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
 363          * we want to get the page from the page tables to make sure
 364          * that we serialize and update with any other user of that
 365          * mapping.
 366          */
 367         if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
 368                 return 0;
 369         /*
 370          * And if we have a fault routine, it's not an anonymous region.
 371          */
 372         return !vma->vm_ops || !vma->vm_ops->fault;
 373 }
 374
 375 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 376
 377 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
 378
 379 #ifdef __HAVE_COLOR_ZERO_PAGE
 380
 381 static inline int swap_is_zero_pfn(unsigned long pfn)
 382 {
 383         unsigned long offset_from_zero_pfn = pfn - swap_zero_pfn;
 384         return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
 385 }
 386
 387 #else /* __HAVE_COLOR_ZERO_PAGE */
 388
 389 static inline int swap_is_zero_pfn(unsigned long pfn)
 390 {
 391         return pfn == swap_zero_pfn;
 392 }
 393 #endif /* __HAVE_COLOR_ZERO_PAGE */
 394
 395 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
 396
 397 static inline int swap_is_zero_pfn(unsigned long pfn)
 398 {
 399 #ifndef is_zero_pfn
 400         return pfn == swap_zero_pfn;
 401 #else /* is_zero_pfn */
 402         return is_zero_pfn(pfn);
 403 #endif /* is_zero_pfn */
 404 }
 405
 406 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
 407
 408 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
 409 {
 410         return stack_guard_page_start(vma, addr) ||
 411                         stack_guard_page_end(vma, addr+PAGE_SIZE);
 412 }
 413
 414 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 415
 416 long __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 417                 unsigned long start, unsigned long nr_pages,
 418                 unsigned int gup_flags, struct page **pages,
 419                 struct vm_area_struct **vmas, int *nonblocking)
 420 {
 421         long i;
 422         unsigned long vm_flags;
 423         unsigned int page_mask;
 424
 425         if (!nr_pages)
 426                 return 0;
 427
 428         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 429
 430         /*
 431          * Require read or write permissions.
 432          * If FOLL_FORCE is set, we only require the "MAY" flags.
 433          */
 434         vm_flags  = (gup_flags & FOLL_WRITE) ?
 435                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 436         vm_flags &= (gup_flags & FOLL_FORCE) ?
 437                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 438
 439         /*
 440          * If FOLL_FORCE and FOLL_NUMA are both set, handle_mm_fault
 441          * would be called on PROT_NONE ranges. We must never invoke
 442          * handle_mm_fault on PROT_NONE ranges or the NUMA hinting
 443          * page faults would unprotect the PROT_NONE ranges if
 444          * _PAGE_NUMA and _PAGE_PROTNONE are sharing the same pte/pmd
 445          * bitflag. So to avoid that, don't set FOLL_NUMA if
 446          * FOLL_FORCE is set.
 447          */
 448         if (!(gup_flags & FOLL_FORCE))
 449                 gup_flags |= FOLL_NUMA;
 450
 451         i = 0;
 452
 453         do {
 454                 struct vm_area_struct *vma;
 455
 456                 vma = swap_find_extend_vma(mm, start);
 457                 if (!vma && swap_in_gate_area(tsk, start)) {
 458                         unsigned long pg = start & PAGE_MASK;
 459                         pgd_t *pgd;
 460                         pud_t *pud;
 461                         pmd_t *pmd;
 462                         pte_t *pte;
 463
 464                         /* user gate pages are read-only */
 465                         if (gup_flags & FOLL_WRITE)
 466                                 return i ? : -EFAULT;
 467                         if (pg > TASK_SIZE)
 468                                 pgd = pgd_offset_k(pg);
 469                         else
 470                                 pgd = pgd_offset_gate(mm, pg);
 471                         BUG_ON(pgd_none(*pgd));
 472                         pud = pud_offset(pgd, pg);
 473                         BUG_ON(pud_none(*pud));
 474                         pmd = pmd_offset(pud, pg);
 475                         if (pmd_none(*pmd))
 476                                 return i ? : -EFAULT;
 477                         VM_BUG_ON(pmd_trans_huge(*pmd));
 478                         pte = pte_offset_map(pmd, pg);
 479                         if (pte_none(*pte)) {
 480                                 pte_unmap(pte);
 481                                 return i ? : -EFAULT;
 482                         }
 483                         vma = swap_get_gate_vma(mm);
 484                         if (pages) {
 485                                 struct page *page;
 486
 487                                 page = swap_vm_normal_page(vma, start, *pte);
 488                                 if (!page) {
 489                                         if (!(gup_flags & FOLL_DUMP) &&
 490                                              swap_is_zero_pfn(pte_pfn(*pte)))
 491                                                 page = pte_page(*pte);
 492                                         else {
 493                                                 pte_unmap(pte);
 494                                                 return i ? : -EFAULT;
 495                                         }
 496                                 }
 497                                 pages[i] = page;
 498                                 get_page(page);
 499                         }
 500                         pte_unmap(pte);
 501                         page_mask = 0;
 502                         goto next_page;
 503                 }
 504
 505                 if (!vma ||
 506                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 507                     !(vm_flags & vma->vm_flags))
 508                         return i ? : -EFAULT;
 509
 510                 if (is_vm_hugetlb_page(vma)) {
 511                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 512                                         &start, &nr_pages, i, gup_flags);
 513                         continue;
 514                 }
 515
 516                 do {
 517                         struct page *page;
 518                         unsigned int foll_flags = gup_flags;
 519                         unsigned int page_increm;
 520
 521                         /*
 522                          * If we have a pending SIGKILL, don't keep faulting
 523                          * pages and potentially allocating memory.
 524                          */
 525                         if (unlikely(fatal_signal_pending(current)))
 526                                 return i ? i : -ERESTARTSYS;
 527
 528                         /* cond_resched(); */
 529                         while (!(page = swap_follow_page_mask(vma, start,
 530                                                 foll_flags, &page_mask))) {
 531                                 int ret;
 532                                 unsigned int fault_flags = 0;
 533
 534                                 /* For mlock, just skip the stack guard page. */
 535                                 if (foll_flags & FOLL_MLOCK) {
 536                                         if (stack_guard_page(vma, start))
 537                                                 goto next_page;
 538                                 }
 539                                 if (foll_flags & FOLL_WRITE)
 540                                         fault_flags |= FAULT_FLAG_WRITE;
 541                                 if (nonblocking)
 542                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 543                                 if (foll_flags & FOLL_NOWAIT)
 544                                         fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT);
 545
 546                                 ret = swap_handle_mm_fault(mm, vma, start,
 547                                                         fault_flags);
 548
 549                                 if (ret & VM_FAULT_ERROR) {
 550                                         if (ret & VM_FAULT_OOM)
 551                                                 return i ? i : -ENOMEM;
 552                                         if (ret & (VM_FAULT_HWPOISON |
 553                                                    VM_FAULT_HWPOISON_LARGE)) {
 554                                                 if (i)
 555                                                         return i;
 556                                                 else if (gup_flags & FOLL_HWPOISON)
 557                                                         return -EHWPOISON;
 558                                                 else
 559                                                         return -EFAULT;
 560                                         }
 561                                         if (ret & VM_FAULT_SIGBUS)
 562                                                 return i ? i : -EFAULT;
 563                                         BUG();
 564                                 }
 565
 566                                 if (tsk) {
 567                                         if (ret & VM_FAULT_MAJOR)
 568                                                 tsk->maj_flt++;
 569                                         else
 570                                                 tsk->min_flt++;
 571                                 }
 572
 573                                 if (ret & VM_FAULT_RETRY) {
 574                                         if (nonblocking)
 575                                                 *nonblocking = 0;
 576                                         return i;
 577                                 }
 578
 579                                 /*
 580                                  * The VM_FAULT_WRITE bit tells us that
 581                                  * do_wp_page has broken COW when necessary,
 582                                  * even if maybe_mkwrite decided not to set
 583                                  * pte_write. We can thus safely do subsequent
 584                                  * page lookups as if they were reads. But only
 585                                  * do so when looping for pte_write is futile:
 586                                  * in some cases userspace may also be wanting
 587                                  * to write to the gotten user page, which a
 588                                  * read fault here might prevent (a readonly
 589                                  * page might get reCOWed by userspace write).
 590                                  */
 591                                 if ((ret & VM_FAULT_WRITE) &&
 592                                     !(vma->vm_flags & VM_WRITE))
 593                                         foll_flags &= ~FOLL_WRITE;
 594
 595                                 /* cond_resched(); */
 596                         }
 597                         if (IS_ERR(page))
 598                                 return i ? i : PTR_ERR(page);
 599                         if (pages) {
 600                                 pages[i] = page;
 601
 602                                 swap_flush_anon_page(vma, page, start);
 603                                 flush_dcache_page(page);
 604                                 page_mask = 0;
 605                         }
 606 next_page:
 607                         if (vmas) {
 608                                 vmas[i] = vma;
 609                                 page_mask = 0;
 610                         }
 611                         page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
 612                         if (page_increm > nr_pages)
 613                                 page_increm = nr_pages;
 614                         i += page_increm;
 615                         start += page_increm * PAGE_SIZE;
 616                         nr_pages -= page_increm;
 617                 } while (nr_pages && start < vma->vm_end);
 618         } while (nr_pages);
 619         return i;
 620 }
 621
 622 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 623
 624 static int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 625                         unsigned long start, int nr_pages, unsigned int gup_flags,
 626                         struct page **pages, struct vm_area_struct **vmas,
 627                         int *nonblocking)
 628 {
 629         int i;
 630         unsigned long vm_flags;
 631
 632         if (nr_pages <= 0) {
 633                 return 0;
 634         }
 635
 636         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 637
 638         /*
 639          * Require read or write permissions.
 640          * If FOLL_FORCE is set, we only require the "MAY" flags.
 641          */
 642         vm_flags  = (gup_flags & FOLL_WRITE) ?
 643                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 644         vm_flags &= (gup_flags & FOLL_FORCE) ?
 645                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 646         i = 0;
 647
 648         do {
 649                 struct vm_area_struct *vma;
 650
 651                 vma = swap_find_extend_vma(mm, start);
 652                 if (!vma && swap_in_gate_area_no_xxx(start)) {
 653                         unsigned long pg = start & PAGE_MASK;
 654                         pgd_t *pgd;
 655                         pud_t *pud;
 656                         pmd_t *pmd;
 657                         pte_t *pte;
 658
 659                         /* user gate pages are read-only */
 660                         if (gup_flags & FOLL_WRITE) {
 661                                 return i ? : -EFAULT;
 662                         }
 663                         if (pg > TASK_SIZE)
 664                                 pgd = pgd_offset_k(pg);
 665                         else
 666                                 pgd = pgd_offset_gate(mm, pg);
 667                         BUG_ON(pgd_none(*pgd));
 668                         pud = pud_offset(pgd, pg);
 669                         BUG_ON(pud_none(*pud));
 670                         pmd = pmd_offset(pud, pg);
 671                         if (pmd_none(*pmd)) {
 672                                 return i ? : -EFAULT;
 673                         }
 674                         VM_BUG_ON(pmd_trans_huge(*pmd));
 675                         pte = pte_offset_map(pmd, pg);
 676                         if (pte_none(*pte)) {
 677                                 pte_unmap(pte);
 678                                 return i ? : -EFAULT;
 679                         }
 680                         vma = swap_get_gate_vma(mm);
 681                         if (pages) {
 682                                 struct page *page;
 683
 684                                 page = swap_vm_normal_page(vma, start, *pte);
 685                                 if (!page) {
 686                                         if (!(gup_flags & FOLL_DUMP) &&
 687                                                 swap_is_zero_pfn(pte_pfn(*pte)))
 688                                                 page = pte_page(*pte);
 689                                         else {
 690                                                 pte_unmap(pte);
 691                                                 return i ? : -EFAULT;
 692                                         }
 693                                 }
 694                                 pages[i] = page;
 695                                 get_page(page);
 696                         }
 697                         pte_unmap(pte);
 698                         goto next_page;
 699                 }
 700
 701                 if (!vma ||
 702                         (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 703                         !(vm_flags & vma->vm_flags)) {
 704                         return i ? : -EFAULT;
 705                 }
 706
 707                 if (is_vm_hugetlb_page(vma)) {
 708                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 709                                         &start, &nr_pages, i, gup_flags);
 710                         continue;
 711                 }
 712
 713                 do {
 714                         struct page *page;
 715                         unsigned int foll_flags = gup_flags;
 716
 717                         /*
 718                          * If we have a pending SIGKILL, don't keep faulting
 719                          * pages and potentially allocating memory.
 720                          */
 721                         if (unlikely(fatal_signal_pending(current))) {
 722                                 return i ? i : -ERESTARTSYS;
 723                         }
 724
 725                         /* cond_resched(); */
 726                         while (!(page = swap_follow_page(vma, start, foll_flags))) {
 727                                 int ret;
 728                                 unsigned int fault_flags = 0;
 729
 730                                 /* For mlock, just skip the stack guard page. */
 731                                 if (foll_flags & FOLL_MLOCK) {
 732                                         if (stack_guard_page(vma, start))
 733                                                 goto next_page;
 734                                 }
 735                                 if (foll_flags & FOLL_WRITE)
 736                                         fault_flags |= FAULT_FLAG_WRITE;
 737                                 if (nonblocking)
 738                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 739                                 if (foll_flags & FOLL_NOWAIT)
 740                                         fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT);
 741
 742                                 ret = swap_handle_mm_fault(mm, vma, start,
 743                                                         fault_flags);
 744
 745                                 if (ret & VM_FAULT_ERROR) {
 746                                         if (ret & VM_FAULT_OOM) {
 747                                                 return i ? i : -ENOMEM;
 748                                         }
 749                                         if (ret & (VM_FAULT_HWPOISON |
 750                                                                 VM_FAULT_HWPOISON_LARGE)) {
 751                                                 if (i) {
 752                                                         return i;
 753                                                 }
 754                                                 else if (gup_flags & FOLL_HWPOISON) {
 755                                                         return -EHWPOISON;
 756                                                 }
 757                                                 else {
 758                                                         return -EFAULT;
 759                                                 }
 760                                         }
 761                                         if (ret & VM_FAULT_SIGBUS) {
 762                                                 return i ? i : -EFAULT;
 763                                         }
 764                                         BUG();
 765                                 }
 766
 767                                 if (tsk) {
 768                                         if (ret & VM_FAULT_MAJOR)
 769                                                 tsk->maj_flt++;
 770                                         else
 771                                                 tsk->min_flt++;
 772                                 }
 773
 774                                 if (ret & VM_FAULT_RETRY) {
 775                                         if (nonblocking)
 776                                                 *nonblocking = 0;
 777                                         return i;
 778                                 }
 779
 780                                 /*
 781                                  * The VM_FAULT_WRITE bit tells us that
 782                                  * do_wp_page has broken COW when necessary,
 783                                  * even if maybe_mkwrite decided not to set
 784                                  * pte_write. We can thus safely do subsequent
 785                                  * page lookups as if they were reads. But only
 786                                  * do so when looping for pte_write is futile:
 787                                  * in some cases userspace may also be wanting
 788                                  * to write to the gotten user page, which a
 789                                  * read fault here might prevent (a readonly
 790                                  * page might get reCOWed by userspace write).
 791                                  */
 792                                 if ((ret & VM_FAULT_WRITE) &&
 793                                         !(vma->vm_flags & VM_WRITE))
 794                                         foll_flags &= ~FOLL_WRITE;
 795
 796                                 /* cond_resched(); */
 797                         }
 798                         if (IS_ERR(page)) {
 799                                 return i ? i : PTR_ERR(page);
 800                         }
 801                         if (pages) {
 802                                 pages[i] = page;
 803
 804                                 swap_flush_anon_page(vma, page, start);
 805                                 flush_dcache_page(page);
 806                         }
 807 next_page:
 808                         if (vmas)
 809                                 vmas[i] = vma;
 810                         i++;
 811                         start += PAGE_SIZE;
 812                         nr_pages--;
 813                 } while (nr_pages && start < vma->vm_end);
 814         } while (nr_pages);
 815
 816         return i;
 817 }
 818
 819 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 820
 821 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 822
 823 static int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 824                 unsigned long start, int len, int flags,
 825                 struct page **pages, struct vm_area_struct **vmas)
 826 {
 827         int i;
 828         unsigned int vm_flags = 0;
 829         int write = !!(flags & GUP_FLAGS_WRITE);
 830         int force = !!(flags & GUP_FLAGS_FORCE);
 831         int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
 832
 833         if (len <= 0)
 834                 return 0;
 835         /*
 836          * Require read or write permissions.
 837          * If 'force' is set, we only require the "MAY" flags.
 838          */
 839         vm_flags  = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 840         vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 841         i = 0;
 842
 843         do {
 844                 struct vm_area_struct *vma;
 845                 unsigned int foll_flags;
 846
 847                 vma = find_vma(mm, start);
 848                 if (!vma && swap_in_gate_area(tsk, start)) {
 849                         unsigned long pg = start & PAGE_MASK;
 850                         struct vm_area_struct *gate_vma = swap_get_gate_vma(tsk);
 851                         pgd_t *pgd;
 852                         pud_t *pud;
 853                         pmd_t *pmd;
 854                         pte_t *pte;
 855
 856                         /* user gate pages are read-only */
 857                         if (!ignore && write)
 858                                 return i ? : -EFAULT;
 859                         if (pg > TASK_SIZE)
 860                                 pgd = pgd_offset_k(pg);
 861                         else
 862                                 pgd = pgd_offset_gate(mm, pg);
 863                         BUG_ON(pgd_none(*pgd));
 864                         pud = pud_offset(pgd, pg);
 865                         BUG_ON(pud_none(*pud));
 866                         pmd = pmd_offset(pud, pg);
 867                         if (pmd_none(*pmd))
 868                                 return i ? : -EFAULT;
 869                         pte = pte_offset_map(pmd, pg);
 870                         if (pte_none(*pte)) {
 871                                 pte_unmap(pte);
 872                                 return i ? : -EFAULT;
 873                         }
 874                         if (pages) {
 875                                 struct page *page = swap_vm_normal_page(gate_vma, start, *pte);
 876                                 pages[i] = page;
 877                                 if (page)
 878                                         get_page(page);
 879                         }
 880                         pte_unmap(pte);
 881                         if (vmas)
 882                                 vmas[i] = gate_vma;
 883                         i++;
 884                         start += PAGE_SIZE;
 885                         len--;
 886                         continue;
 887                 }
 888
 889                 if (!vma ||
 890                         (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 891                         (!ignore && !(vm_flags & vma->vm_flags)))
 892                         return i ? : -EFAULT;
 893
 894                 if (is_vm_hugetlb_page(vma)) {
 895                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 896                                                 &start, &len, i, write);
 897                         continue;
 898                 }
 899
 900                 foll_flags = FOLL_TOUCH;
 901                 if (pages)
 902                         foll_flags |= FOLL_GET;
 903
 904 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 905 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,30)
 906                 if (!write && use_zero_page(vma))
 907                         foll_flags |= FOLL_ANON;
 908 #endif
 909 #endif
 910
 911                 do {
 912                         struct page *page;
 913
 914                         if (write)
 915                                 foll_flags |= FOLL_WRITE;
 916
 917
 918                         //cond_resched();
 919
 920                         DBPRINTF ("pages = %p vma = %p\n", pages, vma);
 921                         while (!(page = swap_follow_page(vma, start, foll_flags))) {
 922                                 int ret;
 923                                 ret = swap_handle_mm_fault(mm, vma, start,
 924                                                 foll_flags & FOLL_WRITE);
 925
 926 #if  LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
 927                                 if (ret & VM_FAULT_WRITE)
 928                                         foll_flags &= ~FOLL_WRITE;
 929
 930                                 switch (ret & ~VM_FAULT_WRITE) {
 931                                 case VM_FAULT_MINOR:
 932                                         tsk->min_flt++;
 933                                         break;
 934                                 case VM_FAULT_MAJOR:
 935                                         tsk->maj_flt++;
 936                                         break;
 937                                 case VM_FAULT_SIGBUS:
 938                                         return i ? i : -EFAULT;
 939                                 case VM_FAULT_OOM:
 940                                         return i ? i : -ENOMEM;
 941                                 default:
 942                                         BUG();
 943                                 }
 944
 945 #else
 946                                 if (ret & VM_FAULT_ERROR) {
 947                                         if (ret & VM_FAULT_OOM)
 948                                                 return i ? i : -ENOMEM;
 949                                         else if (ret & VM_FAULT_SIGBUS)
 950                                                 return i ? i : -EFAULT;
 951                                         BUG();
 952                                 }
 953                                 if (ret & VM_FAULT_MAJOR)
 954                                         tsk->maj_flt++;
 955                                 else
 956                                         tsk->min_flt++;
 957
 958                                 /*
 959                                  * The VM_FAULT_WRITE bit tells us that
 960                                  * do_wp_page has broken COW when necessary,
 961                                  * even if maybe_mkwrite decided not to set
 962                                  * pte_write. We can thus safely do subsequent
 963                                  * page lookups as if they were reads. But only
 964                                  * do so when looping for pte_write is futile:
 965                                  * in some cases userspace may also be wanting
 966                                  * to write to the gotten user page, which a
 967                                  * read fault here might prevent (a readonly
 968                                  * page might get reCOWed by userspace write).
 969                                  */
 970                                 if ((ret & VM_FAULT_WRITE) &&
 971                                                 !(vma->vm_flags & VM_WRITE))
 972                                         foll_flags &= ~FOLL_WRITE;
 973
 974                                 //cond_resched();
 975 #endif
 976
 977                         }
 978
 979                         if (IS_ERR(page))
 980                                 return i ? i : PTR_ERR(page);
 981                         if (pages) {
 982                                 pages[i] = page;
 983
 984                                 swap_flush_anon_page(vma, page, start);
 985                                 flush_dcache_page(page);
 986                         }
 987                         if (vmas)
 988                                 vmas[i] = vma;
 989                         i++;
 990                         start += PAGE_SIZE;
 991                         len--;
 992                 } while (len && start < vma->vm_end);
 993         } while (len);
 994         return i;
 995 }
 996 #endif
 997 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 998
 999 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
1000                 unsigned long start, int len, int write, int force,
1001                 struct page **pages, struct vm_area_struct **vmas)
1002 {
1003 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
1004 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) /* FIXME: must be >= 32! */
1005         int flags = FOLL_TOUCH;
1006
1007         if (pages)
1008                 flags |= FOLL_GET;
1009         if (write)
1010                 flags |= FOLL_WRITE;
1011         if (force)
1012                 flags |= FOLL_FORCE;
1013 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1014         int flags = 0;
1015
1016         if (write)
1017                 flags |= GUP_FLAGS_WRITE;
1018         if (force)
1019                 flags |= GUP_FLAGS_FORCE;
1020 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1021
1022         return __get_user_pages_uprobe(tsk, mm,
1023                                 start, len, flags,
1024 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
1025                                                 pages, vmas, NULL);
1026 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1027                                                 pages, vmas);
1028 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1029 #else
1030         return get_user_pages(tsk, mm, start, len, write, force, pages, vmas);
1031 #endif
1032 }
1033
1034 #define ACCESS_PROCESS_OPTIMIZATION 0
1035
1036 #if ACCESS_PROCESS_OPTIMIZATION
1037
1038 #define GET_STEP_X(LEN, STEP) (((LEN) >= (STEP)) ? (STEP) : (LEN) % (STEP))
1039 #define GET_STEP_4(LEN) GET_STEP_X((LEN), 4)
1040
1041 static void read_data_current(unsigned long addr, void *buf, int len)
1042 {
1043         int step;
1044         int pos = 0;
1045
1046         for (step = GET_STEP_4(len); len; len -= step) {
1047                 switch (GET_STEP_4(len)) {
1048                 case 1:
1049                         get_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
1050                         step = 1;
1051                         break;
1052
1053                 case 2:
1054                 case 3:
1055                         get_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
1056                         step = 2;
1057                         break;
1058
1059                 case 4:
1060                         get_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
1061                         step = 4;
1062                         break;
1063                 }
1064
1065                 pos += step;
1066         }
1067 }
1068
1069 // not working
1070 static void write_data_current(unsigned long addr, void *buf, int len)
1071 {
1072         int step;
1073         int pos = 0;
1074
1075         for (step = GET_STEP_4(len); len; len -= step) {
1076                 switch (GET_STEP_4(len)) {
1077                 case 1:
1078                         put_user(*(u8 *)(buf + pos), (unsigned long *)(addr + pos));
1079                         step = 1;
1080                         break;
1081
1082                 case 2:
1083                 case 3:
1084                         put_user(*(u16 *)(buf + pos), (unsigned long *)(addr + pos));
1085                         step = 2;
1086                         break;
1087
1088                 case 4:
1089                         put_user(*(u32 *)(buf + pos), (unsigned long *)(addr + pos));
1090                         step = 4;
1091                         break;
1092                 }
1093
1094                 pos += step;
1095         }
1096 }
1097 #endif
1098
1099 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
1100 {
1101         struct mm_struct *mm;
1102         struct vm_area_struct *vma;
1103         void *old_buf = buf;
1104         int atomic;
1105
1106         if (len <= 0) {
1107                 return -1;
1108         }
1109
1110 #if ACCESS_PROCESS_OPTIMIZATION
1111         if (write == 0 && tsk == current) {
1112                 read_data_current(addr, buf, len);
1113                 return len;
1114         }
1115 #endif
1116
1117         mm = tsk->mm; /* function 'get_task_mm' is to be called */
1118         if (!mm)
1119                 return 0;
1120
1121         /* FIXME: danger: write memory in atomic context */
1122         atomic = in_atomic();
1123
1124         /* ignore errors, just check how much was successfully transferred */
1125         while (len) {
1126                 int bytes, ret, offset;
1127                 void *maddr;
1128                 struct page *page = NULL;
1129
1130                 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
1131                                                 write, 1, &page, &vma);
1132
1133                 if (ret <= 0) {
1134                         /*
1135                          * Check if this is a VM_IO | VM_PFNMAP VMA, which
1136                          * we can access using slightly different code.
1137                          */
1138 #ifdef CONFIG_HAVE_IOREMAP_PROT
1139                         vma = find_vma(mm, addr);
1140                         if (!vma)
1141                                 break;
1142                         if (vma->vm_ops && vma->vm_ops->access)
1143                                 ret = vma->vm_ops->access(vma, addr, buf,
1144                                                         len, write);
1145                         if (ret <= 0)
1146 #endif
1147                                 break;
1148                         bytes = ret;
1149                 } else {
1150                         bytes = len;
1151                         offset = addr & (PAGE_SIZE-1);
1152                         if (bytes > PAGE_SIZE-offset)
1153                                 bytes = PAGE_SIZE-offset;
1154
1155                         maddr = atomic ? swap_kmap_atomic(page) : kmap(page);
1156
1157                         if (write) {
1158                                 swap_copy_to_user_page(vma, page, addr,
1159                                                         maddr + offset, buf, bytes);
1160                                 set_page_dirty_lock(page);
1161                         } else {
1162                                 copy_from_user_page(vma, page, addr,
1163                                                         buf, maddr + offset, bytes);
1164                         }
1165
1166                         atomic ? swap_kunmap_atomic(maddr) : kunmap(page);
1167                         page_cache_release(page);
1168                 }
1169                 len -= bytes;
1170                 buf += bytes;
1171                 addr += bytes;
1172         }
1173
1174         return buf - old_buf;
1175 }
1176
1177 int page_present (struct mm_struct *mm, unsigned long address)
1178 {
1179         pgd_t *pgd;
1180         pud_t *pud;
1181         pmd_t *pmd;
1182         pte_t *ptep, pte;
1183         unsigned long pfn;
1184
1185         pgd = pgd_offset(mm, address);
1186         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
1187                 goto out;
1188
1189         pud = pud_offset(pgd, address);
1190         if (pud_none(*pud) || unlikely(pud_bad(*pud)))
1191                 goto out;
1192
1193         pmd = pmd_offset(pud, address);
1194         if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
1195                 goto out;
1196
1197         ptep = pte_offset_map(pmd, address);
1198         if (!ptep)
1199                 goto out;
1200
1201         pte = *ptep;
1202         pte_unmap(ptep);
1203         if (pte_present(pte)) {
1204                 pfn = pte_pfn(pte);
1205                 if (pfn_valid(pfn)) {
1206                         return 1;
1207                 }
1208         }
1209
1210 out:
1211         return 0;
1212 }
1213
1214
1215 EXPORT_SYMBOL_GPL (page_present);
1216 EXPORT_SYMBOL_GPL (access_process_vm_atomic);
1217