kernel/swap/kprobe/swap_kprobes_deps.c

   1 /**
   2  * kprobe/swap_kprobes_deps.c
   3  * @author Alexey Gerenkov <a.gerenkov@samsung.com> User-Space Probes initial implementation;
   4  * Support x86/ARM/MIPS for both user and kernel spaces.
   5  * @author Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
   6  *
   7  * @section LICENSE
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  22  *
  23  * @section COPYRIGHT
  24  *
  25  * Copyright (C) Samsung Electronics, 2006-2010
  26  *
  27  * @section DESCRIPTION
  28  *
  29  * SWAP kprobe kernel-dependent dependencies.
  30  */
  31
  32 #include <linux/module.h>
  33 #include <linux/sched.h>
  34
  35 #include <asm/pgtable.h>
  36
  37 #include "swap_kprobes_deps.h"
  38 #include "swap_kdebug.h"
  39
  40
  41 #include <linux/slab.h>
  42 #include <linux/mm.h>
  43
  44 /* kernel define 'pgd_offset_k' redefinition */
  45 #undef pgd_offset_k
  46 #define pgd_offset_k(addr)      pgd_offset(init_task.active_mm, addr)
  47
  48 #ifndef is_zero_pfn
  49
  50 static unsigned long swap_zero_pfn ;
  51
  52 #endif /* is_zero_pfn */
  53
  54 static inline void *swap_kmap_atomic(struct page *page)
  55 {
  56         return kmap_atomic(page);
  57 }
  58 static inline void swap_kunmap_atomic(void *kvaddr)
  59 {
  60         kunmap_atomic(kvaddr);
  61 }
  62
  63 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file,
  64                      unsigned long addr, unsigned long len, unsigned long prot,
  65                      unsigned long flags, unsigned long pgoff,
  66                      unsigned long *populate);
  67 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap_pgoff,
  68                         unsigned long,
  69                         struct file *file, unsigned long addr,
  70                         unsigned long len, unsigned long prot,
  71                         unsigned long flags, unsigned long pgoff,
  72                         unsigned long *populate)
  73 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len,
  74                     prot, flags, pgoff, populate)
  75
  76 EXPORT_SYMBOL_GPL(swap_do_mmap_pgoff);
  77
  78 /* copy_to_user_page */
  79 #ifndef copy_to_user_page
  80 static DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma,
  81                             struct page *page, unsigned long uaddr, void *dst,
  82                             const void *src, unsigned long len);
  83 DECLARE_MOD_DEP_WRAPPER(swap_copy_to_user_page,
  84                         void,
  85                         struct vm_area_struct *vma, struct page *page,
  86                         unsigned long uaddr, void *dst, const void *src,
  87                         unsigned long len)
  88 IMP_MOD_DEP_WRAPPER(copy_to_user_page, vma, page, uaddr, dst, src, len)
  89 #else /* copy_to_user_page */
  90 #define swap_copy_to_user_page copy_to_user_page
  91 #endif /* copy_to_user_page */
  92
  93
  94 static DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *,
  95                             struct mm_struct *mm, unsigned long addr);
  96
  97 static DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm,
  98                             struct vm_area_struct *vma, unsigned long address,
  99                             unsigned int flags);
 100
 101 static DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *,
 102                             struct mm_struct *mm);
 103
 104 #ifdef __HAVE_ARCH_GATE_AREA
 105 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct mm_struct *mm,
 106                      unsigned long addr);
 107 #endif /* __HAVE_ARCH_GATE_AREA */
 108
 109 static DECLARE_MOD_FUNC_DEP(in_gate_area_no_mm, int, unsigned long addr);
 110
 111 static DECLARE_MOD_FUNC_DEP(follow_page_mask, \
 112                 struct page *, struct vm_area_struct *vma, \
 113                 unsigned long address, unsigned int foll_flags, \
 114                 unsigned int *page_mask);
 115 DECLARE_MOD_DEP_WRAPPER(swap_follow_page_mask,
 116                         struct page *,
 117                         struct vm_area_struct *vma, unsigned long address,
 118                         unsigned int foll_flags, unsigned int *page_mask)
 119 IMP_MOD_DEP_WRAPPER(follow_page_mask, vma, address, foll_flags, page_mask)
 120
 121 static DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
 122                 void, struct vm_area_struct *vma, struct page *page, \
 123                 unsigned long vmaddr);
 124 static DECLARE_MOD_FUNC_DEP(vm_normal_page, \
 125                 struct page *, struct vm_area_struct *vma, \
 126                 unsigned long addr, pte_t pte);
 127
 128
 129 static DECLARE_MOD_FUNC_DEP(put_task_struct, \
 130                 void, struct task_struct *tsk);
 131
 132 DECLARE_MOD_DEP_WRAPPER(swap_find_extend_vma,
 133                         struct vm_area_struct *,
 134                         struct mm_struct *mm, unsigned long addr)
 135 IMP_MOD_DEP_WRAPPER(find_extend_vma, mm, addr)
 136
 137 DECLARE_MOD_DEP_WRAPPER(swap_handle_mm_fault,
 138                         int,
 139                         struct mm_struct *mm, struct vm_area_struct *vma,
 140                         unsigned long address, unsigned int flags)
 141 {
 142         if (in_atomic())
 143                 return VM_FAULT_ERROR | VM_FAULT_OOM;
 144
 145         IMP_MOD_DEP_WRAPPER(handle_mm_fault, mm, vma, address, flags)
 146 }
 147
 148 DECLARE_MOD_DEP_WRAPPER(swap_get_gate_vma,
 149                         struct vm_area_struct *,
 150                         struct mm_struct *mm)
 151 IMP_MOD_DEP_WRAPPER(get_gate_vma, mm)
 152
 153 #ifdef CONFIG_HUGETLB_PAGE
 154
 155 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page,                               \
 156                      long,                                              \
 157                      struct mm_struct *mm, struct vm_area_struct *vma,  \
 158                      struct page **pages, struct vm_area_struct **vmas, \
 159                      unsigned long *position, unsigned long *nr_pages,  \
 160                      long i, unsigned int flags);
 161 DECLARE_MOD_DEP_WRAPPER(swap_follow_hugetlb_page,
 162                         long,
 163                         struct mm_struct *mm, struct vm_area_struct *vma,
 164                         struct page **pages, struct vm_area_struct **vmas,
 165                         unsigned long *position, unsigned long *nr_pages,
 166                         long i, unsigned int flags)
 167 IMP_MOD_DEP_WRAPPER(follow_hugetlb_page,                                \
 168                     mm, vma, pages, vmas, position, nr_pages, i, flags)
 169
 170 #else /* CONFIG_HUGETLB_PAGE */
 171 #define swap_follow_hugetlb_page follow_hugetlb_page
 172 #endif /* CONFIG_HUGETLB_PAGE */
 173
 174 static inline int swap_in_gate_area(struct task_struct *task,
 175                                     unsigned long addr)
 176 {
 177 #ifdef __HAVE_ARCH_GATE_AREA
 178         struct mm_struct *mm;
 179
 180         if (task == NULL)
 181                 return 0;
 182
 183         mm = task->mm;
 184         IMP_MOD_DEP_WRAPPER(in_gate_area, mm, addr)
 185 #else /*__HAVE_ARCH_GATE_AREA */
 186         return in_gate_area(task, addr);
 187 #endif/*__HAVE_ARCH_GATE_AREA */
 188 }
 189
 190
 191 DECLARE_MOD_DEP_WRAPPER(swap_in_gate_area_no_mm, int, unsigned long addr)
 192 IMP_MOD_DEP_WRAPPER(in_gate_area_no_mm, addr)
 193
 194 static inline int swap_in_gate_area_no_xxx(unsigned long addr)
 195 {
 196         return swap_in_gate_area_no_mm(addr);
 197 }
 198
 199 DECLARE_MOD_DEP_WRAPPER(swap__flush_anon_page,
 200                         void,
 201                         struct vm_area_struct *vma, struct page *page,
 202                         unsigned long vmaddr)
 203 IMP_MOD_DEP_WRAPPER(__flush_anon_page, vma, page, vmaddr)
 204
 205 static inline void swap_flush_anon_page(struct vm_area_struct *vma,
 206                                         struct page *page,
 207                                         unsigned long vmaddr)
 208 {
 209 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 210         if (PageAnon(page))
 211                 swap__flush_anon_page(vma, page, vmaddr);
 212 #else /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 213         flush_anon_page(vma, page, vmaddr);
 214 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 215 }
 216
 217 DECLARE_MOD_DEP_WRAPPER(swap_vm_normal_page,
 218                         struct page *,
 219                         struct vm_area_struct *vma, unsigned long addr,
 220                         pte_t pte)
 221 IMP_MOD_DEP_WRAPPER(vm_normal_page, vma, addr, pte)
 222
 223
 224
 225 /**
 226  * @brief Initializes module dependencies.
 227  *
 228  * @return 0.
 229  */
 230 int init_module_dependencies(void)
 231 {
 232
 233         INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
 234
 235 #ifndef copy_to_user_page
 236         INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
 237 #endif /* copy_to_user_page */
 238
 239         INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
 240         INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
 241
 242 #ifdef CONFIG_HUGETLB_PAGE
 243         INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
 244 #endif
 245
 246 #ifdef  __HAVE_ARCH_GATE_AREA
 247         INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
 248 #endif
 249
 250         INIT_MOD_DEP_VAR(follow_page_mask, follow_page_mask);
 251
 252
 253 #ifndef is_zero_pfn
 254         swap_zero_pfn = page_to_pfn(ZERO_PAGE(0));
 255 #endif /* is_zero_pfn */
 256
 257         INIT_MOD_DEP_VAR(in_gate_area_no_mm, in_gate_area_no_mm);
 258
 259 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 260         INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
 261 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 262
 263         INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
 264
 265         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
 266         INIT_MOD_DEP_VAR(do_mmap_pgoff, do_mmap_pgoff);
 267
 268         return 0;
 269 }
 270
 271
 272 static inline int use_zero_page(struct vm_area_struct *vma)
 273 {
 274         /*
 275          * We don't want to optimize FOLL_ANON for make_pages_present()
 276          * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
 277          * we want to get the page from the page tables to make sure
 278          * that we serialize and update with any other user of that
 279          * mapping.
 280          */
 281         if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
 282                 return 0;
 283         /*
 284          * And if we have a fault routine, it's not an anonymous region.
 285          */
 286         return !vma->vm_ops || !vma->vm_ops->fault;
 287 }
 288
 289
 290
 291 #ifdef __HAVE_COLOR_ZERO_PAGE
 292
 293 static inline int swap_is_zero_pfn(unsigned long pfn)
 294 {
 295         unsigned long offset_from_zero_pfn = pfn - swap_zero_pfn;
 296         return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
 297 }
 298
 299 #else /* __HAVE_COLOR_ZERO_PAGE */
 300
 301 static inline int swap_is_zero_pfn(unsigned long pfn)
 302 {
 303         return pfn == swap_zero_pfn;
 304 }
 305 #endif /* __HAVE_COLOR_ZERO_PAGE */
 306
 307
 308 static inline int stack_guard_page(struct vm_area_struct *vma,
 309                                    unsigned long addr)
 310 {
 311         return stack_guard_page_start(vma, addr) ||
 312                         stack_guard_page_end(vma, addr+PAGE_SIZE);
 313 }
 314
 315
 316 /**
 317  * @brief Gets user pages uprobe.
 318  *
 319  * @param tsk Pointer to the task_struct.
 320  * @param mm Pointer to the mm_struct.
 321  * @param start Starting address.
 322  * @param nr_pages Pages number.
 323  * @param gup_flags Flags.
 324  * @param pages Pointer to the array of pointers to the target page structs.
 325  * @param vmas Pointer to the array of pointers to the target vm_area_struct.
 326  * @param nonblocking Pointer to int.
 327  * @return negative error code on error, positive result otherwise.
 328  */
 329 long __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 330                 unsigned long start, unsigned long nr_pages,
 331                 unsigned int gup_flags, struct page **pages,
 332                 struct vm_area_struct **vmas, int *nonblocking)
 333 {
 334         long i;
 335         unsigned long vm_flags;
 336         unsigned int page_mask;
 337
 338         if (!nr_pages)
 339                 return 0;
 340
 341         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 342
 343         /*
 344          * Require read or write permissions.
 345          * If FOLL_FORCE is set, we only require the "MAY" flags.
 346          */
 347         vm_flags  = (gup_flags & FOLL_WRITE) ?
 348                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 349         vm_flags &= (gup_flags & FOLL_FORCE) ?
 350                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 351
 352         /*
 353          * If FOLL_FORCE and FOLL_NUMA are both set, handle_mm_fault
 354          * would be called on PROT_NONE ranges. We must never invoke
 355          * handle_mm_fault on PROT_NONE ranges or the NUMA hinting
 356          * page faults would unprotect the PROT_NONE ranges if
 357          * _PAGE_NUMA and _PAGE_PROTNONE are sharing the same pte/pmd
 358          * bitflag. So to avoid that, don't set FOLL_NUMA if
 359          * FOLL_FORCE is set.
 360          */
 361         if (!(gup_flags & FOLL_FORCE))
 362                 gup_flags |= FOLL_NUMA;
 363
 364         i = 0;
 365
 366         do {
 367                 struct vm_area_struct *vma;
 368
 369                 vma = swap_find_extend_vma(mm, start);
 370                 if (!vma && swap_in_gate_area(tsk, start)) {
 371                         unsigned long pg = start & PAGE_MASK;
 372                         pgd_t *pgd;
 373                         pud_t *pud;
 374                         pmd_t *pmd;
 375                         pte_t *pte;
 376
 377                         /* user gate pages are read-only */
 378                         if (gup_flags & FOLL_WRITE)
 379                                 return i ? : -EFAULT;
 380                         if (pg > TASK_SIZE)
 381                                 pgd = pgd_offset_k(pg);
 382                         else
 383                                 pgd = pgd_offset_gate(mm, pg);
 384                         BUG_ON(pgd_none(*pgd));
 385                         pud = pud_offset(pgd, pg);
 386                         BUG_ON(pud_none(*pud));
 387                         pmd = pmd_offset(pud, pg);
 388                         if (pmd_none(*pmd))
 389                                 return i ? : -EFAULT;
 390                         VM_BUG_ON(pmd_trans_huge(*pmd));
 391                         pte = pte_offset_map(pmd, pg);
 392                         if (pte_none(*pte)) {
 393                                 pte_unmap(pte);
 394                                 return i ? : -EFAULT;
 395                         }
 396                         vma = swap_get_gate_vma(mm);
 397                         if (pages) {
 398                                 struct page *page;
 399
 400                                 page = swap_vm_normal_page(vma, start, *pte);
 401                                 if (!page) {
 402                                         if (!(gup_flags & FOLL_DUMP) &&
 403                                              swap_is_zero_pfn(pte_pfn(*pte)))
 404                                                 page = pte_page(*pte);
 405                                         else {
 406                                                 pte_unmap(pte);
 407                                                 return i ? : -EFAULT;
 408                                         }
 409                                 }
 410                                 pages[i] = page;
 411                                 get_page(page);
 412                         }
 413                         pte_unmap(pte);
 414                         page_mask = 0;
 415                         goto next_page;
 416                 }
 417
 418                 if (!vma ||
 419                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 420                     !(vm_flags & vma->vm_flags))
 421                         return i ? : -EFAULT;
 422
 423                 if (is_vm_hugetlb_page(vma)) {
 424                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 425                                         &start, &nr_pages, i, gup_flags);
 426                         continue;
 427                 }
 428
 429                 do {
 430                         struct page *page;
 431                         unsigned int foll_flags = gup_flags;
 432                         unsigned int page_increm;
 433
 434                         /*
 435                          * If we have a pending SIGKILL, don't keep faulting
 436                          * pages and potentially allocating memory.
 437                          */
 438                         if (unlikely(fatal_signal_pending(current)))
 439                                 return i ? i : -ERESTARTSYS;
 440
 441                         /* cond_resched(); */
 442                         while (!(page = swap_follow_page_mask(vma, start,
 443                                                 foll_flags, &page_mask))) {
 444                                 int ret;
 445                                 unsigned int fault_flags = 0;
 446
 447                                 /* For mlock, just skip the stack guard page. */
 448                                 if (foll_flags & FOLL_MLOCK) {
 449                                         if (stack_guard_page(vma, start))
 450                                                 goto next_page;
 451                                 }
 452                                 if (foll_flags & FOLL_WRITE)
 453                                         fault_flags |= FAULT_FLAG_WRITE;
 454                                 if (nonblocking)
 455                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 456                                 if (foll_flags & FOLL_NOWAIT)
 457                                         fault_flags |=
 458                                                 (FAULT_FLAG_ALLOW_RETRY |
 459                                                  FAULT_FLAG_RETRY_NOWAIT);
 460
 461                                 ret = swap_handle_mm_fault(mm, vma, start,
 462                                                         fault_flags);
 463
 464                                 if (ret & VM_FAULT_ERROR) {
 465                                         if (ret & VM_FAULT_OOM)
 466                                                 return i ? i : -ENOMEM;
 467                                         if (ret & (VM_FAULT_HWPOISON |
 468                                                    VM_FAULT_HWPOISON_LARGE)) {
 469                                                 if (i)
 470                                                         return i;
 471                                                 else if (gup_flags &
 472                                                          FOLL_HWPOISON)
 473                                                         return -EHWPOISON;
 474                                                 else
 475                                                         return -EFAULT;
 476                                         }
 477                                         if (ret & VM_FAULT_SIGBUS)
 478                                                 return i ? i : -EFAULT;
 479                                         BUG();
 480                                 }
 481
 482                                 if (tsk) {
 483                                         if (ret & VM_FAULT_MAJOR)
 484                                                 tsk->maj_flt++;
 485                                         else
 486                                                 tsk->min_flt++;
 487                                 }
 488
 489                                 if (ret & VM_FAULT_RETRY) {
 490                                         if (nonblocking)
 491                                                 *nonblocking = 0;
 492                                         return i;
 493                                 }
 494
 495                                 /*
 496                                  * The VM_FAULT_WRITE bit tells us that
 497                                  * do_wp_page has broken COW when necessary,
 498                                  * even if maybe_mkwrite decided not to set
 499                                  * pte_write. We can thus safely do subsequent
 500                                  * page lookups as if they were reads. But only
 501                                  * do so when looping for pte_write is futile:
 502                                  * in some cases userspace may also be wanting
 503                                  * to write to the gotten user page, which a
 504                                  * read fault here might prevent (a readonly
 505                                  * page might get reCOWed by userspace write).
 506                                  */
 507                                 if ((ret & VM_FAULT_WRITE) &&
 508                                     !(vma->vm_flags & VM_WRITE))
 509                                         foll_flags &= ~FOLL_WRITE;
 510
 511                                 /* cond_resched(); */
 512                         }
 513                         if (IS_ERR(page))
 514                                 return i ? i : PTR_ERR(page);
 515                         if (pages) {
 516                                 pages[i] = page;
 517
 518                                 swap_flush_anon_page(vma, page, start);
 519                                 flush_dcache_page(page);
 520                                 page_mask = 0;
 521                         }
 522 next_page:
 523                         if (vmas) {
 524                                 vmas[i] = vma;
 525                                 page_mask = 0;
 526                         }
 527                         page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
 528                         if (page_increm > nr_pages)
 529                                 page_increm = nr_pages;
 530                         i += page_increm;
 531                         start += page_increm * PAGE_SIZE;
 532                         nr_pages -= page_increm;
 533                 } while (nr_pages && start < vma->vm_end);
 534         } while (nr_pages);
 535         return i;
 536 }
 537
 538
 539
 540 /**
 541  * @brief Gets user pages uprobe.
 542  *
 543  * @param tsk Pointer to the task_struct.
 544  * @param mm Pointer to the mm_struct.
 545  * @param start Starting address.
 546  * @param len Length.
 547  * @param write Write flag.
 548  * @param force Force flag.
 549  * @param pages Pointer to the array of pointers to the target page structs.
 550  * @param vmas Pointer to the array of pointers to the target vm_area_struct.
 551  * @return negative error code on error, positive result otherwise.
 552  */
 553 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 554                 unsigned long start, int len, int write, int force,
 555                 struct page **pages, struct vm_area_struct **vmas)
 556 {
 557         int flags = FOLL_TOUCH;
 558
 559         if (pages)
 560                 flags |= FOLL_GET;
 561         if (write)
 562                 flags |= FOLL_WRITE;
 563         if (force)
 564                 flags |= FOLL_FORCE;
 565
 566         return __get_user_pages_uprobe(tsk, mm,
 567                                 start, len, flags,
 568                                                 pages, vmas, NULL);
 569 }
 570
 571 #define ACCESS_PROCESS_OPTIMIZATION 0
 572
 573 #if ACCESS_PROCESS_OPTIMIZATION
 574
 575 #define GET_STEP_X(LEN, STEP) (((LEN) >= (STEP)) ? (STEP) : (LEN) % (STEP))
 576 #define GET_STEP_4(LEN) GET_STEP_X((LEN), 4)
 577
 578 static void read_data_current(unsigned long addr, void *buf, int len)
 579 {
 580         int step;
 581         int pos = 0;
 582
 583         for (step = GET_STEP_4(len); len; len -= step) {
 584                 switch (GET_STEP_4(len)) {
 585                 case 1:
 586                         get_user(*(u8 *)(buf + pos),
 587                                  (unsigned long *)(addr + pos));
 588                         step = 1;
 589                         break;
 590
 591                 case 2:
 592                 case 3:
 593                         get_user(*(u16 *)(buf + pos),
 594                                  (unsigned long *)(addr + pos));
 595                         step = 2;
 596                         break;
 597
 598                 case 4:
 599                         get_user(*(u32 *)(buf + pos),
 600                                  (unsigned long *)(addr + pos));
 601                         step = 4;
 602                         break;
 603                 }
 604
 605                 pos += step;
 606         }
 607 }
 608
 609 /* not working */
 610 static void write_data_current(unsigned long addr, void *buf, int len)
 611 {
 612         int step;
 613         int pos = 0;
 614
 615         for (step = GET_STEP_4(len); len; len -= step) {
 616                 switch (GET_STEP_4(len)) {
 617                 case 1:
 618                         put_user(*(u8 *)(buf + pos),
 619                                  (unsigned long *)(addr + pos));
 620                         step = 1;
 621                         break;
 622
 623                 case 2:
 624                 case 3:
 625                         put_user(*(u16 *)(buf + pos),
 626                                  (unsigned long *)(addr + pos));
 627                         step = 2;
 628                         break;
 629
 630                 case 4:
 631                         put_user(*(u32 *)(buf + pos),
 632                                  (unsigned long *)(addr + pos));
 633                         step = 4;
 634                         break;
 635                 }
 636
 637                 pos += step;
 638         }
 639 }
 640 #endif
 641
 642 /**
 643  * @brief Read-write task memory.
 644  *
 645  * @param tsk Pointer to the target task task_struct.
 646  * @param addr Address to read-write.
 647  * @param buf Pointer to buffer where to put-get data.
 648  * @param len Buffer length.
 649  * @param write Write flag. If 0 - reading, if 1 - writing.
 650  * @return Read-write size, error code on error.
 651  */
 652 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr,
 653                              void *buf, int len, int write)
 654 {
 655         struct mm_struct *mm;
 656         struct vm_area_struct *vma;
 657         void *old_buf = buf;
 658         int atomic;
 659
 660         if (len <= 0)
 661                 return -1;
 662
 663 #if ACCESS_PROCESS_OPTIMIZATION
 664         if (write == 0 && tsk == current) {
 665                 read_data_current(addr, buf, len);
 666                 return len;
 667         }
 668 #endif
 669
 670         mm = tsk->mm; /* function 'get_task_mm' is to be called */
 671         if (!mm)
 672                 return 0;
 673
 674         /* FIXME: danger: write memory in atomic context */
 675         atomic = in_atomic();
 676
 677         /* ignore errors, just check how much was successfully transferred */
 678         while (len) {
 679                 int bytes, ret, offset;
 680                 void *maddr;
 681                 struct page *page = NULL;
 682
 683                 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
 684                                                 write, 1, &page, &vma);
 685
 686                 if (ret <= 0) {
 687                         /*
 688                          * Check if this is a VM_IO | VM_PFNMAP VMA, which
 689                          * we can access using slightly different code.
 690                          */
 691 #ifdef CONFIG_HAVE_IOREMAP_PROT
 692                         vma = find_vma(mm, addr);
 693                         if (!vma)
 694                                 break;
 695                         if (vma->vm_ops && vma->vm_ops->access)
 696                                 ret = vma->vm_ops->access(vma, addr, buf,
 697                                                         len, write);
 698                         if (ret <= 0)
 699 #endif
 700                                 break;
 701                         bytes = ret;
 702                 } else {
 703                         bytes = len;
 704                         offset = addr & (PAGE_SIZE-1);
 705                         if (bytes > PAGE_SIZE-offset)
 706                                 bytes = PAGE_SIZE-offset;
 707
 708                         maddr = atomic ? swap_kmap_atomic(page) : kmap(page);
 709
 710                         if (write) {
 711                                 swap_copy_to_user_page(vma, page, addr,
 712                                                         maddr + offset,
 713                                                        buf, bytes);
 714                                 set_page_dirty_lock(page);
 715                         } else {
 716                                 copy_from_user_page(vma, page, addr,
 717                                                     buf, maddr + offset,
 718                                                     bytes);
 719                         }
 720
 721                         atomic ? swap_kunmap_atomic(maddr) : kunmap(page);
 722                         page_cache_release(page);
 723                 }
 724                 len -= bytes;
 725                 buf += bytes;
 726                 addr += bytes;
 727         }
 728
 729         return buf - old_buf;
 730 }
 731 EXPORT_SYMBOL_GPL(access_process_vm_atomic);
 732
 733 /**
 734  * @brief Page present.
 735  *
 736  * @param mm Pointer to the target mm_struct.
 737  * @param address Address.
 738  */
 739 int page_present(struct mm_struct *mm, unsigned long address)
 740 {
 741         pgd_t *pgd;
 742         pud_t *pud;
 743         pmd_t *pmd;
 744         pte_t *ptep, pte;
 745         unsigned long pfn;
 746
 747         pgd = pgd_offset(mm, address);
 748         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
 749                 goto out;
 750
 751         pud = pud_offset(pgd, address);
 752         if (pud_none(*pud) || unlikely(pud_bad(*pud)))
 753                 goto out;
 754
 755         pmd = pmd_offset(pud, address);
 756         if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
 757                 goto out;
 758
 759         ptep = pte_offset_map(pmd, address);
 760         if (!ptep)
 761                 goto out;
 762
 763         pte = *ptep;
 764         pte_unmap(ptep);
 765         if (pte_present(pte)) {
 766                 pfn = pte_pfn(pte);
 767                 if (pfn_valid(pfn))
 768                         return 1;
 769         }
 770
 771 out:
 772         return 0;
 773 }
 774 EXPORT_SYMBOL_GPL(page_present);
 775