kprobe/swap_kprobes_deps.c

   1 /**
   2  * kprobe/swap_kprobes_deps.c
   3  * @author Alexey Gerenkov <a.gerenkov@samsung.com> User-Space Probes initial implementation;
   4  * Support x86/ARM/MIPS for both user and kernel spaces.
   5  * @author Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
   6  *
   7  * @section LICENSE
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  22  *
  23  * @section COPYRIGHT
  24  *
  25  * Copyright (C) Samsung Electronics, 2006-2010
  26  *
  27  * @section DESCRIPTION
  28  *
  29  * SWAP kprobe kernel-dependent dependencies.
  30  */
  31
  32 #include <linux/module.h>
  33 #include <linux/sched.h>
  34
  35 #include <asm/pgtable.h>
  36
  37 #include "swap_kprobes_deps.h"
  38 #include "swap_kdebug.h"
  39
  40
  41 #include <linux/slab.h>
  42 #include <linux/mm.h>
  43
  44 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
  45 /* kernel define 'pgd_offset_k' redefinition */
  46 #undef pgd_offset_k
  47 #define pgd_offset_k(addr)      pgd_offset(init_task.active_mm, addr)
  48 #endif
  49
  50 #if (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))
  51 #ifndef is_zero_pfn
  52
  53 static unsigned long swap_zero_pfn ;
  54
  55 #endif /* is_zero_pfn */
  56 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)) */
  57
  58 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36)
  59 static inline void *swap_kmap_atomic(struct page *page)
  60 {
  61         return kmap_atomic(page);
  62 }
  63 static inline void swap_kunmap_atomic(void *kvaddr)
  64 {
  65         kunmap_atomic(kvaddr);
  66 }
  67 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36) */
  68 static inline void *swap_kmap_atomic(struct page *page)
  69 {
  70         return kmap_atomic(page, KM_USER0);
  71 }
  72
  73 static inline void swap_kunmap_atomic(void *kvaddr)
  74 {
  75         kunmap_atomic(kvaddr, KM_USER0);
  76 }
  77 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 36) */
  78
  79 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
  80 DECLARE_MOD_FUNC_DEP(do_mmap, unsigned long, struct file *file,
  81                      unsigned long addr, unsigned long len, unsigned long prot,
  82                      unsigned long flags, vm_flags_t vm_flags,
  83                      unsigned long pgoff, unsigned long *populate);
  84 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap,
  85                         unsigned long,
  86                         struct file *file, unsigned long addr,
  87                         unsigned long len, unsigned long prot,
  88                         unsigned long flags, vm_flags_t vm_flags,
  89                         unsigned long pgoff, unsigned long *populate)
  90 IMP_MOD_DEP_WRAPPER(do_mmap, file, addr, len,
  91                     prot, flags, vm_flags, pgoff, populate)
  92 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
  93 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file,
  94                      unsigned long addr, unsigned long len, unsigned long prot,
  95                      unsigned long flags, unsigned long pgoff,
  96                      unsigned long *populate);
  97 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap_pgoff,
  98                         unsigned long,
  99                         struct file *file, unsigned long addr,
 100                         unsigned long len, unsigned long prot,
 101                         unsigned long flags, unsigned long pgoff,
 102                         unsigned long *populate)
 103 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len,
 104                     prot, flags, pgoff, populate)
 105 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
 106 DECLARE_MOD_FUNC_DEP(do_mmap_pgoff, unsigned long, struct file *file,
 107                      unsigned long addr, unsigned long len, unsigned long prot,
 108                      unsigned long flags, unsigned long pgoff);
 109 DECLARE_MOD_DEP_WRAPPER(swap_do_mmap_pgoff,
 110                         unsigned long,
 111                         struct file *file, unsigned long addr,
 112                         unsigned long len, unsigned long prot,
 113                         unsigned long flags, unsigned long pgoff)
 114 IMP_MOD_DEP_WRAPPER(do_mmap_pgoff, file, addr, len, prot, flags, pgoff)
 115 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) */
 116
 117 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
 118 EXPORT_SYMBOL_GPL(swap_do_mmap);
 119 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
 120 EXPORT_SYMBOL_GPL(swap_do_mmap_pgoff);
 121 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) */
 122
 123 /* copy_to_user_page */
 124 #ifndef copy_to_user_page
 125 static DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma,
 126                             struct page *page, unsigned long uaddr, void *dst,
 127                             const void *src, unsigned long len);
 128 DECLARE_MOD_DEP_WRAPPER(swap_copy_to_user_page,
 129                         void,
 130                         struct vm_area_struct *vma, struct page *page,
 131                         unsigned long uaddr, void *dst, const void *src,
 132                         unsigned long len)
 133 IMP_MOD_DEP_WRAPPER(copy_to_user_page, vma, page, uaddr, dst, src, len)
 134 #else /* copy_to_user_page */
 135 #define swap_copy_to_user_page copy_to_user_page
 136 #endif /* copy_to_user_page */
 137
 138
 139 static DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *,
 140                             struct mm_struct *mm, unsigned long addr);
 141
 142 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
 143 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 144 static DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm,
 145                             struct vm_area_struct *vma, unsigned long address,
 146                             int write_access);
 147 #endif
 148 #else
 149 static DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm,
 150                             struct vm_area_struct *vma, unsigned long address,
 151                             unsigned int flags);
 152 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
 153
 154 #ifdef __HAVE_ARCH_GATE_AREA
 155 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 156 static DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *,
 157                             struct mm_struct *mm);
 158 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 159 static DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *,
 160                             struct task_struct *tsk);
 161 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 162
 163 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 164 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct mm_struct *mm,
 165                      unsigned long addr);
 166 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 167 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct task_struct *task,
 168                      unsigned long addr);
 169 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 170
 171 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 172 static DECLARE_MOD_FUNC_DEP(in_gate_area_no_mm, int, unsigned long addr);
 173 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 174 static DECLARE_MOD_FUNC_DEP(in_gate_area_no_task, int, unsigned long addr);
 175 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 176 #endif /* __HAVE_ARCH_GATE_AREA */
 177
 178 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 179 static DECLARE_MOD_FUNC_DEP(follow_page_mask, \
 180                 struct page *, struct vm_area_struct *vma, \
 181                 unsigned long address, unsigned int foll_flags, \
 182                 unsigned int *page_mask);
 183 DECLARE_MOD_DEP_WRAPPER(swap_follow_page_mask,
 184                         struct page *,
 185                         struct vm_area_struct *vma, unsigned long address,
 186                         unsigned int foll_flags, unsigned int *page_mask)
 187 IMP_MOD_DEP_WRAPPER(follow_page_mask, vma, address, foll_flags, page_mask)
 188 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 189 static DECLARE_MOD_FUNC_DEP(follow_page, \
 190                 struct page *, struct vm_area_struct *vma, \
 191                 unsigned long address, unsigned int foll_flags);
 192 DECLARE_MOD_DEP_WRAPPER(swap_follow_page,
 193                         struct page *,
 194                         struct vm_area_struct *vma, unsigned long address,
 195                         unsigned int foll_flags)
 196 IMP_MOD_DEP_WRAPPER(follow_page, vma, address, foll_flags)
 197 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 198
 199 static DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
 200                 void, struct vm_area_struct *vma, struct page *page, \
 201                 unsigned long vmaddr);
 202 static DECLARE_MOD_FUNC_DEP(vm_normal_page, \
 203                 struct page *, struct vm_area_struct *vma, \
 204                 unsigned long addr, pte_t pte);
 205
 206
 207 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
 208 static DECLARE_MOD_FUNC_DEP(put_task_struct, \
 209                 void, struct task_struct *tsk);
 210 #else
 211 static DECLARE_MOD_FUNC_DEP(put_task_struct, \
 212                 void, struct rcu_head *rhp);
 213 #endif
 214
 215 DECLARE_MOD_DEP_WRAPPER(swap_find_extend_vma,
 216                         struct vm_area_struct *,
 217                         struct mm_struct *mm, unsigned long addr)
 218 IMP_MOD_DEP_WRAPPER(find_extend_vma, mm, addr)
 219
 220 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
 221 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 222 DECLARE_MOD_DEP_WRAPPER(swap_handle_mm_fault,
 223                         int,
 224                         struct mm_struct *mm, struct vm_area_struct *vma,
 225                         unsigned long address, int write_access)
 226 {
 227         if (in_atomic())
 228                 return VM_FAULT_ERROR | VM_FAULT_OOM;
 229
 230         IMP_MOD_DEP_WRAPPER(handle_mm_fault, mm, vma, address, write_access)
 231 }
 232 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18) */
 233 #else /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
 234 DECLARE_MOD_DEP_WRAPPER(swap_handle_mm_fault,
 235                         int,
 236                         struct mm_struct *mm, struct vm_area_struct *vma,
 237                         unsigned long address, unsigned int flags)
 238 {
 239         if (in_atomic())
 240                 return VM_FAULT_ERROR | VM_FAULT_OOM;
 241
 242         IMP_MOD_DEP_WRAPPER(handle_mm_fault, mm, vma, address, flags)
 243 }
 244 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
 245
 246 struct vm_area_struct *swap_get_gate_vma(struct mm_struct *mm)
 247 {
 248 #ifdef __HAVE_ARCH_GATE_AREA
 249 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 250 IMP_MOD_DEP_WRAPPER(get_gate_vma, mm)
 251 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 252 IMP_MOD_DEP_WRAPPER(get_gate_vma, tsk)
 253 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 254 #else /* __HAVE_ARCH_GATE_AREA */
 255 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 256         return get_gate_vma(mm);
 257 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 258         return get_gate_vma(tsk);
 259 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 260 #endif /* __HAVE_ARCH_GATE_AREA */
 261 }
 262
 263 #ifdef CONFIG_HUGETLB_PAGE
 264
 265 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0)
 266 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page,                               \
 267                      int,                                               \
 268                      struct mm_struct *mm, struct vm_area_struct *vma,  \
 269                      struct page **pages, struct vm_area_struct **vmas, \
 270                      unsigned long *position, int *length, int i,       \
 271                      unsigned int flags);
 272 DECLARE_MOD_DEP_WRAPPER(swap_follow_hugetlb_page,
 273                         int,
 274                         struct mm_struct *mm, struct vm_area_struct *vma,
 275                         struct page **pages, struct vm_area_struct **vmas,
 276                         unsigned long *position, int *length, int i,
 277                         unsigned int flags)
 278 IMP_MOD_DEP_WRAPPER(follow_hugetlb_page,                                \
 279                     mm, vma, pages, vmas, position, length, i, flags)
 280 #else /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) */
 281 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page,                               \
 282                      long,                                              \
 283                      struct mm_struct *mm, struct vm_area_struct *vma,  \
 284                      struct page **pages, struct vm_area_struct **vmas, \
 285                      unsigned long *position, unsigned long *nr_pages,  \
 286                      long i, unsigned int flags);
 287 DECLARE_MOD_DEP_WRAPPER(swap_follow_hugetlb_page,
 288                         long,
 289                         struct mm_struct *mm, struct vm_area_struct *vma,
 290                         struct page **pages, struct vm_area_struct **vmas,
 291                         unsigned long *position, unsigned long *nr_pages,
 292                         long i, unsigned int flags)
 293 IMP_MOD_DEP_WRAPPER(follow_hugetlb_page,                                \
 294                     mm, vma, pages, vmas, position, nr_pages, i, flags)
 295 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(3, 9, 0) */
 296
 297 #else /* CONFIG_HUGETLB_PAGE */
 298 #define swap_follow_hugetlb_page follow_hugetlb_page
 299 #endif /* CONFIG_HUGETLB_PAGE */
 300
 301 static inline int swap_in_gate_area(struct task_struct *task,
 302                                     unsigned long addr)
 303 {
 304 #ifdef __HAVE_ARCH_GATE_AREA
 305 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 306         struct mm_struct *mm;
 307
 308         if (task == NULL)
 309                 return 0;
 310
 311         mm = task->mm;
 312         IMP_MOD_DEP_WRAPPER(in_gate_area, mm, addr)
 313 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 314         IMP_MOD_DEP_WRAPPER(in_gate_area, task, addr)
 315 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 316 #else /*__HAVE_ARCH_GATE_AREA */
 317 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 318         struct mm_struct *mm;
 319
 320         if (task == NULL)
 321                 return 0;
 322
 323         mm = task->mm;
 324         return in_gate_area(mm, addr);
 325 #else
 326         return in_gate_area(task, addr);
 327 #endif
 328 #endif/*__HAVE_ARCH_GATE_AREA */
 329 }
 330
 331
 332 #ifdef __HAVE_ARCH_GATE_AREA
 333 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 334 DECLARE_MOD_DEP_WRAPPER(swap_in_gate_area_no_mm, int, unsigned long addr)
 335 IMP_MOD_DEP_WRAPPER(in_gate_area_no_mm, addr)
 336 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 337 DECLARE_MOD_DEP_WRAPPER(swap_in_gate_area_no_task, int, unsigned long addr)
 338 IMP_MOD_DEP_WRAPPER(in_gate_area_no_task, addr)
 339 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 340 #endif /* __HAVE_ARCH_GATE_AREA */
 341
 342 static inline int swap_in_gate_area_no_xxx(unsigned long addr)
 343 {
 344 #ifdef __HAVE_ARCH_GATE_AREA
 345 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 346         return swap_in_gate_area_no_mm(addr);
 347 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 348         return swap_in_gate_area_no_task(addr);
 349 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 350 #else /* __HAVE_ARCH_GATE_AREA */
 351 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 352         return in_gate_area_no_mm(addr);
 353 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 354         return in_gate_area_no_task(addr);
 355 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 356 #endif /* __HAVE_ARCH_GATE_AREA */
 357 }
 358
 359 DECLARE_MOD_DEP_WRAPPER(swap__flush_anon_page,
 360                         void,
 361                         struct vm_area_struct *vma, struct page *page,
 362                         unsigned long vmaddr)
 363 IMP_MOD_DEP_WRAPPER(__flush_anon_page, vma, page, vmaddr)
 364
 365 static inline void swap_flush_anon_page(struct vm_area_struct *vma,
 366                                         struct page *page,
 367                                         unsigned long vmaddr)
 368 {
 369 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 370         if (PageAnon(page))
 371                 swap__flush_anon_page(vma, page, vmaddr);
 372 #else /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 373         flush_anon_page(vma, page, vmaddr);
 374 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 375 }
 376
 377 DECLARE_MOD_DEP_WRAPPER(swap_vm_normal_page,
 378                         struct page *,
 379                         struct vm_area_struct *vma, unsigned long addr,
 380                         pte_t pte)
 381 IMP_MOD_DEP_WRAPPER(vm_normal_page, vma, addr, pte)
 382
 383
 384
 385 /**
 386  * @brief Initializes module dependencies.
 387  *
 388  * @return 0.
 389  */
 390 int init_module_dependencies(void)
 391 {
 392
 393 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 394         INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
 395 #endif
 396
 397 #ifndef copy_to_user_page
 398         INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
 399 #endif /* copy_to_user_page */
 400
 401         INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
 402
 403 #ifdef CONFIG_HUGETLB_PAGE
 404         INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
 405 #endif
 406
 407 #ifdef  __HAVE_ARCH_GATE_AREA
 408         INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
 409         INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
 410
 411 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 412         INIT_MOD_DEP_VAR(in_gate_area_no_mm, in_gate_area_no_mm);
 413 #else /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))  */
 414         INIT_MOD_DEP_VAR(in_gate_area_no_task, in_gate_area_no_task);
 415 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))  */
 416 #endif
 417
 418 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 419         INIT_MOD_DEP_VAR(follow_page_mask, follow_page_mask);
 420 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 421         INIT_MOD_DEP_VAR(follow_page, follow_page);
 422 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 423
 424 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 425
 426 #ifndef is_zero_pfn
 427         swap_zero_pfn = page_to_pfn(ZERO_PAGE(0));
 428 #endif /* is_zero_pfn */
 429 #endif /* (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38))  */
 430
 431 #if defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM)
 432         INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
 433 #endif /* defined(ARCH_HAS_FLUSH_ANON_PAGE) && defined(CONFIG_ARM) */
 434
 435         INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
 436
 437 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
 438 # if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
 439         INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
 440 # else
 441         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
 442 # endif
 443 #else /*2.6.16 */
 444         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
 445 #endif
 446 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0)
 447         INIT_MOD_DEP_VAR(do_mmap, do_mmap);
 448 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
 449         INIT_MOD_DEP_VAR(do_mmap_pgoff, do_mmap_pgoff);
 450 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0) */
 451
 452         return 0;
 453 }
 454
 455
 456 #ifdef CONFIG_ARM64
 457
 458 static int do_access_process_vm(struct task_struct *tsk, struct mm_struct *mm,
 459                                 unsigned long addr, void *buf, int len,
 460                                 int write)
 461 {
 462         struct vm_area_struct *vma;
 463         void *old_buf = buf;
 464
 465         while (len) {
 466                 int bytes, ret, offset;
 467                 void *maddr;
 468                 struct page *page = NULL;
 469
 470 # if LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0)
 471                 ret = get_user_pages(tsk, mm, addr, 1, write, 1, &page, &vma);
 472 # else /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) */
 473                 ret = get_user_pages_remote(tsk, mm, addr, 1,
 474                                             FOLL_WRITE | FOLL_FORCE,
 475                                             &page, &vma);
 476 # endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) */
 477
 478                 if (ret <= 0) {
 479 #ifndef CONFIG_HAVE_IOREMAP_PROT
 480                         break;
 481 #else
 482                         /*
 483                          * Check if this is a VM_IO | VM_PFNMAP VMA, which
 484                          * we can access using slightly different code.
 485                          */
 486                         vma = find_vma(mm, addr);
 487                         if (!vma || vma->vm_start > addr)
 488                                 break;
 489                         if (vma->vm_ops && vma->vm_ops->access)
 490                                 ret = vma->vm_ops->access(vma, addr, buf, len,
 491                                                           write);
 492                         if (ret <= 0)
 493                                 break;
 494                         bytes = ret;
 495 #endif
 496                 } else {
 497                         bytes = len;
 498                         offset = addr & (PAGE_SIZE-1);
 499                         if (bytes > PAGE_SIZE-offset)
 500                                 bytes = PAGE_SIZE-offset;
 501
 502                         maddr = kmap(page);
 503                         if (write) {
 504                                 swap_copy_to_user_page(vma, page, addr,
 505                                                        maddr + offset,
 506                                                        buf, bytes);
 507                                 set_page_dirty_lock(page);
 508                         } else {
 509                                 copy_from_user_page(vma, page, addr,
 510                                                     buf, maddr + offset, bytes);
 511                         }
 512                         kunmap(page);
 513                         put_page(page);
 514                 }
 515                 len -= bytes;
 516                 buf += bytes;
 517                 addr += bytes;
 518         }
 519
 520         return buf - old_buf;
 521 }
 522
 523 int swap_access_process_vm(struct task_struct *tsk, unsigned long addr,
 524                            void *buf, int len, int write)
 525 {
 526         int ret;
 527         struct mm_struct *mm;
 528
 529         mm = get_task_mm(tsk);
 530         if (!mm)
 531                 return 0;
 532
 533         ret = do_access_process_vm(tsk, mm, addr, buf, len, write);
 534         mmput(mm);
 535
 536         return ret;
 537 }
 538 EXPORT_SYMBOL_GPL(swap_access_process_vm);
 539
 540 #else /* CONFIG_ARM64 */
 541
 542 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) /* FIXME: must be < 32 */
 543 #define GUP_FLAGS_WRITE                  0x1
 544 #define GUP_FLAGS_FORCE                  0x2
 545 #define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
 546 #define GUP_FLAGS_IGNORE_SIGKILL         0x8
 547 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) */
 548
 549 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 550 static inline int use_zero_page(struct vm_area_struct *vma)
 551 {
 552         /*
 553          * We don't want to optimize FOLL_ANON for make_pages_present()
 554          * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
 555          * we want to get the page from the page tables to make sure
 556          * that we serialize and update with any other user of that
 557          * mapping.
 558          */
 559         if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
 560                 return 0;
 561         /*
 562          * And if we have a fault routine, it's not an anonymous region.
 563          */
 564         return !vma->vm_ops || !vma->vm_ops->fault;
 565 }
 566
 567 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 568
 569 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)
 570
 571 #ifdef __HAVE_COLOR_ZERO_PAGE
 572
 573 static inline int swap_is_zero_pfn(unsigned long pfn)
 574 {
 575         unsigned long offset_from_zero_pfn = pfn - swap_zero_pfn;
 576         return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
 577 }
 578
 579 #else /* __HAVE_COLOR_ZERO_PAGE */
 580
 581 static inline int swap_is_zero_pfn(unsigned long pfn)
 582 {
 583         return pfn == swap_zero_pfn;
 584 }
 585 #endif /* __HAVE_COLOR_ZERO_PAGE */
 586
 587 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
 588
 589 static inline int swap_is_zero_pfn(unsigned long pfn)
 590 {
 591 #ifndef is_zero_pfn
 592         return pfn == swap_zero_pfn;
 593 #else /* is_zero_pfn */
 594         return is_zero_pfn(pfn);
 595 #endif /* is_zero_pfn */
 596 }
 597
 598 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */
 599
 600 static inline int stack_guard_page(struct vm_area_struct *vma,
 601                                    unsigned long addr)
 602 {
 603         return stack_guard_page_start(vma, addr) ||
 604                         stack_guard_page_end(vma, addr+PAGE_SIZE);
 605 }
 606
 607 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 608
 609 /**
 610  * @brief Gets user pages uprobe.
 611  *
 612  * @param tsk Pointer to the task_struct.
 613  * @param mm Pointer to the mm_struct.
 614  * @param start Starting address.
 615  * @param nr_pages Pages number.
 616  * @param gup_flags Flags.
 617  * @param pages Pointer to the array of pointers to the target page structs.
 618  * @param vmas Pointer to the array of pointers to the target vm_area_struct.
 619  * @param nonblocking Pointer to int.
 620  * @return negative error code on error, positive result otherwise.
 621  */
 622 long __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 623                 unsigned long start, unsigned long nr_pages,
 624                 unsigned int gup_flags, struct page **pages,
 625                 struct vm_area_struct **vmas, int *nonblocking)
 626 {
 627         long i;
 628         unsigned long vm_flags;
 629         unsigned int page_mask;
 630
 631         if (!nr_pages)
 632                 return 0;
 633
 634         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 635
 636         /*
 637          * Require read or write permissions.
 638          * If FOLL_FORCE is set, we only require the "MAY" flags.
 639          */
 640         vm_flags  = (gup_flags & FOLL_WRITE) ?
 641                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 642         vm_flags &= (gup_flags & FOLL_FORCE) ?
 643                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 644
 645         /*
 646          * If FOLL_FORCE and FOLL_NUMA are both set, handle_mm_fault
 647          * would be called on PROT_NONE ranges. We must never invoke
 648          * handle_mm_fault on PROT_NONE ranges or the NUMA hinting
 649          * page faults would unprotect the PROT_NONE ranges if
 650          * _PAGE_NUMA and _PAGE_PROTNONE are sharing the same pte/pmd
 651          * bitflag. So to avoid that, don't set FOLL_NUMA if
 652          * FOLL_FORCE is set.
 653          */
 654         if (!(gup_flags & FOLL_FORCE))
 655                 gup_flags |= FOLL_NUMA;
 656
 657         i = 0;
 658
 659         do {
 660                 struct vm_area_struct *vma;
 661
 662                 vma = swap_find_extend_vma(mm, start);
 663                 if (!vma && swap_in_gate_area(tsk, start)) {
 664                         unsigned long pg = start & PAGE_MASK;
 665                         pgd_t *pgd;
 666                         pud_t *pud;
 667                         pmd_t *pmd;
 668                         pte_t *pte;
 669
 670                         /* user gate pages are read-only */
 671                         if (gup_flags & FOLL_WRITE)
 672                                 return i ? : -EFAULT;
 673                         if (pg > TASK_SIZE)
 674                                 pgd = pgd_offset_k(pg);
 675                         else
 676                                 pgd = pgd_offset_gate(mm, pg);
 677                         BUG_ON(pgd_none(*pgd));
 678                         pud = pud_offset(pgd, pg);
 679                         BUG_ON(pud_none(*pud));
 680                         pmd = pmd_offset(pud, pg);
 681                         if (pmd_none(*pmd))
 682                                 return i ? : -EFAULT;
 683                         VM_BUG_ON(pmd_trans_huge(*pmd));
 684                         pte = pte_offset_map(pmd, pg);
 685                         if (pte_none(*pte)) {
 686                                 pte_unmap(pte);
 687                                 return i ? : -EFAULT;
 688                         }
 689                         vma = swap_get_gate_vma(mm);
 690                         if (pages) {
 691                                 struct page *page;
 692
 693                                 page = swap_vm_normal_page(vma, start, *pte);
 694                                 if (!page) {
 695                                         if (!(gup_flags & FOLL_DUMP) &&
 696                                              swap_is_zero_pfn(pte_pfn(*pte)))
 697                                                 page = pte_page(*pte);
 698                                         else {
 699                                                 pte_unmap(pte);
 700                                                 return i ? : -EFAULT;
 701                                         }
 702                                 }
 703                                 pages[i] = page;
 704                                 get_page(page);
 705                         }
 706                         pte_unmap(pte);
 707                         page_mask = 0;
 708                         goto next_page;
 709                 }
 710
 711                 if (!vma ||
 712                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 713                     !(vm_flags & vma->vm_flags))
 714                         return i ? : -EFAULT;
 715
 716                 if (is_vm_hugetlb_page(vma)) {
 717                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 718                                         &start, &nr_pages, i, gup_flags);
 719                         continue;
 720                 }
 721
 722                 do {
 723                         struct page *page;
 724                         unsigned int foll_flags = gup_flags;
 725                         unsigned int page_increm;
 726
 727                         /*
 728                          * If we have a pending SIGKILL, don't keep faulting
 729                          * pages and potentially allocating memory.
 730                          */
 731                         if (unlikely(fatal_signal_pending(current)))
 732                                 return i ? i : -ERESTARTSYS;
 733
 734                         /* cond_resched(); */
 735                         while (!(page = swap_follow_page_mask(vma, start,
 736                                                 foll_flags, &page_mask))) {
 737                                 int ret;
 738                                 unsigned int fault_flags = 0;
 739
 740                                 /* For mlock, just skip the stack guard page. */
 741 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
 742                                 if (foll_flags & FOLL_POPULATE) {
 743 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) */
 744                                 if (foll_flags & FOLL_MLOCK) {
 745 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0) */
 746                                         if (stack_guard_page(vma, start))
 747                                                 goto next_page;
 748                                 }
 749                                 if (foll_flags & FOLL_WRITE)
 750                                         fault_flags |= FAULT_FLAG_WRITE;
 751                                 if (nonblocking)
 752                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 753                                 if (foll_flags & FOLL_NOWAIT)
 754                                         fault_flags |=
 755                                                 (FAULT_FLAG_ALLOW_RETRY |
 756                                                  FAULT_FLAG_RETRY_NOWAIT);
 757
 758                                 ret = swap_handle_mm_fault(mm, vma, start,
 759                                                         fault_flags);
 760
 761                                 if (ret & VM_FAULT_ERROR) {
 762                                         if (ret & VM_FAULT_OOM)
 763                                                 return i ? i : -ENOMEM;
 764                                         if (ret & (VM_FAULT_HWPOISON |
 765                                                    VM_FAULT_HWPOISON_LARGE)) {
 766                                                 if (i)
 767                                                         return i;
 768                                                 else if (gup_flags &
 769                                                          FOLL_HWPOISON)
 770                                                         return -EHWPOISON;
 771                                                 else
 772                                                         return -EFAULT;
 773                                         }
 774                                         if (ret & VM_FAULT_SIGBUS)
 775                                                 return i ? i : -EFAULT;
 776                                         BUG();
 777                                 }
 778
 779                                 if (tsk) {
 780                                         if (ret & VM_FAULT_MAJOR)
 781                                                 tsk->maj_flt++;
 782                                         else
 783                                                 tsk->min_flt++;
 784                                 }
 785
 786                                 if (ret & VM_FAULT_RETRY) {
 787                                         if (nonblocking)
 788                                                 *nonblocking = 0;
 789                                         return i;
 790                                 }
 791
 792                                 /*
 793                                  * The VM_FAULT_WRITE bit tells us that
 794                                  * do_wp_page has broken COW when necessary,
 795                                  * even if maybe_mkwrite decided not to set
 796                                  * pte_write. We can thus safely do subsequent
 797                                  * page lookups as if they were reads. But only
 798                                  * do so when looping for pte_write is futile:
 799                                  * in some cases userspace may also be wanting
 800                                  * to write to the gotten user page, which a
 801                                  * read fault here might prevent (a readonly
 802                                  * page might get reCOWed by userspace write).
 803                                  */
 804                                 if ((ret & VM_FAULT_WRITE) &&
 805                                     !(vma->vm_flags & VM_WRITE))
 806                                         foll_flags &= ~FOLL_WRITE;
 807
 808                                 /* cond_resched(); */
 809                         }
 810                         if (IS_ERR(page))
 811                                 return i ? i : PTR_ERR(page);
 812                         if (pages) {
 813                                 pages[i] = page;
 814
 815                                 swap_flush_anon_page(vma, page, start);
 816                                 flush_dcache_page(page);
 817                                 page_mask = 0;
 818                         }
 819 next_page:
 820                         if (vmas) {
 821                                 vmas[i] = vma;
 822                                 page_mask = 0;
 823                         }
 824                         page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask);
 825                         if (page_increm > nr_pages)
 826                                 page_increm = nr_pages;
 827                         i += page_increm;
 828                         start += page_increm * PAGE_SIZE;
 829                         nr_pages -= page_increm;
 830                 } while (nr_pages && start < vma->vm_end);
 831         } while (nr_pages);
 832         return i;
 833 }
 834
 835 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
 836
 837 static int __get_user_pages_uprobe(struct task_struct *tsk,
 838                                    struct mm_struct *mm, unsigned long start,
 839                                    int nr_pages, unsigned int gup_flags,
 840                                    struct page **pages,
 841                                    struct vm_area_struct **vmas,
 842                                    int *nonblocking)
 843 {
 844         int i;
 845         unsigned long vm_flags;
 846
 847         if (nr_pages <= 0)
 848                 return 0;
 849
 850         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 851
 852         /*
 853          * Require read or write permissions.
 854          * If FOLL_FORCE is set, we only require the "MAY" flags.
 855          */
 856         vm_flags  = (gup_flags & FOLL_WRITE) ?
 857                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 858         vm_flags &= (gup_flags & FOLL_FORCE) ?
 859                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 860         i = 0;
 861
 862         do {
 863                 struct vm_area_struct *vma;
 864
 865                 vma = swap_find_extend_vma(mm, start);
 866                 if (!vma && swap_in_gate_area_no_xxx(start)) {
 867                         unsigned long pg = start & PAGE_MASK;
 868                         pgd_t *pgd;
 869                         pud_t *pud;
 870                         pmd_t *pmd;
 871                         pte_t *pte;
 872
 873                         /* user gate pages are read-only */
 874                         if (gup_flags & FOLL_WRITE)
 875                                 return i ? : -EFAULT;
 876                         if (pg > TASK_SIZE)
 877                                 pgd = pgd_offset_k(pg);
 878                         else
 879                                 pgd = pgd_offset_gate(mm, pg);
 880                         BUG_ON(pgd_none(*pgd));
 881                         pud = pud_offset(pgd, pg);
 882                         BUG_ON(pud_none(*pud));
 883                         pmd = pmd_offset(pud, pg);
 884                         if (pmd_none(*pmd))
 885                                 return i ? : -EFAULT;
 886                         VM_BUG_ON(pmd_trans_huge(*pmd));
 887                         pte = pte_offset_map(pmd, pg);
 888                         if (pte_none(*pte)) {
 889                                 pte_unmap(pte);
 890                                 return i ? : -EFAULT;
 891                         }
 892                         vma = swap_get_gate_vma(mm);
 893                         if (pages) {
 894                                 struct page *page;
 895
 896                                 page = swap_vm_normal_page(vma, start, *pte);
 897                                 if (!page) {
 898                                         if (!(gup_flags & FOLL_DUMP) &&
 899                                                 swap_is_zero_pfn(pte_pfn(*pte)))
 900                                                 page = pte_page(*pte);
 901                                         else {
 902                                                 pte_unmap(pte);
 903                                                 return i ? : -EFAULT;
 904                                         }
 905                                 }
 906                                 pages[i] = page;
 907                                 get_page(page);
 908                         }
 909                         pte_unmap(pte);
 910                         goto next_page;
 911                 }
 912
 913                 if (!vma ||
 914                         (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 915                         !(vm_flags & vma->vm_flags)) {
 916                         return i ? : -EFAULT;
 917                 }
 918
 919                 if (is_vm_hugetlb_page(vma)) {
 920                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
 921                                         &start, &nr_pages, i, gup_flags);
 922                         continue;
 923                 }
 924
 925                 do {
 926                         struct page *page;
 927                         unsigned int foll_flags = gup_flags;
 928
 929                         /*
 930                          * If we have a pending SIGKILL, don't keep faulting
 931                          * pages and potentially allocating memory.
 932                          */
 933                         if (unlikely(fatal_signal_pending(current)))
 934                                 return i ? i : -ERESTARTSYS;
 935
 936                         /* cond_resched(); */
 937                         while (!(page = swap_follow_page(vma, start,
 938                                                          foll_flags))) {
 939                                 int ret;
 940                                 unsigned int fault_flags = 0;
 941
 942                                 /* For mlock, just skip the stack guard page. */
 943                                 if (foll_flags & FOLL_MLOCK) {
 944                                         if (stack_guard_page(vma, start))
 945                                                 goto next_page;
 946                                 }
 947                                 if (foll_flags & FOLL_WRITE)
 948                                         fault_flags |= FAULT_FLAG_WRITE;
 949                                 if (nonblocking)
 950                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 951                                 if (foll_flags & FOLL_NOWAIT)
 952                                         fault_flags |=
 953                                                 (FAULT_FLAG_ALLOW_RETRY |
 954                                                 FAULT_FLAG_RETRY_NOWAIT);
 955
 956                                 ret = swap_handle_mm_fault(mm, vma, start,
 957                                                         fault_flags);
 958
 959                                 if (ret & VM_FAULT_ERROR) {
 960                                         if (ret & VM_FAULT_OOM)
 961                                                 return i ? i : -ENOMEM;
 962                                         if (ret & (VM_FAULT_HWPOISON |
 963                                                    VM_FAULT_HWPOISON_LARGE)) {
 964                                                 if (i)
 965                                                         return i;
 966                                                 else if (gup_flags &
 967                                                          FOLL_HWPOISON)
 968                                                         return -EHWPOISON;
 969                                                 else
 970                                                         return -EFAULT;
 971                                         }
 972                                         if (ret & VM_FAULT_SIGBUS)
 973                                                 return i ? i : -EFAULT;
 974                                         BUG();
 975                                 }
 976
 977                                 if (tsk) {
 978                                         if (ret & VM_FAULT_MAJOR)
 979                                                 tsk->maj_flt++;
 980                                         else
 981                                                 tsk->min_flt++;
 982                                 }
 983
 984                                 if (ret & VM_FAULT_RETRY) {
 985                                         if (nonblocking)
 986                                                 *nonblocking = 0;
 987                                         return i;
 988                                 }
 989
 990                                 /*
 991                                  * The VM_FAULT_WRITE bit tells us that
 992                                  * do_wp_page has broken COW when necessary,
 993                                  * even if maybe_mkwrite decided not to set
 994                                  * pte_write. We can thus safely do subsequent
 995                                  * page lookups as if they were reads. But only
 996                                  * do so when looping for pte_write is futile:
 997                                  * in some cases userspace may also be wanting
 998                                  * to write to the gotten user page, which a
 999                                  * read fault here might prevent (a readonly
1000                                  * page might get reCOWed by userspace write).
1001                                  */
1002                                 if ((ret & VM_FAULT_WRITE) &&
1003                                         !(vma->vm_flags & VM_WRITE))
1004                                         foll_flags &= ~FOLL_WRITE;
1005
1006                                 /* cond_resched(); */
1007                         }
1008                         if (IS_ERR(page))
1009                                 return i ? i : PTR_ERR(page);
1010                         if (pages) {
1011                                 pages[i] = page;
1012
1013                                 swap_flush_anon_page(vma, page, start);
1014                                 flush_dcache_page(page);
1015                         }
1016 next_page:
1017                         if (vmas)
1018                                 vmas[i] = vma;
1019                         i++;
1020                         start += PAGE_SIZE;
1021                         nr_pages--;
1022                 } while (nr_pages && start < vma->vm_end);
1023         } while (nr_pages);
1024
1025         return i;
1026 }
1027
1028 #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0) */
1029
1030 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1031
1032 static int __get_user_pages_uprobe(struct task_struct *tsk,
1033                                    struct mm_struct *mm,
1034                                    unsigned long start, int len, int flags,
1035                                    struct page **pages,
1036                                    struct vm_area_struct **vmas)
1037 {
1038         int i;
1039         unsigned int vm_flags = 0;
1040         int write = !!(flags & GUP_FLAGS_WRITE);
1041         int force = !!(flags & GUP_FLAGS_FORCE);
1042         int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
1043
1044         if (len <= 0)
1045                 return 0;
1046         /*
1047          * Require read or write permissions.
1048          * If 'force' is set, we only require the "MAY" flags.
1049          */
1050         vm_flags  = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1051         vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1052         i = 0;
1053
1054         do {
1055                 struct vm_area_struct *vma;
1056                 unsigned int foll_flags;
1057
1058                 vma = find_vma(mm, start);
1059                 if (!vma && swap_in_gate_area(tsk, start)) {
1060                         unsigned long pg = start & PAGE_MASK;
1061                         struct vm_area_struct *gate_vma =
1062                                 swap_get_gate_vma(tsk);
1063                         pgd_t *pgd;
1064                         pud_t *pud;
1065                         pmd_t *pmd;
1066                         pte_t *pte;
1067
1068                         /* user gate pages are read-only */
1069                         if (!ignore && write)
1070                                 return i ? : -EFAULT;
1071                         if (pg > TASK_SIZE)
1072                                 pgd = pgd_offset_k(pg);
1073                         else
1074                                 pgd = pgd_offset_gate(mm, pg);
1075                         BUG_ON(pgd_none(*pgd));
1076                         pud = pud_offset(pgd, pg);
1077                         BUG_ON(pud_none(*pud));
1078                         pmd = pmd_offset(pud, pg);
1079                         if (pmd_none(*pmd))
1080                                 return i ? : -EFAULT;
1081                         pte = pte_offset_map(pmd, pg);
1082                         if (pte_none(*pte)) {
1083                                 pte_unmap(pte);
1084                                 return i ? : -EFAULT;
1085                         }
1086                         if (pages) {
1087                                 struct page *page =
1088                                         swap_vm_normal_page(gate_vma, start,
1089                                                             *pte);
1090                                 pages[i] = page;
1091                                 if (page)
1092                                         get_page(page);
1093                         }
1094                         pte_unmap(pte);
1095                         if (vmas)
1096                                 vmas[i] = gate_vma;
1097                         i++;
1098                         start += PAGE_SIZE;
1099                         len--;
1100                         continue;
1101                 }
1102
1103                 if (!vma ||
1104                         (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1105                         (!ignore && !(vm_flags & vma->vm_flags)))
1106                         return i ? : -EFAULT;
1107
1108                 if (is_vm_hugetlb_page(vma)) {
1109                         i = swap_follow_hugetlb_page(mm, vma, pages, vmas,
1110                                                 &start, &len, i, write);
1111                         continue;
1112                 }
1113
1114                 foll_flags = FOLL_TOUCH;
1115                 if (pages)
1116                         foll_flags |= FOLL_GET;
1117
1118 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
1119 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
1120                 if (!write && use_zero_page(vma))
1121                         foll_flags |= FOLL_ANON;
1122 #endif
1123 #endif
1124
1125                 do {
1126                         struct page *page;
1127
1128                         if (write)
1129                                 foll_flags |= FOLL_WRITE;
1130
1131
1132                         /* cond_resched(); */
1133
1134                         DBPRINTF("pages = %p vma = %p\n", pages, vma);
1135                         while (!(page = swap_follow_page(vma, start,
1136                                                          foll_flags))) {
1137                                 int ret;
1138                                 ret = swap_handle_mm_fault(mm, vma, start,
1139                                                 foll_flags & FOLL_WRITE);
1140
1141 #if  LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 18)
1142                                 if (ret & VM_FAULT_WRITE)
1143                                         foll_flags &= ~FOLL_WRITE;
1144
1145                                 switch (ret & ~VM_FAULT_WRITE) {
1146                                 case VM_FAULT_MINOR:
1147                                         tsk->min_flt++;
1148                                         break;
1149                                 case VM_FAULT_MAJOR:
1150                                         tsk->maj_flt++;
1151                                         break;
1152                                 case VM_FAULT_SIGBUS:
1153                                         return i ? i : -EFAULT;
1154                                 case VM_FAULT_OOM:
1155                                         return i ? i : -ENOMEM;
1156                                 default:
1157                                         BUG();
1158                                 }
1159
1160 #else
1161                                 if (ret & VM_FAULT_ERROR) {
1162                                         if (ret & VM_FAULT_OOM)
1163                                                 return i ? i : -ENOMEM;
1164                                         else if (ret & VM_FAULT_SIGBUS)
1165                                                 return i ? i : -EFAULT;
1166                                         BUG();
1167                                 }
1168                                 if (ret & VM_FAULT_MAJOR)
1169                                         tsk->maj_flt++;
1170                                 else
1171                                         tsk->min_flt++;
1172
1173                                 /*
1174                                  * The VM_FAULT_WRITE bit tells us that
1175                                  * do_wp_page has broken COW when necessary,
1176                                  * even if maybe_mkwrite decided not to set
1177                                  * pte_write. We can thus safely do subsequent
1178                                  * page lookups as if they were reads. But only
1179                                  * do so when looping for pte_write is futile:
1180                                  * in some cases userspace may also be wanting
1181                                  * to write to the gotten user page, which a
1182                                  * read fault here might prevent (a readonly
1183                                  * page might get reCOWed by userspace write).
1184                                  */
1185                                 if ((ret & VM_FAULT_WRITE) &&
1186                                                 !(vma->vm_flags & VM_WRITE))
1187                                         foll_flags &= ~FOLL_WRITE;
1188
1189                                 /* cond_resched(); */
1190 #endif
1191
1192                         }
1193
1194                         if (IS_ERR(page))
1195                                 return i ? i : PTR_ERR(page);
1196                         if (pages) {
1197                                 pages[i] = page;
1198
1199                                 swap_flush_anon_page(vma, page, start);
1200                                 flush_dcache_page(page);
1201                         }
1202                         if (vmas)
1203                                 vmas[i] = vma;
1204                         i++;
1205                         start += PAGE_SIZE;
1206                         len--;
1207                 } while (len && start < vma->vm_end);
1208         } while (len);
1209         return i;
1210 }
1211 #endif
1212 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1213
1214 /**
1215  * @brief Gets user pages uprobe.
1216  *
1217  * @param tsk Pointer to the task_struct.
1218  * @param mm Pointer to the mm_struct.
1219  * @param start Starting address.
1220  * @param len Length.
1221  * @param write Write flag.
1222  * @param force Force flag.
1223  * @param pages Pointer to the array of pointers to the target page structs.
1224  * @param vmas Pointer to the array of pointers to the target vm_area_struct.
1225  * @return negative error code on error, positive result otherwise.
1226  */
1227 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
1228                 unsigned long start, int len, int write, int force,
1229                 struct page **pages, struct vm_area_struct **vmas)
1230 {
1231 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
1232 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) /* FIXME: must be >= 32! */
1233         int flags = FOLL_TOUCH;
1234
1235         if (pages)
1236                 flags |= FOLL_GET;
1237         if (write)
1238                 flags |= FOLL_WRITE;
1239         if (force)
1240                 flags |= FOLL_FORCE;
1241 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1242         int flags = 0;
1243
1244         if (write)
1245                 flags |= GUP_FLAGS_WRITE;
1246         if (force)
1247                 flags |= GUP_FLAGS_FORCE;
1248 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1249
1250         return __get_user_pages_uprobe(tsk, mm,
1251                                 start, len, flags,
1252 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
1253                                                 pages, vmas, NULL);
1254 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1255                                                 pages, vmas);
1256 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
1257 #else
1258         return get_user_pages(tsk, mm, start, len, write, force, pages, vmas);
1259 #endif
1260 }
1261
1262 #define ACCESS_PROCESS_OPTIMIZATION 0
1263
1264 #if ACCESS_PROCESS_OPTIMIZATION
1265
1266 #define GET_STEP_X(LEN, STEP) (((LEN) >= (STEP)) ? (STEP) : (LEN) % (STEP))
1267 #define GET_STEP_4(LEN) GET_STEP_X((LEN), 4)
1268
1269 static void read_data_current(unsigned long addr, void *buf, int len)
1270 {
1271         int step;
1272         int pos = 0;
1273
1274         for (step = GET_STEP_4(len); len; len -= step) {
1275                 switch (GET_STEP_4(len)) {
1276                 case 1:
1277                         get_user(*(u8 *)(buf + pos),
1278                                  (unsigned long *)(addr + pos));
1279                         step = 1;
1280                         break;
1281
1282                 case 2:
1283                 case 3:
1284                         get_user(*(u16 *)(buf + pos),
1285                                  (unsigned long *)(addr + pos));
1286                         step = 2;
1287                         break;
1288
1289                 case 4:
1290                         get_user(*(u32 *)(buf + pos),
1291                                  (unsigned long *)(addr + pos));
1292                         step = 4;
1293                         break;
1294                 }
1295
1296                 pos += step;
1297         }
1298 }
1299
1300 /* not working */
1301 static void write_data_current(unsigned long addr, void *buf, int len)
1302 {
1303         int step;
1304         int pos = 0;
1305
1306         for (step = GET_STEP_4(len); len; len -= step) {
1307                 switch (GET_STEP_4(len)) {
1308                 case 1:
1309                         put_user(*(u8 *)(buf + pos),
1310                                  (unsigned long *)(addr + pos));
1311                         step = 1;
1312                         break;
1313
1314                 case 2:
1315                 case 3:
1316                         put_user(*(u16 *)(buf + pos),
1317                                  (unsigned long *)(addr + pos));
1318                         step = 2;
1319                         break;
1320
1321                 case 4:
1322                         put_user(*(u32 *)(buf + pos),
1323                                  (unsigned long *)(addr + pos));
1324                         step = 4;
1325                         break;
1326                 }
1327
1328                 pos += step;
1329         }
1330 }
1331 #endif
1332
1333 /**
1334  * @brief Read-write task memory.
1335  *
1336  * @param tsk Pointer to the target task task_struct.
1337  * @param addr Address to read-write.
1338  * @param buf Pointer to buffer where to put-get data.
1339  * @param len Buffer length.
1340  * @param write Write flag. If 0 - reading, if 1 - writing.
1341  * @return Read-write size, error code on error.
1342  */
1343 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr,
1344                              void *buf, int len, int write)
1345 {
1346         struct mm_struct *mm;
1347         struct vm_area_struct *vma;
1348         void *old_buf = buf;
1349         int atomic;
1350
1351         if (len <= 0)
1352                 return -1;
1353
1354 #if ACCESS_PROCESS_OPTIMIZATION
1355         if (write == 0 && tsk == current) {
1356                 read_data_current(addr, buf, len);
1357                 return len;
1358         }
1359 #endif
1360
1361         mm = tsk->mm; /* function 'get_task_mm' is to be called */
1362         if (!mm)
1363                 return 0;
1364
1365         /* FIXME: danger: write memory in atomic context */
1366         atomic = in_atomic();
1367
1368         /* ignore errors, just check how much was successfully transferred */
1369         while (len) {
1370                 int bytes, ret, offset;
1371                 void *maddr;
1372                 struct page *page = NULL;
1373
1374                 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
1375                                                 write, 1, &page, &vma);
1376
1377                 if (ret <= 0) {
1378                         /*
1379                          * Check if this is a VM_IO | VM_PFNMAP VMA, which
1380                          * we can access using slightly different code.
1381                          */
1382 #ifdef CONFIG_HAVE_IOREMAP_PROT
1383                         vma = find_vma(mm, addr);
1384                         if (!vma)
1385                                 break;
1386                         if (vma->vm_ops && vma->vm_ops->access)
1387                                 ret = vma->vm_ops->access(vma, addr, buf,
1388                                                         len, write);
1389                         if (ret <= 0)
1390 #endif
1391                                 break;
1392                         bytes = ret;
1393                 } else {
1394                         bytes = len;
1395                         offset = addr & (PAGE_SIZE-1);
1396                         if (bytes > PAGE_SIZE-offset)
1397                                 bytes = PAGE_SIZE-offset;
1398
1399                         maddr = atomic ? swap_kmap_atomic(page) : kmap(page);
1400
1401                         if (write) {
1402                                 swap_copy_to_user_page(vma, page, addr,
1403                                                         maddr + offset,
1404                                                        buf, bytes);
1405                                 set_page_dirty_lock(page);
1406                         } else {
1407                                 copy_from_user_page(vma, page, addr,
1408                                                     buf, maddr + offset,
1409                                                     bytes);
1410                         }
1411
1412                         atomic ? swap_kunmap_atomic(maddr) : kunmap(page);
1413                         page_cache_release(page);
1414                 }
1415                 len -= bytes;
1416                 buf += bytes;
1417                 addr += bytes;
1418         }
1419
1420         return buf - old_buf;
1421 }
1422 EXPORT_SYMBOL_GPL(access_process_vm_atomic);
1423
1424 #endif /* CONFIG_ARM64 */
1425
1426 /**
1427  * @brief Page present.
1428  *
1429  * @param mm Pointer to the target mm_struct.
1430  * @param address Address.
1431  */
1432 int page_present(struct mm_struct *mm, unsigned long address)
1433 {
1434         pgd_t *pgd;
1435         pud_t *pud;
1436         pmd_t *pmd;
1437         pte_t *ptep, pte;
1438         unsigned long pfn;
1439
1440         pgd = pgd_offset(mm, address);
1441         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
1442                 goto out;
1443
1444         pud = pud_offset(pgd, address);
1445         if (pud_none(*pud) || unlikely(pud_bad(*pud)))
1446                 goto out;
1447
1448         pmd = pmd_offset(pud, address);
1449         if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
1450                 goto out;
1451
1452         ptep = pte_offset_map(pmd, address);
1453         if (pte_none(*ptep)) {
1454                 pte_unmap(ptep);
1455                 goto out;
1456         }
1457
1458         pte = *ptep;
1459         pte_unmap(ptep);
1460         if (pte_present(pte)) {
1461                 pfn = pte_pfn(pte);
1462                 if (pfn_valid(pfn))
1463                         return 1;
1464         }
1465
1466 out:
1467         return 0;
1468 }
1469 EXPORT_SYMBOL_GPL(page_present);
1470