kprobe/dbi_kprobes_deps.c

   1 /*
   2  *  Dynamic Binary Instrumentation Module based on KProbes
   3  *  modules/kprobe/dbi_kprobes_deps.h
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * Copyright (C) Samsung Electronics, 2006-2010
  20  *
  21  * 2008-2009    Alexey Gerenkov <a.gerenkov@samsung.com> User-Space
  22  *              Probes initial implementation; Support x86/ARM/MIPS for both user and kernel spaces.
  23  * 2010         Ekaterina Gorelkina <e.gorelkina@samsung.com>: redesign module for separating core and arch parts
  24  *
  25
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/sched.h>
  30
  31 #include <asm/pgtable.h>
  32
  33 #include "dbi_kprobes_deps.h"
  34 #include "dbi_kdebug.h"
  35
  36
  37 #include <linux/slab.h>
  38 #include <linux/mm.h>
  39
  40 unsigned int *sched_addr;
  41 unsigned int *fork_addr;
  42
  43 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
  44 struct mm_struct* init_mm_ptr;
  45 struct mm_struct init_mm;
  46 #endif
  47
  48
  49 DECLARE_MOD_CB_DEP(kallsyms_search, unsigned long, const char *name);
  50 DECLARE_MOD_FUNC_DEP(access_process_vm, int, struct task_struct * tsk, unsigned long addr, void *buf, int len, int write);
  51 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)
  52 DECLARE_MOD_FUNC_DEP(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len);
  53 #endif
  54
  55 DECLARE_MOD_FUNC_DEP(find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr);
  56
  57 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
  58 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
  59 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access);
  60 #endif
  61 #else
  62 DECLARE_MOD_FUNC_DEP(handle_mm_fault, int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags);
  63 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30) */
  64
  65 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
  66 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct mm_struct *mm);
  67 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
  68 DECLARE_MOD_FUNC_DEP(get_gate_vma, struct vm_area_struct *, struct task_struct *tsk);
  69 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
  70
  71 #ifdef CONFIG_HUGETLB_PAGE
  72 DECLARE_MOD_FUNC_DEP(follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, int write);
  73 #endif
  74
  75 #ifdef  __HAVE_ARCH_GATE_AREA
  76 DECLARE_MOD_FUNC_DEP(in_gate_area, int, struct task_struct *tsk,unsigned long addr);
  77 #else
  78 DECLARE_MOD_FUNC_DEP(in_gate_area_no_task, int, unsigned long addr);
  79 #endif
  80 DECLARE_MOD_FUNC_DEP(follow_page, \
  81                 struct page *, struct vm_area_struct * vma, \
  82                 unsigned long address, unsigned int foll_flags);
  83 DECLARE_MOD_FUNC_DEP(__flush_anon_page, \
  84                 void, struct vm_area_struct *vma, struct page *page, \
  85                 unsigned long vmaddr);
  86 DECLARE_MOD_FUNC_DEP(vm_normal_page, \
  87                 struct page *, struct vm_area_struct *vma, \
  88                 unsigned long addr, pte_t pte);
  89
  90 DECLARE_MOD_FUNC_DEP(flush_ptrace_access, \
  91                 void, struct vm_area_struct *vma, struct page *page, \
  92                 unsigned long uaddr, void *kaddr, unsigned long len, int write);
  93
  94
  95 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
  96 DECLARE_MOD_FUNC_DEP(put_task_struct, \
  97                 void, struct task_struct *tsk);
  98 #else
  99 DECLARE_MOD_FUNC_DEP(put_task_struct, \
 100                 void, struct rcu_head * rhp);
 101 #endif
 102
 103         DECLARE_MOD_DEP_WRAPPER(access_process_vm, int, struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
 104 IMP_MOD_DEP_WRAPPER (access_process_vm, tsk, addr, buf, len, write)
 105
 106         DECLARE_MOD_DEP_WRAPPER (find_extend_vma, struct vm_area_struct *, struct mm_struct * mm, unsigned long addr)
 107 IMP_MOD_DEP_WRAPPER (find_extend_vma, mm, addr)
 108
 109 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 30)
 110 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 111         DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
 112                         int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access)
 113 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, write_access)
 114 #endif
 115 #else
 116         DECLARE_MOD_DEP_WRAPPER (handle_mm_fault, \
 117                         int, struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags)
 118 IMP_MOD_DEP_WRAPPER (handle_mm_fault, mm, vma, address, flags)
 119 #endif
 120
 121 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 122         DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
 123                         struct vm_area_struct *, struct mm_struct *mm)
 124 IMP_MOD_DEP_WRAPPER (get_gate_vma, mm)
 125 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 126         DECLARE_MOD_DEP_WRAPPER (get_gate_vma, \
 127                         struct vm_area_struct *, struct task_struct *tsk)
 128 IMP_MOD_DEP_WRAPPER (get_gate_vma, tsk)
 129 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 130
 131 #ifdef CONFIG_HUGETLB_PAGE
 132         DECLARE_MOD_DEP_WRAPPER (follow_hugetlb_page, int, struct mm_struct *mm, struct vm_area_struct *vma, struct page **pages, struct vm_area_struct **vmas, unsigned long *position, int *length, int i, unsigned int write)
 133         IMP_MOD_DEP_WRAPPER (follow_hugetlb_page, mm, vma, pages, vmas, position, length, i, write)
 134 #endif
 135
 136 #ifdef  __HAVE_ARCH_GATE_AREA
 137         DECLARE_MOD_DEP_WRAPPER (in_gate_area, int, struct task_struct *tsk, unsigned long addr)
 138         IMP_MOD_DEP_WRAPPER (in_gate_area, tsk, addr)
 139 #else
 140         DECLARE_MOD_DEP_WRAPPER (in_gate_area_no_task, int, unsigned long addr)
 141 IMP_MOD_DEP_WRAPPER (in_gate_area_no_task, addr)
 142 #endif
 143
 144 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 11))
 145         DECLARE_MOD_DEP_WRAPPER (follow_page, \
 146                         struct page *, struct vm_area_struct * vma, \
 147                         unsigned long address, unsigned int foll_flags)
 148 IMP_MOD_DEP_WRAPPER (follow_page, vma, address, foll_flags)
 149 #endif
 150         DECLARE_MOD_DEP_WRAPPER (__flush_anon_page, \
 151                         void, struct vm_area_struct *vma, \
 152                         struct page *page, unsigned long vmaddr)
 153 IMP_MOD_DEP_WRAPPER (__flush_anon_page, vma, page, vmaddr)
 154
 155         DECLARE_MOD_DEP_WRAPPER(vm_normal_page, \
 156                         struct page *, struct vm_area_struct *vma, \
 157                         unsigned long addr, pte_t pte)
 158 IMP_MOD_DEP_WRAPPER (vm_normal_page, vma, addr, pte)
 159
 160         DECLARE_MOD_DEP_WRAPPER (flush_ptrace_access, \
 161                         void, struct vm_area_struct *vma, struct page *page, \
 162                         unsigned long uaddr, void *kaddr, unsigned long len, int write)
 163 IMP_MOD_DEP_WRAPPER (flush_ptrace_access, vma, page, uaddr, kaddr, len, write)
 164
 165 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34)
 166         DECLARE_MOD_DEP_WRAPPER(copy_to_user_page, void, struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len)
 167         IMP_MOD_DEP_WRAPPER (copy_to_user_page, vma, page, uaddr, dst, src, len)
 168 #endif
 169
 170
 171 int init_module_dependencies()
 172 {
 173
 174 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 29)
 175         init_mm_ptr = (struct mm_struct*) kallsyms_search ("init_mm");
 176         memcmp(init_mm_ptr, &init_mm, sizeof(struct mm_struct));
 177 #endif
 178
 179 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 18)
 180         INIT_MOD_DEP_VAR(handle_mm_fault, handle_mm_fault);
 181 #endif
 182
 183         INIT_MOD_DEP_VAR(flush_ptrace_access, flush_ptrace_access);
 184         INIT_MOD_DEP_VAR(find_extend_vma, find_extend_vma);
 185         INIT_MOD_DEP_VAR(get_gate_vma, get_gate_vma);
 186
 187 #ifdef CONFIG_HUGETLB_PAGE
 188         INIT_MOD_DEP_VAR(follow_hugetlb_page, follow_hugetlb_page);
 189 #endif
 190
 191 #ifdef  __HAVE_ARCH_GATE_AREA
 192         INIT_MOD_DEP_VAR(in_gate_area, in_gate_area);
 193 #else
 194         INIT_MOD_DEP_VAR(in_gate_area_no_task, in_gate_area_no_task);
 195 #endif
 196         INIT_MOD_DEP_VAR(follow_page, follow_page);
 197
 198         INIT_MOD_DEP_VAR(__flush_anon_page, __flush_anon_page);
 199         INIT_MOD_DEP_VAR(vm_normal_page, vm_normal_page);
 200         INIT_MOD_DEP_VAR(access_process_vm, access_process_vm);
 201
 202 #if (LINUX_VERSION_CODE != KERNEL_VERSION(2, 6, 16))
 203 # if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11))
 204         INIT_MOD_DEP_VAR(put_task_struct, put_task_struct);
 205 # else
 206         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct);
 207 # endif
 208 #else /*2.6.16 */
 209         INIT_MOD_DEP_VAR(put_task_struct, __put_task_struct_cb);
 210 #endif
 211
 212
 213 // for 2.6.32.9-iboot (tegra-froyo)
 214 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34))
 215         INIT_MOD_DEP_VAR(copy_to_user_page, copy_to_user_page);
 216 #endif
 217
 218         return 0;
 219 }
 220
 221 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) /* FIXME: must be < 32 */
 222 #define GUP_FLAGS_WRITE                  0x1
 223 #define GUP_FLAGS_FORCE                  0x2
 224 #define GUP_FLAGS_IGNORE_VMA_PERMISSIONS 0x4
 225 #define GUP_FLAGS_IGNORE_SIGKILL         0x8
 226 #endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 38) */
 227
 228 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 229 static inline int use_zero_page(struct vm_area_struct *vma)
 230 {
 231         /*
 232          * We don't want to optimize FOLL_ANON for make_pages_present()
 233          * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
 234          * we want to get the page from the page tables to make sure
 235          * that we serialize and update with any other user of that
 236          * mapping.
 237          */
 238         if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
 239                 return 0;
 240         /*
 241          * And if we have a fault routine, it's not an anonymous region.
 242          */
 243         return !vma->vm_ops || !vma->vm_ops->fault;
 244 }
 245
 246 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 247 extern unsigned long (* in_gate_area_fp)(unsigned long);
 248 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 249
 250 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 251 unsigned long zero_pfn __read_mostly;
 252
 253 #ifndef is_zero_pfn
 254 static inline int is_zero_pfn(unsigned long pfn)
 255 {
 256         return pfn == zero_pfn;
 257 }
 258 #endif
 259
 260 static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long addr)
 261 {
 262         return stack_guard_page_start(vma, addr) ||
 263                stack_guard_page_end(vma, addr+PAGE_SIZE);
 264 }
 265
 266 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 267                      unsigned long start, int nr_pages, unsigned int gup_flags,
 268                      struct page **pages, struct vm_area_struct **vmas,
 269                      int *nonblocking)
 270 {
 271         int i;
 272         unsigned long vm_flags;
 273
 274         if (nr_pages <= 0) {
 275                 return 0;
 276         }
 277
 278         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 279
 280         /*
 281          * Require read or write permissions.
 282          * If FOLL_FORCE is set, we only require the "MAY" flags.
 283          */
 284         vm_flags  = (gup_flags & FOLL_WRITE) ?
 285                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 286         vm_flags &= (gup_flags & FOLL_FORCE) ?
 287                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 288         i = 0;
 289
 290         do {
 291                 struct vm_area_struct *vma;
 292
 293                 vma = find_extend_vma(mm, start);
 294                 if (!vma && in_gate_area_fp(start)) {
 295                         unsigned long pg = start & PAGE_MASK;
 296                         pgd_t *pgd;
 297                         pud_t *pud;
 298                         pmd_t *pmd;
 299                         pte_t *pte;
 300
 301                         /* user gate pages are read-only */
 302                         if (gup_flags & FOLL_WRITE) {
 303                                 return i ? : -EFAULT;
 304                         }
 305                         if (pg > TASK_SIZE)
 306                                 pgd = pgd_offset_k(pg);
 307                         else
 308                                 pgd = pgd_offset_gate(mm, pg);
 309                         BUG_ON(pgd_none(*pgd));
 310                         pud = pud_offset(pgd, pg);
 311                         BUG_ON(pud_none(*pud));
 312                         pmd = pmd_offset(pud, pg);
 313                         if (pmd_none(*pmd)) {
 314                                 return i ? : -EFAULT;
 315                         }
 316                         VM_BUG_ON(pmd_trans_huge(*pmd));
 317                         pte = pte_offset_map(pmd, pg);
 318                         if (pte_none(*pte)) {
 319                                 pte_unmap(pte);
 320                                 return i ? : -EFAULT;
 321                         }
 322                         vma = get_gate_vma(mm);
 323                         if (pages) {
 324                                 struct page *page;
 325
 326                                 page = vm_normal_page(vma, start, *pte);
 327                                 if (!page) {
 328                                         if (!(gup_flags & FOLL_DUMP) &&
 329                                              is_zero_pfn(pte_pfn(*pte)))
 330                                                 page = pte_page(*pte);
 331                                         else {
 332                                                 pte_unmap(pte);
 333                                                 return i ? : -EFAULT;
 334                                         }
 335                                 }
 336                                 pages[i] = page;
 337                                 get_page(page);
 338                         }
 339                         pte_unmap(pte);
 340                         goto next_page;
 341                 }
 342
 343                 if (!vma ||
 344                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 345                     !(vm_flags & vma->vm_flags)) {
 346                         return i ? : -EFAULT;
 347                 }
 348
 349                 if (is_vm_hugetlb_page(vma)) {
 350                         i = follow_hugetlb_page(mm, vma, pages, vmas,
 351                                         &start, &nr_pages, i, gup_flags);
 352                         continue;
 353                 }
 354
 355                 do {
 356                         struct page *page;
 357                         unsigned int foll_flags = gup_flags;
 358
 359                         /*
 360                          * If we have a pending SIGKILL, don't keep faulting
 361                          * pages and potentially allocating memory.
 362                          */
 363                         if (unlikely(fatal_signal_pending(current))) {
 364                                 return i ? i : -ERESTARTSYS;
 365                         }
 366
 367                         /* cond_resched(); */
 368                         while (!(page = follow_page(vma, start, foll_flags))) {
 369                                 int ret;
 370                                 unsigned int fault_flags = 0;
 371
 372                                 /* For mlock, just skip the stack guard page. */
 373                                 if (foll_flags & FOLL_MLOCK) {
 374                                         if (stack_guard_page(vma, start))
 375                                                 goto next_page;
 376                                 }
 377                                 if (foll_flags & FOLL_WRITE)
 378                                         fault_flags |= FAULT_FLAG_WRITE;
 379                                 if (nonblocking)
 380                                         fault_flags |= FAULT_FLAG_ALLOW_RETRY;
 381                                 if (foll_flags & FOLL_NOWAIT)
 382                                         fault_flags |= (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT);
 383
 384                                 ret = handle_mm_fault(mm, vma, start,
 385                                                         fault_flags);
 386
 387                                 if (ret & VM_FAULT_ERROR) {
 388                                         if (ret & VM_FAULT_OOM) {
 389                                                 return i ? i : -ENOMEM;
 390                                         }
 391                                         if (ret & (VM_FAULT_HWPOISON |
 392                                                    VM_FAULT_HWPOISON_LARGE)) {
 393                                                 if (i) {
 394                                                         return i;
 395                                                 }
 396                                                 else if (gup_flags & FOLL_HWPOISON) {
 397                                                         return -EHWPOISON;
 398                                                 }
 399                                                 else {
 400                                                         return -EFAULT;
 401                                                 }
 402                                         }
 403                                         if (ret & VM_FAULT_SIGBUS) {
 404                                                 return i ? i : -EFAULT;
 405                                         }
 406                                         BUG();
 407                                 }
 408
 409                                 if (tsk) {
 410                                         if (ret & VM_FAULT_MAJOR)
 411                                                 tsk->maj_flt++;
 412                                         else
 413                                                 tsk->min_flt++;
 414                                 }
 415
 416                                 if (ret & VM_FAULT_RETRY) {
 417                                         if (nonblocking)
 418                                                 *nonblocking = 0;
 419                                         return i;
 420                                 }
 421
 422                                 /*
 423                                  * The VM_FAULT_WRITE bit tells us that
 424                                  * do_wp_page has broken COW when necessary,
 425                                  * even if maybe_mkwrite decided not to set
 426                                  * pte_write. We can thus safely do subsequent
 427                                  * page lookups as if they were reads. But only
 428                                  * do so when looping for pte_write is futile:
 429                                  * in some cases userspace may also be wanting
 430                                  * to write to the gotten user page, which a
 431                                  * read fault here might prevent (a readonly
 432                                  * page might get reCOWed by userspace write).
 433                                  */
 434                                 if ((ret & VM_FAULT_WRITE) &&
 435                                     !(vma->vm_flags & VM_WRITE))
 436                                         foll_flags &= ~FOLL_WRITE;
 437
 438                                 /* cond_resched(); */
 439                         }
 440                         if (IS_ERR(page)) {
 441                                 return i ? i : PTR_ERR(page);
 442                         }
 443                         if (pages) {
 444                                 pages[i] = page;
 445
 446                                 flush_anon_page(vma, page, start);
 447                                 flush_dcache_page(page);
 448                         }
 449 next_page:
 450                         if (vmas)
 451                                 vmas[i] = vma;
 452                         i++;
 453                         start += PAGE_SIZE;
 454                         nr_pages--;
 455                 } while (nr_pages && start < vma->vm_end);
 456         } while (nr_pages);
 457
 458         return i;
 459 }
 460 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 461
 462 int __get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 463                      unsigned long start, int len, int flags,
 464                 struct page **pages, struct vm_area_struct **vmas)
 465 {
 466         int i;
 467         unsigned int vm_flags = 0;
 468         int write = !!(flags & GUP_FLAGS_WRITE);
 469         int force = !!(flags & GUP_FLAGS_FORCE);
 470         int ignore = !!(flags & GUP_FLAGS_IGNORE_VMA_PERMISSIONS);
 471         int ignore_sigkill = !!(flags & GUP_FLAGS_IGNORE_SIGKILL);
 472
 473         if (len <= 0)
 474                 return 0;
 475         /*
 476          * Require read or write permissions.
 477          * If 'force' is set, we only require the "MAY" flags.
 478          */
 479         vm_flags  = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 480         vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 481         i = 0;
 482
 483         do {
 484                 struct vm_area_struct *vma;
 485                 unsigned int foll_flags;
 486
 487                 vma = find_vma(mm, start);
 488                 if (!vma && in_gate_area(tsk, start)) {
 489                         unsigned long pg = start & PAGE_MASK;
 490                         struct vm_area_struct *gate_vma = get_gate_vma(tsk);
 491                         pgd_t *pgd;
 492                         pud_t *pud;
 493                         pmd_t *pmd;
 494                         pte_t *pte;
 495
 496                         /* user gate pages are read-only */
 497                         if (!ignore && write)
 498                                 return i ? : -EFAULT;
 499                         if (pg > TASK_SIZE)
 500                                 pgd = pgd_offset_k(pg);
 501                         else
 502                                 pgd = pgd_offset_gate(mm, pg);
 503                         BUG_ON(pgd_none(*pgd));
 504                         pud = pud_offset(pgd, pg);
 505                         BUG_ON(pud_none(*pud));
 506                         pmd = pmd_offset(pud, pg);
 507                         if (pmd_none(*pmd))
 508                                 return i ? : -EFAULT;
 509                         pte = pte_offset_map(pmd, pg);
 510                         if (pte_none(*pte)) {
 511                                 pte_unmap(pte);
 512                                 return i ? : -EFAULT;
 513                         }
 514                         if (pages) {
 515                                 struct page *page = vm_normal_page(gate_vma, start, *pte);
 516                                 pages[i] = page;
 517                                 if (page)
 518                                         get_page(page);
 519                         }
 520                         pte_unmap(pte);
 521                         if (vmas)
 522                                 vmas[i] = gate_vma;
 523                         i++;
 524                         start += PAGE_SIZE;
 525                         len--;
 526                         continue;
 527                 }
 528
 529                 if (!vma ||
 530                     (vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
 531                     (!ignore && !(vm_flags & vma->vm_flags)))
 532                         return i ? : -EFAULT;
 533
 534                 if (is_vm_hugetlb_page(vma)) {
 535 #if  LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
 536                         i = follow_hugetlb_page(mm, vma, pages, vmas,
 537                                                 &start, &len, i);
 538 #else
 539                         i = follow_hugetlb_page(mm, vma, pages, vmas,
 540                                                 &start, &len, i, write);
 541 #endif
 542                         continue;
 543                 }
 544
 545                 foll_flags = FOLL_TOUCH;
 546                 if (pages)
 547                         foll_flags |= FOLL_GET;
 548
 549 #if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
 550 #if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,30)
 551                 if (!write && use_zero_page(vma))
 552                   foll_flags |= FOLL_ANON;
 553 #endif
 554 #endif
 555
 556                 do {
 557                         struct page *page;
 558
 559 #if 0
 560                         /*
 561                          * If we have a pending SIGKILL, don't keep faulting
 562                          * pages and potentially allocating memory, unless
 563                          * current is handling munlock--e.g., on exit. In
 564                          * that case, we are not allocating memory.  Rather,
 565                          * we're only unlocking already resident/mapped pages.
 566                          */
 567                         if (unlikely(!ignore_sigkill &&
 568                                         fatal_signal_pending(current)))
 569                                 return i ? i : -ERESTARTSYS;
 570 #endif
 571
 572                         if (write)
 573                                 foll_flags |= FOLL_WRITE;
 574
 575
 576                         //cond_resched();
 577
 578                         DBPRINTF ("pages = %p vma = %p\n", pages, vma);
 579                         while (!(page = follow_page(vma, start, foll_flags))) {
 580                                 int ret;
 581                                 ret = handle_mm_fault(mm, vma, start,
 582                                                 foll_flags & FOLL_WRITE);
 583
 584 #if  LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
 585                                 if (ret & VM_FAULT_WRITE)
 586                                   foll_flags &= ~FOLL_WRITE;
 587
 588                                 switch (ret & ~VM_FAULT_WRITE) {
 589                                 case VM_FAULT_MINOR:
 590                                   tsk->min_flt++;
 591                                   break;
 592                                 case VM_FAULT_MAJOR:
 593                                   tsk->maj_flt++;
 594                                   break;
 595                                 case VM_FAULT_SIGBUS:
 596                                   return i ? i : -EFAULT;
 597                                 case VM_FAULT_OOM:
 598                                   return i ? i : -ENOMEM;
 599                                 default:
 600                                   BUG();
 601                                 }
 602
 603 #else
 604                                 if (ret & VM_FAULT_ERROR) {
 605                                   if (ret & VM_FAULT_OOM)
 606                                     return i ? i : -ENOMEM;
 607                                   else if (ret & VM_FAULT_SIGBUS)
 608                                     return i ? i : -EFAULT;
 609                                   BUG();
 610                                 }
 611                                 if (ret & VM_FAULT_MAJOR)
 612                                   tsk->maj_flt++;
 613                                 else
 614                                   tsk->min_flt++;
 615
 616                                 /*
 617                                  * The VM_FAULT_WRITE bit tells us that
 618                                  * do_wp_page has broken COW when necessary,
 619                                  * even if maybe_mkwrite decided not to set
 620                                  * pte_write. We can thus safely do subsequent
 621                                  * page lookups as if they were reads. But only
 622                                  * do so when looping for pte_write is futile:
 623                                  * in some cases userspace may also be wanting
 624                                  * to write to the gotten user page, which a
 625                                  * read fault here might prevent (a readonly
 626                                  * page might get reCOWed by userspace write).
 627                                  */
 628                                 if ((ret & VM_FAULT_WRITE) &&
 629                                     !(vma->vm_flags & VM_WRITE))
 630                                   foll_flags &= ~FOLL_WRITE;
 631
 632                                 //cond_resched();
 633 #endif
 634
 635                         }
 636
 637                         if (IS_ERR(page))
 638                                 return i ? i : PTR_ERR(page);
 639                         if (pages) {
 640                                 pages[i] = page;
 641
 642 #if  LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,18)
 643                                 flush_anon_page(page, start);
 644 #else
 645                                 flush_anon_page(vma, page, start);
 646 #endif
 647                                 flush_dcache_page(page);
 648                         }
 649                         if (vmas)
 650                                 vmas[i] = vma;
 651                         i++;
 652                         start += PAGE_SIZE;
 653                         len--;
 654                 } while (len && start < vma->vm_end);
 655         } while (len);
 656         return i;
 657 }
 658 #endif
 659 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 660
 661 int get_user_pages_uprobe(struct task_struct *tsk, struct mm_struct *mm,
 662                 unsigned long start, int len, int write, int force,
 663                 struct page **pages, struct vm_area_struct **vmas)
 664 {
 665 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
 666 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) /* FIXME: must be >= 32! */
 667         int flags = FOLL_TOUCH;
 668
 669         if (pages)
 670                 flags |= FOLL_GET;
 671         if (write)
 672                 flags |= FOLL_WRITE;
 673         if (force)
 674                 flags |= FOLL_FORCE;
 675 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 676         int flags = 0;
 677
 678         if (write)
 679                 flags |= GUP_FLAGS_WRITE;
 680         if (force)
 681                 flags |= GUP_FLAGS_FORCE;
 682 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 683
 684         return __get_user_pages_uprobe(tsk, mm,
 685                                 start, len, flags,
 686 #if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38)
 687                                        pages, vmas, 0);
 688 #else /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 689                                        pages, vmas);
 690 #endif /* LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 38) */
 691 #else
 692         return get_user_pages(tsk, mm, start, len, write, force, pages, vmas);
 693 #endif
 694 }
 695
 696
 697 int access_process_vm_atomic(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
 698 {
 699         struct mm_struct *mm;
 700         struct vm_area_struct *vma;
 701         void *old_buf = buf;
 702         unsigned long addr1 = addr;
 703         unsigned int* inst_buf = (unsigned int*)old_buf;
 704
 705
 706         mm = get_task_mm(tsk);
 707         if (!mm)
 708                 return 0;
 709
 710         /* down_read(&mm->mmap_sem); */
 711         /* ignore errors, just check how much was successfully transferred */
 712         while (len) {
 713                 int bytes, ret, offset;
 714                 void *maddr;
 715                 struct page *page = NULL;
 716
 717                 ret = get_user_pages_uprobe(tsk, mm, addr, 1,
 718                                             write, 1, &page, &vma);
 719
 720                 if (ret <= 0) {
 721                         /*
 722                          * Check if this is a VM_IO | VM_PFNMAP VMA, which
 723                          * we can access using slightly different code.
 724                          */
 725 #ifdef CONFIG_HAVE_IOREMAP_PROT
 726                         vma = find_vma(mm, addr);
 727                         if (!vma)
 728                                 break;
 729                         if (vma->vm_ops && vma->vm_ops->access)
 730                                 ret = vma->vm_ops->access(vma, addr, buf,
 731                                                           len, write);
 732                         if (ret <= 0)
 733 #endif
 734                                 break;
 735                         bytes = ret;
 736                 } else {
 737                         bytes = len;
 738                         offset = addr & (PAGE_SIZE-1);
 739                         if (bytes > PAGE_SIZE-offset)
 740                                 bytes = PAGE_SIZE-offset;
 741
 742                         maddr = kmap(page);
 743                         if (write) {
 744                                 copy_to_user_page(vma, page, addr,
 745                                                   maddr + offset, buf, bytes);
 746                                 set_page_dirty_lock(page);
 747                         } else {
 748                                 copy_from_user_page(vma, page, addr,
 749                                                     buf, maddr + offset, bytes);
 750                         }
 751                         kunmap(page);
 752                         page_cache_release(page);
 753                 }
 754                 len -= bytes;
 755                 buf += bytes;
 756                 addr += bytes;
 757         }
 758         /* up_read(&mm->mmap_sem); */
 759         mmput(mm);
 760
 761         return buf - old_buf;
 762 }
 763
 764 int page_present (struct mm_struct *mm, unsigned long address)
 765 {
 766         pgd_t *pgd;
 767         pud_t *pud;
 768         pmd_t *pmd;
 769         pte_t *ptep, pte;
 770         unsigned long pfn;
 771
 772         pgd = pgd_offset(mm, address);
 773         if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
 774                 goto out;
 775
 776         pud = pud_offset(pgd, address);
 777         if (pud_none(*pud) || unlikely(pud_bad(*pud)))
 778                 goto out;
 779
 780         pmd = pmd_offset(pud, address);
 781         if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
 782                 goto out;
 783
 784         ptep = pte_offset_map(pmd, address);
 785         if (!ptep)
 786                 goto out;
 787
 788         pte = *ptep;
 789         pte_unmap(ptep);
 790         if (pte_present(pte)) {
 791                 pfn = pte_pfn(pte);
 792                 if (pfn_valid(pfn)) {
 793                         return 1;
 794                 }
 795         }
 796
 797 out:
 798         return 0;
 799 }
 800
 801
 802 EXPORT_SYMBOL_GPL (page_present);
 803 EXPORT_SYMBOL_GPL (get_user_pages_uprobe);
 804 EXPORT_SYMBOL_GPL (access_process_vm_atomic);
 805