mmap: handle mlocked pages during map, remap, unmap

Originally by Nick Piggin <npiggin@suse.de>

Remove mlocked pages from the LRU using "unevictable infrastructure"
during mmap(), munmap(), mremap() and truncate().  Try to move back to
normal LRU lists on munmap() when last mlocked mapping removed.  Remove
PageMlocked() status when page truncated from file.

[akpm@linux-foundation.org: cleanup]
[kamezawa.hiroyu@jp.fujitsu.com: fix double unlock_page()]
[kosaki.motohiro@jp.fujitsu.com: split LRU: munlock rework]
[lee.schermerhorn@hp.com: mlock: fix __mlock_vma_pages_range comment block]
[akpm@linux-foundation.org: remove bogus kerneldoc token]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: KAMEZAWA Hiroyuki <kamewzawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/fremap.c b/mm/fremap.c
index 7881638e4a12bd8c5489c8d08b5779b5ae5a0080..7d12ca70ef7bf22c7e1efcdee186adad87a6c8c8 100644 (file)
--- a/mm/fremap.c
+++ b/mm/fremap.c
@@ -21,6 +21,8 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 

+#include "internal.h"
+

 static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma,
                        unsigned long addr, pte_t *ptep)
 {
 static void zap_pte(struct mm_struct *mm, struct vm_area_struct *vma,
                        unsigned long addr, pte_t *ptep)
 {


@@ -215,15 +217,31 @@ asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size,
                spin_unlock(&mapping->i_mmap_lock);
        }
 
                spin_unlock(&mapping->i_mmap_lock);
        }
 

+       if (vma->vm_flags & VM_LOCKED) {
+               /*
+                * drop PG_Mlocked flag for over-mapped range
+                */
+               unsigned int saved_flags = vma->vm_flags;
+               munlock_vma_pages_range(vma, start, start + size);
+               vma->vm_flags = saved_flags;
+       }
+

        mmu_notifier_invalidate_range_start(mm, start, start + size);
        err = populate_range(mm, vma, start, size, pgoff);
        mmu_notifier_invalidate_range_end(mm, start, start + size);
        if (!err && !(flags & MAP_NONBLOCK)) {
        mmu_notifier_invalidate_range_start(mm, start, start + size);
        err = populate_range(mm, vma, start, size, pgoff);
        mmu_notifier_invalidate_range_end(mm, start, start + size);
        if (!err && !(flags & MAP_NONBLOCK)) {

-               if (unlikely(has_write_lock)) {
-                       downgrade_write(&mm->mmap_sem);
-                       has_write_lock = 0;
+               if (vma->vm_flags & VM_LOCKED) {
+                       /*
+                        * might be mapping previously unmapped range of file
+                        */
+                       mlock_vma_pages_range(vma, start, start + size);
+               } else {
+                       if (unlikely(has_write_lock)) {
+                               downgrade_write(&mm->mmap_sem);
+                               has_write_lock = 0;
+                       }
+                       make_pages_present(start, start+size);

                }
                }

-               make_pages_present(start, start+size);

        }
 
        /*
        }
 
        /*


@@ -240,4 +258,3 @@ out:
 
        return err;
 }
 
        return err;
 }

-

diff --git a/mm/internal.h b/mm/internal.h
index 4ebf0bef9a39f0bed64dd40e3167ae02b7891446..48e32f790571f8868e69771e2c91c8c2fd3a34ae 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -61,9 +61,14 @@ static inline unsigned long page_order(struct page *page)
        return page_private(page);
 }
 
        return page_private(page);
 }
 

-extern int mlock_vma_pages_range(struct vm_area_struct *vma,
+extern long mlock_vma_pages_range(struct vm_area_struct *vma,

                        unsigned long start, unsigned long end);
                        unsigned long start, unsigned long end);

-extern void munlock_vma_pages_all(struct vm_area_struct *vma);
+extern void munlock_vma_pages_range(struct vm_area_struct *vma,
+                       unsigned long start, unsigned long end);
+static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
+{
+       munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
+}

 
 #ifdef CONFIG_UNEVICTABLE_LRU
 /*
 
 #ifdef CONFIG_UNEVICTABLE_LRU
 /*

diff --git a/mm/mlock.c b/mm/mlock.c
index c83896a725042d45694777305107fd33ba9c9c28..8b478350a2a1528487ea60113db54b5e6976a4cb 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -112,26 +112,49 @@ static void munlock_vma_page(struct page *page)
        }
 }
 
        }
 }
 

-/*
- * mlock a range of pages in the vma.
+/**
+ * __mlock_vma_pages_range() -  mlock/munlock a range of pages in the vma.
+ * @vma:   target vma
+ * @start: start address
+ * @end:   end address
+ * @mlock: 0 indicate munlock, otherwise mlock.
+ *
+ * If @mlock == 0, unlock an mlocked range;
+ * else mlock the range of pages.  This takes care of making the pages present ,
+ * too.

  *
  *

- * This takes care of making the pages present too.
+ * return 0 on success, negative error code on error.

  *
  *

- * vma->vm_mm->mmap_sem must be held for write.
+ * vma->vm_mm->mmap_sem must be held for at least read.

  */
  */

-static int __mlock_vma_pages_range(struct vm_area_struct *vma,
-                       unsigned long start, unsigned long end)
+static long __mlock_vma_pages_range(struct vm_area_struct *vma,
+                                  unsigned long start, unsigned long end,
+                                  int mlock)

 {
        struct mm_struct *mm = vma->vm_mm;
        unsigned long addr = start;
        struct page *pages[16]; /* 16 gives a reasonable batch */
 {
        struct mm_struct *mm = vma->vm_mm;
        unsigned long addr = start;
        struct page *pages[16]; /* 16 gives a reasonable batch */

-       int write = !!(vma->vm_flags & VM_WRITE);

        int nr_pages = (end - start) / PAGE_SIZE;
        int ret;
        int nr_pages = (end - start) / PAGE_SIZE;
        int ret;

+       int gup_flags = 0;

 
 

-       VM_BUG_ON(start & ~PAGE_MASK || end & ~PAGE_MASK);
-       VM_BUG_ON(start < vma->vm_start || end > vma->vm_end);
-       VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
+       VM_BUG_ON(start & ~PAGE_MASK);
+       VM_BUG_ON(end   & ~PAGE_MASK);
+       VM_BUG_ON(start < vma->vm_start);
+       VM_BUG_ON(end   > vma->vm_end);
+       VM_BUG_ON((!rwsem_is_locked(&mm->mmap_sem)) &&
+                 (atomic_read(&mm->mm_users) != 0));
+
+       /*
+        * mlock:   don't page populate if page has PROT_NONE permission.
+        * munlock: the pages always do munlock althrough
+        *          its has PROT_NONE permission.
+        */
+       if (!mlock)
+               gup_flags |= GUP_FLAGS_IGNORE_VMA_PERMISSIONS;
+
+       if (vma->vm_flags & VM_WRITE)
+               gup_flags |= GUP_FLAGS_WRITE;

 
        lru_add_drain_all();    /* push cached pages to LRU */
 
 
        lru_add_drain_all();    /* push cached pages to LRU */
 


@@ -146,9 +169,9 @@ static int __mlock_vma_pages_range(struct vm_area_struct *vma,
                 * disable migration of this page.  However, page may
                 * still be truncated out from under us.
                 */
                 * disable migration of this page.  However, page may
                 * still be truncated out from under us.
                 */

-               ret = get_user_pages(current, mm, addr,
+               ret = __get_user_pages(current, mm, addr,

                                min_t(int, nr_pages, ARRAY_SIZE(pages)),
                                min_t(int, nr_pages, ARRAY_SIZE(pages)),

-                               write, 0, pages, NULL);
+                               gup_flags, pages, NULL);

                /*
                 * This can happen for, e.g., VM_NONLINEAR regions before
                 * a page has been allocated and mapped at a given offset,
                /*
                 * This can happen for, e.g., VM_NONLINEAR regions before
                 * a page has been allocated and mapped at a given offset,


@@ -178,8 +201,12 @@ static int __mlock_vma_pages_range(struct vm_area_struct *vma,
                         * by the elevated reference, we need only check for
                         * page truncation (file-cache only).
                         */
                         * by the elevated reference, we need only check for
                         * page truncation (file-cache only).
                         */

-                       if (page->mapping)
-                               mlock_vma_page(page);
+                       if (page->mapping) {
+                               if (mlock)
+                                       mlock_vma_page(page);
+                               else
+                                       munlock_vma_page(page);
+                       }

                        unlock_page(page);
                        put_page(page);         /* ref from get_user_pages() */
 
                        unlock_page(page);
                        put_page(page);         /* ref from get_user_pages() */
 


@@ -197,125 +224,38 @@ static int __mlock_vma_pages_range(struct vm_area_struct *vma,
        return 0;       /* count entire vma as locked_vm */
 }
 
        return 0;       /* count entire vma as locked_vm */
 }
 

-/*
- * private structure for munlock page table walk
- */
-struct munlock_page_walk {
-       struct vm_area_struct *vma;
-       pmd_t                 *pmd; /* for migration_entry_wait() */
-};
-
-/*
- * munlock normal pages for present ptes
- */
-static int __munlock_pte_handler(pte_t *ptep, unsigned long addr,
-                                  unsigned long end, struct mm_walk *walk)
-{
-       struct munlock_page_walk *mpw = walk->private;
-       swp_entry_t entry;
-       struct page *page;
-       pte_t pte;
-
-retry:
-       pte = *ptep;
-       /*
-        * If it's a swap pte, we might be racing with page migration.
-        */
-       if (unlikely(!pte_present(pte))) {
-               if (!is_swap_pte(pte))
-                       goto out;
-               entry = pte_to_swp_entry(pte);
-               if (is_migration_entry(entry)) {
-                       migration_entry_wait(mpw->vma->vm_mm, mpw->pmd, addr);
-                       goto retry;
-               }
-               goto out;
-       }
-
-       page = vm_normal_page(mpw->vma, addr, pte);
-       if (!page)
-               goto out;
-
-       lock_page(page);
-       if (!page->mapping) {
-               unlock_page(page);
-               goto retry;
-       }
-       munlock_vma_page(page);
-       unlock_page(page);
-
-out:
-       return 0;
-}
-
-/*
- * Save pmd for pte handler for waiting on migration entries
- */
-static int __munlock_pmd_handler(pmd_t *pmd, unsigned long addr,
-                                unsigned long end, struct mm_walk *walk)
-{
-       struct munlock_page_walk *mpw = walk->private;
-
-       mpw->pmd = pmd;
-       return 0;
-}
-
-
-/*
- * munlock a range of pages in the vma using standard page table walk.
- *
- * vma->vm_mm->mmap_sem must be held for write.
- */
-static void __munlock_vma_pages_range(struct vm_area_struct *vma,
-                             unsigned long start, unsigned long end)
-{
-       struct mm_struct *mm = vma->vm_mm;
-       struct munlock_page_walk mpw = {
-               .vma = vma,
-       };
-       struct mm_walk munlock_page_walk = {
-               .pmd_entry = __munlock_pmd_handler,
-               .pte_entry = __munlock_pte_handler,
-               .private = &mpw,
-               .mm = mm,
-       };
-
-       VM_BUG_ON(start & ~PAGE_MASK || end & ~PAGE_MASK);
-       VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
-       VM_BUG_ON(start < vma->vm_start);
-       VM_BUG_ON(end > vma->vm_end);
-
-       lru_add_drain_all();    /* push cached pages to LRU */
-       walk_page_range(start, end, &munlock_page_walk);
-       lru_add_drain_all();    /* to update stats */
-}
-

 #else /* CONFIG_UNEVICTABLE_LRU */
 
 /*
  * Just make pages present if VM_LOCKED.  No-op if unlocking.
  */
 #else /* CONFIG_UNEVICTABLE_LRU */
 
 /*
  * Just make pages present if VM_LOCKED.  No-op if unlocking.
  */

-static int __mlock_vma_pages_range(struct vm_area_struct *vma,
-                       unsigned long start, unsigned long end)
+static long __mlock_vma_pages_range(struct vm_area_struct *vma,
+                                  unsigned long start, unsigned long end,
+                                  int mlock)

 {
 {

-       if (vma->vm_flags & VM_LOCKED)
+       if (mlock && (vma->vm_flags & VM_LOCKED))

                make_pages_present(start, end);
        return 0;
 }
                make_pages_present(start, end);
        return 0;
 }

-
-/*
- * munlock a range of pages in the vma -- no-op.
- */
-static void __munlock_vma_pages_range(struct vm_area_struct *vma,
-                             unsigned long start, unsigned long end)
-{
-}

 #endif /* CONFIG_UNEVICTABLE_LRU */
 
 #endif /* CONFIG_UNEVICTABLE_LRU */
 

-/*
- * mlock all pages in this vma range.  For mmap()/mremap()/...
+/**
+ * mlock_vma_pages_range() - mlock pages in specified vma range.
+ * @vma - the vma containing the specfied address range
+ * @start - starting address in @vma to mlock
+ * @end   - end address [+1] in @vma to mlock
+ *
+ * For mmap()/mremap()/expansion of mlocked vma.
+ *
+ * return 0 on success for "normal" vmas.
+ *
+ * return number of pages [> 0] to be removed from locked_vm on success
+ * of "special" vmas.
+ *
+ * return negative error if vma spanning @start-@range disappears while
+ * mmap semaphore is dropped.  Unlikely?

  */
  */

-int mlock_vma_pages_range(struct vm_area_struct *vma,
+long mlock_vma_pages_range(struct vm_area_struct *vma,

                        unsigned long start, unsigned long end)
 {
        struct mm_struct *mm = vma->vm_mm;
                        unsigned long start, unsigned long end)
 {
        struct mm_struct *mm = vma->vm_mm;


@@ -331,8 +271,10 @@ int mlock_vma_pages_range(struct vm_area_struct *vma,
        if (!((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) ||
                        is_vm_hugetlb_page(vma) ||
                        vma == get_gate_vma(current))) {
        if (!((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) ||
                        is_vm_hugetlb_page(vma) ||
                        vma == get_gate_vma(current))) {

+               long error;

                downgrade_write(&mm->mmap_sem);
                downgrade_write(&mm->mmap_sem);

-               nr_pages = __mlock_vma_pages_range(vma, start, end);
+
+               error = __mlock_vma_pages_range(vma, start, end, 1);

 
                up_read(&mm->mmap_sem);
                /* vma can change or disappear */
 
                up_read(&mm->mmap_sem);
                /* vma can change or disappear */


@@ -340,8 +282,9 @@ int mlock_vma_pages_range(struct vm_area_struct *vma,
                vma = find_vma(mm, start);
                /* non-NULL vma must contain @start, but need to check @end */
                if (!vma ||  end > vma->vm_end)
                vma = find_vma(mm, start);
                /* non-NULL vma must contain @start, but need to check @end */
                if (!vma ||  end > vma->vm_end)

-                       return -EAGAIN;
-               return nr_pages;
+                       return -ENOMEM;
+
+               return 0;       /* hide other errors from mmap(), et al */

        }
 
        /*
        }
 
        /*


@@ -356,17 +299,33 @@ int mlock_vma_pages_range(struct vm_area_struct *vma,
 
 no_mlock:
        vma->vm_flags &= ~VM_LOCKED;    /* and don't come back! */
 
 no_mlock:
        vma->vm_flags &= ~VM_LOCKED;    /* and don't come back! */

-       return nr_pages;                /* pages NOT mlocked */
+       return nr_pages;                /* error or pages NOT mlocked */

 }
 
 
 /*
 }
 
 
 /*

- * munlock all pages in vma.   For munmap() and exit().
+ * munlock_vma_pages_range() - munlock all pages in the vma range.'
+ * @vma - vma containing range to be munlock()ed.
+ * @start - start address in @vma of the range
+ * @end - end of range in @vma.
+ *
+ *  For mremap(), munmap() and exit().
+ *
+ * Called with @vma VM_LOCKED.
+ *
+ * Returns with VM_LOCKED cleared.  Callers must be prepared to
+ * deal with this.
+ *
+ * We don't save and restore VM_LOCKED here because pages are
+ * still on lru.  In unmap path, pages might be scanned by reclaim
+ * and re-mlocked by try_to_{munlock|unmap} before we unmap and
+ * free them.  This will result in freeing mlocked pages.

  */
  */

-void munlock_vma_pages_all(struct vm_area_struct *vma)
+void munlock_vma_pages_range(struct vm_area_struct *vma,
+                          unsigned long start, unsigned long end)

 {
        vma->vm_flags &= ~VM_LOCKED;
 {
        vma->vm_flags &= ~VM_LOCKED;

-       __munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
+       __mlock_vma_pages_range(vma, start, end, 0);

 }
 
 /*
 }
 
 /*


@@ -443,7 +402,7 @@ success:
                 */
                downgrade_write(&mm->mmap_sem);
 
                 */
                downgrade_write(&mm->mmap_sem);
 

-               ret = __mlock_vma_pages_range(vma, start, end);
+               ret = __mlock_vma_pages_range(vma, start, end, 1);

                if (ret > 0) {
                        mm->locked_vm -= ret;
                        ret = 0;
                if (ret > 0) {
                        mm->locked_vm -= ret;
                        ret = 0;


@@ -460,7 +419,7 @@ success:
                *prev = find_vma(mm, start);
                /* non-NULL *prev must contain @start, but need to check @end */
                if (!(*prev) || end > (*prev)->vm_end)
                *prev = find_vma(mm, start);
                /* non-NULL *prev must contain @start, but need to check @end */
                if (!(*prev) || end > (*prev)->vm_end)

-                       ret = -EAGAIN;
+                       ret = -ENOMEM;

        } else {
                /*
                 * TODO:  for unlocking, pages will already be resident, so
        } else {
                /*
                 * TODO:  for unlocking, pages will already be resident, so


@@ -469,7 +428,7 @@ success:
                 * while.  Should we downgrade the semaphore for both lock
                 * AND unlock ?
                 */
                 * while.  Should we downgrade the semaphore for both lock
                 * AND unlock ?
                 */

-               __munlock_vma_pages_range(vma, start, end);
+               __mlock_vma_pages_range(vma, start, end, 0);

        }
 
 out:
        }
 
 out:

diff --git a/mm/mmap.c b/mm/mmap.c
index 7bdfd2661f173a273d34eaeef7e93244d584990c..505a454f365e35074cf21f3bed2fb1066cb68c15 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -970,6 +970,7 @@ unsigned long do_mmap_pgoff(struct file * file, unsigned long addr,
                        return -EPERM;
                vm_flags |= VM_LOCKED;
        }
                        return -EPERM;
                vm_flags |= VM_LOCKED;
        }

+

        /* mlock MCL_FUTURE? */
        if (vm_flags & VM_LOCKED) {
                unsigned long locked, lock_limit;
        /* mlock MCL_FUTURE? */
        if (vm_flags & VM_LOCKED) {
                unsigned long locked, lock_limit;


@@ -1137,10 +1138,12 @@ munmap_back:
         * The VM_SHARED test is necessary because shmem_zero_setup
         * will create the file object for a shared anonymous map below.
         */
         * The VM_SHARED test is necessary because shmem_zero_setup
         * will create the file object for a shared anonymous map below.
         */

-       if (!file && !(vm_flags & VM_SHARED) &&
-           vma_merge(mm, prev, addr, addr + len, vm_flags,
-                                       NULL, NULL, pgoff, NULL))
-               goto out;
+       if (!file && !(vm_flags & VM_SHARED)) {
+               vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
+                                       NULL, NULL, pgoff, NULL);
+               if (vma)
+                       goto out;
+       }

 
        /*
         * Determine the object being mapped and call the appropriate
 
        /*
         * Determine the object being mapped and call the appropriate


@@ -1222,10 +1225,14 @@ out:
        mm->total_vm += len >> PAGE_SHIFT;
        vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
        if (vm_flags & VM_LOCKED) {
        mm->total_vm += len >> PAGE_SHIFT;
        vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
        if (vm_flags & VM_LOCKED) {

-               mm->locked_vm += len >> PAGE_SHIFT;
-               make_pages_present(addr, addr + len);
-       }
-       if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
+               /*
+                * makes pages present; downgrades, drops, reacquires mmap_sem
+                */
+               long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
+               if (nr_pages < 0)
+                       return nr_pages;        /* vma gone! */
+               mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
+       } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))

                make_pages_present(addr, addr + len);
        return addr;
 
                make_pages_present(addr, addr + len);
        return addr;
 


@@ -1698,8 +1705,10 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
                return vma;
        if (!prev || expand_stack(prev, addr))
                return NULL;
                return vma;
        if (!prev || expand_stack(prev, addr))
                return NULL;

-       if (prev->vm_flags & VM_LOCKED)
-               make_pages_present(addr, prev->vm_end);
+       if (prev->vm_flags & VM_LOCKED) {
+               if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
+                       return NULL;    /* vma gone! */
+       }

        return prev;
 }
 #else
        return prev;
 }
 #else


@@ -1725,8 +1734,10 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
        start = vma->vm_start;
        if (expand_stack(vma, addr))
                return NULL;
        start = vma->vm_start;
        if (expand_stack(vma, addr))
                return NULL;

-       if (vma->vm_flags & VM_LOCKED)
-               make_pages_present(addr, start);
+       if (vma->vm_flags & VM_LOCKED) {
+               if (mlock_vma_pages_range(vma, addr, start) < 0)
+                       return NULL;    /* vma gone! */
+       }

        return vma;
 }
 #endif
        return vma;
 }
 #endif


@@ -1745,8 +1756,6 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
                long nrpages = vma_pages(vma);
 
                mm->total_vm -= nrpages;
                long nrpages = vma_pages(vma);
 
                mm->total_vm -= nrpages;

-               if (vma->vm_flags & VM_LOCKED)
-                       mm->locked_vm -= nrpages;

                vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
                vma = remove_vma(vma);
        } while (vma);
                vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
                vma = remove_vma(vma);
        } while (vma);


@@ -1911,6 +1920,20 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
        }
        vma = prev? prev->vm_next: mm->mmap;
 
        }
        vma = prev? prev->vm_next: mm->mmap;
 

+       /*
+        * unlock any mlock()ed ranges before detaching vmas
+        */
+       if (mm->locked_vm) {
+               struct vm_area_struct *tmp = vma;
+               while (tmp && tmp->vm_start < end) {
+                       if (tmp->vm_flags & VM_LOCKED) {
+                               mm->locked_vm -= vma_pages(tmp);
+                               munlock_vma_pages_all(tmp);
+                       }
+                       tmp = tmp->vm_next;
+               }
+       }
+

        /*
         * Remove the vma's, and unmap the actual pages
         */
        /*
         * Remove the vma's, and unmap the actual pages
         */


@@ -2023,8 +2046,9 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                return -ENOMEM;
 
        /* Can we just expand an old private anonymous mapping? */
                return -ENOMEM;
 
        /* Can we just expand an old private anonymous mapping? */

-       if (vma_merge(mm, prev, addr, addr + len, flags,
-                                       NULL, NULL, pgoff, NULL))
+       vma = vma_merge(mm, prev, addr, addr + len, flags,
+                                       NULL, NULL, pgoff, NULL);
+       if (vma)

                goto out;
 
        /*
                goto out;
 
        /*


@@ -2046,8 +2070,8 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
 out:
        mm->total_vm += len >> PAGE_SHIFT;
        if (flags & VM_LOCKED) {
 out:
        mm->total_vm += len >> PAGE_SHIFT;
        if (flags & VM_LOCKED) {

-               mm->locked_vm += len >> PAGE_SHIFT;
-               make_pages_present(addr, addr + len);
+               if (!mlock_vma_pages_range(vma, addr, addr + len))
+                       mm->locked_vm += (len >> PAGE_SHIFT);

        }
        return addr;
 }
        }
        return addr;
 }


@@ -2058,7 +2082,7 @@ EXPORT_SYMBOL(do_brk);
 void exit_mmap(struct mm_struct *mm)
 {
        struct mmu_gather *tlb;
 void exit_mmap(struct mm_struct *mm)
 {
        struct mmu_gather *tlb;

-       struct vm_area_struct *vma = mm->mmap;
+       struct vm_area_struct *vma;

        unsigned long nr_accounted = 0;
        unsigned long end;
 
        unsigned long nr_accounted = 0;
        unsigned long end;
 


@@ -2066,6 +2090,15 @@ void exit_mmap(struct mm_struct *mm)
        arch_exit_mmap(mm);
        mmu_notifier_release(mm);
 
        arch_exit_mmap(mm);
        mmu_notifier_release(mm);
 

+       if (mm->locked_vm) {
+               vma = mm->mmap;
+               while (vma) {
+                       if (vma->vm_flags & VM_LOCKED)
+                               munlock_vma_pages_all(vma);
+                       vma = vma->vm_next;
+               }
+       }
+       vma = mm->mmap;

        lru_add_drain();
        flush_cache_mm(mm);
        tlb = tlb_gather_mmu(mm, 1);
        lru_add_drain();
        flush_cache_mm(mm);
        tlb = tlb_gather_mmu(mm, 1);

diff --git a/mm/mremap.c b/mm/mremap.c
index 1a7743923c8c42c536d79ecb64f45457b2a3f994..58a2908f42f500002e49adc218adbca0e8602c26 100644 (file)
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -24,6 +24,8 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 

+#include "internal.h"
+

 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
 {
        pgd_t *pgd;
 static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
 {
        pgd_t *pgd;


@@ -238,8 +240,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
        if (vm_flags & VM_LOCKED) {
                mm->locked_vm += new_len >> PAGE_SHIFT;
                if (new_len > old_len)
        if (vm_flags & VM_LOCKED) {
                mm->locked_vm += new_len >> PAGE_SHIFT;
                if (new_len > old_len)

-                       make_pages_present(new_addr + old_len,
-                                          new_addr + new_len);
+                       mlock_vma_pages_range(new_vma, new_addr + old_len,
+                                                      new_addr + new_len);

        }
 
        return new_addr;
        }
 
        return new_addr;


@@ -379,7 +381,7 @@ unsigned long do_mremap(unsigned long addr,
                        vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
                        if (vma->vm_flags & VM_LOCKED) {
                                mm->locked_vm += pages;
                        vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
                        if (vma->vm_flags & VM_LOCKED) {
                                mm->locked_vm += pages;

-                               make_pages_present(addr + old_len,
+                               mlock_vma_pages_range(vma, addr + old_len,

                                                   addr + new_len);
                        }
                        ret = addr;
                                                   addr + new_len);
                        }
                        ret = addr;

diff --git a/mm/truncate.c b/mm/truncate.c
index e83e4b114ef12b78c74480cd0ddaa6b4369d3b4c..1229211104f84beb0e05e5df891f1780d392e418 100644 (file)
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -18,6 +18,7 @@
 #include <linux/task_io_accounting_ops.h>
 #include <linux/buffer_head.h> /* grr. try_to_release_page,
                                   do_invalidatepage */
 #include <linux/task_io_accounting_ops.h>
 #include <linux/buffer_head.h> /* grr. try_to_release_page,
                                   do_invalidatepage */

+#include "internal.h"

 
 
 /**
 
 
 /**


@@ -103,6 +104,7 @@ truncate_complete_page(struct address_space *mapping, struct page *page)
 
        cancel_dirty_page(page, PAGE_CACHE_SIZE);
 
 
        cancel_dirty_page(page, PAGE_CACHE_SIZE);
 

+       clear_page_mlock(page);

        remove_from_page_cache(page);
        ClearPageMappedToDisk(page);
        page_cache_release(page);       /* pagecache ref */
        remove_from_page_cache(page);
        ClearPageMappedToDisk(page);
        page_cache_release(page);       /* pagecache ref */


@@ -127,6 +129,7 @@ invalidate_complete_page(struct address_space *mapping, struct page *page)
        if (PagePrivate(page) && !try_to_release_page(page, 0))
                return 0;
 
        if (PagePrivate(page) && !try_to_release_page(page, 0))
                return 0;
 

+       clear_page_mlock(page);

        ret = remove_mapping(mapping, page);
 
        return ret;
        ret = remove_mapping(mapping, page);
 
        return ret;


@@ -352,6 +355,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
        if (PageDirty(page))
                goto failed;
 
        if (PageDirty(page))
                goto failed;
 

+       clear_page_mlock(page);

        BUG_ON(PagePrivate(page));
        __remove_from_page_cache(page);
        spin_unlock_irq(&mapping->tree_lock);
        BUG_ON(PagePrivate(page));
        __remove_from_page_cache(page);
        spin_unlock_irq(&mapping->tree_lock);