memory-hotplug: integrated __remove_section() of CONFIG_SPARSEMEM_VMEMMAP.

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / mmap.c

diff --git a/mm/mmap.c b/mm/mmap.c
index 35730ee..44bb4d8 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -32,6 +32,7 @@
 #include <linux/khugepaged.h>
 #include <linux/uprobes.h>
 #include <linux/rbtree_augmented.h>
+#include <linux/sched/sysctl.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -255,6 +256,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
        unsigned long newbrk, oldbrk;
        struct mm_struct *mm = current->mm;
        unsigned long min_brk;
+       bool populate;
 
        down_write(&mm->mmap_sem);
 
@@ -304,8 +306,15 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
        /* Ok, looks good - let it rip. */
        if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
                goto out;
+
 set_brk:
        mm->brk = brk;
+       populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
+       up_write(&mm->mmap_sem);
+       if (populate)
+               mm_populate(oldbrk, newbrk - oldbrk);
+       return brk;
+
 out:
        retval = mm->brk;
        up_write(&mm->mmap_sem);
@@ -1153,12 +1162,15 @@ static inline unsigned long round_hint_to_min(unsigned long hint)
 
 unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
                        unsigned long len, unsigned long prot,
-                       unsigned long flags, unsigned long pgoff)
+                       unsigned long flags, unsigned long pgoff,
+                       unsigned long *populate)
 {
        struct mm_struct * mm = current->mm;
        struct inode *inode;
        vm_flags_t vm_flags;
 
+       *populate = 0;
+
        /*
         * Does the application expect PROT_READ to imply PROT_EXEC?
         *
@@ -1279,7 +1291,24 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
                }
        }
 
-       return mmap_region(file, addr, len, flags, vm_flags, pgoff);
+       /*
+        * Set 'VM_NORESERVE' if we should not account for the
+        * memory use of this mapping.
+        */
+       if (flags & MAP_NORESERVE) {
+               /* We honor MAP_NORESERVE if allowed to overcommit */
+               if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
+                       vm_flags |= VM_NORESERVE;
+
+               /* hugetlb applies strict overcommit unless MAP_NORESERVE */
+               if (file && is_file_hugepages(file))
+                       vm_flags |= VM_NORESERVE;
+       }
+
+       addr = mmap_region(file, addr, len, vm_flags, pgoff);
+       if (!IS_ERR_VALUE(addr) && (vm_flags & VM_POPULATE))
+               *populate = len;
+       return addr;
 }
 
 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
@@ -1394,8 +1423,7 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
 }
 
 unsigned long mmap_region(struct file *file, unsigned long addr,
-                         unsigned long len, unsigned long flags,
-                         vm_flags_t vm_flags, unsigned long pgoff)
+               unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
 {
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma, *prev;
@@ -1419,20 +1447,6 @@ munmap_back:
                return -ENOMEM;
 
        /*
-        * Set 'VM_NORESERVE' if we should not account for the
-        * memory use of this mapping.
-        */
-       if ((flags & MAP_NORESERVE)) {
-               /* We honor MAP_NORESERVE if allowed to overcommit */
-               if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
-                       vm_flags |= VM_NORESERVE;
-
-               /* hugetlb applies strict overcommit unless MAP_NORESERVE */
-               if (file && is_file_hugepages(file))
-                       vm_flags |= VM_NORESERVE;
-       }
-
-       /*
         * Private writable mapping: check memory availability
         */
        if (accountable_mapping(file, vm_flags)) {
@@ -1530,10 +1544,12 @@ out:
 
        vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
        if (vm_flags & VM_LOCKED) {
-               if (!mlock_vma_pages_range(vma, addr, addr + len))
+               if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
+                                       vma == get_gate_vma(current->mm)))
                        mm->locked_vm += (len >> PAGE_SHIFT);
-       } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
-               make_pages_present(addr, addr + len);
+               else
+                       vma->vm_flags &= ~VM_LOCKED;
+       }
 
        if (file)
                uprobe_mmap(vma);
@@ -2186,9 +2202,8 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
                return vma;
        if (!prev || expand_stack(prev, addr))
                return NULL;
-       if (prev->vm_flags & VM_LOCKED) {
-               mlock_vma_pages_range(prev, addr, prev->vm_end);
-       }
+       if (prev->vm_flags & VM_LOCKED)
+               __mlock_vma_pages_range(prev, addr, prev->vm_end, NULL);
        return prev;
 }
 #else
@@ -2214,9 +2229,8 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
        start = vma->vm_start;
        if (expand_stack(vma, addr))
                return NULL;
-       if (vma->vm_flags & VM_LOCKED) {
-               mlock_vma_pages_range(vma, addr, start);
-       }
+       if (vma->vm_flags & VM_LOCKED)
+               __mlock_vma_pages_range(vma, addr, start, NULL);
        return vma;
 }
 #endif
@@ -2589,10 +2603,8 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
 out:
        perf_event_mmap(vma);
        mm->total_vm += len >> PAGE_SHIFT;
-       if (flags & VM_LOCKED) {
-               if (!mlock_vma_pages_range(vma, addr, addr + len))
-                       mm->locked_vm += (len >> PAGE_SHIFT);
-       }
+       if (flags & VM_LOCKED)
+               mm->locked_vm += (len >> PAGE_SHIFT);
        return addr;
 }
 
@@ -2600,10 +2612,14 @@ unsigned long vm_brk(unsigned long addr, unsigned long len)
 {
        struct mm_struct *mm = current->mm;
        unsigned long ret;
+       bool populate;
 
        down_write(&mm->mmap_sem);
        ret = do_brk(addr, len);
+       populate = ((mm->def_flags & VM_LOCKED) != 0);
        up_write(&mm->mmap_sem);
+       if (populate)
+               mm_populate(addr, len);
        return ret;
 }
 EXPORT_SYMBOL(vm_brk);
@@ -2943,7 +2959,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
  * vma in this mm is backed by the same anon_vma or address_space.
  *
  * We can take all the locks in random order because the VM code
- * taking i_mmap_mutex or anon_vma->mutex outside the mmap_sem never
+ * taking i_mmap_mutex or anon_vma->rwsem outside the mmap_sem never
  * takes more than one of them in a row. Secondly we're protected
  * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
  *