mm/compaction: do not call suitable_migration_target() on every page

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / memory.c

diff --git a/mm/memory.c b/mm/memory.c
index 6768ce9..492e36f 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -59,6 +59,7 @@
 #include <linux/gfp.h>
 #include <linux/migrate.h>
 #include <linux/string.h>
+#include <linux/dma-debug.h>
 
 #include <asm/io.h>
 #include <asm/pgalloc.h>
@@ -288,7 +289,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
                        return 0;
                batch = tlb->active;
        }
-       VM_BUG_ON(batch->nr > batch->max);
+       VM_BUG_ON_PAGE(batch->nr > batch->max, page);
 
        return batch->max - batch->nr;
 }
@@ -670,7 +671,7 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
                current->comm,
                (long long)pte_val(pte), (long long)pmd_val(*pmd));
        if (page)
-               dump_page(page);
+               dump_page(page, "bad pte");
        printk(KERN_ALERT
                "addr:%p vm_flags:%08lx anon_vma:%p mapping:%p index:%lx\n",
                (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
@@ -1119,7 +1120,7 @@ again:
                                                addr) != page->index) {
                                pte_t ptfile = pgoff_to_pte(page->index);
                                if (pte_soft_dirty(ptent))
-                                       pte_file_mksoft_dirty(ptfile);
+                                       ptfile = pte_file_mksoft_dirty(ptfile);
                                set_pte_at(mm, addr, pte, ptfile);
                        }
                        if (PageAnon(page))
@@ -1928,12 +1929,17 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
                     unsigned long address, unsigned int fault_flags)
 {
        struct vm_area_struct *vma;
+       vm_flags_t vm_flags;
        int ret;
 
        vma = find_extend_vma(mm, address);
        if (!vma || address < vma->vm_start)
                return -EFAULT;
 
+       vm_flags = (fault_flags & FAULT_FLAG_WRITE) ? VM_WRITE : VM_READ;
+       if (!(vm_flags & vma->vm_flags))
+               return -EFAULT;
+
        ret = handle_mm_fault(mm, vma, address, fault_flags);
        if (ret & VM_FAULT_ERROR) {
                if (ret & VM_FAULT_OOM)
@@ -2559,6 +2565,8 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
 
 static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
 {
+       debug_dma_assert_idle(src);
+
        /*
         * If the source page was a PFN mapping, we don't have
         * a "struct page" for it. We do a best-effort copy by
@@ -2699,7 +2707,7 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
                                        goto unwritable_page;
                                }
                        } else
-                               VM_BUG_ON(!PageLocked(old_page));
+                               VM_BUG_ON_PAGE(!PageLocked(old_page), old_page);
 
                        /*
                         * Since we dropped the lock we need to revalidate
@@ -3345,6 +3353,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                if (ret & VM_FAULT_LOCKED)
                        unlock_page(vmf.page);
                ret = VM_FAULT_HWPOISON;
+               page_cache_release(vmf.page);
                goto uncharge_out;
        }
 
@@ -3355,7 +3364,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        if (unlikely(!(ret & VM_FAULT_LOCKED)))
                lock_page(vmf.page);
        else
-               VM_BUG_ON(!PageLocked(vmf.page));
+               VM_BUG_ON_PAGE(!PageLocked(vmf.page), vmf.page);
 
        /*
         * Should we do an early C-O-W break?
@@ -3392,7 +3401,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                                                goto unwritable_page;
                                        }
                                } else
-                                       VM_BUG_ON(!PageLocked(page));
+                                       VM_BUG_ON_PAGE(!PageLocked(page), page);
                                page_mkwrite = 1;
                        }
                }
@@ -3700,7 +3709,6 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        if (unlikely(is_vm_hugetlb_page(vma)))
                return hugetlb_fault(mm, vma, address, flags);
 
-retry:
        pgd = pgd_offset(mm, address);
        pud = pud_alloc(mm, pgd, address);
        if (!pud)
@@ -3738,26 +3746,16 @@ retry:
                        if (dirty && !pmd_write(orig_pmd)) {
                                ret = do_huge_pmd_wp_page(mm, vma, address, pmd,
                                                          orig_pmd);
-                               /*
-                                * If COW results in an oom, the huge pmd will
-                                * have been split, so retry the fault on the
-                                * pte for a smaller charge.
-                                */
-                               if (unlikely(ret & VM_FAULT_OOM))
-                                       goto retry;
-                               return ret;
+                               if (!(ret & VM_FAULT_FALLBACK))
+                                       return ret;
                        } else {
                                huge_pmd_set_accessed(mm, vma, address, pmd,
                                                      orig_pmd, dirty);
+                               return 0;
                        }
-
-                       return 0;
                }
        }
 
-       /* THP should already have been handled */
-       BUG_ON(pmd_numa(*pmd));
-
        /*
         * Use __pte_alloc instead of pte_alloc_map, because we can't
         * run pte_offset_map on the pmd, if an huge pmd could
@@ -4272,11 +4270,20 @@ void copy_user_huge_page(struct page *dst, struct page *src,
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
 #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS
+
+static struct kmem_cache *page_ptl_cachep;
+
+void __init ptlock_cache_init(void)
+{
+       page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
+                       SLAB_PANIC, NULL);
+}
+
 bool ptlock_alloc(struct page *page)
 {
        spinlock_t *ptl;
 
-       ptl = kmalloc(sizeof(spinlock_t), GFP_KERNEL);
+       ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
        if (!ptl)
                return false;
        page->ptl = ptl;
@@ -4285,6 +4292,6 @@ bool ptlock_alloc(struct page *page)
 
 void ptlock_free(struct page *page)
 {
-       kfree(page->ptl);
+       kmem_cache_free(page_ptl_cachep, page->ptl);
 }
 #endif