mm: set page->pfmemalloc in prep_new_page()

[platform/kernel/linux-rpi.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8e20f9c..30a3250 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -552,17 +552,15 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
                return 0;
 
        if (page_is_guard(buddy) && page_order(buddy) == order) {
-               VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
-
                if (page_zone_id(page) != page_zone_id(buddy))
                        return 0;
 
+               VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
+
                return 1;
        }
 
        if (PageBuddy(buddy) && page_order(buddy) == order) {
-               VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
-
                /*
                 * zone check is done late to avoid uselessly
                 * calculating zone/node ids for pages that could
@@ -571,6 +569,8 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
                if (page_zone_id(page) != page_zone_id(buddy))
                        return 0;
 
+               VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
+
                return 1;
        }
        return 0;
@@ -970,7 +970,8 @@ static inline int check_new_page(struct page *page)
        return 0;
 }
 
-static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags)
+static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
+                                                               int alloc_flags)
 {
        int i;
 
@@ -994,6 +995,14 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags)
 
        set_page_owner(page, order, gfp_flags);
 
+       /*
+        * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was necessary to
+        * allocate the page. The expectation is that the caller is taking
+        * steps that will free more memory. The caller should avoid the page
+        * being used for !PFMEMALLOC purposes.
+        */
+       page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
+
        return 0;
 }
 
@@ -1642,9 +1651,7 @@ int split_free_page(struct page *page)
 }
 
 /*
- * Really, prep_compound_page() should be called from __rmqueue_bulk().  But
- * we cheat by calling it from here, in the order > 0 path.  Saves a branch
- * or two.
+ * Allocate a page from the given zone. Use pcplists for order-0 allocations.
  */
 static inline
 struct page *buffered_rmqueue(struct zone *preferred_zone,
@@ -1655,7 +1662,6 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
        struct page *page;
        bool cold = ((gfp_flags & __GFP_COLD) != 0);
 
-again:
        if (likely(order == 0)) {
                struct per_cpu_pages *pcp;
                struct list_head *list;
@@ -1711,8 +1717,6 @@ again:
        local_irq_restore(flags);
 
        VM_BUG_ON_PAGE(bad_range(zone, page), page);
-       if (prep_new_page(page, order, gfp_flags))
-               goto again;
        return page;
 
 failed:
@@ -2177,25 +2181,16 @@ zonelist_scan:
 try_this_zone:
                page = buffered_rmqueue(preferred_zone, zone, order,
                                                gfp_mask, migratetype);
-               if (page)
-                       break;
+               if (page) {
+                       if (prep_new_page(page, order, gfp_mask, alloc_flags))
+                               goto try_this_zone;
+                       return page;
+               }
 this_zone_full:
                if (IS_ENABLED(CONFIG_NUMA) && zlc_active)
                        zlc_mark_zone_full(zonelist, z);
        }
 
-       if (page) {
-               /*
-                * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
-                * necessary to allocate the page. The expectation is
-                * that the caller is taking steps that will free more
-                * memory. The caller should avoid the page being used
-                * for !PFMEMALLOC purposes.
-                */
-               page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS);
-               return page;
-       }
-
        /*
         * The first pass makes sure allocations are spread fairly within the
         * local node.  However, the local node might have free pages left
@@ -2842,11 +2837,7 @@ retry:
 
 nopage:
        warn_alloc_failed(gfp_mask, order, NULL);
-       return page;
 got_pg:
-       if (kmemcheck_enabled)
-               kmemcheck_pagealloc_alloc(page, order, gfp_mask);
-
        return page;
 }
 
@@ -2865,6 +2856,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
        unsigned int cpuset_mems_cookie;
        int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
        int classzone_idx;
+       gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */
 
        gfp_mask &= gfp_allowed_mask;
 
@@ -2898,22 +2890,27 @@ retry_cpuset:
        classzone_idx = zonelist_zone_idx(preferred_zoneref);
 
        /* First allocation attempt */
-       page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
-                       zonelist, high_zoneidx, alloc_flags,
-                       preferred_zone, classzone_idx, migratetype);
+       alloc_mask = gfp_mask|__GFP_HARDWALL;
+       page = get_page_from_freelist(alloc_mask, nodemask, order, zonelist,
+                       high_zoneidx, alloc_flags, preferred_zone,
+                       classzone_idx, migratetype);
        if (unlikely(!page)) {
                /*
                 * Runtime PM, block IO and its error handling path
                 * can deadlock because I/O on the device might not
                 * complete.
                 */
-               gfp_mask = memalloc_noio_flags(gfp_mask);
-               page = __alloc_pages_slowpath(gfp_mask, order,
+               alloc_mask = memalloc_noio_flags(gfp_mask);
+
+               page = __alloc_pages_slowpath(alloc_mask, order,
                                zonelist, high_zoneidx, nodemask,
                                preferred_zone, classzone_idx, migratetype);
        }
 
-       trace_mm_page_alloc(page, order, gfp_mask, migratetype);
+       if (kmemcheck_enabled && page)
+               kmemcheck_pagealloc_alloc(page, order, gfp_mask);
+
+       trace_mm_page_alloc(page, order, alloc_mask, migratetype);
 
 out:
        /*