mm: vmscan: call NUMA-unaware shrinkers irrespective of nodemask

[platform/adaptation/renesas_rcar/renesas_kernel.git] / mm / vmscan.c

diff --git a/mm/vmscan.c b/mm/vmscan.c
index eea668d..90c4075 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -281,17 +281,34 @@ shrink_slab_node(struct shrink_control *shrinkctl, struct shrinker *shrinker,
                                nr_pages_scanned, lru_pages,
                                max_pass, delta, total_scan);
 
-       while (total_scan >= batch_size) {
+       /*
+        * Normally, we should not scan less than batch_size objects in one
+        * pass to avoid too frequent shrinker calls, but if the slab has less
+        * than batch_size objects in total and we are really tight on memory,
+        * we will try to reclaim all available objects, otherwise we can end
+        * up failing allocations although there are plenty of reclaimable
+        * objects spread over several slabs with usage less than the
+        * batch_size.
+        *
+        * We detect the "tight on memory" situations by looking at the total
+        * number of objects we want to scan (total_scan). If it is greater
+        * than the total number of objects on slab (max_pass), we must be
+        * scanning at high prio and therefore should try to reclaim as much as
+        * possible.
+        */
+       while (total_scan >= batch_size ||
+              total_scan >= max_pass) {
                unsigned long ret;
+               unsigned long nr_to_scan = min(batch_size, total_scan);
 
-               shrinkctl->nr_to_scan = batch_size;
+               shrinkctl->nr_to_scan = nr_to_scan;
                ret = shrinker->scan_objects(shrinker, shrinkctl);
                if (ret == SHRINK_STOP)
                        break;
                freed += ret;
 
-               count_vm_events(SLABS_SCANNED, batch_size);
-               total_scan -= batch_size;
+               count_vm_events(SLABS_SCANNED, nr_to_scan);
+               total_scan -= nr_to_scan;
 
                cond_resched();
        }
@@ -352,16 +369,17 @@ unsigned long shrink_slab(struct shrink_control *shrinkctl,
        }
 
        list_for_each_entry(shrinker, &shrinker_list, list) {
-               for_each_node_mask(shrinkctl->nid, shrinkctl->nodes_to_scan) {
-                       if (!node_online(shrinkctl->nid))
-                               continue;
-
-                       if (!(shrinker->flags & SHRINKER_NUMA_AWARE) &&
-                           (shrinkctl->nid != 0))
-                               break;
-
+               if (!(shrinker->flags & SHRINKER_NUMA_AWARE)) {
+                       shrinkctl->nid = 0;
                        freed += shrink_slab_node(shrinkctl, shrinker,
-                                nr_pages_scanned, lru_pages);
+                                       nr_pages_scanned, lru_pages);
+                       continue;
+               }
+
+               for_each_node_mask(shrinkctl->nid, shrinkctl->nodes_to_scan) {
+                       if (node_online(shrinkctl->nid))
+                               freed += shrink_slab_node(shrinkctl, shrinker,
+                                               nr_pages_scanned, lru_pages);
 
                }
        }
@@ -603,7 +621,7 @@ void putback_lru_page(struct page *page)
        bool is_unevictable;
        int was_unevictable = PageUnevictable(page);
 
-       VM_BUG_ON(PageLRU(page));
+       VM_BUG_ON_PAGE(PageLRU(page), page);
 
 redo:
        ClearPageUnevictable(page);
@@ -794,8 +812,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
                if (!trylock_page(page))
                        goto keep;
 
-               VM_BUG_ON(PageActive(page));
-               VM_BUG_ON(page_zone(page) != zone);
+               VM_BUG_ON_PAGE(PageActive(page), page);
+               VM_BUG_ON_PAGE(page_zone(page) != zone, page);
 
                sc->nr_scanned++;
 
@@ -1079,14 +1097,14 @@ activate_locked:
                /* Not a candidate for swapping, so reclaim swap space. */
                if (PageSwapCache(page) && vm_swap_full())
                        try_to_free_swap(page);
-               VM_BUG_ON(PageActive(page));
+               VM_BUG_ON_PAGE(PageActive(page), page);
                SetPageActive(page);
                pgactivate++;
 keep_locked:
                unlock_page(page);
 keep:
                list_add(&page->lru, &ret_pages);
-               VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
+               VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
        }
 
        free_hot_cold_page_list(&free_pages, 1);
@@ -1240,7 +1258,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                page = lru_to_page(src);
                prefetchw_prev_lru_page(page, src, flags);
 
-               VM_BUG_ON(!PageLRU(page));
+               VM_BUG_ON_PAGE(!PageLRU(page), page);
 
                switch (__isolate_lru_page(page, mode)) {
                case 0:
@@ -1295,7 +1313,7 @@ int isolate_lru_page(struct page *page)
 {
        int ret = -EBUSY;
 
-       VM_BUG_ON(!page_count(page));
+       VM_BUG_ON_PAGE(!page_count(page), page);
 
        if (PageLRU(page)) {
                struct zone *zone = page_zone(page);
@@ -1366,7 +1384,7 @@ putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
                struct page *page = lru_to_page(page_list);
                int lru;
 
-               VM_BUG_ON(PageLRU(page));
+               VM_BUG_ON_PAGE(PageLRU(page), page);
                list_del(&page->lru);
                if (unlikely(!page_evictable(page))) {
                        spin_unlock_irq(&zone->lru_lock);
@@ -1586,7 +1604,7 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,
                page = lru_to_page(list);
                lruvec = mem_cgroup_page_lruvec(page, zone);
 
-               VM_BUG_ON(PageLRU(page));
+               VM_BUG_ON_PAGE(PageLRU(page), page);
                SetPageLRU(page);
 
                nr_pages = hpage_nr_pages(page);
@@ -3701,7 +3719,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)
                if (page_evictable(page)) {
                        enum lru_list lru = page_lru_base_type(page);
 
-                       VM_BUG_ON(PageActive(page));
+                       VM_BUG_ON_PAGE(PageActive(page), page);
                        ClearPageUnevictable(page);
                        del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
                        add_page_to_lru_list(page, lruvec, lru);