mm: balance LRU lists based on relative thrashing

Since the LRUs were split into anon and file lists, the VM has been
balancing between page cache and anonymous pages based on per-list ratios
of scanned vs.  rotated pages.  In most cases that tips page reclaim
towards the list that is easier to reclaim and has the fewest actively
used pages, but there are a few problems with it:

1. Refaults and LRU rotations are weighted the same way, even though
   one costs IO and the other costs a bit of CPU.

2. The less we scan an LRU list based on already observed rotations,
   the more we increase the sampling interval for new references, and
   rotations become even more likely on that list. This can enter a
   death spiral in which we stop looking at one list completely until
   the other one is all but annihilated by page reclaim.

Since commit a528910e12ec ("mm: thrash detection-based file cache sizing")
we have refault detection for the page cache.  Along with swapin events,
they are good indicators of when the file or anon list, respectively, is
too small for its workingset and needs to grow.

For example, if the page cache is thrashing, the cache pages need more
time in memory, while there may be colder pages on the anonymous list.
Likewise, if swapped pages are faulting back in, it indicates that we
reclaim anonymous pages too aggressively and should back off.

Replace LRU rotations with refaults and swapins as the basis for relative
reclaim cost of the two LRUs.  This will have the VM target list balances
that incur the least amount of IO on aggregate.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Rik van Riel <riel@surriel.com>
Link: http://lkml.kernel.org/r/20200520232525.798933-12-hannes@cmpxchg.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/swap.h b/include/linux/swap.h
index ce3c55747006e382eedbaa3f811184bb32d291de..0b71bf75fb679f276ed6be3624665f2ef26fbbab 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -334,8 +334,7 @@ extern unsigned long nr_free_pagecache_pages(void);
 
 
 /* linux/mm/swap.c */
-extern void lru_note_cost(struct lruvec *lruvec, bool file,
-                         unsigned int nr_pages);
+extern void lru_note_cost(struct page *);
 extern void lru_cache_add(struct page *);
 extern void lru_add_page_tail(struct page *page, struct page *page_tail,
                         struct lruvec *lruvec, struct list_head *head);

diff --git a/mm/swap.c b/mm/swap.c
index 7d552af257975971310b09bf8b10e231389d164a..2dc7d392642f3fe09438af5e7387a139b6fdb6b8 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -278,12 +278,15 @@ void rotate_reclaimable_page(struct page *page)
        }
 }
 
-void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages)
+void lru_note_cost(struct page *page)
 {
-       if (file)
-               lruvec->file_cost += nr_pages;
+       struct lruvec *lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
+
+       /* Record new data point */
+       if (page_is_file_lru(page))
+               lruvec->file_cost++;
        else
-               lruvec->anon_cost += nr_pages;
+               lruvec->anon_cost++;
 }
 
 static void __activate_page(struct page *page, struct lruvec *lruvec,

diff --git a/mm/swap_state.c b/mm/swap_state.c
index fa089002125f91795c8f84206da74a4847b55084..1cd0b345ff7e1e394c05ef4acc59ca8e81bc4210 100644 (file)
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -440,6 +440,11 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                goto fail_unlock;
        }
 
+       /* XXX: Move to lru_cache_add() when it supports new vs putback */
+       spin_lock_irq(&page_pgdat(page)->lru_lock);
+       lru_note_cost(page);
+       spin_unlock_irq(&page_pgdat(page)->lru_lock);
+
        /* Caller will initiate read into locked page */
        SetPageWorkingset(page);
        lru_cache_add(page);

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3c89eac629f30c61ba7dac20684351df2a74afe4..76e823db21a72c3d93902870a50e17c4d9872fe7 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1958,12 +1958,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
        move_pages_to_lru(lruvec, &page_list);
 
        __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
-       /*
-        * Rotating pages costs CPU without actually
-        * progressing toward the reclaim goal.
-        */
-       lru_note_cost(lruvec, 0, stat.nr_activate[0]);
-       lru_note_cost(lruvec, 1, stat.nr_activate[1]);
        item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
        if (!cgroup_reclaim(sc))
                __count_vm_events(item, nr_reclaimed);
@@ -2079,11 +2073,6 @@ static void shrink_active_list(unsigned long nr_to_scan,
         * Move pages back to the lru list.
         */
        spin_lock_irq(&pgdat->lru_lock);
-       /*
-        * Rotating pages costs CPU without actually
-        * progressing toward the reclaim goal.
-        */
-       lru_note_cost(lruvec, file, nr_rotated);
 
        nr_activate = move_pages_to_lru(lruvec, &l_active);
        nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
@@ -2298,22 +2287,23 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
        scan_balance = SCAN_FRACT;
 
        /*
-        * With swappiness at 100, anonymous and file have the same priority.
-        * This scanning priority is essentially the inverse of IO cost.
+        * Calculate the pressure balance between anon and file pages.
+        *
+        * The amount of pressure we put on each LRU is inversely
+        * proportional to the cost of reclaiming each list, as
+        * determined by the share of pages that are refaulting, times
+        * the relative IO cost of bringing back a swapped out
+        * anonymous page vs reloading a filesystem page (swappiness).
+        *
+        * With swappiness at 100, anon and file have equal IO cost.
         */
        anon_prio = swappiness;
        file_prio = 200 - anon_prio;
 
        /*
-        * OK, so we have swap space and a fair amount of page cache
-        * pages.  We use the recently rotated / recently scanned
-        * ratios to determine how valuable each cache is.
-        *
         * Because workloads change over time (and to avoid overflow)
         * we keep these statistics as a floating average, which ends
-        * up weighing recent references more than old ones.
-        *
-        * anon in [0], file in [1]
+        * up weighing recent refaults more than old ones.
         */
 
        anon  = lruvec_lru_size(lruvec, LRU_ACTIVE_ANON, MAX_NR_ZONES) +
@@ -2328,15 +2318,6 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
                lruvec->file_cost /= 2;
                totalcost /= 2;
        }
-
-       /*
-        * The amount of pressure on anon vs file pages is inversely
-        * proportional to the assumed cost of reclaiming each list,
-        * as determined by the share of pages that are likely going
-        * to refault or rotate on each list (recently referenced),
-        * times the relative IO cost of bringing back a swapped out
-        * anonymous page vs reloading a filesystem page (swappiness).
-        */
        ap = anon_prio * (totalcost + 1);
        ap /= lruvec->anon_cost + 1;
 

diff --git a/mm/workingset.c b/mm/workingset.c
index e69865739539abf0c1df0b97e369118fe4dcd162..a6a2a740ed0ba2bea4d0e5bcb221a1a717aba3c0 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -365,6 +365,10 @@ void workingset_refault(struct page *page, void *shadow)
        /* Page was active prior to eviction */
        if (workingset) {
                SetPageWorkingset(page);
+               /* XXX: Move to lru_cache_add() when it supports new vs putback */
+               spin_lock_irq(&page_pgdat(page)->lru_lock);
+               lru_note_cost(page);
+               spin_unlock_irq(&page_pgdat(page)->lru_lock);
                inc_lruvec_state(lruvec, WORKINGSET_RESTORE);
        }
 out: