BACKPORT: mm: multi-gen LRU: minimal implementation

[platform/kernel/linux-rpi.git] / mm / workingset.c

diff --git a/mm/workingset.c b/mm/workingset.c
index 474186b..745fd72 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -15,6 +15,7 @@
 #include <linux/dax.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
+#include <linux/mm_inline.h>
 
 /*
  *             Double CLOCK lists
@@ -184,7 +185,6 @@ static unsigned int bucket_order __read_mostly;
 static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction,
                         bool workingset)
 {
-       eviction >>= bucket_order;
        eviction &= EVICTION_MASK;
        eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid;
        eviction = (eviction << NODES_SHIFT) | pgdat->node_id;
@@ -209,11 +209,109 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
 
        *memcgidp = memcgid;
        *pgdat = NODE_DATA(nid);
-       *evictionp = entry << bucket_order;
+       *evictionp = entry;
        *workingsetp = workingset;
 }
 
+#ifdef CONFIG_LRU_GEN
+
+static void *lru_gen_eviction(struct page *page)
+{
+       int hist;
+       unsigned long token;
+       unsigned long min_seq;
+       struct lruvec *lruvec;
+       struct lru_gen_struct *lrugen;
+       int type = page_is_file_cache(page);
+       int delta = hpage_nr_pages(page);
+       int refs = page_lru_refs(page);
+       int tier = lru_tier_from_refs(refs);
+       struct mem_cgroup *memcg = page_memcg(page);
+       struct pglist_data *pgdat = page_pgdat(page);
+
+       BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT);
+
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
+       lrugen = &lruvec->lrugen;
+       min_seq = READ_ONCE(lrugen->min_seq[type]);
+       token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
+
+       hist = lru_hist_from_seq(min_seq);
+       atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
+
+       return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs);
+}
+
+static void lru_gen_refault(struct page *page, void *shadow)
+{
+       int hist, tier, refs;
+       int memcg_id;
+       bool workingset;
+       unsigned long token;
+       unsigned long min_seq;
+       struct lruvec *lruvec;
+       struct lru_gen_struct *lrugen;
+       struct mem_cgroup *memcg;
+       struct pglist_data *pgdat;
+       int type = page_is_file_cache(page);
+       int delta = hpage_nr_pages(page);
+
+       unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset);
+
+       if (pgdat != page_pgdat(page))
+               return;
+
+       rcu_read_lock();
+
+       memcg = page_memcg_rcu(page);
+       if (memcg_id != mem_cgroup_id(memcg))
+               goto unlock;
+
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
+       lrugen = &lruvec->lrugen;
+
+       min_seq = READ_ONCE(lrugen->min_seq[type]);
+       if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH)))
+               goto unlock;
+
+       hist = lru_hist_from_seq(min_seq);
+       /* see the comment in page_lru_refs() */
+       refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset;
+       tier = lru_tier_from_refs(refs);
+
+       atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
+       mod_lruvec_state(lruvec, WORKINGSET_REFAULT, delta);
+
+       /*
+        * Count the following two cases as stalls:
+        * 1. For pages accessed through page tables, hotter pages pushed out
+        *    hot pages which refaulted immediately.
+        * 2. For pages accessed multiple times through file descriptors,
+        *    numbers of accesses might have been out of the range.
+        */
+       if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) {
+               SetPageWorkingset(page);
+               mod_lruvec_state(lruvec, WORKINGSET_RESTORE, delta);
+       }
+unlock:
+       rcu_read_unlock();
+}
+
+#else /* !CONFIG_LRU_GEN */
+
+static void *lru_gen_eviction(struct page *page)
+{
+       return NULL;
+}
+
+static void lru_gen_refault(struct page *page, void *shadow)
+{
+}
+
+#endif /* CONFIG_LRU_GEN */
+
 static void advance_inactive_age(struct mem_cgroup *memcg, pg_data_t *pgdat)
+
 {
        /*
         * Reclaiming a cgroup means reclaiming all its children in a
@@ -254,12 +352,16 @@ void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
        VM_BUG_ON_PAGE(page_count(page), page);
        VM_BUG_ON_PAGE(!PageLocked(page), page);
 
+       if (lru_gen_enabled())
+               return lru_gen_eviction(page);
+
        advance_inactive_age(page_memcg(page), pgdat);
 
        lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
        /* XXX: target_memcg can be NULL, go through lruvec */
        memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
        eviction = atomic_long_read(&lruvec->inactive_age);
+       eviction >>= bucket_order;
        return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
 }
 
@@ -286,7 +388,13 @@ void workingset_refault(struct page *page, void *shadow)
        bool workingset;
        int memcgid;
 
+       if (lru_gen_enabled()) {
+               lru_gen_refault(page, shadow);
+               return;
+       }
+
        unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset);
+       eviction <<= bucket_order;
 
        rcu_read_lock();
        /*