mm: multi-gen LRU: section for memcg LRU

Move memcg LRU code into a dedicated section.  Improve the design doc to
outline its architecture.

Link: https://lkml.kernel.org/r/20230118001827.1040870-5-talumbau@google.com
Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst
index 770b5d5..5f1f6ec 100644 (file)
--- a/Documentation/mm/multigen_lru.rst
+++ b/Documentation/mm/multigen_lru.rst
@@ -186,9 +186,40 @@ is false positive, the cost is an additional scan of a range of PTEs,
 which may yield hot pages anyway. Parameters of the filter itself can
 control the false positive rate in the limit.
 
+Memcg LRU
+---------
+An memcg LRU is a per-node LRU of memcgs. It is also an LRU of LRUs,
+since each node and memcg combination has an LRU of folios (see
+``mem_cgroup_lruvec()``). Its goal is to improve the scalability of
+global reclaim, which is critical to system-wide memory overcommit in
+data centers. Note that memcg LRU only applies to global reclaim.
+
+The basic structure of an memcg LRU can be understood by an analogy to
+the active/inactive LRU (of folios):
+
+1. It has the young and the old (generations), i.e., the counterparts
+   to the active and the inactive;
+2. The increment of ``max_seq`` triggers promotion, i.e., the
+   counterpart to activation;
+3. Other events trigger similar operations, e.g., offlining an memcg
+   triggers demotion, i.e., the counterpart to deactivation.
+
+In terms of global reclaim, it has two distinct features:
+
+1. Sharding, which allows each thread to start at a random memcg (in
+   the old generation) and improves parallelism;
+2. Eventual fairness, which allows direct reclaim to bail out at will
+   and reduces latency without affecting fairness over some time.
+
+In terms of traversing memcgs during global reclaim, it improves the
+best-case complexity from O(n) to O(1) and does not affect the
+worst-case complexity O(n). Therefore, on average, it has a sublinear
+complexity.
+
 Summary
 -------
-The multi-gen LRU can be disassembled into the following parts:
+The multi-gen LRU (of folios) can be disassembled into the following
+parts:
 
 * Generations
 * Rmap walks

diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 26dcbda..de1e622 100644 (file)
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -122,18 +122,6 @@ static inline bool lru_gen_in_fault(void)
        return current->in_lru_fault;
 }
 
-#ifdef CONFIG_MEMCG
-static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
-{
-       return READ_ONCE(lruvec->lrugen.seg);
-}
-#else
-static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
-{
-       return 0;
-}
-#endif
-
 static inline int lru_gen_from_seq(unsigned long seq)
 {
        return seq % MAX_NR_GENS;
@@ -309,11 +297,6 @@ static inline bool lru_gen_in_fault(void)
        return false;
 }
 
-static inline int lru_gen_memcg_seg(struct lruvec *lruvec)
-{
-       return 0;
-}
-
 static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
 {
        return false;

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 815c7c2..977be52 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -368,15 +368,6 @@ struct page_vma_mapped_walk;
 #define LRU_GEN_MASK           ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF)
 #define LRU_REFS_MASK          ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF)
 
-/* see the comment on MEMCG_NR_GENS */
-enum {
-       MEMCG_LRU_NOP,
-       MEMCG_LRU_HEAD,
-       MEMCG_LRU_TAIL,
-       MEMCG_LRU_OLD,
-       MEMCG_LRU_YOUNG,
-};
-
 #ifdef CONFIG_LRU_GEN
 
 enum {
@@ -557,7 +548,7 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg);
 void lru_gen_online_memcg(struct mem_cgroup *memcg);
 void lru_gen_offline_memcg(struct mem_cgroup *memcg);
 void lru_gen_release_memcg(struct mem_cgroup *memcg);
-void lru_gen_rotate_memcg(struct lruvec *lruvec, int op);
+void lru_gen_soft_reclaim(struct lruvec *lruvec);
 
 #else /* !CONFIG_MEMCG */
 
@@ -608,7 +599,7 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg)
 {
 }
 
-static inline void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
+static inline void lru_gen_soft_reclaim(struct lruvec *lruvec)
 {
 }
 

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 893427a..1733545 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -476,12 +476,8 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, int nid)
        struct mem_cgroup_tree_per_node *mctz;
 
        if (lru_gen_enabled()) {
-               struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
-
-               /* see the comment on MEMCG_NR_GENS */
-               if (soft_limit_excess(memcg) && lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD)
-                       lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
-
+               if (soft_limit_excess(memcg))
+                       lru_gen_soft_reclaim(&memcg->nodeinfo[nid]->lruvec);
                return;
        }
 

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1be9120..796d4ca 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4706,6 +4706,148 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw)
 }
 
 /******************************************************************************
+ *                          memcg LRU
+ ******************************************************************************/
+
+/* see the comment on MEMCG_NR_GENS */
+enum {
+       MEMCG_LRU_NOP,
+       MEMCG_LRU_HEAD,
+       MEMCG_LRU_TAIL,
+       MEMCG_LRU_OLD,
+       MEMCG_LRU_YOUNG,
+};
+
+#ifdef CONFIG_MEMCG
+
+static int lru_gen_memcg_seg(struct lruvec *lruvec)
+{
+       return READ_ONCE(lruvec->lrugen.seg);
+}
+
+static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
+{
+       int seg;
+       int old, new;
+       int bin = get_random_u32_below(MEMCG_NR_BINS);
+       struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+
+       spin_lock(&pgdat->memcg_lru.lock);
+
+       VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
+
+       seg = 0;
+       new = old = lruvec->lrugen.gen;
+
+       /* see the comment on MEMCG_NR_GENS */
+       if (op == MEMCG_LRU_HEAD)
+               seg = MEMCG_LRU_HEAD;
+       else if (op == MEMCG_LRU_TAIL)
+               seg = MEMCG_LRU_TAIL;
+       else if (op == MEMCG_LRU_OLD)
+               new = get_memcg_gen(pgdat->memcg_lru.seq);
+       else if (op == MEMCG_LRU_YOUNG)
+               new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
+       else
+               VM_WARN_ON_ONCE(true);
+
+       hlist_nulls_del_rcu(&lruvec->lrugen.list);
+
+       if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
+               hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
+       else
+               hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
+
+       pgdat->memcg_lru.nr_memcgs[old]--;
+       pgdat->memcg_lru.nr_memcgs[new]++;
+
+       lruvec->lrugen.gen = new;
+       WRITE_ONCE(lruvec->lrugen.seg, seg);
+
+       if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
+               WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+       spin_unlock(&pgdat->memcg_lru.lock);
+}
+
+void lru_gen_online_memcg(struct mem_cgroup *memcg)
+{
+       int gen;
+       int nid;
+       int bin = get_random_u32_below(MEMCG_NR_BINS);
+
+       for_each_node(nid) {
+               struct pglist_data *pgdat = NODE_DATA(nid);
+               struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+               spin_lock(&pgdat->memcg_lru.lock);
+
+               VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
+
+               gen = get_memcg_gen(pgdat->memcg_lru.seq);
+
+               hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
+               pgdat->memcg_lru.nr_memcgs[gen]++;
+
+               lruvec->lrugen.gen = gen;
+
+               spin_unlock(&pgdat->memcg_lru.lock);
+       }
+}
+
+void lru_gen_offline_memcg(struct mem_cgroup *memcg)
+{
+       int nid;
+
+       for_each_node(nid) {
+               struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+               lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
+       }
+}
+
+void lru_gen_release_memcg(struct mem_cgroup *memcg)
+{
+       int gen;
+       int nid;
+
+       for_each_node(nid) {
+               struct pglist_data *pgdat = NODE_DATA(nid);
+               struct lruvec *lruvec = get_lruvec(memcg, nid);
+
+               spin_lock(&pgdat->memcg_lru.lock);
+
+               VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
+
+               gen = lruvec->lrugen.gen;
+
+               hlist_nulls_del_rcu(&lruvec->lrugen.list);
+               pgdat->memcg_lru.nr_memcgs[gen]--;
+
+               if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
+                       WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
+
+               spin_unlock(&pgdat->memcg_lru.lock);
+       }
+}
+
+void lru_gen_soft_reclaim(struct lruvec *lruvec)
+{
+       /* see the comment on MEMCG_NR_GENS */
+       if (lru_gen_memcg_seg(lruvec) != MEMCG_LRU_HEAD)
+               lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
+}
+
+#else /* !CONFIG_MEMCG */
+
+static int lru_gen_memcg_seg(struct lruvec *lruvec)
+{
+       return 0;
+}
+
+#endif
+
+/******************************************************************************
  *                          the eviction
  ******************************************************************************/
 
@@ -5397,53 +5539,6 @@ done:
        pgdat->kswapd_failures = 0;
 }
 
-#ifdef CONFIG_MEMCG
-void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
-{
-       int seg;
-       int old, new;
-       int bin = get_random_u32_below(MEMCG_NR_BINS);
-       struct pglist_data *pgdat = lruvec_pgdat(lruvec);
-
-       spin_lock(&pgdat->memcg_lru.lock);
-
-       VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
-
-       seg = 0;
-       new = old = lruvec->lrugen.gen;
-
-       /* see the comment on MEMCG_NR_GENS */
-       if (op == MEMCG_LRU_HEAD)
-               seg = MEMCG_LRU_HEAD;
-       else if (op == MEMCG_LRU_TAIL)
-               seg = MEMCG_LRU_TAIL;
-       else if (op == MEMCG_LRU_OLD)
-               new = get_memcg_gen(pgdat->memcg_lru.seq);
-       else if (op == MEMCG_LRU_YOUNG)
-               new = get_memcg_gen(pgdat->memcg_lru.seq + 1);
-       else
-               VM_WARN_ON_ONCE(true);
-
-       hlist_nulls_del_rcu(&lruvec->lrugen.list);
-
-       if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD)
-               hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
-       else
-               hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
-
-       pgdat->memcg_lru.nr_memcgs[old]--;
-       pgdat->memcg_lru.nr_memcgs[new]++;
-
-       lruvec->lrugen.gen = new;
-       WRITE_ONCE(lruvec->lrugen.seg, seg);
-
-       if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq))
-               WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
-
-       spin_unlock(&pgdat->memcg_lru.lock);
-}
-#endif
-
 /******************************************************************************
  *                          state change
  ******************************************************************************/
@@ -6086,67 +6181,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
        }
 }
 
-void lru_gen_online_memcg(struct mem_cgroup *memcg)
-{
-       int gen;
-       int nid;
-       int bin = get_random_u32_below(MEMCG_NR_BINS);
-
-       for_each_node(nid) {
-               struct pglist_data *pgdat = NODE_DATA(nid);
-               struct lruvec *lruvec = get_lruvec(memcg, nid);
-
-               spin_lock(&pgdat->memcg_lru.lock);
-
-               VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
-
-               gen = get_memcg_gen(pgdat->memcg_lru.seq);
-
-               hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
-               pgdat->memcg_lru.nr_memcgs[gen]++;
-
-               lruvec->lrugen.gen = gen;
-
-               spin_unlock(&pgdat->memcg_lru.lock);
-       }
-}
-
-void lru_gen_offline_memcg(struct mem_cgroup *memcg)
-{
-       int nid;
-
-       for_each_node(nid) {
-               struct lruvec *lruvec = get_lruvec(memcg, nid);
-
-               lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
-       }
-}
-
-void lru_gen_release_memcg(struct mem_cgroup *memcg)
-{
-       int gen;
-       int nid;
-
-       for_each_node(nid) {
-               struct pglist_data *pgdat = NODE_DATA(nid);
-               struct lruvec *lruvec = get_lruvec(memcg, nid);
-
-               spin_lock(&pgdat->memcg_lru.lock);
-
-               VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
-
-               gen = lruvec->lrugen.gen;
-
-               hlist_nulls_del_rcu(&lruvec->lrugen.list);
-               pgdat->memcg_lru.nr_memcgs[gen]--;
-
-               if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq))
-                       WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1);
-
-               spin_unlock(&pgdat->memcg_lru.lock);
-       }
-}
-
 #endif /* CONFIG_MEMCG */
 
 static int __init init_lru_gen(void)