mm: clean up and clarify lruvec lookup procedure

There is a per-memcg lruvec and a NUMA node lruvec.  Which one is being
used is somewhat confusing right now, and it's easy to make mistakes -
especially when it comes to global reclaim.

How it works: when memory cgroups are enabled, we always use the
root_mem_cgroup's per-node lruvecs.  When memory cgroups are not compiled
in or disabled at runtime, we use pgdat->lruvec.

Document that in a comment.

Due to the way the reclaim code is generalized, all lookups use the
mem_cgroup_lruvec() helper function, and nobody should have to find the
right lruvec manually right now.  But to avoid future mistakes, rename the
pgdat->lruvec member to pgdat->__lruvec and delete the convenience wrapper
that suggests it's a commonly accessed member.

While in this area, swap the mem_cgroup_lruvec() argument order.  The name
suggests a memcg operation, yet it takes a pgdat first and a memcg second.
I have to double take every time I call this.  Fix that.

Link: http://lkml.kernel.org/r/20191022144803.302233-3-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Michal Hocko <mhocko@suse.com>
Reviewed-by: Shakeel Butt <shakeelb@google.com>
Cc: Roman Gushchin <guro@fb.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 239e752a78177d7e1fc206cca2171cf83d80663f..feeb2c76f5681d8e244d4368a2400b619d34c341 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -385,21 +385,21 @@ mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
 }
 
 /**
- * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
- * @node: node of the wanted lruvec
+ * mem_cgroup_lruvec - get the lru list vector for a memcg & node
  * @memcg: memcg of the wanted lruvec
  *
- * Returns the lru list vector holding pages for a given @node or a given
- * @memcg. This can be the node lruvec, if the memory controller is disabled.
+ * Returns the lru list vector holding pages for a given @memcg &
+ * @node combination. This can be the node lruvec, if the memory
+ * controller is disabled.
  */
-static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
-                               struct mem_cgroup *memcg)
+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
+                                              struct pglist_data *pgdat)
 {
        struct mem_cgroup_per_node *mz;
        struct lruvec *lruvec;
 
        if (mem_cgroup_disabled()) {
-               lruvec = node_lruvec(pgdat);
+               lruvec = &pgdat->__lruvec;
                goto out;
        }
 
@@ -718,7 +718,7 @@ static inline void __mod_lruvec_page_state(struct page *page,
                return;
        }
 
-       lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
+       lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat);
        __mod_lruvec_state(lruvec, idx, val);
 }
 
@@ -889,16 +889,16 @@ static inline void mem_cgroup_migrate(struct page *old, struct page *new)
 {
 }
 
-static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
-                               struct mem_cgroup *memcg)
+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
+                                              struct pglist_data *pgdat)
 {
-       return node_lruvec(pgdat);
+       return &pgdat->__lruvec;
 }
 
 static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
                                                    struct pglist_data *pgdat)
 {
-       return &pgdat->lruvec;
+       return &pgdat->__lruvec;
 }
 
 static inline bool mm_match_cgroup(struct mm_struct *mm,

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index c7fb21f19eddb8da7df53a07d34e3b622498dad0..cc8232a100bd0e53f8af572762780168844950df 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -777,7 +777,13 @@ typedef struct pglist_data {
 #endif
 
        /* Fields commonly accessed by the page reclaim scanner */
-       struct lruvec           lruvec;
+
+       /*
+        * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
+        *
+        * Use mem_cgroup_lruvec() to look up lruvecs.
+        */
+       struct lruvec           __lruvec;
 
        unsigned long           flags;
 
@@ -800,11 +806,6 @@ typedef struct pglist_data {
 #define node_start_pfn(nid)    (NODE_DATA(nid)->node_start_pfn)
 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
 
-static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
-{
-       return &pgdat->lruvec;
-}
-
 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
 {
        return pgdat->node_start_pfn + pgdat->node_spanned_pages;
@@ -842,7 +843,7 @@ static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
 #ifdef CONFIG_MEMCG
        return lruvec->pgdat;
 #else
-       return container_of(lruvec, struct pglist_data, lruvec);
+       return container_of(lruvec, struct pglist_data, __lruvec);
 #endif
 }
 

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 529e12a59131974a850262cafdb7f293b5b41008..bc01423277c58aeedcb6892acbc550c951a135cb 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -777,7 +777,7 @@ void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
        if (!memcg || memcg == root_mem_cgroup) {
                __mod_node_page_state(pgdat, idx, val);
        } else {
-               lruvec = mem_cgroup_lruvec(pgdat, memcg);
+               lruvec = mem_cgroup_lruvec(memcg, pgdat);
                __mod_lruvec_state(lruvec, idx, val);
        }
        rcu_read_unlock();
@@ -1221,7 +1221,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd
        struct lruvec *lruvec;
 
        if (mem_cgroup_disabled()) {
-               lruvec = &pgdat->lruvec;
+               lruvec = &pgdat->__lruvec;
                goto out;
        }
 
@@ -3634,7 +3634,7 @@ static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
                                           int nid, unsigned int lru_mask)
 {
-       struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
+       struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
        unsigned long nr = 0;
        enum lru_list lru;
 
@@ -5338,8 +5338,8 @@ static int mem_cgroup_move_account(struct page *page,
        anon = PageAnon(page);
 
        pgdat = page_pgdat(page);
-       from_vec = mem_cgroup_lruvec(pgdat, from);
-       to_vec = mem_cgroup_lruvec(pgdat, to);
+       from_vec = mem_cgroup_lruvec(from, pgdat);
+       to_vec = mem_cgroup_lruvec(to, pgdat);
 
        spin_lock_irqsave(&from->move_lock, flags);
 

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e3a69ba5ec533a2a727e745ac779176f1b4614c7..4785a8a2040eeccaa996e93597b30ddf4626e838 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6713,7 +6713,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
 
        pgdat_page_ext_init(pgdat);
        spin_lock_init(&pgdat->lru_lock);
-       lruvec_init(node_lruvec(pgdat));
+       lruvec_init(&pgdat->__lruvec);
 }
 
 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,

diff --git a/mm/slab.h b/mm/slab.h
index 8b77f973a6abc3c088b4325cfac473fcef84ec08..7e94700aa78c6e4fd9f26dcfa559688a7920fc3f 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -369,7 +369,7 @@ static __always_inline int memcg_charge_slab(struct page *page,
        if (ret)
                goto out;
 
-       lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
+       lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
        mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);
 
        /* transer try_charge() page references to kmem_cache */
@@ -393,7 +393,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order,
        rcu_read_lock();
        memcg = READ_ONCE(s->memcg_params.memcg);
        if (likely(!mem_cgroup_is_root(memcg))) {
-               lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
+               lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
                mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));
                memcg_kmem_uncharge_memcg(page, order, memcg);
        } else {

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 266620f7c8143bd729cd4f543bbe02b1e60fd572..94d73725813d3be8d72c4624fe7a3bec0631385a 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2545,7 +2545,7 @@ out:
 static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
                              struct scan_control *sc)
 {
-       struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
+       struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
        unsigned long nr[NR_LRU_LISTS];
        unsigned long targets[NR_LRU_LISTS];
        unsigned long nr_to_scan;
@@ -3023,7 +3023,7 @@ static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat)
                unsigned long refaults;
                struct lruvec *lruvec;
 
-               lruvec = mem_cgroup_lruvec(pgdat, memcg);
+               lruvec = mem_cgroup_lruvec(memcg, pgdat);
                refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
                lruvec->refaults = refaults;
        } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
@@ -3379,7 +3379,7 @@ static void age_active_anon(struct pglist_data *pgdat,
 
        memcg = mem_cgroup_iter(NULL, NULL, NULL);
        do {
-               struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
+               struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
 
                if (inactive_list_is_low(lruvec, false, sc, true))
                        shrink_active_list(SWAP_CLUSTER_MAX, lruvec,

diff --git a/mm/workingset.c b/mm/workingset.c
index c963831d354f8058447888c049c41178fe9a4d55..e8212123c1c3d3aa6c2a09a2ce74155b64f3a663 100644 (file)
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -233,7 +233,7 @@ void *workingset_eviction(struct page *page)
        VM_BUG_ON_PAGE(page_count(page), page);
        VM_BUG_ON_PAGE(!PageLocked(page), page);
 
-       lruvec = mem_cgroup_lruvec(pgdat, memcg);
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
        eviction = atomic_long_inc_return(&lruvec->inactive_age);
        return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
 }
@@ -280,7 +280,7 @@ void workingset_refault(struct page *page, void *shadow)
        memcg = mem_cgroup_from_id(memcgid);
        if (!mem_cgroup_disabled() && !memcg)
                goto out;
-       lruvec = mem_cgroup_lruvec(pgdat, memcg);
+       lruvec = mem_cgroup_lruvec(memcg, pgdat);
        refault = atomic_long_read(&lruvec->inactive_age);
        active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES);
 
@@ -345,7 +345,7 @@ void workingset_activation(struct page *page)
        memcg = page_memcg_rcu(page);
        if (!mem_cgroup_disabled() && !memcg)
                goto out;
-       lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
+       lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
        atomic_long_inc(&lruvec->inactive_age);
 out:
        rcu_read_unlock();
@@ -426,7 +426,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker,
                struct lruvec *lruvec;
                int i;
 
-               lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);
+               lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
                for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
                        pages += lruvec_page_state_local(lruvec,
                                                         NR_LRU_BASE + i);