percpu: make pcpu_nr_empty_pop_pages per chunk type

nr_empty_pop_pages is used to guarantee that there are some free
populated pages to satisfy atomic allocations. Accounted and
non-accounted allocations are using separate sets of chunks,
so both need to have a surplus of empty pages.

This commit makes pcpu_nr_empty_pop_pages and the corresponding logic
per chunk type.

[Dennis]
This issue came up as I was reviewing [1] and realized I missed this.
Simultaneously, it was reported btrfs was seeing failed atomic
allocations in fsstress tests [2] and [3].

[1] https://lore.kernel.org/linux-mm/20210324190626.564297-1-guro@fb.com/
[2] https://lore.kernel.org/linux-mm/20210401185158.3275.409509F4@e16-tech.com/
[3] https://lore.kernel.org/linux-mm/CAL3q7H5RNBjCi708GH7jnczAOe0BLnacT9C+OBgA-Dx9jhB6SQ@mail.gmail.com/

Fixes: 3c7be18ac9a0 ("mm: memcg/percpu: account percpu memory to memory cgroups")
Cc: stable@vger.kernel.org # 5.9+
Signed-off-by: Roman Gushchin <guro@fb.com>
Tested-by: Filipe Manana <fdmanana@suse.com>
Signed-off-by: Dennis Zhou <dennis@kernel.org>

diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 18b768ac7dcae26326cab9417a976274e1efb8b9..095d7eaa0db426541a5b1ee10cba8eede97dffdc 100644 (file)
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -87,7 +87,7 @@ extern spinlock_t pcpu_lock;
 
 extern struct list_head *pcpu_chunk_lists;
 extern int pcpu_nr_slots;
-extern int pcpu_nr_empty_pop_pages;
+extern int pcpu_nr_empty_pop_pages[];
 
 extern struct pcpu_chunk *pcpu_first_chunk;
 extern struct pcpu_chunk *pcpu_reserved_chunk;

diff --git a/mm/percpu-stats.c b/mm/percpu-stats.c
index c8400a2adbc2b226ba01b39087d6b40610d12f42..f6026dbcdf6b3beca15ae315dcb858b6361849da 100644 (file)
--- a/mm/percpu-stats.c
+++ b/mm/percpu-stats.c
@@ -145,6 +145,7 @@ static int percpu_stats_show(struct seq_file *m, void *v)
        int slot, max_nr_alloc;
        int *buffer;
        enum pcpu_chunk_type type;
+       int nr_empty_pop_pages;
 
 alloc_buffer:
        spin_lock_irq(&pcpu_lock);
@@ -165,7 +166,11 @@ alloc_buffer:
                goto alloc_buffer;
        }
 
-#define PL(X) \
+       nr_empty_pop_pages = 0;
+       for (type = 0; type < PCPU_NR_CHUNK_TYPES; type++)
+               nr_empty_pop_pages += pcpu_nr_empty_pop_pages[type];
+
+#define PL(X)                                                          \
        seq_printf(m, "  %-20s: %12lld\n", #X, (long long int)pcpu_stats_ai.X)
 
        seq_printf(m,
@@ -196,7 +201,7 @@ alloc_buffer:
        PU(nr_max_chunks);
        PU(min_alloc_size);
        PU(max_alloc_size);
-       P("empty_pop_pages", pcpu_nr_empty_pop_pages);
+       P("empty_pop_pages", nr_empty_pop_pages);
        seq_putc(m, '\n');
 
 #undef PU

diff --git a/mm/percpu.c b/mm/percpu.c
index 6596a0a4286e7de91ca5dca1c1366884a32ebc22..23308113a5ff0b72b96f49b04887d2b05f19e405 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -173,10 +173,10 @@ struct list_head *pcpu_chunk_lists __ro_after_init; /* chunk list slots */
 static LIST_HEAD(pcpu_map_extend_chunks);
 
 /*
- * The number of empty populated pages, protected by pcpu_lock.  The
- * reserved chunk doesn't contribute to the count.
+ * The number of empty populated pages by chunk type, protected by pcpu_lock.
+ * The reserved chunk doesn't contribute to the count.
  */
-int pcpu_nr_empty_pop_pages;
+int pcpu_nr_empty_pop_pages[PCPU_NR_CHUNK_TYPES];
 
 /*
  * The number of populated pages in use by the allocator, protected by
@@ -556,7 +556,7 @@ static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
 {
        chunk->nr_empty_pop_pages += nr;
        if (chunk != pcpu_reserved_chunk)
-               pcpu_nr_empty_pop_pages += nr;
+               pcpu_nr_empty_pop_pages[pcpu_chunk_type(chunk)] += nr;
 }
 
 /*
@@ -1832,7 +1832,7 @@ area_found:
                mutex_unlock(&pcpu_alloc_mutex);
        }
 
-       if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW)
+       if (pcpu_nr_empty_pop_pages[type] < PCPU_EMPTY_POP_PAGES_LOW)
                pcpu_schedule_balance_work();
 
        /* clear the areas and return address relative to base address */
@@ -2000,7 +2000,7 @@ retry_pop:
                pcpu_atomic_alloc_failed = false;
        } else {
                nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -
-                                 pcpu_nr_empty_pop_pages,
+                                 pcpu_nr_empty_pop_pages[type],
                                  0, PCPU_EMPTY_POP_PAGES_HIGH);
        }
 
@@ -2580,7 +2580,7 @@ void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
        /* link the first chunk in */
        pcpu_first_chunk = chunk;
-       pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
+       pcpu_nr_empty_pop_pages[PCPU_CHUNK_ROOT] = pcpu_first_chunk->nr_empty_pop_pages;
        pcpu_chunk_relocate(pcpu_first_chunk, -1);
 
        /* include all regions of the first chunk */