mm: multi-gen LRU: improve walk_pmd_range()

Improve readability of walk_pmd_range() and walk_pmd_range_locked().

Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com
Signed-off-by: T.J. Alumbaugh <talumbau@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/vmscan.c b/mm/vmscan.c
index c2e6ad5..ff3b4aa 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3999,8 +3999,8 @@ restart:
 }
 
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
-static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
-                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
+static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
+                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
 {
        int i;
        pmd_t *pmd;
@@ -4013,18 +4013,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
        VM_WARN_ON_ONCE(pud_leaf(*pud));
 
        /* try to batch at most 1+MIN_LRU_BATCH+1 entries */
-       if (*start == -1) {
-               *start = next;
+       if (*first == -1) {
+               *first = addr;
+               bitmap_zero(bitmap, MIN_LRU_BATCH);
                return;
        }
 
-       i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);
+       i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);
        if (i && i <= MIN_LRU_BATCH) {
                __set_bit(i - 1, bitmap);
                return;
        }
 
-       pmd = pmd_offset(pud, *start);
+       pmd = pmd_offset(pud, *first);
 
        ptl = pmd_lockptr(args->mm, pmd);
        if (!spin_trylock(ptl))
@@ -4035,15 +4036,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area
        do {
                unsigned long pfn;
                struct folio *folio;
-               unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;
+
+               /* don't round down the first address */
+               addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;
 
                pfn = get_pmd_pfn(pmd[i], vma, addr);
                if (pfn == -1)
                        goto next;
 
                if (!pmd_trans_huge(pmd[i])) {
-                       if (arch_has_hw_nonleaf_pmd_young() &&
-                           get_cap(LRU_GEN_NONLEAF_YOUNG))
+                       if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))
                                pmdp_test_and_clear_young(vma, addr, pmd + i);
                        goto next;
                }
@@ -4072,12 +4074,11 @@ next:
        arch_leave_lazy_mmu_mode();
        spin_unlock(ptl);
 done:
-       *start = -1;
-       bitmap_zero(bitmap, MIN_LRU_BATCH);
+       *first = -1;
 }
 #else
-static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,
-                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *start)
+static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,
+                                 struct mm_walk *args, unsigned long *bitmap, unsigned long *first)
 {
 }
 #endif
@@ -4090,9 +4091,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end,
        unsigned long next;
        unsigned long addr;
        struct vm_area_struct *vma;
-       unsigned long pos = -1;
+       unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];
+       unsigned long first = -1;
        struct lru_gen_mm_walk *walk = args->private;
-       unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};
 
        VM_WARN_ON_ONCE(pud_leaf(*pud));
 
@@ -4131,18 +4132,17 @@ restart:
                        if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))
                                continue;
 
-                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
+                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
                        continue;
                }
 #endif
                walk->mm_stats[MM_NONLEAF_TOTAL]++;
 
-               if (arch_has_hw_nonleaf_pmd_young() &&
-                   get_cap(LRU_GEN_NONLEAF_YOUNG)) {
+               if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {
                        if (!pmd_young(val))
                                continue;
 
-                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);
+                       walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
                }
 
                if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
@@ -4159,7 +4159,7 @@ restart:
                update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);
        }
 
-       walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);
+       walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
 
        if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))
                goto restart;