mm: compaction: convert to use a folio in isolate_migratepages_block()

Directly use a folio instead of page_folio() when page successfully
isolated (hugepage and movable page) and after folio_get_nontail_page(),
which removes several calls to compound_head().

Link: https://lkml.kernel.org/r/20230619110718.65679-1-wangkefeng.wang@huawei.com
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: James Gowans <jgowans@amazon.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/compaction.c b/mm/compaction.c
index 0fb3b89..dbc9f86 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -822,6 +822,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
        struct lruvec *lruvec;
        unsigned long flags = 0;
        struct lruvec *locked = NULL;
+       struct folio *folio = NULL;
        struct page *page = NULL, *valid_page = NULL;
        struct address_space *mapping;
        unsigned long start_pfn = low_pfn;
@@ -918,7 +919,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                if (!valid_page && pageblock_aligned(low_pfn)) {
                        if (!isolation_suitable(cc, page)) {
                                low_pfn = end_pfn;
-                               page = NULL;
+                               folio = NULL;
                                goto isolate_abort;
                        }
                        valid_page = page;
@@ -950,7 +951,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                                 * Hugepage was successfully isolated and placed
                                 * on the cc->migratepages list.
                                 */
-                               low_pfn += compound_nr(page) - 1;
+                               folio = page_folio(page);
+                               low_pfn += folio_nr_pages(folio) - 1;
                                goto isolate_success_no_list;
                        }
 
@@ -1018,8 +1020,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                                        locked = NULL;
                                }
 
-                               if (isolate_movable_page(page, mode))
+                               if (isolate_movable_page(page, mode)) {
+                                       folio = page_folio(page);
                                        goto isolate_success;
+                               }
                        }
 
                        goto isolate_fail;
@@ -1030,7 +1034,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 * sure the page is not being freed elsewhere -- the
                 * page release code relies on it.
                 */
-               if (unlikely(!get_page_unless_zero(page)))
+               folio = folio_get_nontail_page(page);
+               if (unlikely(!folio))
                        goto isolate_fail;
 
                /*
@@ -1038,8 +1043,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 * so avoid taking lru_lock and isolating it unnecessarily in an
                 * admittedly racy check.
                 */
-               mapping = page_mapping(page);
-               if (!mapping && (page_count(page) - 1) > total_mapcount(page))
+               mapping = folio_mapping(folio);
+               if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))
                        goto isolate_fail_put;
 
                /*
@@ -1050,11 +1055,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        goto isolate_fail_put;
 
                /* Only take pages on LRU: a check now makes later tests safe */
-               if (!PageLRU(page))
+               if (!folio_test_lru(folio))
                        goto isolate_fail_put;
 
                /* Compaction might skip unevictable pages but CMA takes them */
-               if (!(mode & ISOLATE_UNEVICTABLE) && PageUnevictable(page))
+               if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio))
                        goto isolate_fail_put;
 
                /*
@@ -1063,10 +1068,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                 * it will be able to migrate without blocking - clean pages
                 * for the most part.  PageWriteback would require blocking.
                 */
-               if ((mode & ISOLATE_ASYNC_MIGRATE) && PageWriteback(page))
+               if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
                        goto isolate_fail_put;
 
-               if ((mode & ISOLATE_ASYNC_MIGRATE) && PageDirty(page)) {
+               if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) {
                        bool migrate_dirty;
 
                        /*
@@ -1078,22 +1083,22 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                         * the page lock until after the page is removed
                         * from the page cache.
                         */
-                       if (!trylock_page(page))
+                       if (!folio_trylock(folio))
                                goto isolate_fail_put;
 
-                       mapping = page_mapping(page);
+                       mapping = folio_mapping(folio);
                        migrate_dirty = !mapping ||
                                        mapping->a_ops->migrate_folio;
-                       unlock_page(page);
+                       folio_unlock(folio);
                        if (!migrate_dirty)
                                goto isolate_fail_put;
                }
 
-               /* Try isolate the page */
-               if (!TestClearPageLRU(page))
+               /* Try isolate the folio */
+               if (!folio_test_clear_lru(folio))
                        goto isolate_fail_put;
 
-               lruvec = folio_lruvec(page_folio(page));
+               lruvec = folio_lruvec(folio);
 
                /* If we already hold the lock, we can skip some rechecking */
                if (lruvec != locked) {
@@ -1103,7 +1108,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
                        locked = lruvec;
 
-                       lruvec_memcg_debug(lruvec, page_folio(page));
+                       lruvec_memcg_debug(lruvec, folio);
 
                        /*
                         * Try get exclusive access under lock. If marked for
@@ -1119,34 +1124,33 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        }
 
                        /*
-                        * Page become compound since the non-locked check,
-                        * and it's on LRU. It can only be a THP so the order
-                        * is safe to read and it's 0 for tail pages.
+                        * folio become large since the non-locked check,
+                        * and it's on LRU.
                         */
-                       if (unlikely(PageCompound(page) && !cc->alloc_contig)) {
-                               low_pfn += compound_nr(page) - 1;
-                               nr_scanned += compound_nr(page) - 1;
-                               SetPageLRU(page);
+                       if (unlikely(folio_test_large(folio) && !cc->alloc_contig)) {
+                               low_pfn += folio_nr_pages(folio) - 1;
+                               nr_scanned += folio_nr_pages(folio) - 1;
+                               folio_set_lru(folio);
                                goto isolate_fail_put;
                        }
                }
 
-               /* The whole page is taken off the LRU; skip the tail pages. */
-               if (PageCompound(page))
-                       low_pfn += compound_nr(page) - 1;
+               /* The folio is taken off the LRU */
+               if (folio_test_large(folio))
+                       low_pfn += folio_nr_pages(folio) - 1;
 
                /* Successfully isolated */
-               del_page_from_lru_list(page, lruvec);
-               mod_node_page_state(page_pgdat(page),
-                               NR_ISOLATED_ANON + page_is_file_lru(page),
-                               thp_nr_pages(page));
+               lruvec_del_folio(lruvec, folio);
+               node_stat_mod_folio(folio,
+                               NR_ISOLATED_ANON + folio_is_file_lru(folio),
+                               folio_nr_pages(folio));
 
 isolate_success:
-               list_add(&page->lru, &cc->migratepages);
+               list_add(&folio->lru, &cc->migratepages);
 isolate_success_no_list:
-               cc->nr_migratepages += compound_nr(page);
-               nr_isolated += compound_nr(page);
-               nr_scanned += compound_nr(page) - 1;
+               cc->nr_migratepages += folio_nr_pages(folio);
+               nr_isolated += folio_nr_pages(folio);
+               nr_scanned += folio_nr_pages(folio) - 1;
 
                /*
                 * Avoid isolating too much unless this block is being
@@ -1168,7 +1172,7 @@ isolate_fail_put:
                        unlock_page_lruvec_irqrestore(locked, flags);
                        locked = NULL;
                }
-               put_page(page);
+               folio_put(folio);
 
 isolate_fail:
                if (!skip_on_failure && ret != -ENOMEM)
@@ -1209,14 +1213,14 @@ isolate_fail:
        if (unlikely(low_pfn > end_pfn))
                low_pfn = end_pfn;
 
-       page = NULL;
+       folio = NULL;
 
 isolate_abort:
        if (locked)
                unlock_page_lruvec_irqrestore(locked, flags);
-       if (page) {
-               SetPageLRU(page);
-               put_page(page);
+       if (folio) {
+               folio_set_lru(folio);
+               folio_put(folio);
        }
 
        /*