mm/swap: Add folio_rotate_reclaimable()

Convert rotate_reclaimable_page() to folio_rotate_reclaimable().  This
eliminates all five of the calls to compound_head() in this function,
saving 75 bytes at the cost of adding 15 bytes to its one caller,
end_page_writeback().  We also save 36 bytes from pagevec_move_tail_fn()
due to using folios there.  Net 96 bytes savings.

Also move its declaration to mm/internal.h as it's only used by filemap.c.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Mike Rapoport <rppt@linux.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 85607c6..c7ecd3a 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -371,7 +371,6 @@ extern void lru_add_drain(void);
 extern void lru_add_drain_cpu(int cpu);
 extern void lru_add_drain_cpu_zone(struct zone *zone);
 extern void lru_add_drain_all(void);
-extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_file_page(struct page *page);
 extern void deactivate_page(struct page *page);
 extern void mark_page_lazyfree(struct page *page);

diff --git a/mm/filemap.c b/mm/filemap.c
index 706d121..6a5bdb4 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1584,8 +1584,9 @@ void end_page_writeback(struct page *page)
         * ever page writeback.
         */
        if (PageReclaim(page)) {
+               struct folio *folio = page_folio(page);
                ClearPageReclaim(page);
-               rotate_reclaimable_page(page);
+               folio_rotate_reclaimable(folio);
        }
 
        /*

diff --git a/mm/internal.h b/mm/internal.h
index cf3cb93..1a84484 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -35,6 +35,7 @@
 void page_writeback_init(void);
 
 vm_fault_t do_swap_page(struct vm_fault *vmf);
+void folio_rotate_reclaimable(struct folio *folio);
 
 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
                unsigned long floor, unsigned long ceiling);

diff --git a/mm/page_io.c b/mm/page_io.c
index c493ce9..d597bc6 100644 (file)
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -38,7 +38,7 @@ void end_swap_bio_write(struct bio *bio)
                 * Also print a dire warning that things will go BAD (tm)
                 * very quickly.
                 *
-                * Also clear PG_reclaim to avoid rotate_reclaimable_page()
+                * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
                 */
                set_page_dirty(page);
                pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
@@ -317,7 +317,7 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
                         * temporary failure if the system has limited
                         * memory for allocating transmit buffers.
                         * Mark the page dirty and avoid
-                        * rotate_reclaimable_page but rate-limit the
+                        * folio_rotate_reclaimable but rate-limit the
                         * messages but do not flag PageError like
                         * the normal direct-to-bio case as it could
                         * be temporary.

diff --git a/mm/swap.c b/mm/swap.c
index af3cad4..0edbcb9 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -206,11 +206,13 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,
 
 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec)
 {
-       if (!PageUnevictable(page)) {
-               del_page_from_lru_list(page, lruvec);
-               ClearPageActive(page);
-               add_page_to_lru_list_tail(page, lruvec);
-               __count_vm_events(PGROTATED, thp_nr_pages(page));
+       struct folio *folio = page_folio(page);
+
+       if (!folio_test_unevictable(folio)) {
+               lruvec_del_folio(lruvec, folio);
+               folio_clear_active(folio);
+               lruvec_add_folio_tail(lruvec, folio);
+               __count_vm_events(PGROTATED, folio_nr_pages(folio));
        }
 }
 
@@ -227,23 +229,23 @@ static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page)
 }
 
 /*
- * Writeback is about to end against a page which has been marked for immediate
- * reclaim.  If it still appears to be reclaimable, move it to the tail of the
- * inactive list.
+ * Writeback is about to end against a folio which has been marked for
+ * immediate reclaim.  If it still appears to be reclaimable, move it
+ * to the tail of the inactive list.
  *
- * rotate_reclaimable_page() must disable IRQs, to prevent nasty races.
+ * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
  */
-void rotate_reclaimable_page(struct page *page)
+void folio_rotate_reclaimable(struct folio *folio)
 {
-       if (!PageLocked(page) && !PageDirty(page) &&
-           !PageUnevictable(page) && PageLRU(page)) {
+       if (!folio_test_locked(folio) && !folio_test_dirty(folio) &&
+           !folio_test_unevictable(folio) && folio_test_lru(folio)) {
                struct pagevec *pvec;
                unsigned long flags;
 
-               get_page(page);
+               folio_get(folio);
                local_lock_irqsave(&lru_rotate.lock, flags);
                pvec = this_cpu_ptr(&lru_rotate.pvec);
-               if (pagevec_add_and_need_flush(pvec, page))
+               if (pagevec_add_and_need_flush(pvec, &folio->page))
                        pagevec_lru_move_fn(pvec, pagevec_move_tail_fn);
                local_unlock_irqrestore(&lru_rotate.lock, flags);
        }