mm/filemap: Add folio_wait_bit()

Rename wait_on_page_bit() to folio_wait_bit().  We must always wait on
the folio, otherwise we won't be woken up due to the tail page hashing
to a different bucket from the head page.

This commit shrinks the kernel by 770 bytes, mostly due to moving
the page waitqueue lookup into folio_wait_bit_common().

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Acked-by: Jeff Layton <jlayton@kernel.org>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Acked-by: Mike Rapoport <rppt@linux.ibm.com>

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index ee39ad7..2f48132 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -728,11 +728,11 @@ static inline bool lock_page_or_retry(struct page *page, struct mm_struct *mm,
 }
 
 /*
- * This is exported only for wait_on_page_locked/wait_on_page_writeback, etc.,
+ * This is exported only for folio_wait_locked/folio_wait_writeback, etc.,
  * and should not be used directly.
  */
-extern void wait_on_page_bit(struct page *page, int bit_nr);
-extern int wait_on_page_bit_killable(struct page *page, int bit_nr);
+void folio_wait_bit(struct folio *folio, int bit_nr);
+int folio_wait_bit_killable(struct folio *folio, int bit_nr);
 
 /* 
  * Wait for a folio to be unlocked.
@@ -744,14 +744,14 @@ extern int wait_on_page_bit_killable(struct page *page, int bit_nr);
 static inline void folio_wait_locked(struct folio *folio)
 {
        if (folio_test_locked(folio))
-               wait_on_page_bit(&folio->page, PG_locked);
+               folio_wait_bit(folio, PG_locked);
 }
 
 static inline int folio_wait_locked_killable(struct folio *folio)
 {
        if (!folio_test_locked(folio))
                return 0;
-       return wait_on_page_bit_killable(&folio->page, PG_locked);
+       return folio_wait_bit_killable(folio, PG_locked);
 }
 
 static inline void wait_on_page_locked(struct page *page)

diff --git a/mm/filemap.c b/mm/filemap.c
index 4e41bfd..13663f0 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1157,7 +1157,7 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync,
         *
         * So update the flags atomically, and wake up the waiter
         * afterwards to avoid any races. This store-release pairs
-        * with the load-acquire in wait_on_page_bit_common().
+        * with the load-acquire in folio_wait_bit_common().
         */
        smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
        wake_up_state(wait->private, mode);
@@ -1238,7 +1238,7 @@ static void folio_wake(struct folio *folio, int bit)
 }
 
 /*
- * A choice of three behaviors for wait_on_page_bit_common():
+ * A choice of three behaviors for folio_wait_bit_common():
  */
 enum behavior {
        EXCLUSIVE,      /* Hold ref to page and take the bit when woken, like
@@ -1253,16 +1253,16 @@ enum behavior {
 };
 
 /*
- * Attempt to check (or get) the page bit, and mark us done
+ * Attempt to check (or get) the folio flag, and mark us done
  * if successful.
  */
-static inline bool trylock_page_bit_common(struct page *page, int bit_nr,
+static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
                                        struct wait_queue_entry *wait)
 {
        if (wait->flags & WQ_FLAG_EXCLUSIVE) {
-               if (test_and_set_bit(bit_nr, &page->flags))
+               if (test_and_set_bit(bit_nr, &folio->flags))
                        return false;
-       } else if (test_bit(bit_nr, &page->flags))
+       } else if (test_bit(bit_nr, &folio->flags))
                return false;
 
        wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
@@ -1272,9 +1272,10 @@ static inline bool trylock_page_bit_common(struct page *page, int bit_nr,
 /* How many times do we accept lock stealing from under a waiter? */
 int sysctl_page_lock_unfairness = 5;
 
-static inline int wait_on_page_bit_common(wait_queue_head_t *q,
-       struct page *page, int bit_nr, int state, enum behavior behavior)
+static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
+               int state, enum behavior behavior)
 {
+       wait_queue_head_t *q = page_waitqueue(&folio->page);
        int unfairness = sysctl_page_lock_unfairness;
        struct wait_page_queue wait_page;
        wait_queue_entry_t *wait = &wait_page.wait;
@@ -1283,8 +1284,8 @@ static inline int wait_on_page_bit_common(wait_queue_head_t *q,
        unsigned long pflags;
 
        if (bit_nr == PG_locked &&
-           !PageUptodate(page) && PageWorkingset(page)) {
-               if (!PageSwapBacked(page)) {
+           !folio_test_uptodate(folio) && folio_test_workingset(folio)) {
+               if (!folio_test_swapbacked(folio)) {
                        delayacct_thrashing_start();
                        delayacct = true;
                }
@@ -1294,7 +1295,7 @@ static inline int wait_on_page_bit_common(wait_queue_head_t *q,
 
        init_wait(wait);
        wait->func = wake_page_function;
-       wait_page.page = page;
+       wait_page.page = &folio->page;
        wait_page.bit_nr = bit_nr;
 
 repeat:
@@ -1309,7 +1310,7 @@ repeat:
         * Do one last check whether we can get the
         * page bit synchronously.
         *
-        * Do the SetPageWaiters() marking before that
+        * Do the folio_set_waiters() marking before that
         * to let any waker we _just_ missed know they
         * need to wake us up (otherwise they'll never
         * even go to the slow case that looks at the
@@ -1320,8 +1321,8 @@ repeat:
         * lock to avoid races.
         */
        spin_lock_irq(&q->lock);
-       SetPageWaiters(page);
-       if (!trylock_page_bit_common(page, bit_nr, wait))
+       folio_set_waiters(folio);
+       if (!folio_trylock_flag(folio, bit_nr, wait))
                __add_wait_queue_entry_tail(q, wait);
        spin_unlock_irq(&q->lock);
 
@@ -1331,10 +1332,10 @@ repeat:
         * see whether the page bit testing has already
         * been done by the wake function.
         *
-        * We can drop our reference to the page.
+        * We can drop our reference to the folio.
         */
        if (behavior == DROP)
-               put_page(page);
+               folio_put(folio);
 
        /*
         * Note that until the "finish_wait()", or until
@@ -1371,7 +1372,7 @@ repeat:
                 *
                 * And if that fails, we'll have to retry this all.
                 */
-               if (unlikely(test_and_set_bit(bit_nr, &page->flags)))
+               if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0))))
                        goto repeat;
 
                wait->flags |= WQ_FLAG_DONE;
@@ -1380,7 +1381,7 @@ repeat:
 
        /*
         * If a signal happened, this 'finish_wait()' may remove the last
-        * waiter from the wait-queues, but the PageWaiters bit will remain
+        * waiter from the wait-queues, but the folio waiters bit will remain
         * set. That's ok. The next wakeup will take care of it, and trying
         * to do it here would be difficult and prone to races.
         */
@@ -1411,19 +1412,17 @@ repeat:
        return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
 }
 
-void wait_on_page_bit(struct page *page, int bit_nr)
+void folio_wait_bit(struct folio *folio, int bit_nr)
 {
-       wait_queue_head_t *q = page_waitqueue(page);
-       wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
+       folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED);
 }
-EXPORT_SYMBOL(wait_on_page_bit);
+EXPORT_SYMBOL(folio_wait_bit);
 
-int wait_on_page_bit_killable(struct page *page, int bit_nr)
+int folio_wait_bit_killable(struct folio *folio, int bit_nr)
 {
-       wait_queue_head_t *q = page_waitqueue(page);
-       return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, SHARED);
+       return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED);
 }
-EXPORT_SYMBOL(wait_on_page_bit_killable);
+EXPORT_SYMBOL(folio_wait_bit_killable);
 
 /**
  * put_and_wait_on_page_locked - Drop a reference and wait for it to be unlocked
@@ -1440,11 +1439,8 @@ EXPORT_SYMBOL(wait_on_page_bit_killable);
  */
 int put_and_wait_on_page_locked(struct page *page, int state)
 {
-       wait_queue_head_t *q;
-
-       page = compound_head(page);
-       q = page_waitqueue(page);
-       return wait_on_page_bit_common(q, page, PG_locked, state, DROP);
+       return folio_wait_bit_common(page_folio(page), PG_locked, state,
+                       DROP);
 }
 
 /**
@@ -1538,9 +1534,10 @@ EXPORT_SYMBOL(end_page_private_2);
  */
 void wait_on_page_private_2(struct page *page)
 {
-       page = compound_head(page);
-       while (PagePrivate2(page))
-               wait_on_page_bit(page, PG_private_2);
+       struct folio *folio = page_folio(page);
+
+       while (folio_test_private_2(folio))
+               folio_wait_bit(folio, PG_private_2);
 }
 EXPORT_SYMBOL(wait_on_page_private_2);
 
@@ -1557,11 +1554,11 @@ EXPORT_SYMBOL(wait_on_page_private_2);
  */
 int wait_on_page_private_2_killable(struct page *page)
 {
+       struct folio *folio = page_folio(page);
        int ret = 0;
 
-       page = compound_head(page);
-       while (PagePrivate2(page)) {
-               ret = wait_on_page_bit_killable(page, PG_private_2);
+       while (folio_test_private_2(folio)) {
+               ret = folio_wait_bit_killable(folio, PG_private_2);
                if (ret < 0)
                        break;
        }
@@ -1638,16 +1635,14 @@ EXPORT_SYMBOL_GPL(page_endio);
  */
 void __folio_lock(struct folio *folio)
 {
-       wait_queue_head_t *q = page_waitqueue(&folio->page);
-       wait_on_page_bit_common(q, &folio->page, PG_locked, TASK_UNINTERRUPTIBLE,
+       folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE,
                                EXCLUSIVE);
 }
 EXPORT_SYMBOL(__folio_lock);
 
 int __folio_lock_killable(struct folio *folio)
 {
-       wait_queue_head_t *q = page_waitqueue(&folio->page);
-       return wait_on_page_bit_common(q, &folio->page, PG_locked, TASK_KILLABLE,
+       return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE,
                                        EXCLUSIVE);
 }
 EXPORT_SYMBOL_GPL(__folio_lock_killable);

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 0b0b7cd..1d8f2ee 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2889,7 +2889,7 @@ void folio_wait_writeback(struct folio *folio)
 {
        while (folio_test_writeback(folio)) {
                trace_wait_on_page_writeback(&folio->page, folio_mapping(folio));
-               wait_on_page_bit(&folio->page, PG_writeback);
+               folio_wait_bit(folio, PG_writeback);
        }
 }
 EXPORT_SYMBOL_GPL(folio_wait_writeback);
@@ -2911,7 +2911,7 @@ int folio_wait_writeback_killable(struct folio *folio)
 {
        while (folio_test_writeback(folio)) {
                trace_wait_on_page_writeback(&folio->page, folio_mapping(folio));
-               if (wait_on_page_bit_killable(&folio->page, PG_writeback))
+               if (folio_wait_bit_killable(folio, PG_writeback))
                        return -EINTR;
        }