Merge tag 'kbuild-fixes-v6.1' of git://git.kernel.org/pub/scm/linux/kernel/git/masahi...
[platform/kernel/linux-starfive.git] / fs / buffer.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/fs/buffer.c
4  *
5  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
6  */
7
8 /*
9  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10  *
11  * Removed a lot of unnecessary code and simplified things now that
12  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13  *
14  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
15  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
16  *
17  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18  *
19  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
20  */
21
22 #include <linux/kernel.h>
23 #include <linux/sched/signal.h>
24 #include <linux/syscalls.h>
25 #include <linux/fs.h>
26 #include <linux/iomap.h>
27 #include <linux/mm.h>
28 #include <linux/percpu.h>
29 #include <linux/slab.h>
30 #include <linux/capability.h>
31 #include <linux/blkdev.h>
32 #include <linux/file.h>
33 #include <linux/quotaops.h>
34 #include <linux/highmem.h>
35 #include <linux/export.h>
36 #include <linux/backing-dev.h>
37 #include <linux/writeback.h>
38 #include <linux/hash.h>
39 #include <linux/suspend.h>
40 #include <linux/buffer_head.h>
41 #include <linux/task_io_accounting_ops.h>
42 #include <linux/bio.h>
43 #include <linux/cpu.h>
44 #include <linux/bitops.h>
45 #include <linux/mpage.h>
46 #include <linux/bit_spinlock.h>
47 #include <linux/pagevec.h>
48 #include <linux/sched/mm.h>
49 #include <trace/events/block.h>
50 #include <linux/fscrypt.h>
51
52 #include "internal.h"
53
54 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
55 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
56                           struct writeback_control *wbc);
57
58 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
59
60 inline void touch_buffer(struct buffer_head *bh)
61 {
62         trace_block_touch_buffer(bh);
63         mark_page_accessed(bh->b_page);
64 }
65 EXPORT_SYMBOL(touch_buffer);
66
67 void __lock_buffer(struct buffer_head *bh)
68 {
69         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
70 }
71 EXPORT_SYMBOL(__lock_buffer);
72
73 void unlock_buffer(struct buffer_head *bh)
74 {
75         clear_bit_unlock(BH_Lock, &bh->b_state);
76         smp_mb__after_atomic();
77         wake_up_bit(&bh->b_state, BH_Lock);
78 }
79 EXPORT_SYMBOL(unlock_buffer);
80
81 /*
82  * Returns if the folio has dirty or writeback buffers. If all the buffers
83  * are unlocked and clean then the folio_test_dirty information is stale. If
84  * any of the buffers are locked, it is assumed they are locked for IO.
85  */
86 void buffer_check_dirty_writeback(struct folio *folio,
87                                      bool *dirty, bool *writeback)
88 {
89         struct buffer_head *head, *bh;
90         *dirty = false;
91         *writeback = false;
92
93         BUG_ON(!folio_test_locked(folio));
94
95         head = folio_buffers(folio);
96         if (!head)
97                 return;
98
99         if (folio_test_writeback(folio))
100                 *writeback = true;
101
102         bh = head;
103         do {
104                 if (buffer_locked(bh))
105                         *writeback = true;
106
107                 if (buffer_dirty(bh))
108                         *dirty = true;
109
110                 bh = bh->b_this_page;
111         } while (bh != head);
112 }
113 EXPORT_SYMBOL(buffer_check_dirty_writeback);
114
115 /*
116  * Block until a buffer comes unlocked.  This doesn't stop it
117  * from becoming locked again - you have to lock it yourself
118  * if you want to preserve its state.
119  */
120 void __wait_on_buffer(struct buffer_head * bh)
121 {
122         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
123 }
124 EXPORT_SYMBOL(__wait_on_buffer);
125
126 static void buffer_io_error(struct buffer_head *bh, char *msg)
127 {
128         if (!test_bit(BH_Quiet, &bh->b_state))
129                 printk_ratelimited(KERN_ERR
130                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
131                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
132 }
133
134 /*
135  * End-of-IO handler helper function which does not touch the bh after
136  * unlocking it.
137  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
138  * a race there is benign: unlock_buffer() only use the bh's address for
139  * hashing after unlocking the buffer, so it doesn't actually touch the bh
140  * itself.
141  */
142 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
143 {
144         if (uptodate) {
145                 set_buffer_uptodate(bh);
146         } else {
147                 /* This happens, due to failed read-ahead attempts. */
148                 clear_buffer_uptodate(bh);
149         }
150         unlock_buffer(bh);
151 }
152
153 /*
154  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
155  * unlock the buffer.
156  */
157 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
158 {
159         __end_buffer_read_notouch(bh, uptodate);
160         put_bh(bh);
161 }
162 EXPORT_SYMBOL(end_buffer_read_sync);
163
164 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
165 {
166         if (uptodate) {
167                 set_buffer_uptodate(bh);
168         } else {
169                 buffer_io_error(bh, ", lost sync page write");
170                 mark_buffer_write_io_error(bh);
171                 clear_buffer_uptodate(bh);
172         }
173         unlock_buffer(bh);
174         put_bh(bh);
175 }
176 EXPORT_SYMBOL(end_buffer_write_sync);
177
178 /*
179  * Various filesystems appear to want __find_get_block to be non-blocking.
180  * But it's the page lock which protects the buffers.  To get around this,
181  * we get exclusion from try_to_free_buffers with the blockdev mapping's
182  * private_lock.
183  *
184  * Hack idea: for the blockdev mapping, private_lock contention
185  * may be quite high.  This code could TryLock the page, and if that
186  * succeeds, there is no need to take private_lock.
187  */
188 static struct buffer_head *
189 __find_get_block_slow(struct block_device *bdev, sector_t block)
190 {
191         struct inode *bd_inode = bdev->bd_inode;
192         struct address_space *bd_mapping = bd_inode->i_mapping;
193         struct buffer_head *ret = NULL;
194         pgoff_t index;
195         struct buffer_head *bh;
196         struct buffer_head *head;
197         struct page *page;
198         int all_mapped = 1;
199         static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
200
201         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
202         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
203         if (!page)
204                 goto out;
205
206         spin_lock(&bd_mapping->private_lock);
207         if (!page_has_buffers(page))
208                 goto out_unlock;
209         head = page_buffers(page);
210         bh = head;
211         do {
212                 if (!buffer_mapped(bh))
213                         all_mapped = 0;
214                 else if (bh->b_blocknr == block) {
215                         ret = bh;
216                         get_bh(bh);
217                         goto out_unlock;
218                 }
219                 bh = bh->b_this_page;
220         } while (bh != head);
221
222         /* we might be here because some of the buffers on this page are
223          * not mapped.  This is due to various races between
224          * file io on the block device and getblk.  It gets dealt with
225          * elsewhere, don't buffer_error if we had some unmapped buffers
226          */
227         ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
228         if (all_mapped && __ratelimit(&last_warned)) {
229                 printk("__find_get_block_slow() failed. block=%llu, "
230                        "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
231                        "device %pg blocksize: %d\n",
232                        (unsigned long long)block,
233                        (unsigned long long)bh->b_blocknr,
234                        bh->b_state, bh->b_size, bdev,
235                        1 << bd_inode->i_blkbits);
236         }
237 out_unlock:
238         spin_unlock(&bd_mapping->private_lock);
239         put_page(page);
240 out:
241         return ret;
242 }
243
244 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
245 {
246         unsigned long flags;
247         struct buffer_head *first;
248         struct buffer_head *tmp;
249         struct page *page;
250         int page_uptodate = 1;
251
252         BUG_ON(!buffer_async_read(bh));
253
254         page = bh->b_page;
255         if (uptodate) {
256                 set_buffer_uptodate(bh);
257         } else {
258                 clear_buffer_uptodate(bh);
259                 buffer_io_error(bh, ", async page read");
260                 SetPageError(page);
261         }
262
263         /*
264          * Be _very_ careful from here on. Bad things can happen if
265          * two buffer heads end IO at almost the same time and both
266          * decide that the page is now completely done.
267          */
268         first = page_buffers(page);
269         spin_lock_irqsave(&first->b_uptodate_lock, flags);
270         clear_buffer_async_read(bh);
271         unlock_buffer(bh);
272         tmp = bh;
273         do {
274                 if (!buffer_uptodate(tmp))
275                         page_uptodate = 0;
276                 if (buffer_async_read(tmp)) {
277                         BUG_ON(!buffer_locked(tmp));
278                         goto still_busy;
279                 }
280                 tmp = tmp->b_this_page;
281         } while (tmp != bh);
282         spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
283
284         /*
285          * If all of the buffers are uptodate then we can set the page
286          * uptodate.
287          */
288         if (page_uptodate)
289                 SetPageUptodate(page);
290         unlock_page(page);
291         return;
292
293 still_busy:
294         spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
295         return;
296 }
297
298 struct decrypt_bh_ctx {
299         struct work_struct work;
300         struct buffer_head *bh;
301 };
302
303 static void decrypt_bh(struct work_struct *work)
304 {
305         struct decrypt_bh_ctx *ctx =
306                 container_of(work, struct decrypt_bh_ctx, work);
307         struct buffer_head *bh = ctx->bh;
308         int err;
309
310         err = fscrypt_decrypt_pagecache_blocks(bh->b_page, bh->b_size,
311                                                bh_offset(bh));
312         end_buffer_async_read(bh, err == 0);
313         kfree(ctx);
314 }
315
316 /*
317  * I/O completion handler for block_read_full_folio() - pages
318  * which come unlocked at the end of I/O.
319  */
320 static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
321 {
322         /* Decrypt if needed */
323         if (uptodate &&
324             fscrypt_inode_uses_fs_layer_crypto(bh->b_page->mapping->host)) {
325                 struct decrypt_bh_ctx *ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC);
326
327                 if (ctx) {
328                         INIT_WORK(&ctx->work, decrypt_bh);
329                         ctx->bh = bh;
330                         fscrypt_enqueue_decrypt_work(&ctx->work);
331                         return;
332                 }
333                 uptodate = 0;
334         }
335         end_buffer_async_read(bh, uptodate);
336 }
337
338 /*
339  * Completion handler for block_write_full_page() - pages which are unlocked
340  * during I/O, and which have PageWriteback cleared upon I/O completion.
341  */
342 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
343 {
344         unsigned long flags;
345         struct buffer_head *first;
346         struct buffer_head *tmp;
347         struct page *page;
348
349         BUG_ON(!buffer_async_write(bh));
350
351         page = bh->b_page;
352         if (uptodate) {
353                 set_buffer_uptodate(bh);
354         } else {
355                 buffer_io_error(bh, ", lost async page write");
356                 mark_buffer_write_io_error(bh);
357                 clear_buffer_uptodate(bh);
358                 SetPageError(page);
359         }
360
361         first = page_buffers(page);
362         spin_lock_irqsave(&first->b_uptodate_lock, flags);
363
364         clear_buffer_async_write(bh);
365         unlock_buffer(bh);
366         tmp = bh->b_this_page;
367         while (tmp != bh) {
368                 if (buffer_async_write(tmp)) {
369                         BUG_ON(!buffer_locked(tmp));
370                         goto still_busy;
371                 }
372                 tmp = tmp->b_this_page;
373         }
374         spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
375         end_page_writeback(page);
376         return;
377
378 still_busy:
379         spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
380         return;
381 }
382 EXPORT_SYMBOL(end_buffer_async_write);
383
384 /*
385  * If a page's buffers are under async readin (end_buffer_async_read
386  * completion) then there is a possibility that another thread of
387  * control could lock one of the buffers after it has completed
388  * but while some of the other buffers have not completed.  This
389  * locked buffer would confuse end_buffer_async_read() into not unlocking
390  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
391  * that this buffer is not under async I/O.
392  *
393  * The page comes unlocked when it has no locked buffer_async buffers
394  * left.
395  *
396  * PageLocked prevents anyone starting new async I/O reads any of
397  * the buffers.
398  *
399  * PageWriteback is used to prevent simultaneous writeout of the same
400  * page.
401  *
402  * PageLocked prevents anyone from starting writeback of a page which is
403  * under read I/O (PageWriteback is only ever set against a locked page).
404  */
405 static void mark_buffer_async_read(struct buffer_head *bh)
406 {
407         bh->b_end_io = end_buffer_async_read_io;
408         set_buffer_async_read(bh);
409 }
410
411 static void mark_buffer_async_write_endio(struct buffer_head *bh,
412                                           bh_end_io_t *handler)
413 {
414         bh->b_end_io = handler;
415         set_buffer_async_write(bh);
416 }
417
418 void mark_buffer_async_write(struct buffer_head *bh)
419 {
420         mark_buffer_async_write_endio(bh, end_buffer_async_write);
421 }
422 EXPORT_SYMBOL(mark_buffer_async_write);
423
424
425 /*
426  * fs/buffer.c contains helper functions for buffer-backed address space's
427  * fsync functions.  A common requirement for buffer-based filesystems is
428  * that certain data from the backing blockdev needs to be written out for
429  * a successful fsync().  For example, ext2 indirect blocks need to be
430  * written back and waited upon before fsync() returns.
431  *
432  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
433  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
434  * management of a list of dependent buffers at ->i_mapping->private_list.
435  *
436  * Locking is a little subtle: try_to_free_buffers() will remove buffers
437  * from their controlling inode's queue when they are being freed.  But
438  * try_to_free_buffers() will be operating against the *blockdev* mapping
439  * at the time, not against the S_ISREG file which depends on those buffers.
440  * So the locking for private_list is via the private_lock in the address_space
441  * which backs the buffers.  Which is different from the address_space 
442  * against which the buffers are listed.  So for a particular address_space,
443  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
444  * mapping->private_list will always be protected by the backing blockdev's
445  * ->private_lock.
446  *
447  * Which introduces a requirement: all buffers on an address_space's
448  * ->private_list must be from the same address_space: the blockdev's.
449  *
450  * address_spaces which do not place buffers at ->private_list via these
451  * utility functions are free to use private_lock and private_list for
452  * whatever they want.  The only requirement is that list_empty(private_list)
453  * be true at clear_inode() time.
454  *
455  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
456  * filesystems should do that.  invalidate_inode_buffers() should just go
457  * BUG_ON(!list_empty).
458  *
459  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
460  * take an address_space, not an inode.  And it should be called
461  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
462  * queued up.
463  *
464  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
465  * list if it is already on a list.  Because if the buffer is on a list,
466  * it *must* already be on the right one.  If not, the filesystem is being
467  * silly.  This will save a ton of locking.  But first we have to ensure
468  * that buffers are taken *off* the old inode's list when they are freed
469  * (presumably in truncate).  That requires careful auditing of all
470  * filesystems (do it inside bforget()).  It could also be done by bringing
471  * b_inode back.
472  */
473
474 /*
475  * The buffer's backing address_space's private_lock must be held
476  */
477 static void __remove_assoc_queue(struct buffer_head *bh)
478 {
479         list_del_init(&bh->b_assoc_buffers);
480         WARN_ON(!bh->b_assoc_map);
481         bh->b_assoc_map = NULL;
482 }
483
484 int inode_has_buffers(struct inode *inode)
485 {
486         return !list_empty(&inode->i_data.private_list);
487 }
488
489 /*
490  * osync is designed to support O_SYNC io.  It waits synchronously for
491  * all already-submitted IO to complete, but does not queue any new
492  * writes to the disk.
493  *
494  * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer
495  * as you dirty the buffers, and then use osync_inode_buffers to wait for
496  * completion.  Any other dirty buffers which are not yet queued for
497  * write will not be flushed to disk by the osync.
498  */
499 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
500 {
501         struct buffer_head *bh;
502         struct list_head *p;
503         int err = 0;
504
505         spin_lock(lock);
506 repeat:
507         list_for_each_prev(p, list) {
508                 bh = BH_ENTRY(p);
509                 if (buffer_locked(bh)) {
510                         get_bh(bh);
511                         spin_unlock(lock);
512                         wait_on_buffer(bh);
513                         if (!buffer_uptodate(bh))
514                                 err = -EIO;
515                         brelse(bh);
516                         spin_lock(lock);
517                         goto repeat;
518                 }
519         }
520         spin_unlock(lock);
521         return err;
522 }
523
524 void emergency_thaw_bdev(struct super_block *sb)
525 {
526         while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
527                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
528 }
529
530 /**
531  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
532  * @mapping: the mapping which wants those buffers written
533  *
534  * Starts I/O against the buffers at mapping->private_list, and waits upon
535  * that I/O.
536  *
537  * Basically, this is a convenience function for fsync().
538  * @mapping is a file or directory which needs those buffers to be written for
539  * a successful fsync().
540  */
541 int sync_mapping_buffers(struct address_space *mapping)
542 {
543         struct address_space *buffer_mapping = mapping->private_data;
544
545         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
546                 return 0;
547
548         return fsync_buffers_list(&buffer_mapping->private_lock,
549                                         &mapping->private_list);
550 }
551 EXPORT_SYMBOL(sync_mapping_buffers);
552
553 /*
554  * Called when we've recently written block `bblock', and it is known that
555  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
556  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
557  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
558  */
559 void write_boundary_block(struct block_device *bdev,
560                         sector_t bblock, unsigned blocksize)
561 {
562         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
563         if (bh) {
564                 if (buffer_dirty(bh))
565                         write_dirty_buffer(bh, 0);
566                 put_bh(bh);
567         }
568 }
569
570 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
571 {
572         struct address_space *mapping = inode->i_mapping;
573         struct address_space *buffer_mapping = bh->b_page->mapping;
574
575         mark_buffer_dirty(bh);
576         if (!mapping->private_data) {
577                 mapping->private_data = buffer_mapping;
578         } else {
579                 BUG_ON(mapping->private_data != buffer_mapping);
580         }
581         if (!bh->b_assoc_map) {
582                 spin_lock(&buffer_mapping->private_lock);
583                 list_move_tail(&bh->b_assoc_buffers,
584                                 &mapping->private_list);
585                 bh->b_assoc_map = mapping;
586                 spin_unlock(&buffer_mapping->private_lock);
587         }
588 }
589 EXPORT_SYMBOL(mark_buffer_dirty_inode);
590
591 /*
592  * Add a page to the dirty page list.
593  *
594  * It is a sad fact of life that this function is called from several places
595  * deeply under spinlocking.  It may not sleep.
596  *
597  * If the page has buffers, the uptodate buffers are set dirty, to preserve
598  * dirty-state coherency between the page and the buffers.  It the page does
599  * not have buffers then when they are later attached they will all be set
600  * dirty.
601  *
602  * The buffers are dirtied before the page is dirtied.  There's a small race
603  * window in which a writepage caller may see the page cleanness but not the
604  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
605  * before the buffers, a concurrent writepage caller could clear the page dirty
606  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
607  * page on the dirty page list.
608  *
609  * We use private_lock to lock against try_to_free_buffers while using the
610  * page's buffer list.  Also use this to protect against clean buffers being
611  * added to the page after it was set dirty.
612  *
613  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
614  * address_space though.
615  */
616 bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
617 {
618         struct buffer_head *head;
619         bool newly_dirty;
620
621         spin_lock(&mapping->private_lock);
622         head = folio_buffers(folio);
623         if (head) {
624                 struct buffer_head *bh = head;
625
626                 do {
627                         set_buffer_dirty(bh);
628                         bh = bh->b_this_page;
629                 } while (bh != head);
630         }
631         /*
632          * Lock out page's memcg migration to keep PageDirty
633          * synchronized with per-memcg dirty page counters.
634          */
635         folio_memcg_lock(folio);
636         newly_dirty = !folio_test_set_dirty(folio);
637         spin_unlock(&mapping->private_lock);
638
639         if (newly_dirty)
640                 __folio_mark_dirty(folio, mapping, 1);
641
642         folio_memcg_unlock(folio);
643
644         if (newly_dirty)
645                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
646
647         return newly_dirty;
648 }
649 EXPORT_SYMBOL(block_dirty_folio);
650
651 /*
652  * Write out and wait upon a list of buffers.
653  *
654  * We have conflicting pressures: we want to make sure that all
655  * initially dirty buffers get waited on, but that any subsequently
656  * dirtied buffers don't.  After all, we don't want fsync to last
657  * forever if somebody is actively writing to the file.
658  *
659  * Do this in two main stages: first we copy dirty buffers to a
660  * temporary inode list, queueing the writes as we go.  Then we clean
661  * up, waiting for those writes to complete.
662  * 
663  * During this second stage, any subsequent updates to the file may end
664  * up refiling the buffer on the original inode's dirty list again, so
665  * there is a chance we will end up with a buffer queued for write but
666  * not yet completed on that list.  So, as a final cleanup we go through
667  * the osync code to catch these locked, dirty buffers without requeuing
668  * any newly dirty buffers for write.
669  */
670 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
671 {
672         struct buffer_head *bh;
673         struct list_head tmp;
674         struct address_space *mapping;
675         int err = 0, err2;
676         struct blk_plug plug;
677
678         INIT_LIST_HEAD(&tmp);
679         blk_start_plug(&plug);
680
681         spin_lock(lock);
682         while (!list_empty(list)) {
683                 bh = BH_ENTRY(list->next);
684                 mapping = bh->b_assoc_map;
685                 __remove_assoc_queue(bh);
686                 /* Avoid race with mark_buffer_dirty_inode() which does
687                  * a lockless check and we rely on seeing the dirty bit */
688                 smp_mb();
689                 if (buffer_dirty(bh) || buffer_locked(bh)) {
690                         list_add(&bh->b_assoc_buffers, &tmp);
691                         bh->b_assoc_map = mapping;
692                         if (buffer_dirty(bh)) {
693                                 get_bh(bh);
694                                 spin_unlock(lock);
695                                 /*
696                                  * Ensure any pending I/O completes so that
697                                  * write_dirty_buffer() actually writes the
698                                  * current contents - it is a noop if I/O is
699                                  * still in flight on potentially older
700                                  * contents.
701                                  */
702                                 write_dirty_buffer(bh, REQ_SYNC);
703
704                                 /*
705                                  * Kick off IO for the previous mapping. Note
706                                  * that we will not run the very last mapping,
707                                  * wait_on_buffer() will do that for us
708                                  * through sync_buffer().
709                                  */
710                                 brelse(bh);
711                                 spin_lock(lock);
712                         }
713                 }
714         }
715
716         spin_unlock(lock);
717         blk_finish_plug(&plug);
718         spin_lock(lock);
719
720         while (!list_empty(&tmp)) {
721                 bh = BH_ENTRY(tmp.prev);
722                 get_bh(bh);
723                 mapping = bh->b_assoc_map;
724                 __remove_assoc_queue(bh);
725                 /* Avoid race with mark_buffer_dirty_inode() which does
726                  * a lockless check and we rely on seeing the dirty bit */
727                 smp_mb();
728                 if (buffer_dirty(bh)) {
729                         list_add(&bh->b_assoc_buffers,
730                                  &mapping->private_list);
731                         bh->b_assoc_map = mapping;
732                 }
733                 spin_unlock(lock);
734                 wait_on_buffer(bh);
735                 if (!buffer_uptodate(bh))
736                         err = -EIO;
737                 brelse(bh);
738                 spin_lock(lock);
739         }
740         
741         spin_unlock(lock);
742         err2 = osync_buffers_list(lock, list);
743         if (err)
744                 return err;
745         else
746                 return err2;
747 }
748
749 /*
750  * Invalidate any and all dirty buffers on a given inode.  We are
751  * probably unmounting the fs, but that doesn't mean we have already
752  * done a sync().  Just drop the buffers from the inode list.
753  *
754  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
755  * assumes that all the buffers are against the blockdev.  Not true
756  * for reiserfs.
757  */
758 void invalidate_inode_buffers(struct inode *inode)
759 {
760         if (inode_has_buffers(inode)) {
761                 struct address_space *mapping = &inode->i_data;
762                 struct list_head *list = &mapping->private_list;
763                 struct address_space *buffer_mapping = mapping->private_data;
764
765                 spin_lock(&buffer_mapping->private_lock);
766                 while (!list_empty(list))
767                         __remove_assoc_queue(BH_ENTRY(list->next));
768                 spin_unlock(&buffer_mapping->private_lock);
769         }
770 }
771 EXPORT_SYMBOL(invalidate_inode_buffers);
772
773 /*
774  * Remove any clean buffers from the inode's buffer list.  This is called
775  * when we're trying to free the inode itself.  Those buffers can pin it.
776  *
777  * Returns true if all buffers were removed.
778  */
779 int remove_inode_buffers(struct inode *inode)
780 {
781         int ret = 1;
782
783         if (inode_has_buffers(inode)) {
784                 struct address_space *mapping = &inode->i_data;
785                 struct list_head *list = &mapping->private_list;
786                 struct address_space *buffer_mapping = mapping->private_data;
787
788                 spin_lock(&buffer_mapping->private_lock);
789                 while (!list_empty(list)) {
790                         struct buffer_head *bh = BH_ENTRY(list->next);
791                         if (buffer_dirty(bh)) {
792                                 ret = 0;
793                                 break;
794                         }
795                         __remove_assoc_queue(bh);
796                 }
797                 spin_unlock(&buffer_mapping->private_lock);
798         }
799         return ret;
800 }
801
802 /*
803  * Create the appropriate buffers when given a page for data area and
804  * the size of each buffer.. Use the bh->b_this_page linked list to
805  * follow the buffers created.  Return NULL if unable to create more
806  * buffers.
807  *
808  * The retry flag is used to differentiate async IO (paging, swapping)
809  * which may not fail from ordinary buffer allocations.
810  */
811 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
812                 bool retry)
813 {
814         struct buffer_head *bh, *head;
815         gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
816         long offset;
817         struct mem_cgroup *memcg, *old_memcg;
818
819         if (retry)
820                 gfp |= __GFP_NOFAIL;
821
822         /* The page lock pins the memcg */
823         memcg = page_memcg(page);
824         old_memcg = set_active_memcg(memcg);
825
826         head = NULL;
827         offset = PAGE_SIZE;
828         while ((offset -= size) >= 0) {
829                 bh = alloc_buffer_head(gfp);
830                 if (!bh)
831                         goto no_grow;
832
833                 bh->b_this_page = head;
834                 bh->b_blocknr = -1;
835                 head = bh;
836
837                 bh->b_size = size;
838
839                 /* Link the buffer to its page */
840                 set_bh_page(bh, page, offset);
841         }
842 out:
843         set_active_memcg(old_memcg);
844         return head;
845 /*
846  * In case anything failed, we just free everything we got.
847  */
848 no_grow:
849         if (head) {
850                 do {
851                         bh = head;
852                         head = head->b_this_page;
853                         free_buffer_head(bh);
854                 } while (head);
855         }
856
857         goto out;
858 }
859 EXPORT_SYMBOL_GPL(alloc_page_buffers);
860
861 static inline void
862 link_dev_buffers(struct page *page, struct buffer_head *head)
863 {
864         struct buffer_head *bh, *tail;
865
866         bh = head;
867         do {
868                 tail = bh;
869                 bh = bh->b_this_page;
870         } while (bh);
871         tail->b_this_page = head;
872         attach_page_private(page, head);
873 }
874
875 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
876 {
877         sector_t retval = ~((sector_t)0);
878         loff_t sz = bdev_nr_bytes(bdev);
879
880         if (sz) {
881                 unsigned int sizebits = blksize_bits(size);
882                 retval = (sz >> sizebits);
883         }
884         return retval;
885 }
886
887 /*
888  * Initialise the state of a blockdev page's buffers.
889  */ 
890 static sector_t
891 init_page_buffers(struct page *page, struct block_device *bdev,
892                         sector_t block, int size)
893 {
894         struct buffer_head *head = page_buffers(page);
895         struct buffer_head *bh = head;
896         int uptodate = PageUptodate(page);
897         sector_t end_block = blkdev_max_block(bdev, size);
898
899         do {
900                 if (!buffer_mapped(bh)) {
901                         bh->b_end_io = NULL;
902                         bh->b_private = NULL;
903                         bh->b_bdev = bdev;
904                         bh->b_blocknr = block;
905                         if (uptodate)
906                                 set_buffer_uptodate(bh);
907                         if (block < end_block)
908                                 set_buffer_mapped(bh);
909                 }
910                 block++;
911                 bh = bh->b_this_page;
912         } while (bh != head);
913
914         /*
915          * Caller needs to validate requested block against end of device.
916          */
917         return end_block;
918 }
919
920 /*
921  * Create the page-cache page that contains the requested block.
922  *
923  * This is used purely for blockdev mappings.
924  */
925 static int
926 grow_dev_page(struct block_device *bdev, sector_t block,
927               pgoff_t index, int size, int sizebits, gfp_t gfp)
928 {
929         struct inode *inode = bdev->bd_inode;
930         struct page *page;
931         struct buffer_head *bh;
932         sector_t end_block;
933         int ret = 0;
934         gfp_t gfp_mask;
935
936         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
937
938         /*
939          * XXX: __getblk_slow() can not really deal with failure and
940          * will endlessly loop on improvised global reclaim.  Prefer
941          * looping in the allocator rather than here, at least that
942          * code knows what it's doing.
943          */
944         gfp_mask |= __GFP_NOFAIL;
945
946         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
947
948         BUG_ON(!PageLocked(page));
949
950         if (page_has_buffers(page)) {
951                 bh = page_buffers(page);
952                 if (bh->b_size == size) {
953                         end_block = init_page_buffers(page, bdev,
954                                                 (sector_t)index << sizebits,
955                                                 size);
956                         goto done;
957                 }
958                 if (!try_to_free_buffers(page_folio(page)))
959                         goto failed;
960         }
961
962         /*
963          * Allocate some buffers for this page
964          */
965         bh = alloc_page_buffers(page, size, true);
966
967         /*
968          * Link the page to the buffers and initialise them.  Take the
969          * lock to be atomic wrt __find_get_block(), which does not
970          * run under the page lock.
971          */
972         spin_lock(&inode->i_mapping->private_lock);
973         link_dev_buffers(page, bh);
974         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
975                         size);
976         spin_unlock(&inode->i_mapping->private_lock);
977 done:
978         ret = (block < end_block) ? 1 : -ENXIO;
979 failed:
980         unlock_page(page);
981         put_page(page);
982         return ret;
983 }
984
985 /*
986  * Create buffers for the specified block device block's page.  If
987  * that page was dirty, the buffers are set dirty also.
988  */
989 static int
990 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
991 {
992         pgoff_t index;
993         int sizebits;
994
995         sizebits = PAGE_SHIFT - __ffs(size);
996         index = block >> sizebits;
997
998         /*
999          * Check for a block which wants to lie outside our maximum possible
1000          * pagecache index.  (this comparison is done using sector_t types).
1001          */
1002         if (unlikely(index != block >> sizebits)) {
1003                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1004                         "device %pg\n",
1005                         __func__, (unsigned long long)block,
1006                         bdev);
1007                 return -EIO;
1008         }
1009
1010         /* Create a page with the proper size buffers.. */
1011         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1012 }
1013
1014 static struct buffer_head *
1015 __getblk_slow(struct block_device *bdev, sector_t block,
1016              unsigned size, gfp_t gfp)
1017 {
1018         /* Size must be multiple of hard sectorsize */
1019         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1020                         (size < 512 || size > PAGE_SIZE))) {
1021                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1022                                         size);
1023                 printk(KERN_ERR "logical block size: %d\n",
1024                                         bdev_logical_block_size(bdev));
1025
1026                 dump_stack();
1027                 return NULL;
1028         }
1029
1030         for (;;) {
1031                 struct buffer_head *bh;
1032                 int ret;
1033
1034                 bh = __find_get_block(bdev, block, size);
1035                 if (bh)
1036                         return bh;
1037
1038                 ret = grow_buffers(bdev, block, size, gfp);
1039                 if (ret < 0)
1040                         return NULL;
1041         }
1042 }
1043
1044 /*
1045  * The relationship between dirty buffers and dirty pages:
1046  *
1047  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1048  * the page is tagged dirty in the page cache.
1049  *
1050  * At all times, the dirtiness of the buffers represents the dirtiness of
1051  * subsections of the page.  If the page has buffers, the page dirty bit is
1052  * merely a hint about the true dirty state.
1053  *
1054  * When a page is set dirty in its entirety, all its buffers are marked dirty
1055  * (if the page has buffers).
1056  *
1057  * When a buffer is marked dirty, its page is dirtied, but the page's other
1058  * buffers are not.
1059  *
1060  * Also.  When blockdev buffers are explicitly read with bread(), they
1061  * individually become uptodate.  But their backing page remains not
1062  * uptodate - even if all of its buffers are uptodate.  A subsequent
1063  * block_read_full_folio() against that folio will discover all the uptodate
1064  * buffers, will set the folio uptodate and will perform no I/O.
1065  */
1066
1067 /**
1068  * mark_buffer_dirty - mark a buffer_head as needing writeout
1069  * @bh: the buffer_head to mark dirty
1070  *
1071  * mark_buffer_dirty() will set the dirty bit against the buffer, then set
1072  * its backing page dirty, then tag the page as dirty in the page cache
1073  * and then attach the address_space's inode to its superblock's dirty
1074  * inode list.
1075  *
1076  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1077  * i_pages lock and mapping->host->i_lock.
1078  */
1079 void mark_buffer_dirty(struct buffer_head *bh)
1080 {
1081         WARN_ON_ONCE(!buffer_uptodate(bh));
1082
1083         trace_block_dirty_buffer(bh);
1084
1085         /*
1086          * Very *carefully* optimize the it-is-already-dirty case.
1087          *
1088          * Don't let the final "is it dirty" escape to before we
1089          * perhaps modified the buffer.
1090          */
1091         if (buffer_dirty(bh)) {
1092                 smp_mb();
1093                 if (buffer_dirty(bh))
1094                         return;
1095         }
1096
1097         if (!test_set_buffer_dirty(bh)) {
1098                 struct page *page = bh->b_page;
1099                 struct address_space *mapping = NULL;
1100
1101                 lock_page_memcg(page);
1102                 if (!TestSetPageDirty(page)) {
1103                         mapping = page_mapping(page);
1104                         if (mapping)
1105                                 __set_page_dirty(page, mapping, 0);
1106                 }
1107                 unlock_page_memcg(page);
1108                 if (mapping)
1109                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1110         }
1111 }
1112 EXPORT_SYMBOL(mark_buffer_dirty);
1113
1114 void mark_buffer_write_io_error(struct buffer_head *bh)
1115 {
1116         struct super_block *sb;
1117
1118         set_buffer_write_io_error(bh);
1119         /* FIXME: do we need to set this in both places? */
1120         if (bh->b_page && bh->b_page->mapping)
1121                 mapping_set_error(bh->b_page->mapping, -EIO);
1122         if (bh->b_assoc_map)
1123                 mapping_set_error(bh->b_assoc_map, -EIO);
1124         rcu_read_lock();
1125         sb = READ_ONCE(bh->b_bdev->bd_super);
1126         if (sb)
1127                 errseq_set(&sb->s_wb_err, -EIO);
1128         rcu_read_unlock();
1129 }
1130 EXPORT_SYMBOL(mark_buffer_write_io_error);
1131
1132 /*
1133  * Decrement a buffer_head's reference count.  If all buffers against a page
1134  * have zero reference count, are clean and unlocked, and if the page is clean
1135  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1136  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1137  * a page but it ends up not being freed, and buffers may later be reattached).
1138  */
1139 void __brelse(struct buffer_head * buf)
1140 {
1141         if (atomic_read(&buf->b_count)) {
1142                 put_bh(buf);
1143                 return;
1144         }
1145         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1146 }
1147 EXPORT_SYMBOL(__brelse);
1148
1149 /*
1150  * bforget() is like brelse(), except it discards any
1151  * potentially dirty data.
1152  */
1153 void __bforget(struct buffer_head *bh)
1154 {
1155         clear_buffer_dirty(bh);
1156         if (bh->b_assoc_map) {
1157                 struct address_space *buffer_mapping = bh->b_page->mapping;
1158
1159                 spin_lock(&buffer_mapping->private_lock);
1160                 list_del_init(&bh->b_assoc_buffers);
1161                 bh->b_assoc_map = NULL;
1162                 spin_unlock(&buffer_mapping->private_lock);
1163         }
1164         __brelse(bh);
1165 }
1166 EXPORT_SYMBOL(__bforget);
1167
1168 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1169 {
1170         lock_buffer(bh);
1171         if (buffer_uptodate(bh)) {
1172                 unlock_buffer(bh);
1173                 return bh;
1174         } else {
1175                 get_bh(bh);
1176                 bh->b_end_io = end_buffer_read_sync;
1177                 submit_bh(REQ_OP_READ, bh);
1178                 wait_on_buffer(bh);
1179                 if (buffer_uptodate(bh))
1180                         return bh;
1181         }
1182         brelse(bh);
1183         return NULL;
1184 }
1185
1186 /*
1187  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1188  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1189  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1190  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1191  * CPU's LRUs at the same time.
1192  *
1193  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1194  * sb_find_get_block().
1195  *
1196  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1197  * a local interrupt disable for that.
1198  */
1199
1200 #define BH_LRU_SIZE     16
1201
1202 struct bh_lru {
1203         struct buffer_head *bhs[BH_LRU_SIZE];
1204 };
1205
1206 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1207
1208 #ifdef CONFIG_SMP
1209 #define bh_lru_lock()   local_irq_disable()
1210 #define bh_lru_unlock() local_irq_enable()
1211 #else
1212 #define bh_lru_lock()   preempt_disable()
1213 #define bh_lru_unlock() preempt_enable()
1214 #endif
1215
1216 static inline void check_irqs_on(void)
1217 {
1218 #ifdef irqs_disabled
1219         BUG_ON(irqs_disabled());
1220 #endif
1221 }
1222
1223 /*
1224  * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
1225  * inserted at the front, and the buffer_head at the back if any is evicted.
1226  * Or, if already in the LRU it is moved to the front.
1227  */
1228 static void bh_lru_install(struct buffer_head *bh)
1229 {
1230         struct buffer_head *evictee = bh;
1231         struct bh_lru *b;
1232         int i;
1233
1234         check_irqs_on();
1235         bh_lru_lock();
1236
1237         /*
1238          * the refcount of buffer_head in bh_lru prevents dropping the
1239          * attached page(i.e., try_to_free_buffers) so it could cause
1240          * failing page migration.
1241          * Skip putting upcoming bh into bh_lru until migration is done.
1242          */
1243         if (lru_cache_disabled()) {
1244                 bh_lru_unlock();
1245                 return;
1246         }
1247
1248         b = this_cpu_ptr(&bh_lrus);
1249         for (i = 0; i < BH_LRU_SIZE; i++) {
1250                 swap(evictee, b->bhs[i]);
1251                 if (evictee == bh) {
1252                         bh_lru_unlock();
1253                         return;
1254                 }
1255         }
1256
1257         get_bh(bh);
1258         bh_lru_unlock();
1259         brelse(evictee);
1260 }
1261
1262 /*
1263  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1264  */
1265 static struct buffer_head *
1266 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1267 {
1268         struct buffer_head *ret = NULL;
1269         unsigned int i;
1270
1271         check_irqs_on();
1272         bh_lru_lock();
1273         for (i = 0; i < BH_LRU_SIZE; i++) {
1274                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1275
1276                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1277                     bh->b_size == size) {
1278                         if (i) {
1279                                 while (i) {
1280                                         __this_cpu_write(bh_lrus.bhs[i],
1281                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1282                                         i--;
1283                                 }
1284                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1285                         }
1286                         get_bh(bh);
1287                         ret = bh;
1288                         break;
1289                 }
1290         }
1291         bh_lru_unlock();
1292         return ret;
1293 }
1294
1295 /*
1296  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1297  * it in the LRU and mark it as accessed.  If it is not present then return
1298  * NULL
1299  */
1300 struct buffer_head *
1301 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1302 {
1303         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1304
1305         if (bh == NULL) {
1306                 /* __find_get_block_slow will mark the page accessed */
1307                 bh = __find_get_block_slow(bdev, block);
1308                 if (bh)
1309                         bh_lru_install(bh);
1310         } else
1311                 touch_buffer(bh);
1312
1313         return bh;
1314 }
1315 EXPORT_SYMBOL(__find_get_block);
1316
1317 /*
1318  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1319  * which corresponds to the passed block_device, block and size. The
1320  * returned buffer has its reference count incremented.
1321  *
1322  * __getblk_gfp() will lock up the machine if grow_dev_page's
1323  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1324  */
1325 struct buffer_head *
1326 __getblk_gfp(struct block_device *bdev, sector_t block,
1327              unsigned size, gfp_t gfp)
1328 {
1329         struct buffer_head *bh = __find_get_block(bdev, block, size);
1330
1331         might_sleep();
1332         if (bh == NULL)
1333                 bh = __getblk_slow(bdev, block, size, gfp);
1334         return bh;
1335 }
1336 EXPORT_SYMBOL(__getblk_gfp);
1337
1338 /*
1339  * Do async read-ahead on a buffer..
1340  */
1341 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1342 {
1343         struct buffer_head *bh = __getblk(bdev, block, size);
1344         if (likely(bh)) {
1345                 bh_readahead(bh, REQ_RAHEAD);
1346                 brelse(bh);
1347         }
1348 }
1349 EXPORT_SYMBOL(__breadahead);
1350
1351 /**
1352  *  __bread_gfp() - reads a specified block and returns the bh
1353  *  @bdev: the block_device to read from
1354  *  @block: number of block
1355  *  @size: size (in bytes) to read
1356  *  @gfp: page allocation flag
1357  *
1358  *  Reads a specified block, and returns buffer head that contains it.
1359  *  The page cache can be allocated from non-movable area
1360  *  not to prevent page migration if you set gfp to zero.
1361  *  It returns NULL if the block was unreadable.
1362  */
1363 struct buffer_head *
1364 __bread_gfp(struct block_device *bdev, sector_t block,
1365                    unsigned size, gfp_t gfp)
1366 {
1367         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1368
1369         if (likely(bh) && !buffer_uptodate(bh))
1370                 bh = __bread_slow(bh);
1371         return bh;
1372 }
1373 EXPORT_SYMBOL(__bread_gfp);
1374
1375 static void __invalidate_bh_lrus(struct bh_lru *b)
1376 {
1377         int i;
1378
1379         for (i = 0; i < BH_LRU_SIZE; i++) {
1380                 brelse(b->bhs[i]);
1381                 b->bhs[i] = NULL;
1382         }
1383 }
1384 /*
1385  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1386  * This doesn't race because it runs in each cpu either in irq
1387  * or with preempt disabled.
1388  */
1389 static void invalidate_bh_lru(void *arg)
1390 {
1391         struct bh_lru *b = &get_cpu_var(bh_lrus);
1392
1393         __invalidate_bh_lrus(b);
1394         put_cpu_var(bh_lrus);
1395 }
1396
1397 bool has_bh_in_lru(int cpu, void *dummy)
1398 {
1399         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1400         int i;
1401         
1402         for (i = 0; i < BH_LRU_SIZE; i++) {
1403                 if (b->bhs[i])
1404                         return true;
1405         }
1406
1407         return false;
1408 }
1409
1410 void invalidate_bh_lrus(void)
1411 {
1412         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
1413 }
1414 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1415
1416 /*
1417  * It's called from workqueue context so we need a bh_lru_lock to close
1418  * the race with preemption/irq.
1419  */
1420 void invalidate_bh_lrus_cpu(void)
1421 {
1422         struct bh_lru *b;
1423
1424         bh_lru_lock();
1425         b = this_cpu_ptr(&bh_lrus);
1426         __invalidate_bh_lrus(b);
1427         bh_lru_unlock();
1428 }
1429
1430 void set_bh_page(struct buffer_head *bh,
1431                 struct page *page, unsigned long offset)
1432 {
1433         bh->b_page = page;
1434         BUG_ON(offset >= PAGE_SIZE);
1435         if (PageHighMem(page))
1436                 /*
1437                  * This catches illegal uses and preserves the offset:
1438                  */
1439                 bh->b_data = (char *)(0 + offset);
1440         else
1441                 bh->b_data = page_address(page) + offset;
1442 }
1443 EXPORT_SYMBOL(set_bh_page);
1444
1445 /*
1446  * Called when truncating a buffer on a page completely.
1447  */
1448
1449 /* Bits that are cleared during an invalidate */
1450 #define BUFFER_FLAGS_DISCARD \
1451         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1452          1 << BH_Delay | 1 << BH_Unwritten)
1453
1454 static void discard_buffer(struct buffer_head * bh)
1455 {
1456         unsigned long b_state;
1457
1458         lock_buffer(bh);
1459         clear_buffer_dirty(bh);
1460         bh->b_bdev = NULL;
1461         b_state = READ_ONCE(bh->b_state);
1462         do {
1463         } while (!try_cmpxchg(&bh->b_state, &b_state,
1464                               b_state & ~BUFFER_FLAGS_DISCARD));
1465         unlock_buffer(bh);
1466 }
1467
1468 /**
1469  * block_invalidate_folio - Invalidate part or all of a buffer-backed folio.
1470  * @folio: The folio which is affected.
1471  * @offset: start of the range to invalidate
1472  * @length: length of the range to invalidate
1473  *
1474  * block_invalidate_folio() is called when all or part of the folio has been
1475  * invalidated by a truncate operation.
1476  *
1477  * block_invalidate_folio() does not have to release all buffers, but it must
1478  * ensure that no dirty buffer is left outside @offset and that no I/O
1479  * is underway against any of the blocks which are outside the truncation
1480  * point.  Because the caller is about to free (and possibly reuse) those
1481  * blocks on-disk.
1482  */
1483 void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)
1484 {
1485         struct buffer_head *head, *bh, *next;
1486         size_t curr_off = 0;
1487         size_t stop = length + offset;
1488
1489         BUG_ON(!folio_test_locked(folio));
1490
1491         /*
1492          * Check for overflow
1493          */
1494         BUG_ON(stop > folio_size(folio) || stop < length);
1495
1496         head = folio_buffers(folio);
1497         if (!head)
1498                 return;
1499
1500         bh = head;
1501         do {
1502                 size_t next_off = curr_off + bh->b_size;
1503                 next = bh->b_this_page;
1504
1505                 /*
1506                  * Are we still fully in range ?
1507                  */
1508                 if (next_off > stop)
1509                         goto out;
1510
1511                 /*
1512                  * is this block fully invalidated?
1513                  */
1514                 if (offset <= curr_off)
1515                         discard_buffer(bh);
1516                 curr_off = next_off;
1517                 bh = next;
1518         } while (bh != head);
1519
1520         /*
1521          * We release buffers only if the entire folio is being invalidated.
1522          * The get_block cached value has been unconditionally invalidated,
1523          * so real IO is not possible anymore.
1524          */
1525         if (length == folio_size(folio))
1526                 filemap_release_folio(folio, 0);
1527 out:
1528         return;
1529 }
1530 EXPORT_SYMBOL(block_invalidate_folio);
1531
1532
1533 /*
1534  * We attach and possibly dirty the buffers atomically wrt
1535  * block_dirty_folio() via private_lock.  try_to_free_buffers
1536  * is already excluded via the page lock.
1537  */
1538 void create_empty_buffers(struct page *page,
1539                         unsigned long blocksize, unsigned long b_state)
1540 {
1541         struct buffer_head *bh, *head, *tail;
1542
1543         head = alloc_page_buffers(page, blocksize, true);
1544         bh = head;
1545         do {
1546                 bh->b_state |= b_state;
1547                 tail = bh;
1548                 bh = bh->b_this_page;
1549         } while (bh);
1550         tail->b_this_page = head;
1551
1552         spin_lock(&page->mapping->private_lock);
1553         if (PageUptodate(page) || PageDirty(page)) {
1554                 bh = head;
1555                 do {
1556                         if (PageDirty(page))
1557                                 set_buffer_dirty(bh);
1558                         if (PageUptodate(page))
1559                                 set_buffer_uptodate(bh);
1560                         bh = bh->b_this_page;
1561                 } while (bh != head);
1562         }
1563         attach_page_private(page, head);
1564         spin_unlock(&page->mapping->private_lock);
1565 }
1566 EXPORT_SYMBOL(create_empty_buffers);
1567
1568 /**
1569  * clean_bdev_aliases: clean a range of buffers in block device
1570  * @bdev: Block device to clean buffers in
1571  * @block: Start of a range of blocks to clean
1572  * @len: Number of blocks to clean
1573  *
1574  * We are taking a range of blocks for data and we don't want writeback of any
1575  * buffer-cache aliases starting from return from this function and until the
1576  * moment when something will explicitly mark the buffer dirty (hopefully that
1577  * will not happen until we will free that block ;-) We don't even need to mark
1578  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1579  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1580  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1581  * would confuse anyone who might pick it with bread() afterwards...
1582  *
1583  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1584  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1585  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1586  * need to.  That happens here.
1587  */
1588 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1589 {
1590         struct inode *bd_inode = bdev->bd_inode;
1591         struct address_space *bd_mapping = bd_inode->i_mapping;
1592         struct folio_batch fbatch;
1593         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1594         pgoff_t end;
1595         int i, count;
1596         struct buffer_head *bh;
1597         struct buffer_head *head;
1598
1599         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1600         folio_batch_init(&fbatch);
1601         while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
1602                 count = folio_batch_count(&fbatch);
1603                 for (i = 0; i < count; i++) {
1604                         struct folio *folio = fbatch.folios[i];
1605
1606                         if (!folio_buffers(folio))
1607                                 continue;
1608                         /*
1609                          * We use folio lock instead of bd_mapping->private_lock
1610                          * to pin buffers here since we can afford to sleep and
1611                          * it scales better than a global spinlock lock.
1612                          */
1613                         folio_lock(folio);
1614                         /* Recheck when the folio is locked which pins bhs */
1615                         head = folio_buffers(folio);
1616                         if (!head)
1617                                 goto unlock_page;
1618                         bh = head;
1619                         do {
1620                                 if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1621                                         goto next;
1622                                 if (bh->b_blocknr >= block + len)
1623                                         break;
1624                                 clear_buffer_dirty(bh);
1625                                 wait_on_buffer(bh);
1626                                 clear_buffer_req(bh);
1627 next:
1628                                 bh = bh->b_this_page;
1629                         } while (bh != head);
1630 unlock_page:
1631                         folio_unlock(folio);
1632                 }
1633                 folio_batch_release(&fbatch);
1634                 cond_resched();
1635                 /* End of range already reached? */
1636                 if (index > end || !index)
1637                         break;
1638         }
1639 }
1640 EXPORT_SYMBOL(clean_bdev_aliases);
1641
1642 /*
1643  * Size is a power-of-two in the range 512..PAGE_SIZE,
1644  * and the case we care about most is PAGE_SIZE.
1645  *
1646  * So this *could* possibly be written with those
1647  * constraints in mind (relevant mostly if some
1648  * architecture has a slow bit-scan instruction)
1649  */
1650 static inline int block_size_bits(unsigned int blocksize)
1651 {
1652         return ilog2(blocksize);
1653 }
1654
1655 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1656 {
1657         BUG_ON(!PageLocked(page));
1658
1659         if (!page_has_buffers(page))
1660                 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1661                                      b_state);
1662         return page_buffers(page);
1663 }
1664
1665 /*
1666  * NOTE! All mapped/uptodate combinations are valid:
1667  *
1668  *      Mapped  Uptodate        Meaning
1669  *
1670  *      No      No              "unknown" - must do get_block()
1671  *      No      Yes             "hole" - zero-filled
1672  *      Yes     No              "allocated" - allocated on disk, not read in
1673  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1674  *
1675  * "Dirty" is valid only with the last case (mapped+uptodate).
1676  */
1677
1678 /*
1679  * While block_write_full_page is writing back the dirty buffers under
1680  * the page lock, whoever dirtied the buffers may decide to clean them
1681  * again at any time.  We handle that by only looking at the buffer
1682  * state inside lock_buffer().
1683  *
1684  * If block_write_full_page() is called for regular writeback
1685  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1686  * locked buffer.   This only can happen if someone has written the buffer
1687  * directly, with submit_bh().  At the address_space level PageWriteback
1688  * prevents this contention from occurring.
1689  *
1690  * If block_write_full_page() is called with wbc->sync_mode ==
1691  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1692  * causes the writes to be flagged as synchronous writes.
1693  */
1694 int __block_write_full_page(struct inode *inode, struct page *page,
1695                         get_block_t *get_block, struct writeback_control *wbc,
1696                         bh_end_io_t *handler)
1697 {
1698         int err;
1699         sector_t block;
1700         sector_t last_block;
1701         struct buffer_head *bh, *head;
1702         unsigned int blocksize, bbits;
1703         int nr_underway = 0;
1704         blk_opf_t write_flags = wbc_to_write_flags(wbc);
1705
1706         head = create_page_buffers(page, inode,
1707                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1708
1709         /*
1710          * Be very careful.  We have no exclusion from block_dirty_folio
1711          * here, and the (potentially unmapped) buffers may become dirty at
1712          * any time.  If a buffer becomes dirty here after we've inspected it
1713          * then we just miss that fact, and the page stays dirty.
1714          *
1715          * Buffers outside i_size may be dirtied by block_dirty_folio;
1716          * handle that here by just cleaning them.
1717          */
1718
1719         bh = head;
1720         blocksize = bh->b_size;
1721         bbits = block_size_bits(blocksize);
1722
1723         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1724         last_block = (i_size_read(inode) - 1) >> bbits;
1725
1726         /*
1727          * Get all the dirty buffers mapped to disk addresses and
1728          * handle any aliases from the underlying blockdev's mapping.
1729          */
1730         do {
1731                 if (block > last_block) {
1732                         /*
1733                          * mapped buffers outside i_size will occur, because
1734                          * this page can be outside i_size when there is a
1735                          * truncate in progress.
1736                          */
1737                         /*
1738                          * The buffer was zeroed by block_write_full_page()
1739                          */
1740                         clear_buffer_dirty(bh);
1741                         set_buffer_uptodate(bh);
1742                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1743                            buffer_dirty(bh)) {
1744                         WARN_ON(bh->b_size != blocksize);
1745                         err = get_block(inode, block, bh, 1);
1746                         if (err)
1747                                 goto recover;
1748                         clear_buffer_delay(bh);
1749                         if (buffer_new(bh)) {
1750                                 /* blockdev mappings never come here */
1751                                 clear_buffer_new(bh);
1752                                 clean_bdev_bh_alias(bh);
1753                         }
1754                 }
1755                 bh = bh->b_this_page;
1756                 block++;
1757         } while (bh != head);
1758
1759         do {
1760                 if (!buffer_mapped(bh))
1761                         continue;
1762                 /*
1763                  * If it's a fully non-blocking write attempt and we cannot
1764                  * lock the buffer then redirty the page.  Note that this can
1765                  * potentially cause a busy-wait loop from writeback threads
1766                  * and kswapd activity, but those code paths have their own
1767                  * higher-level throttling.
1768                  */
1769                 if (wbc->sync_mode != WB_SYNC_NONE) {
1770                         lock_buffer(bh);
1771                 } else if (!trylock_buffer(bh)) {
1772                         redirty_page_for_writepage(wbc, page);
1773                         continue;
1774                 }
1775                 if (test_clear_buffer_dirty(bh)) {
1776                         mark_buffer_async_write_endio(bh, handler);
1777                 } else {
1778                         unlock_buffer(bh);
1779                 }
1780         } while ((bh = bh->b_this_page) != head);
1781
1782         /*
1783          * The page and its buffers are protected by PageWriteback(), so we can
1784          * drop the bh refcounts early.
1785          */
1786         BUG_ON(PageWriteback(page));
1787         set_page_writeback(page);
1788
1789         do {
1790                 struct buffer_head *next = bh->b_this_page;
1791                 if (buffer_async_write(bh)) {
1792                         submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
1793                         nr_underway++;
1794                 }
1795                 bh = next;
1796         } while (bh != head);
1797         unlock_page(page);
1798
1799         err = 0;
1800 done:
1801         if (nr_underway == 0) {
1802                 /*
1803                  * The page was marked dirty, but the buffers were
1804                  * clean.  Someone wrote them back by hand with
1805                  * write_dirty_buffer/submit_bh.  A rare case.
1806                  */
1807                 end_page_writeback(page);
1808
1809                 /*
1810                  * The page and buffer_heads can be released at any time from
1811                  * here on.
1812                  */
1813         }
1814         return err;
1815
1816 recover:
1817         /*
1818          * ENOSPC, or some other error.  We may already have added some
1819          * blocks to the file, so we need to write these out to avoid
1820          * exposing stale data.
1821          * The page is currently locked and not marked for writeback
1822          */
1823         bh = head;
1824         /* Recovery: lock and submit the mapped buffers */
1825         do {
1826                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1827                     !buffer_delay(bh)) {
1828                         lock_buffer(bh);
1829                         mark_buffer_async_write_endio(bh, handler);
1830                 } else {
1831                         /*
1832                          * The buffer may have been set dirty during
1833                          * attachment to a dirty page.
1834                          */
1835                         clear_buffer_dirty(bh);
1836                 }
1837         } while ((bh = bh->b_this_page) != head);
1838         SetPageError(page);
1839         BUG_ON(PageWriteback(page));
1840         mapping_set_error(page->mapping, err);
1841         set_page_writeback(page);
1842         do {
1843                 struct buffer_head *next = bh->b_this_page;
1844                 if (buffer_async_write(bh)) {
1845                         clear_buffer_dirty(bh);
1846                         submit_bh_wbc(REQ_OP_WRITE | write_flags, bh, wbc);
1847                         nr_underway++;
1848                 }
1849                 bh = next;
1850         } while (bh != head);
1851         unlock_page(page);
1852         goto done;
1853 }
1854 EXPORT_SYMBOL(__block_write_full_page);
1855
1856 /*
1857  * If a page has any new buffers, zero them out here, and mark them uptodate
1858  * and dirty so they'll be written out (in order to prevent uninitialised
1859  * block data from leaking). And clear the new bit.
1860  */
1861 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1862 {
1863         unsigned int block_start, block_end;
1864         struct buffer_head *head, *bh;
1865
1866         BUG_ON(!PageLocked(page));
1867         if (!page_has_buffers(page))
1868                 return;
1869
1870         bh = head = page_buffers(page);
1871         block_start = 0;
1872         do {
1873                 block_end = block_start + bh->b_size;
1874
1875                 if (buffer_new(bh)) {
1876                         if (block_end > from && block_start < to) {
1877                                 if (!PageUptodate(page)) {
1878                                         unsigned start, size;
1879
1880                                         start = max(from, block_start);
1881                                         size = min(to, block_end) - start;
1882
1883                                         zero_user(page, start, size);
1884                                         set_buffer_uptodate(bh);
1885                                 }
1886
1887                                 clear_buffer_new(bh);
1888                                 mark_buffer_dirty(bh);
1889                         }
1890                 }
1891
1892                 block_start = block_end;
1893                 bh = bh->b_this_page;
1894         } while (bh != head);
1895 }
1896 EXPORT_SYMBOL(page_zero_new_buffers);
1897
1898 static void
1899 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1900                 const struct iomap *iomap)
1901 {
1902         loff_t offset = block << inode->i_blkbits;
1903
1904         bh->b_bdev = iomap->bdev;
1905
1906         /*
1907          * Block points to offset in file we need to map, iomap contains
1908          * the offset at which the map starts. If the map ends before the
1909          * current block, then do not map the buffer and let the caller
1910          * handle it.
1911          */
1912         BUG_ON(offset >= iomap->offset + iomap->length);
1913
1914         switch (iomap->type) {
1915         case IOMAP_HOLE:
1916                 /*
1917                  * If the buffer is not up to date or beyond the current EOF,
1918                  * we need to mark it as new to ensure sub-block zeroing is
1919                  * executed if necessary.
1920                  */
1921                 if (!buffer_uptodate(bh) ||
1922                     (offset >= i_size_read(inode)))
1923                         set_buffer_new(bh);
1924                 break;
1925         case IOMAP_DELALLOC:
1926                 if (!buffer_uptodate(bh) ||
1927                     (offset >= i_size_read(inode)))
1928                         set_buffer_new(bh);
1929                 set_buffer_uptodate(bh);
1930                 set_buffer_mapped(bh);
1931                 set_buffer_delay(bh);
1932                 break;
1933         case IOMAP_UNWRITTEN:
1934                 /*
1935                  * For unwritten regions, we always need to ensure that regions
1936                  * in the block we are not writing to are zeroed. Mark the
1937                  * buffer as new to ensure this.
1938                  */
1939                 set_buffer_new(bh);
1940                 set_buffer_unwritten(bh);
1941                 fallthrough;
1942         case IOMAP_MAPPED:
1943                 if ((iomap->flags & IOMAP_F_NEW) ||
1944                     offset >= i_size_read(inode))
1945                         set_buffer_new(bh);
1946                 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1947                                 inode->i_blkbits;
1948                 set_buffer_mapped(bh);
1949                 break;
1950         }
1951 }
1952
1953 int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
1954                 get_block_t *get_block, const struct iomap *iomap)
1955 {
1956         unsigned from = pos & (PAGE_SIZE - 1);
1957         unsigned to = from + len;
1958         struct inode *inode = folio->mapping->host;
1959         unsigned block_start, block_end;
1960         sector_t block;
1961         int err = 0;
1962         unsigned blocksize, bbits;
1963         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1964
1965         BUG_ON(!folio_test_locked(folio));
1966         BUG_ON(from > PAGE_SIZE);
1967         BUG_ON(to > PAGE_SIZE);
1968         BUG_ON(from > to);
1969
1970         head = create_page_buffers(&folio->page, inode, 0);
1971         blocksize = head->b_size;
1972         bbits = block_size_bits(blocksize);
1973
1974         block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
1975
1976         for(bh = head, block_start = 0; bh != head || !block_start;
1977             block++, block_start=block_end, bh = bh->b_this_page) {
1978                 block_end = block_start + blocksize;
1979                 if (block_end <= from || block_start >= to) {
1980                         if (folio_test_uptodate(folio)) {
1981                                 if (!buffer_uptodate(bh))
1982                                         set_buffer_uptodate(bh);
1983                         }
1984                         continue;
1985                 }
1986                 if (buffer_new(bh))
1987                         clear_buffer_new(bh);
1988                 if (!buffer_mapped(bh)) {
1989                         WARN_ON(bh->b_size != blocksize);
1990                         if (get_block) {
1991                                 err = get_block(inode, block, bh, 1);
1992                                 if (err)
1993                                         break;
1994                         } else {
1995                                 iomap_to_bh(inode, block, bh, iomap);
1996                         }
1997
1998                         if (buffer_new(bh)) {
1999                                 clean_bdev_bh_alias(bh);
2000                                 if (folio_test_uptodate(folio)) {
2001                                         clear_buffer_new(bh);
2002                                         set_buffer_uptodate(bh);
2003                                         mark_buffer_dirty(bh);
2004                                         continue;
2005                                 }
2006                                 if (block_end > to || block_start < from)
2007                                         folio_zero_segments(folio,
2008                                                 to, block_end,
2009                                                 block_start, from);
2010                                 continue;
2011                         }
2012                 }
2013                 if (folio_test_uptodate(folio)) {
2014                         if (!buffer_uptodate(bh))
2015                                 set_buffer_uptodate(bh);
2016                         continue; 
2017                 }
2018                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2019                     !buffer_unwritten(bh) &&
2020                      (block_start < from || block_end > to)) {
2021                         bh_read_nowait(bh, 0);
2022                         *wait_bh++=bh;
2023                 }
2024         }
2025         /*
2026          * If we issued read requests - let them complete.
2027          */
2028         while(wait_bh > wait) {
2029                 wait_on_buffer(*--wait_bh);
2030                 if (!buffer_uptodate(*wait_bh))
2031                         err = -EIO;
2032         }
2033         if (unlikely(err))
2034                 page_zero_new_buffers(&folio->page, from, to);
2035         return err;
2036 }
2037
2038 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2039                 get_block_t *get_block)
2040 {
2041         return __block_write_begin_int(page_folio(page), pos, len, get_block,
2042                                        NULL);
2043 }
2044 EXPORT_SYMBOL(__block_write_begin);
2045
2046 static int __block_commit_write(struct inode *inode, struct page *page,
2047                 unsigned from, unsigned to)
2048 {
2049         unsigned block_start, block_end;
2050         int partial = 0;
2051         unsigned blocksize;
2052         struct buffer_head *bh, *head;
2053
2054         bh = head = page_buffers(page);
2055         blocksize = bh->b_size;
2056
2057         block_start = 0;
2058         do {
2059                 block_end = block_start + blocksize;
2060                 if (block_end <= from || block_start >= to) {
2061                         if (!buffer_uptodate(bh))
2062                                 partial = 1;
2063                 } else {
2064                         set_buffer_uptodate(bh);
2065                         mark_buffer_dirty(bh);
2066                 }
2067                 if (buffer_new(bh))
2068                         clear_buffer_new(bh);
2069
2070                 block_start = block_end;
2071                 bh = bh->b_this_page;
2072         } while (bh != head);
2073
2074         /*
2075          * If this is a partial write which happened to make all buffers
2076          * uptodate then we can optimize away a bogus read_folio() for
2077          * the next read(). Here we 'discover' whether the page went
2078          * uptodate as a result of this (potentially partial) write.
2079          */
2080         if (!partial)
2081                 SetPageUptodate(page);
2082         return 0;
2083 }
2084
2085 /*
2086  * block_write_begin takes care of the basic task of block allocation and
2087  * bringing partial write blocks uptodate first.
2088  *
2089  * The filesystem needs to handle block truncation upon failure.
2090  */
2091 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2092                 struct page **pagep, get_block_t *get_block)
2093 {
2094         pgoff_t index = pos >> PAGE_SHIFT;
2095         struct page *page;
2096         int status;
2097
2098         page = grab_cache_page_write_begin(mapping, index);
2099         if (!page)
2100                 return -ENOMEM;
2101
2102         status = __block_write_begin(page, pos, len, get_block);
2103         if (unlikely(status)) {
2104                 unlock_page(page);
2105                 put_page(page);
2106                 page = NULL;
2107         }
2108
2109         *pagep = page;
2110         return status;
2111 }
2112 EXPORT_SYMBOL(block_write_begin);
2113
2114 int block_write_end(struct file *file, struct address_space *mapping,
2115                         loff_t pos, unsigned len, unsigned copied,
2116                         struct page *page, void *fsdata)
2117 {
2118         struct inode *inode = mapping->host;
2119         unsigned start;
2120
2121         start = pos & (PAGE_SIZE - 1);
2122
2123         if (unlikely(copied < len)) {
2124                 /*
2125                  * The buffers that were written will now be uptodate, so
2126                  * we don't have to worry about a read_folio reading them
2127                  * and overwriting a partial write. However if we have
2128                  * encountered a short write and only partially written
2129                  * into a buffer, it will not be marked uptodate, so a
2130                  * read_folio might come in and destroy our partial write.
2131                  *
2132                  * Do the simplest thing, and just treat any short write to a
2133                  * non uptodate page as a zero-length write, and force the
2134                  * caller to redo the whole thing.
2135                  */
2136                 if (!PageUptodate(page))
2137                         copied = 0;
2138
2139                 page_zero_new_buffers(page, start+copied, start+len);
2140         }
2141         flush_dcache_page(page);
2142
2143         /* This could be a short (even 0-length) commit */
2144         __block_commit_write(inode, page, start, start+copied);
2145
2146         return copied;
2147 }
2148 EXPORT_SYMBOL(block_write_end);
2149
2150 int generic_write_end(struct file *file, struct address_space *mapping,
2151                         loff_t pos, unsigned len, unsigned copied,
2152                         struct page *page, void *fsdata)
2153 {
2154         struct inode *inode = mapping->host;
2155         loff_t old_size = inode->i_size;
2156         bool i_size_changed = false;
2157
2158         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2159
2160         /*
2161          * No need to use i_size_read() here, the i_size cannot change under us
2162          * because we hold i_rwsem.
2163          *
2164          * But it's important to update i_size while still holding page lock:
2165          * page writeout could otherwise come in and zero beyond i_size.
2166          */
2167         if (pos + copied > inode->i_size) {
2168                 i_size_write(inode, pos + copied);
2169                 i_size_changed = true;
2170         }
2171
2172         unlock_page(page);
2173         put_page(page);
2174
2175         if (old_size < pos)
2176                 pagecache_isize_extended(inode, old_size, pos);
2177         /*
2178          * Don't mark the inode dirty under page lock. First, it unnecessarily
2179          * makes the holding time of page lock longer. Second, it forces lock
2180          * ordering of page lock and transaction start for journaling
2181          * filesystems.
2182          */
2183         if (i_size_changed)
2184                 mark_inode_dirty(inode);
2185         return copied;
2186 }
2187 EXPORT_SYMBOL(generic_write_end);
2188
2189 /*
2190  * block_is_partially_uptodate checks whether buffers within a folio are
2191  * uptodate or not.
2192  *
2193  * Returns true if all buffers which correspond to the specified part
2194  * of the folio are uptodate.
2195  */
2196 bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)
2197 {
2198         unsigned block_start, block_end, blocksize;
2199         unsigned to;
2200         struct buffer_head *bh, *head;
2201         bool ret = true;
2202
2203         head = folio_buffers(folio);
2204         if (!head)
2205                 return false;
2206         blocksize = head->b_size;
2207         to = min_t(unsigned, folio_size(folio) - from, count);
2208         to = from + to;
2209         if (from < blocksize && to > folio_size(folio) - blocksize)
2210                 return false;
2211
2212         bh = head;
2213         block_start = 0;
2214         do {
2215                 block_end = block_start + blocksize;
2216                 if (block_end > from && block_start < to) {
2217                         if (!buffer_uptodate(bh)) {
2218                                 ret = false;
2219                                 break;
2220                         }
2221                         if (block_end >= to)
2222                                 break;
2223                 }
2224                 block_start = block_end;
2225                 bh = bh->b_this_page;
2226         } while (bh != head);
2227
2228         return ret;
2229 }
2230 EXPORT_SYMBOL(block_is_partially_uptodate);
2231
2232 /*
2233  * Generic "read_folio" function for block devices that have the normal
2234  * get_block functionality. This is most of the block device filesystems.
2235  * Reads the folio asynchronously --- the unlock_buffer() and
2236  * set/clear_buffer_uptodate() functions propagate buffer state into the
2237  * folio once IO has completed.
2238  */
2239 int block_read_full_folio(struct folio *folio, get_block_t *get_block)
2240 {
2241         struct inode *inode = folio->mapping->host;
2242         sector_t iblock, lblock;
2243         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2244         unsigned int blocksize, bbits;
2245         int nr, i;
2246         int fully_mapped = 1;
2247         bool page_error = false;
2248
2249         VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2250
2251         head = create_page_buffers(&folio->page, inode, 0);
2252         blocksize = head->b_size;
2253         bbits = block_size_bits(blocksize);
2254
2255         iblock = (sector_t)folio->index << (PAGE_SHIFT - bbits);
2256         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2257         bh = head;
2258         nr = 0;
2259         i = 0;
2260
2261         do {
2262                 if (buffer_uptodate(bh))
2263                         continue;
2264
2265                 if (!buffer_mapped(bh)) {
2266                         int err = 0;
2267
2268                         fully_mapped = 0;
2269                         if (iblock < lblock) {
2270                                 WARN_ON(bh->b_size != blocksize);
2271                                 err = get_block(inode, iblock, bh, 0);
2272                                 if (err) {
2273                                         folio_set_error(folio);
2274                                         page_error = true;
2275                                 }
2276                         }
2277                         if (!buffer_mapped(bh)) {
2278                                 folio_zero_range(folio, i * blocksize,
2279                                                 blocksize);
2280                                 if (!err)
2281                                         set_buffer_uptodate(bh);
2282                                 continue;
2283                         }
2284                         /*
2285                          * get_block() might have updated the buffer
2286                          * synchronously
2287                          */
2288                         if (buffer_uptodate(bh))
2289                                 continue;
2290                 }
2291                 arr[nr++] = bh;
2292         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2293
2294         if (fully_mapped)
2295                 folio_set_mappedtodisk(folio);
2296
2297         if (!nr) {
2298                 /*
2299                  * All buffers are uptodate - we can set the folio uptodate
2300                  * as well. But not if get_block() returned an error.
2301                  */
2302                 if (!page_error)
2303                         folio_mark_uptodate(folio);
2304                 folio_unlock(folio);
2305                 return 0;
2306         }
2307
2308         /* Stage two: lock the buffers */
2309         for (i = 0; i < nr; i++) {
2310                 bh = arr[i];
2311                 lock_buffer(bh);
2312                 mark_buffer_async_read(bh);
2313         }
2314
2315         /*
2316          * Stage 3: start the IO.  Check for uptodateness
2317          * inside the buffer lock in case another process reading
2318          * the underlying blockdev brought it uptodate (the sct fix).
2319          */
2320         for (i = 0; i < nr; i++) {
2321                 bh = arr[i];
2322                 if (buffer_uptodate(bh))
2323                         end_buffer_async_read(bh, 1);
2324                 else
2325                         submit_bh(REQ_OP_READ, bh);
2326         }
2327         return 0;
2328 }
2329 EXPORT_SYMBOL(block_read_full_folio);
2330
2331 /* utility function for filesystems that need to do work on expanding
2332  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2333  * deal with the hole.  
2334  */
2335 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2336 {
2337         struct address_space *mapping = inode->i_mapping;
2338         const struct address_space_operations *aops = mapping->a_ops;
2339         struct page *page;
2340         void *fsdata = NULL;
2341         int err;
2342
2343         err = inode_newsize_ok(inode, size);
2344         if (err)
2345                 goto out;
2346
2347         err = aops->write_begin(NULL, mapping, size, 0, &page, &fsdata);
2348         if (err)
2349                 goto out;
2350
2351         err = aops->write_end(NULL, mapping, size, 0, 0, page, fsdata);
2352         BUG_ON(err > 0);
2353
2354 out:
2355         return err;
2356 }
2357 EXPORT_SYMBOL(generic_cont_expand_simple);
2358
2359 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2360                             loff_t pos, loff_t *bytes)
2361 {
2362         struct inode *inode = mapping->host;
2363         const struct address_space_operations *aops = mapping->a_ops;
2364         unsigned int blocksize = i_blocksize(inode);
2365         struct page *page;
2366         void *fsdata = NULL;
2367         pgoff_t index, curidx;
2368         loff_t curpos;
2369         unsigned zerofrom, offset, len;
2370         int err = 0;
2371
2372         index = pos >> PAGE_SHIFT;
2373         offset = pos & ~PAGE_MASK;
2374
2375         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2376                 zerofrom = curpos & ~PAGE_MASK;
2377                 if (zerofrom & (blocksize-1)) {
2378                         *bytes |= (blocksize-1);
2379                         (*bytes)++;
2380                 }
2381                 len = PAGE_SIZE - zerofrom;
2382
2383                 err = aops->write_begin(file, mapping, curpos, len,
2384                                             &page, &fsdata);
2385                 if (err)
2386                         goto out;
2387                 zero_user(page, zerofrom, len);
2388                 err = aops->write_end(file, mapping, curpos, len, len,
2389                                                 page, fsdata);
2390                 if (err < 0)
2391                         goto out;
2392                 BUG_ON(err != len);
2393                 err = 0;
2394
2395                 balance_dirty_pages_ratelimited(mapping);
2396
2397                 if (fatal_signal_pending(current)) {
2398                         err = -EINTR;
2399                         goto out;
2400                 }
2401         }
2402
2403         /* page covers the boundary, find the boundary offset */
2404         if (index == curidx) {
2405                 zerofrom = curpos & ~PAGE_MASK;
2406                 /* if we will expand the thing last block will be filled */
2407                 if (offset <= zerofrom) {
2408                         goto out;
2409                 }
2410                 if (zerofrom & (blocksize-1)) {
2411                         *bytes |= (blocksize-1);
2412                         (*bytes)++;
2413                 }
2414                 len = offset - zerofrom;
2415
2416                 err = aops->write_begin(file, mapping, curpos, len,
2417                                             &page, &fsdata);
2418                 if (err)
2419                         goto out;
2420                 zero_user(page, zerofrom, len);
2421                 err = aops->write_end(file, mapping, curpos, len, len,
2422                                                 page, fsdata);
2423                 if (err < 0)
2424                         goto out;
2425                 BUG_ON(err != len);
2426                 err = 0;
2427         }
2428 out:
2429         return err;
2430 }
2431
2432 /*
2433  * For moronic filesystems that do not allow holes in file.
2434  * We may have to extend the file.
2435  */
2436 int cont_write_begin(struct file *file, struct address_space *mapping,
2437                         loff_t pos, unsigned len,
2438                         struct page **pagep, void **fsdata,
2439                         get_block_t *get_block, loff_t *bytes)
2440 {
2441         struct inode *inode = mapping->host;
2442         unsigned int blocksize = i_blocksize(inode);
2443         unsigned int zerofrom;
2444         int err;
2445
2446         err = cont_expand_zero(file, mapping, pos, bytes);
2447         if (err)
2448                 return err;
2449
2450         zerofrom = *bytes & ~PAGE_MASK;
2451         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2452                 *bytes |= (blocksize-1);
2453                 (*bytes)++;
2454         }
2455
2456         return block_write_begin(mapping, pos, len, pagep, get_block);
2457 }
2458 EXPORT_SYMBOL(cont_write_begin);
2459
2460 int block_commit_write(struct page *page, unsigned from, unsigned to)
2461 {
2462         struct inode *inode = page->mapping->host;
2463         __block_commit_write(inode,page,from,to);
2464         return 0;
2465 }
2466 EXPORT_SYMBOL(block_commit_write);
2467
2468 /*
2469  * block_page_mkwrite() is not allowed to change the file size as it gets
2470  * called from a page fault handler when a page is first dirtied. Hence we must
2471  * be careful to check for EOF conditions here. We set the page up correctly
2472  * for a written page which means we get ENOSPC checking when writing into
2473  * holes and correct delalloc and unwritten extent mapping on filesystems that
2474  * support these features.
2475  *
2476  * We are not allowed to take the i_mutex here so we have to play games to
2477  * protect against truncate races as the page could now be beyond EOF.  Because
2478  * truncate writes the inode size before removing pages, once we have the
2479  * page lock we can determine safely if the page is beyond EOF. If it is not
2480  * beyond EOF, then the page is guaranteed safe against truncation until we
2481  * unlock the page.
2482  *
2483  * Direct callers of this function should protect against filesystem freezing
2484  * using sb_start_pagefault() - sb_end_pagefault() functions.
2485  */
2486 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2487                          get_block_t get_block)
2488 {
2489         struct page *page = vmf->page;
2490         struct inode *inode = file_inode(vma->vm_file);
2491         unsigned long end;
2492         loff_t size;
2493         int ret;
2494
2495         lock_page(page);
2496         size = i_size_read(inode);
2497         if ((page->mapping != inode->i_mapping) ||
2498             (page_offset(page) > size)) {
2499                 /* We overload EFAULT to mean page got truncated */
2500                 ret = -EFAULT;
2501                 goto out_unlock;
2502         }
2503
2504         /* page is wholly or partially inside EOF */
2505         if (((page->index + 1) << PAGE_SHIFT) > size)
2506                 end = size & ~PAGE_MASK;
2507         else
2508                 end = PAGE_SIZE;
2509
2510         ret = __block_write_begin(page, 0, end, get_block);
2511         if (!ret)
2512                 ret = block_commit_write(page, 0, end);
2513
2514         if (unlikely(ret < 0))
2515                 goto out_unlock;
2516         set_page_dirty(page);
2517         wait_for_stable_page(page);
2518         return 0;
2519 out_unlock:
2520         unlock_page(page);
2521         return ret;
2522 }
2523 EXPORT_SYMBOL(block_page_mkwrite);
2524
2525 int block_truncate_page(struct address_space *mapping,
2526                         loff_t from, get_block_t *get_block)
2527 {
2528         pgoff_t index = from >> PAGE_SHIFT;
2529         unsigned offset = from & (PAGE_SIZE-1);
2530         unsigned blocksize;
2531         sector_t iblock;
2532         unsigned length, pos;
2533         struct inode *inode = mapping->host;
2534         struct page *page;
2535         struct buffer_head *bh;
2536         int err;
2537
2538         blocksize = i_blocksize(inode);
2539         length = offset & (blocksize - 1);
2540
2541         /* Block boundary? Nothing to do */
2542         if (!length)
2543                 return 0;
2544
2545         length = blocksize - length;
2546         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2547         
2548         page = grab_cache_page(mapping, index);
2549         err = -ENOMEM;
2550         if (!page)
2551                 goto out;
2552
2553         if (!page_has_buffers(page))
2554                 create_empty_buffers(page, blocksize, 0);
2555
2556         /* Find the buffer that contains "offset" */
2557         bh = page_buffers(page);
2558         pos = blocksize;
2559         while (offset >= pos) {
2560                 bh = bh->b_this_page;
2561                 iblock++;
2562                 pos += blocksize;
2563         }
2564
2565         err = 0;
2566         if (!buffer_mapped(bh)) {
2567                 WARN_ON(bh->b_size != blocksize);
2568                 err = get_block(inode, iblock, bh, 0);
2569                 if (err)
2570                         goto unlock;
2571                 /* unmapped? It's a hole - nothing to do */
2572                 if (!buffer_mapped(bh))
2573                         goto unlock;
2574         }
2575
2576         /* Ok, it's mapped. Make sure it's up-to-date */
2577         if (PageUptodate(page))
2578                 set_buffer_uptodate(bh);
2579
2580         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2581                 err = bh_read(bh, 0);
2582                 /* Uhhuh. Read error. Complain and punt. */
2583                 if (err < 0)
2584                         goto unlock;
2585         }
2586
2587         zero_user(page, offset, length);
2588         mark_buffer_dirty(bh);
2589         err = 0;
2590
2591 unlock:
2592         unlock_page(page);
2593         put_page(page);
2594 out:
2595         return err;
2596 }
2597 EXPORT_SYMBOL(block_truncate_page);
2598
2599 /*
2600  * The generic ->writepage function for buffer-backed address_spaces
2601  */
2602 int block_write_full_page(struct page *page, get_block_t *get_block,
2603                         struct writeback_control *wbc)
2604 {
2605         struct inode * const inode = page->mapping->host;
2606         loff_t i_size = i_size_read(inode);
2607         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2608         unsigned offset;
2609
2610         /* Is the page fully inside i_size? */
2611         if (page->index < end_index)
2612                 return __block_write_full_page(inode, page, get_block, wbc,
2613                                                end_buffer_async_write);
2614
2615         /* Is the page fully outside i_size? (truncate in progress) */
2616         offset = i_size & (PAGE_SIZE-1);
2617         if (page->index >= end_index+1 || !offset) {
2618                 unlock_page(page);
2619                 return 0; /* don't care */
2620         }
2621
2622         /*
2623          * The page straddles i_size.  It must be zeroed out on each and every
2624          * writepage invocation because it may be mmapped.  "A file is mapped
2625          * in multiples of the page size.  For a file that is not a multiple of
2626          * the  page size, the remaining memory is zeroed when mapped, and
2627          * writes to that region are not written out to the file."
2628          */
2629         zero_user_segment(page, offset, PAGE_SIZE);
2630         return __block_write_full_page(inode, page, get_block, wbc,
2631                                                         end_buffer_async_write);
2632 }
2633 EXPORT_SYMBOL(block_write_full_page);
2634
2635 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2636                             get_block_t *get_block)
2637 {
2638         struct inode *inode = mapping->host;
2639         struct buffer_head tmp = {
2640                 .b_size = i_blocksize(inode),
2641         };
2642
2643         get_block(inode, block, &tmp, 0);
2644         return tmp.b_blocknr;
2645 }
2646 EXPORT_SYMBOL(generic_block_bmap);
2647
2648 static void end_bio_bh_io_sync(struct bio *bio)
2649 {
2650         struct buffer_head *bh = bio->bi_private;
2651
2652         if (unlikely(bio_flagged(bio, BIO_QUIET)))
2653                 set_bit(BH_Quiet, &bh->b_state);
2654
2655         bh->b_end_io(bh, !bio->bi_status);
2656         bio_put(bio);
2657 }
2658
2659 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,
2660                           struct writeback_control *wbc)
2661 {
2662         const enum req_op op = opf & REQ_OP_MASK;
2663         struct bio *bio;
2664
2665         BUG_ON(!buffer_locked(bh));
2666         BUG_ON(!buffer_mapped(bh));
2667         BUG_ON(!bh->b_end_io);
2668         BUG_ON(buffer_delay(bh));
2669         BUG_ON(buffer_unwritten(bh));
2670
2671         /*
2672          * Only clear out a write error when rewriting
2673          */
2674         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
2675                 clear_buffer_write_io_error(bh);
2676
2677         if (buffer_meta(bh))
2678                 opf |= REQ_META;
2679         if (buffer_prio(bh))
2680                 opf |= REQ_PRIO;
2681
2682         bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO);
2683
2684         fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
2685
2686         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
2687
2688         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
2689         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
2690
2691         bio->bi_end_io = end_bio_bh_io_sync;
2692         bio->bi_private = bh;
2693
2694         /* Take care of bh's that straddle the end of the device */
2695         guard_bio_eod(bio);
2696
2697         if (wbc) {
2698                 wbc_init_bio(wbc, bio);
2699                 wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
2700         }
2701
2702         submit_bio(bio);
2703 }
2704
2705 void submit_bh(blk_opf_t opf, struct buffer_head *bh)
2706 {
2707         submit_bh_wbc(opf, bh, NULL);
2708 }
2709 EXPORT_SYMBOL(submit_bh);
2710
2711 void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2712 {
2713         lock_buffer(bh);
2714         if (!test_clear_buffer_dirty(bh)) {
2715                 unlock_buffer(bh);
2716                 return;
2717         }
2718         bh->b_end_io = end_buffer_write_sync;
2719         get_bh(bh);
2720         submit_bh(REQ_OP_WRITE | op_flags, bh);
2721 }
2722 EXPORT_SYMBOL(write_dirty_buffer);
2723
2724 /*
2725  * For a data-integrity writeout, we need to wait upon any in-progress I/O
2726  * and then start new I/O and then wait upon it.  The caller must have a ref on
2727  * the buffer_head.
2728  */
2729 int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags)
2730 {
2731         WARN_ON(atomic_read(&bh->b_count) < 1);
2732         lock_buffer(bh);
2733         if (test_clear_buffer_dirty(bh)) {
2734                 /*
2735                  * The bh should be mapped, but it might not be if the
2736                  * device was hot-removed. Not much we can do but fail the I/O.
2737                  */
2738                 if (!buffer_mapped(bh)) {
2739                         unlock_buffer(bh);
2740                         return -EIO;
2741                 }
2742
2743                 get_bh(bh);
2744                 bh->b_end_io = end_buffer_write_sync;
2745                 submit_bh(REQ_OP_WRITE | op_flags, bh);
2746                 wait_on_buffer(bh);
2747                 if (!buffer_uptodate(bh))
2748                         return -EIO;
2749         } else {
2750                 unlock_buffer(bh);
2751         }
2752         return 0;
2753 }
2754 EXPORT_SYMBOL(__sync_dirty_buffer);
2755
2756 int sync_dirty_buffer(struct buffer_head *bh)
2757 {
2758         return __sync_dirty_buffer(bh, REQ_SYNC);
2759 }
2760 EXPORT_SYMBOL(sync_dirty_buffer);
2761
2762 /*
2763  * try_to_free_buffers() checks if all the buffers on this particular folio
2764  * are unused, and releases them if so.
2765  *
2766  * Exclusion against try_to_free_buffers may be obtained by either
2767  * locking the folio or by holding its mapping's private_lock.
2768  *
2769  * If the folio is dirty but all the buffers are clean then we need to
2770  * be sure to mark the folio clean as well.  This is because the folio
2771  * may be against a block device, and a later reattachment of buffers
2772  * to a dirty folio will set *all* buffers dirty.  Which would corrupt
2773  * filesystem data on the same device.
2774  *
2775  * The same applies to regular filesystem folios: if all the buffers are
2776  * clean then we set the folio clean and proceed.  To do that, we require
2777  * total exclusion from block_dirty_folio().  That is obtained with
2778  * private_lock.
2779  *
2780  * try_to_free_buffers() is non-blocking.
2781  */
2782 static inline int buffer_busy(struct buffer_head *bh)
2783 {
2784         return atomic_read(&bh->b_count) |
2785                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
2786 }
2787
2788 static bool
2789 drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)
2790 {
2791         struct buffer_head *head = folio_buffers(folio);
2792         struct buffer_head *bh;
2793
2794         bh = head;
2795         do {
2796                 if (buffer_busy(bh))
2797                         goto failed;
2798                 bh = bh->b_this_page;
2799         } while (bh != head);
2800
2801         do {
2802                 struct buffer_head *next = bh->b_this_page;
2803
2804                 if (bh->b_assoc_map)
2805                         __remove_assoc_queue(bh);
2806                 bh = next;
2807         } while (bh != head);
2808         *buffers_to_free = head;
2809         folio_detach_private(folio);
2810         return true;
2811 failed:
2812         return false;
2813 }
2814
2815 bool try_to_free_buffers(struct folio *folio)
2816 {
2817         struct address_space * const mapping = folio->mapping;
2818         struct buffer_head *buffers_to_free = NULL;
2819         bool ret = 0;
2820
2821         BUG_ON(!folio_test_locked(folio));
2822         if (folio_test_writeback(folio))
2823                 return false;
2824
2825         if (mapping == NULL) {          /* can this still happen? */
2826                 ret = drop_buffers(folio, &buffers_to_free);
2827                 goto out;
2828         }
2829
2830         spin_lock(&mapping->private_lock);
2831         ret = drop_buffers(folio, &buffers_to_free);
2832
2833         /*
2834          * If the filesystem writes its buffers by hand (eg ext3)
2835          * then we can have clean buffers against a dirty folio.  We
2836          * clean the folio here; otherwise the VM will never notice
2837          * that the filesystem did any IO at all.
2838          *
2839          * Also, during truncate, discard_buffer will have marked all
2840          * the folio's buffers clean.  We discover that here and clean
2841          * the folio also.
2842          *
2843          * private_lock must be held over this entire operation in order
2844          * to synchronise against block_dirty_folio and prevent the
2845          * dirty bit from being lost.
2846          */
2847         if (ret)
2848                 folio_cancel_dirty(folio);
2849         spin_unlock(&mapping->private_lock);
2850 out:
2851         if (buffers_to_free) {
2852                 struct buffer_head *bh = buffers_to_free;
2853
2854                 do {
2855                         struct buffer_head *next = bh->b_this_page;
2856                         free_buffer_head(bh);
2857                         bh = next;
2858                 } while (bh != buffers_to_free);
2859         }
2860         return ret;
2861 }
2862 EXPORT_SYMBOL(try_to_free_buffers);
2863
2864 /*
2865  * Buffer-head allocation
2866  */
2867 static struct kmem_cache *bh_cachep __read_mostly;
2868
2869 /*
2870  * Once the number of bh's in the machine exceeds this level, we start
2871  * stripping them in writeback.
2872  */
2873 static unsigned long max_buffer_heads;
2874
2875 int buffer_heads_over_limit;
2876
2877 struct bh_accounting {
2878         int nr;                 /* Number of live bh's */
2879         int ratelimit;          /* Limit cacheline bouncing */
2880 };
2881
2882 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
2883
2884 static void recalc_bh_state(void)
2885 {
2886         int i;
2887         int tot = 0;
2888
2889         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
2890                 return;
2891         __this_cpu_write(bh_accounting.ratelimit, 0);
2892         for_each_online_cpu(i)
2893                 tot += per_cpu(bh_accounting, i).nr;
2894         buffer_heads_over_limit = (tot > max_buffer_heads);
2895 }
2896
2897 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
2898 {
2899         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
2900         if (ret) {
2901                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
2902                 spin_lock_init(&ret->b_uptodate_lock);
2903                 preempt_disable();
2904                 __this_cpu_inc(bh_accounting.nr);
2905                 recalc_bh_state();
2906                 preempt_enable();
2907         }
2908         return ret;
2909 }
2910 EXPORT_SYMBOL(alloc_buffer_head);
2911
2912 void free_buffer_head(struct buffer_head *bh)
2913 {
2914         BUG_ON(!list_empty(&bh->b_assoc_buffers));
2915         kmem_cache_free(bh_cachep, bh);
2916         preempt_disable();
2917         __this_cpu_dec(bh_accounting.nr);
2918         recalc_bh_state();
2919         preempt_enable();
2920 }
2921 EXPORT_SYMBOL(free_buffer_head);
2922
2923 static int buffer_exit_cpu_dead(unsigned int cpu)
2924 {
2925         int i;
2926         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
2927
2928         for (i = 0; i < BH_LRU_SIZE; i++) {
2929                 brelse(b->bhs[i]);
2930                 b->bhs[i] = NULL;
2931         }
2932         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
2933         per_cpu(bh_accounting, cpu).nr = 0;
2934         return 0;
2935 }
2936
2937 /**
2938  * bh_uptodate_or_lock - Test whether the buffer is uptodate
2939  * @bh: struct buffer_head
2940  *
2941  * Return true if the buffer is up-to-date and false,
2942  * with the buffer locked, if not.
2943  */
2944 int bh_uptodate_or_lock(struct buffer_head *bh)
2945 {
2946         if (!buffer_uptodate(bh)) {
2947                 lock_buffer(bh);
2948                 if (!buffer_uptodate(bh))
2949                         return 0;
2950                 unlock_buffer(bh);
2951         }
2952         return 1;
2953 }
2954 EXPORT_SYMBOL(bh_uptodate_or_lock);
2955
2956 /**
2957  * __bh_read - Submit read for a locked buffer
2958  * @bh: struct buffer_head
2959  * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
2960  * @wait: wait until reading finish
2961  *
2962  * Returns zero on success or don't wait, and -EIO on error.
2963  */
2964 int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait)
2965 {
2966         int ret = 0;
2967
2968         BUG_ON(!buffer_locked(bh));
2969
2970         get_bh(bh);
2971         bh->b_end_io = end_buffer_read_sync;
2972         submit_bh(REQ_OP_READ | op_flags, bh);
2973         if (wait) {
2974                 wait_on_buffer(bh);
2975                 if (!buffer_uptodate(bh))
2976                         ret = -EIO;
2977         }
2978         return ret;
2979 }
2980 EXPORT_SYMBOL(__bh_read);
2981
2982 /**
2983  * __bh_read_batch - Submit read for a batch of unlocked buffers
2984  * @nr: entry number of the buffer batch
2985  * @bhs: a batch of struct buffer_head
2986  * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ
2987  * @force_lock: force to get a lock on the buffer if set, otherwise drops any
2988  *              buffer that cannot lock.
2989  *
2990  * Returns zero on success or don't wait, and -EIO on error.
2991  */
2992 void __bh_read_batch(int nr, struct buffer_head *bhs[],
2993                      blk_opf_t op_flags, bool force_lock)
2994 {
2995         int i;
2996
2997         for (i = 0; i < nr; i++) {
2998                 struct buffer_head *bh = bhs[i];
2999
3000                 if (buffer_uptodate(bh))
3001                         continue;
3002
3003                 if (force_lock)
3004                         lock_buffer(bh);
3005                 else
3006                         if (!trylock_buffer(bh))
3007                                 continue;
3008
3009                 if (buffer_uptodate(bh)) {
3010                         unlock_buffer(bh);
3011                         continue;
3012                 }
3013
3014                 bh->b_end_io = end_buffer_read_sync;
3015                 get_bh(bh);
3016                 submit_bh(REQ_OP_READ | op_flags, bh);
3017         }
3018 }
3019 EXPORT_SYMBOL(__bh_read_batch);
3020
3021 void __init buffer_init(void)
3022 {
3023         unsigned long nrpages;
3024         int ret;
3025
3026         bh_cachep = kmem_cache_create("buffer_head",
3027                         sizeof(struct buffer_head), 0,
3028                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3029                                 SLAB_MEM_SPREAD),
3030                                 NULL);
3031
3032         /*
3033          * Limit the bh occupancy to 10% of ZONE_NORMAL
3034          */
3035         nrpages = (nr_free_buffer_pages() * 10) / 100;
3036         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3037         ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3038                                         NULL, buffer_exit_cpu_dead);
3039         WARN_ON(ret < 0);
3040 }