1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
7 #include <linux/spinlock.h>
8 #include <linux/completion.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/gfs2_ondisk.h>
12 #include <linux/crc32.h>
13 #include <linux/iomap.h>
14 #include <linux/ktime.h>
30 #include "trace_gfs2.h"
32 /* This doesn't need to be that large as max 64 bit pointers in a 4k
33 * block is 512, so __u16 is fine for that. It saves stack space to
37 struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT];
38 __u16 mp_list[GFS2_MAX_META_HEIGHT];
39 int mp_fheight; /* find_metapath height */
40 int mp_aheight; /* actual height (lookup height) */
43 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length);
46 * gfs2_unstuffer_page - unstuff a stuffed inode into a block cached by a page
48 * @dibh: the dinode buffer
49 * @block: the block number that was allocated
50 * @page: The (optional) page. This is looked up if @page is NULL
55 static int gfs2_unstuffer_page(struct gfs2_inode *ip, struct buffer_head *dibh,
56 u64 block, struct page *page)
58 struct inode *inode = &ip->i_inode;
60 if (!PageUptodate(page)) {
61 void *kaddr = kmap(page);
62 u64 dsize = i_size_read(inode);
64 if (dsize > gfs2_max_stuffed_size(ip))
65 dsize = gfs2_max_stuffed_size(ip);
67 memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize);
68 memset(kaddr + dsize, 0, PAGE_SIZE - dsize);
71 SetPageUptodate(page);
74 if (gfs2_is_jdata(ip)) {
75 struct buffer_head *bh;
77 if (!page_has_buffers(page))
78 create_empty_buffers(page, BIT(inode->i_blkbits),
81 bh = page_buffers(page);
82 if (!buffer_mapped(bh))
83 map_bh(bh, inode->i_sb, block);
85 set_buffer_uptodate(bh);
86 gfs2_trans_add_data(ip->i_gl, bh);
89 gfs2_ordered_add_inode(ip);
95 static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct page *page)
97 struct buffer_head *bh, *dibh;
98 struct gfs2_dinode *di;
100 int isdir = gfs2_is_dir(ip);
103 error = gfs2_meta_inode_buffer(ip, &dibh);
107 if (i_size_read(&ip->i_inode)) {
108 /* Get a free block, fill it with the stuffed data,
109 and write it out to disk */
112 error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
116 gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
117 error = gfs2_dir_get_new_buffer(ip, block, &bh);
120 gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header),
121 dibh, sizeof(struct gfs2_dinode));
124 error = gfs2_unstuffer_page(ip, dibh, block, page);
130 /* Set up the pointer to the new block */
132 gfs2_trans_add_meta(ip->i_gl, dibh);
133 di = (struct gfs2_dinode *)dibh->b_data;
134 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
136 if (i_size_read(&ip->i_inode)) {
137 *(__be64 *)(di + 1) = cpu_to_be64(block);
138 gfs2_add_inode_blocks(&ip->i_inode, 1);
139 di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
143 di->di_height = cpu_to_be16(1);
151 * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big
152 * @ip: The GFS2 inode to unstuff
154 * This routine unstuffs a dinode and returns it to a "normal" state such
155 * that the height can be grown in the traditional way.
160 int gfs2_unstuff_dinode(struct gfs2_inode *ip)
162 struct inode *inode = &ip->i_inode;
166 down_write(&ip->i_rw_mutex);
167 page = find_or_create_page(inode->i_mapping, 0, GFP_NOFS);
171 error = __gfs2_unstuff_inode(ip, page);
175 up_write(&ip->i_rw_mutex);
180 * find_metapath - Find path through the metadata tree
181 * @sdp: The superblock
182 * @block: The disk block to look up
183 * @mp: The metapath to return the result in
184 * @height: The pre-calculated height of the metadata tree
186 * This routine returns a struct metapath structure that defines a path
187 * through the metadata of inode "ip" to get to block "block".
190 * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a
191 * filesystem with a blocksize of 4096.
193 * find_metapath() would return a struct metapath structure set to:
194 * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165.
196 * That means that in order to get to the block containing the byte at
197 * offset 101342453, we would load the indirect block pointed to by pointer
198 * 0 in the dinode. We would then load the indirect block pointed to by
199 * pointer 48 in that indirect block. We would then load the data block
200 * pointed to by pointer 165 in that indirect block.
202 * ----------------------------------------
207 * ----------------------------------------
211 * ----------------------------------------
215 * |0 5 6 7 8 9 0 1 2|
216 * ----------------------------------------
220 * ----------------------------------------
225 * ----------------------------------------
229 * ----------------------------------------
230 * | Data block containing offset |
234 * ----------------------------------------
238 static void find_metapath(const struct gfs2_sbd *sdp, u64 block,
239 struct metapath *mp, unsigned int height)
243 mp->mp_fheight = height;
244 for (i = height; i--;)
245 mp->mp_list[i] = do_div(block, sdp->sd_inptrs);
248 static inline unsigned int metapath_branch_start(const struct metapath *mp)
250 if (mp->mp_list[0] == 0)
256 * metaptr1 - Return the first possible metadata pointer in a metapath buffer
257 * @height: The metadata height (0 = dinode)
260 static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp)
262 struct buffer_head *bh = mp->mp_bh[height];
264 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode)));
265 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)));
269 * metapointer - Return pointer to start of metadata in a buffer
270 * @height: The metadata height (0 = dinode)
273 * Return a pointer to the block number of the next height of the metadata
274 * tree given a buffer containing the pointer to the current height of the
278 static inline __be64 *metapointer(unsigned int height, const struct metapath *mp)
280 __be64 *p = metaptr1(height, mp);
281 return p + mp->mp_list[height];
284 static inline const __be64 *metaend(unsigned int height, const struct metapath *mp)
286 const struct buffer_head *bh = mp->mp_bh[height];
287 return (const __be64 *)(bh->b_data + bh->b_size);
290 static void clone_metapath(struct metapath *clone, struct metapath *mp)
295 for (hgt = 0; hgt < mp->mp_aheight; hgt++)
296 get_bh(clone->mp_bh[hgt]);
299 static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end)
303 for (t = start; t < end; t++) {
304 struct buffer_head *rabh;
309 rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE);
310 if (trylock_buffer(rabh)) {
311 if (!buffer_uptodate(rabh)) {
312 rabh->b_end_io = end_buffer_read_sync;
313 submit_bh(REQ_OP_READ,
314 REQ_RAHEAD | REQ_META | REQ_PRIO,
324 static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp,
325 unsigned int x, unsigned int h)
328 __be64 *ptr = metapointer(x, mp);
329 u64 dblock = be64_to_cpu(*ptr);
334 ret = gfs2_meta_buffer(ip, GFS2_METATYPE_IN, dblock, &mp->mp_bh[x + 1]);
338 mp->mp_aheight = x + 1;
343 * lookup_metapath - Walk the metadata tree to a specific point
347 * Assumes that the inode's buffer has already been looked up and
348 * hooked onto mp->mp_bh[0] and that the metapath has been initialised
349 * by find_metapath().
351 * If this function encounters part of the tree which has not been
352 * allocated, it returns the current height of the tree at the point
353 * at which it found the unallocated block. Blocks which are found are
354 * added to the mp->mp_bh[] list.
359 static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp)
361 return __fillup_metapath(ip, mp, 0, ip->i_height - 1);
365 * fillup_metapath - fill up buffers for the metadata path to a specific height
368 * @h: The height to which it should be mapped
370 * Similar to lookup_metapath, but does lookups for a range of heights
372 * Returns: error or the number of buffers filled
375 static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h)
381 /* find the first buffer we need to look up. */
382 for (x = h - 1; x > 0; x--) {
387 ret = __fillup_metapath(ip, mp, x, h);
390 return mp->mp_aheight - x - 1;
393 static sector_t metapath_to_block(struct gfs2_sbd *sdp, struct metapath *mp)
395 sector_t factor = 1, block = 0;
398 for (hgt = mp->mp_fheight - 1; hgt >= 0; hgt--) {
399 if (hgt < mp->mp_aheight)
400 block += mp->mp_list[hgt] * factor;
401 factor *= sdp->sd_inptrs;
406 static void release_metapath(struct metapath *mp)
410 for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) {
411 if (mp->mp_bh[i] == NULL)
413 brelse(mp->mp_bh[i]);
419 * gfs2_extent_length - Returns length of an extent of blocks
420 * @bh: The metadata block
421 * @ptr: Current position in @bh
422 * @limit: Max extent length to return
423 * @eob: Set to 1 if we hit "end of block"
425 * Returns: The length of the extent (minimum of one block)
428 static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, size_t limit, int *eob)
430 const __be64 *end = (__be64 *)(bh->b_data + bh->b_size);
431 const __be64 *first = ptr;
432 u64 d = be64_to_cpu(*ptr);
440 } while(be64_to_cpu(*ptr) == d);
446 enum walker_status { WALK_STOP, WALK_FOLLOW, WALK_CONTINUE };
449 * gfs2_metadata_walker - walk an indirect block
450 * @mp: Metapath to indirect block
451 * @ptrs: Number of pointers to look at
453 * When returning WALK_FOLLOW, the walker must update @mp to point at the right
454 * indirect block to follow.
456 typedef enum walker_status (*gfs2_metadata_walker)(struct metapath *mp,
460 * gfs2_walk_metadata - walk a tree of indirect blocks
462 * @mp: Starting point of walk
463 * @max_len: Maximum number of blocks to walk
464 * @walker: Called during the walk
466 * Returns 1 if the walk was stopped by @walker, 0 if we went past @max_len or
467 * past the end of metadata, and a negative error code otherwise.
470 static int gfs2_walk_metadata(struct inode *inode, struct metapath *mp,
471 u64 max_len, gfs2_metadata_walker walker)
473 struct gfs2_inode *ip = GFS2_I(inode);
474 struct gfs2_sbd *sdp = GFS2_SB(inode);
480 * The walk starts in the lowest allocated indirect block, which may be
481 * before the position indicated by @mp. Adjust @max_len accordingly
482 * to avoid a short walk.
484 for (hgt = mp->mp_fheight - 1; hgt >= mp->mp_aheight; hgt--) {
485 max_len += mp->mp_list[hgt] * factor;
486 mp->mp_list[hgt] = 0;
487 factor *= sdp->sd_inptrs;
491 u16 start = mp->mp_list[hgt];
492 enum walker_status status;
496 /* Walk indirect block. */
497 ptrs = (hgt >= 1 ? sdp->sd_inptrs : sdp->sd_diptrs) - start;
500 ptrs = DIV_ROUND_UP_ULL(max_len, factor);
501 status = walker(mp, ptrs);
506 BUG_ON(mp->mp_aheight == mp->mp_fheight);
507 ptrs = mp->mp_list[hgt] - start;
516 if (status == WALK_FOLLOW)
517 goto fill_up_metapath;
520 /* Decrease height of metapath. */
521 brelse(mp->mp_bh[hgt]);
522 mp->mp_bh[hgt] = NULL;
523 mp->mp_list[hgt] = 0;
527 factor *= sdp->sd_inptrs;
529 /* Advance in metadata tree. */
530 (mp->mp_list[hgt])++;
532 if (mp->mp_list[hgt] >= sdp->sd_inptrs)
535 if (mp->mp_list[hgt] >= sdp->sd_diptrs)
540 /* Increase height of metapath. */
541 ret = fillup_metapath(ip, mp, ip->i_height - 1);
546 do_div(factor, sdp->sd_inptrs);
547 mp->mp_aheight = hgt + 1;
552 static enum walker_status gfs2_hole_walker(struct metapath *mp,
555 const __be64 *start, *ptr, *end;
558 hgt = mp->mp_aheight - 1;
559 start = metapointer(hgt, mp);
562 for (ptr = start; ptr < end; ptr++) {
564 mp->mp_list[hgt] += ptr - start;
565 if (mp->mp_aheight == mp->mp_fheight)
570 return WALK_CONTINUE;
574 * gfs2_hole_size - figure out the size of a hole
576 * @lblock: The logical starting block number
577 * @len: How far to look (in blocks)
578 * @mp: The metapath at lblock
579 * @iomap: The iomap to store the hole size in
581 * This function modifies @mp.
583 * Returns: errno on error
585 static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len,
586 struct metapath *mp, struct iomap *iomap)
588 struct metapath clone;
592 clone_metapath(&clone, mp);
593 ret = gfs2_walk_metadata(inode, &clone, len, gfs2_hole_walker);
598 hole_size = metapath_to_block(GFS2_SB(inode), &clone) - lblock;
601 iomap->length = hole_size << inode->i_blkbits;
605 release_metapath(&clone);
609 static inline __be64 *gfs2_indirect_init(struct metapath *mp,
610 struct gfs2_glock *gl, unsigned int i,
611 unsigned offset, u64 bn)
613 __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data +
614 ((i > 1) ? sizeof(struct gfs2_meta_header) :
615 sizeof(struct gfs2_dinode)));
617 BUG_ON(mp->mp_bh[i] != NULL);
618 mp->mp_bh[i] = gfs2_meta_new(gl, bn);
619 gfs2_trans_add_meta(gl, mp->mp_bh[i]);
620 gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN);
621 gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header));
623 *ptr = cpu_to_be64(bn);
629 ALLOC_GROW_DEPTH = 1,
630 ALLOC_GROW_HEIGHT = 2,
631 /* ALLOC_UNSTUFF = 3, TBD and rather complicated */
635 * __gfs2_iomap_alloc - Build a metadata tree of the requested height
636 * @inode: The GFS2 inode
637 * @iomap: The iomap structure
638 * @mp: The metapath, with proper height information calculated
640 * In this routine we may have to alloc:
641 * i) Indirect blocks to grow the metadata tree height
642 * ii) Indirect blocks to fill in lower part of the metadata tree
645 * This function is called after __gfs2_iomap_get, which works out the
646 * total number of blocks which we need via gfs2_alloc_size.
648 * We then do the actual allocation asking for an extent at a time (if
649 * enough contiguous free blocks are available, there will only be one
650 * allocation request per call) and uses the state machine to initialise
651 * the blocks in order.
653 * Right now, this function will allocate at most one indirect block
654 * worth of data -- with a default block size of 4K, that's slightly
655 * less than 2M. If this limitation is ever removed to allow huge
656 * allocations, we would probably still want to limit the iomap size we
657 * return to avoid stalling other tasks during huge writes; the next
658 * iomap iteration would then find the blocks already allocated.
660 * Returns: errno on error
663 static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
666 struct gfs2_inode *ip = GFS2_I(inode);
667 struct gfs2_sbd *sdp = GFS2_SB(inode);
668 struct buffer_head *dibh = mp->mp_bh[0];
670 unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0;
671 size_t dblks = iomap->length >> inode->i_blkbits;
672 const unsigned end_of_metadata = mp->mp_fheight - 1;
674 enum alloc_state state;
678 BUG_ON(mp->mp_aheight < 1);
679 BUG_ON(dibh == NULL);
682 gfs2_trans_add_meta(ip->i_gl, dibh);
684 down_write(&ip->i_rw_mutex);
686 if (mp->mp_fheight == mp->mp_aheight) {
687 /* Bottom indirect block exists */
690 /* Need to allocate indirect blocks */
691 if (mp->mp_fheight == ip->i_height) {
692 /* Writing into existing tree, extend tree down */
693 iblks = mp->mp_fheight - mp->mp_aheight;
694 state = ALLOC_GROW_DEPTH;
696 /* Building up tree height */
697 state = ALLOC_GROW_HEIGHT;
698 iblks = mp->mp_fheight - ip->i_height;
699 branch_start = metapath_branch_start(mp);
700 iblks += (mp->mp_fheight - branch_start);
704 /* start of the second part of the function (state machine) */
706 blks = dblks + iblks;
710 ret = gfs2_alloc_blocks(ip, &bn, &n, 0, NULL);
714 if (state != ALLOC_DATA || gfs2_is_jdata(ip))
715 gfs2_trans_remove_revoke(sdp, bn, n);
717 /* Growing height of tree */
718 case ALLOC_GROW_HEIGHT:
720 ptr = (__be64 *)(dibh->b_data +
721 sizeof(struct gfs2_dinode));
724 for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
726 gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
727 if (i - 1 == mp->mp_fheight - ip->i_height) {
729 gfs2_buffer_copy_tail(mp->mp_bh[i],
730 sizeof(struct gfs2_meta_header),
731 dibh, sizeof(struct gfs2_dinode));
732 gfs2_buffer_clear_tail(dibh,
733 sizeof(struct gfs2_dinode) +
735 ptr = (__be64 *)(mp->mp_bh[i]->b_data +
736 sizeof(struct gfs2_meta_header));
738 state = ALLOC_GROW_DEPTH;
739 for(i = branch_start; i < mp->mp_fheight; i++) {
740 if (mp->mp_bh[i] == NULL)
742 brelse(mp->mp_bh[i]);
749 fallthrough; /* To branching from existing tree */
750 case ALLOC_GROW_DEPTH:
751 if (i > 1 && i < mp->mp_fheight)
752 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
753 for (; i < mp->mp_fheight && n > 0; i++, n--)
754 gfs2_indirect_init(mp, ip->i_gl, i,
755 mp->mp_list[i-1], bn++);
756 if (i == mp->mp_fheight)
760 fallthrough; /* To tree complete, adding data blocks */
763 BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
764 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
766 ptr = metapointer(end_of_metadata, mp);
767 iomap->addr = bn << inode->i_blkbits;
768 iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW;
770 *ptr++ = cpu_to_be64(bn++);
773 } while (iomap->addr == IOMAP_NULL_ADDR);
775 iomap->type = IOMAP_MAPPED;
776 iomap->length = (u64)dblks << inode->i_blkbits;
777 ip->i_height = mp->mp_fheight;
778 gfs2_add_inode_blocks(&ip->i_inode, alloced);
779 gfs2_dinode_out(ip, dibh->b_data);
781 up_write(&ip->i_rw_mutex);
785 #define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE
788 * gfs2_alloc_size - Compute the maximum allocation size
791 * @size: Requested size in blocks
793 * Compute the maximum size of the next allocation at @mp.
795 * Returns: size in blocks
797 static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size)
799 struct gfs2_inode *ip = GFS2_I(inode);
800 struct gfs2_sbd *sdp = GFS2_SB(inode);
801 const __be64 *first, *ptr, *end;
804 * For writes to stuffed files, this function is called twice via
805 * __gfs2_iomap_get, before and after unstuffing. The size we return the
806 * first time needs to be large enough to get the reservation and
807 * allocation sizes right. The size we return the second time must
808 * be exact or else __gfs2_iomap_alloc won't do the right thing.
811 if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) {
812 unsigned int maxsize = mp->mp_fheight > 1 ?
813 sdp->sd_inptrs : sdp->sd_diptrs;
814 maxsize -= mp->mp_list[mp->mp_fheight - 1];
820 first = metapointer(ip->i_height - 1, mp);
821 end = metaend(ip->i_height - 1, mp);
822 if (end - first > size)
824 for (ptr = first; ptr < end; ptr++) {
832 * __gfs2_iomap_get - Map blocks from an inode to disk blocks
834 * @pos: Starting position in bytes
835 * @length: Length to map, in bytes
836 * @flags: iomap flags
837 * @iomap: The iomap structure
842 static int __gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
843 unsigned flags, struct iomap *iomap,
846 struct gfs2_inode *ip = GFS2_I(inode);
847 struct gfs2_sbd *sdp = GFS2_SB(inode);
848 loff_t size = i_size_read(inode);
851 sector_t lblock_stop;
855 struct buffer_head *dibh = NULL, *bh;
861 down_read(&ip->i_rw_mutex);
863 ret = gfs2_meta_inode_buffer(ip, &dibh);
868 if (gfs2_is_stuffed(ip)) {
869 if (flags & IOMAP_WRITE) {
870 loff_t max_size = gfs2_max_stuffed_size(ip);
872 if (pos + length > max_size)
874 iomap->length = max_size;
877 if (flags & IOMAP_REPORT) {
882 iomap->length = length;
886 iomap->length = size;
888 iomap->addr = (ip->i_no_addr << inode->i_blkbits) +
889 sizeof(struct gfs2_dinode);
890 iomap->type = IOMAP_INLINE;
891 iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode);
896 lblock = pos >> inode->i_blkbits;
897 iomap->offset = lblock << inode->i_blkbits;
898 lblock_stop = (pos + length - 1) >> inode->i_blkbits;
899 len = lblock_stop - lblock + 1;
900 iomap->length = len << inode->i_blkbits;
902 height = ip->i_height;
903 while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height])
905 find_metapath(sdp, lblock, mp, height);
906 if (height > ip->i_height || gfs2_is_stuffed(ip))
909 ret = lookup_metapath(ip, mp);
913 if (mp->mp_aheight != ip->i_height)
916 ptr = metapointer(ip->i_height - 1, mp);
920 bh = mp->mp_bh[ip->i_height - 1];
921 len = gfs2_extent_length(bh, ptr, len, &eob);
923 iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits;
924 iomap->length = len << inode->i_blkbits;
925 iomap->type = IOMAP_MAPPED;
926 iomap->flags |= IOMAP_F_MERGED;
928 iomap->flags |= IOMAP_F_GFS2_BOUNDARY;
931 iomap->bdev = inode->i_sb->s_bdev;
933 up_read(&ip->i_rw_mutex);
937 if (flags & IOMAP_REPORT) {
940 else if (height == ip->i_height)
941 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
943 iomap->length = size - iomap->offset;
944 } else if (flags & IOMAP_WRITE) {
947 if (flags & IOMAP_DIRECT)
948 goto out; /* (see gfs2_file_direct_write) */
950 len = gfs2_alloc_size(inode, mp, len);
951 alloc_size = len << inode->i_blkbits;
952 if (alloc_size < iomap->length)
953 iomap->length = alloc_size;
955 if (pos < size && height == ip->i_height)
956 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
959 iomap->addr = IOMAP_NULL_ADDR;
960 iomap->type = IOMAP_HOLE;
964 static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
967 unsigned int blockmask = i_blocksize(inode) - 1;
968 struct gfs2_sbd *sdp = GFS2_SB(inode);
971 blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits;
972 return gfs2_trans_begin(sdp, RES_DINODE + blocks, 0);
975 static void gfs2_iomap_page_done(struct inode *inode, loff_t pos,
976 unsigned copied, struct page *page)
978 struct gfs2_trans *tr = current->journal_info;
979 struct gfs2_inode *ip = GFS2_I(inode);
980 struct gfs2_sbd *sdp = GFS2_SB(inode);
982 if (page && !gfs2_is_stuffed(ip))
983 gfs2_page_add_databufs(ip, page, offset_in_page(pos), copied);
985 if (tr->tr_num_buf_new)
986 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
991 static const struct iomap_page_ops gfs2_iomap_page_ops = {
992 .page_prepare = gfs2_iomap_page_prepare,
993 .page_done = gfs2_iomap_page_done,
996 static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
997 loff_t length, unsigned flags,
1001 struct gfs2_inode *ip = GFS2_I(inode);
1002 struct gfs2_sbd *sdp = GFS2_SB(inode);
1006 unstuff = gfs2_is_stuffed(ip) &&
1007 pos + length > gfs2_max_stuffed_size(ip);
1009 if (unstuff || iomap->type == IOMAP_HOLE) {
1010 unsigned int data_blocks, ind_blocks;
1011 struct gfs2_alloc_parms ap = {};
1012 unsigned int rblocks;
1013 struct gfs2_trans *tr;
1015 gfs2_write_calc_reserv(ip, iomap->length, &data_blocks,
1017 ap.target = data_blocks + ind_blocks;
1018 ret = gfs2_quota_lock_check(ip, &ap);
1022 ret = gfs2_inplace_reserve(ip, &ap);
1026 rblocks = RES_DINODE + ind_blocks;
1027 if (gfs2_is_jdata(ip))
1028 rblocks += data_blocks;
1029 if (ind_blocks || data_blocks)
1030 rblocks += RES_STATFS + RES_QUOTA;
1031 if (inode == sdp->sd_rindex)
1032 rblocks += 2 * RES_STATFS;
1033 rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
1035 ret = gfs2_trans_begin(sdp, rblocks,
1036 iomap->length >> inode->i_blkbits);
1038 goto out_trans_fail;
1041 ret = gfs2_unstuff_dinode(ip);
1044 release_metapath(mp);
1045 ret = __gfs2_iomap_get(inode, iomap->offset,
1046 iomap->length, flags, iomap, mp);
1051 if (iomap->type == IOMAP_HOLE) {
1052 ret = __gfs2_iomap_alloc(inode, iomap, mp);
1054 gfs2_trans_end(sdp);
1055 gfs2_inplace_release(ip);
1056 punch_hole(ip, iomap->offset, iomap->length);
1061 tr = current->journal_info;
1062 if (tr->tr_num_buf_new)
1063 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1065 gfs2_trans_end(sdp);
1068 if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
1069 iomap->page_ops = &gfs2_iomap_page_ops;
1073 gfs2_trans_end(sdp);
1075 gfs2_inplace_release(ip);
1077 gfs2_quota_unlock(ip);
1081 static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
1082 unsigned flags, struct iomap *iomap,
1083 struct iomap *srcmap)
1085 struct gfs2_inode *ip = GFS2_I(inode);
1086 struct metapath mp = { .mp_aheight = 1, };
1089 if (gfs2_is_jdata(ip))
1090 iomap->flags |= IOMAP_F_BUFFER_HEAD;
1092 trace_gfs2_iomap_start(ip, pos, length, flags);
1093 ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
1097 switch(flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1099 if (flags & IOMAP_DIRECT) {
1101 * Silently fall back to buffered I/O for stuffed files
1102 * or if we've got a hole (see gfs2_file_direct_write).
1104 if (iomap->type != IOMAP_MAPPED)
1110 if (iomap->type == IOMAP_HOLE)
1117 ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
1120 release_metapath(&mp);
1121 trace_gfs2_iomap_end(ip, iomap, ret);
1125 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
1126 ssize_t written, unsigned flags, struct iomap *iomap)
1128 struct gfs2_inode *ip = GFS2_I(inode);
1129 struct gfs2_sbd *sdp = GFS2_SB(inode);
1131 switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1133 if (flags & IOMAP_DIRECT)
1137 if (iomap->type == IOMAP_HOLE)
1144 if (!gfs2_is_stuffed(ip))
1145 gfs2_ordered_add_inode(ip);
1147 if (inode == sdp->sd_rindex)
1148 adjust_fs_space(inode);
1150 gfs2_inplace_release(ip);
1152 if (ip->i_qadata && ip->i_qadata->qa_qd_num)
1153 gfs2_quota_unlock(ip);
1155 if (length != written && (iomap->flags & IOMAP_F_NEW)) {
1156 /* Deallocate blocks that were just allocated. */
1157 loff_t blockmask = i_blocksize(inode) - 1;
1158 loff_t end = (pos + length) & ~blockmask;
1160 pos = (pos + written + blockmask) & ~blockmask;
1162 truncate_pagecache_range(inode, pos, end - 1);
1163 punch_hole(ip, pos, end - pos);
1167 if (unlikely(!written))
1170 if (iomap->flags & IOMAP_F_SIZE_CHANGED)
1171 mark_inode_dirty(inode);
1172 set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
1176 const struct iomap_ops gfs2_iomap_ops = {
1177 .iomap_begin = gfs2_iomap_begin,
1178 .iomap_end = gfs2_iomap_end,
1182 * gfs2_block_map - Map one or more blocks of an inode to a disk block
1184 * @lblock: The logical block number
1185 * @bh_map: The bh to be mapped
1186 * @create: True if its ok to alloc blocks to satify the request
1188 * The size of the requested mapping is defined in bh_map->b_size.
1190 * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged
1191 * when @lblock is not mapped. Sets buffer_mapped(bh_map) and
1192 * bh_map->b_size to indicate the size of the mapping when @lblock and
1193 * successive blocks are mapped, up to the requested size.
1195 * Sets buffer_boundary() if a read of metadata will be required
1196 * before the next block can be mapped. Sets buffer_new() if new
1197 * blocks were allocated.
1202 int gfs2_block_map(struct inode *inode, sector_t lblock,
1203 struct buffer_head *bh_map, int create)
1205 struct gfs2_inode *ip = GFS2_I(inode);
1206 loff_t pos = (loff_t)lblock << inode->i_blkbits;
1207 loff_t length = bh_map->b_size;
1208 struct iomap iomap = { };
1211 clear_buffer_mapped(bh_map);
1212 clear_buffer_new(bh_map);
1213 clear_buffer_boundary(bh_map);
1214 trace_gfs2_bmap(ip, bh_map, lblock, create, 1);
1217 ret = gfs2_iomap_get(inode, pos, length, &iomap);
1219 ret = gfs2_iomap_alloc(inode, pos, length, &iomap);
1223 if (iomap.length > bh_map->b_size) {
1224 iomap.length = bh_map->b_size;
1225 iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY;
1227 if (iomap.addr != IOMAP_NULL_ADDR)
1228 map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits);
1229 bh_map->b_size = iomap.length;
1230 if (iomap.flags & IOMAP_F_GFS2_BOUNDARY)
1231 set_buffer_boundary(bh_map);
1232 if (iomap.flags & IOMAP_F_NEW)
1233 set_buffer_new(bh_map);
1236 trace_gfs2_bmap(ip, bh_map, lblock, create, ret);
1240 int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock,
1241 unsigned int *extlen)
1243 unsigned int blkbits = inode->i_blkbits;
1244 struct iomap iomap = { };
1248 ret = gfs2_iomap_get(inode, lblock << blkbits, *extlen << blkbits,
1252 if (iomap.type != IOMAP_MAPPED)
1254 *dblock = iomap.addr >> blkbits;
1255 len = iomap.length >> blkbits;
1261 int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock,
1262 unsigned int *extlen, bool *new)
1264 unsigned int blkbits = inode->i_blkbits;
1265 struct iomap iomap = { };
1269 ret = gfs2_iomap_alloc(inode, lblock << blkbits, *extlen << blkbits,
1273 if (iomap.type != IOMAP_MAPPED)
1275 *dblock = iomap.addr >> blkbits;
1276 len = iomap.length >> blkbits;
1279 *new = iomap.flags & IOMAP_F_NEW;
1284 * NOTE: Never call gfs2_block_zero_range with an open transaction because it
1285 * uses iomap write to perform its actions, which begin their own transactions
1286 * (iomap_begin, page_prepare, etc.)
1288 static int gfs2_block_zero_range(struct inode *inode, loff_t from,
1289 unsigned int length)
1291 BUG_ON(current->journal_info);
1292 return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops);
1295 #define GFS2_JTRUNC_REVOKES 8192
1298 * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files
1299 * @inode: The inode being truncated
1300 * @oldsize: The original (larger) size
1301 * @newsize: The new smaller size
1303 * With jdata files, we have to journal a revoke for each block which is
1304 * truncated. As a result, we need to split this into separate transactions
1305 * if the number of pages being truncated gets too large.
1308 static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize)
1310 struct gfs2_sbd *sdp = GFS2_SB(inode);
1311 u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
1315 while (oldsize != newsize) {
1316 struct gfs2_trans *tr;
1319 chunk = oldsize - newsize;
1320 if (chunk > max_chunk)
1323 offs = oldsize & ~PAGE_MASK;
1324 if (offs && chunk > PAGE_SIZE)
1325 chunk = offs + ((chunk - offs) & PAGE_MASK);
1327 truncate_pagecache(inode, oldsize - chunk);
1330 tr = current->journal_info;
1331 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
1334 gfs2_trans_end(sdp);
1335 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
1343 static int trunc_start(struct inode *inode, u64 newsize)
1345 struct gfs2_inode *ip = GFS2_I(inode);
1346 struct gfs2_sbd *sdp = GFS2_SB(inode);
1347 struct buffer_head *dibh = NULL;
1348 int journaled = gfs2_is_jdata(ip);
1349 u64 oldsize = inode->i_size;
1352 if (!gfs2_is_stuffed(ip)) {
1353 unsigned int blocksize = i_blocksize(inode);
1354 unsigned int offs = newsize & (blocksize - 1);
1356 error = gfs2_block_zero_range(inode, newsize,
1363 error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES);
1365 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1369 error = gfs2_meta_inode_buffer(ip, &dibh);
1373 gfs2_trans_add_meta(ip->i_gl, dibh);
1375 if (gfs2_is_stuffed(ip))
1376 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
1378 ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG;
1380 i_size_write(inode, newsize);
1381 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1382 gfs2_dinode_out(ip, dibh->b_data);
1385 error = gfs2_journaled_truncate(inode, oldsize, newsize);
1387 truncate_pagecache(inode, newsize);
1391 if (current->journal_info)
1392 gfs2_trans_end(sdp);
1396 int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
1397 struct iomap *iomap)
1399 struct metapath mp = { .mp_aheight = 1, };
1402 ret = __gfs2_iomap_get(inode, pos, length, 0, iomap, &mp);
1403 release_metapath(&mp);
1407 int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length,
1408 struct iomap *iomap)
1410 struct metapath mp = { .mp_aheight = 1, };
1413 ret = __gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp);
1414 if (!ret && iomap->type == IOMAP_HOLE)
1415 ret = __gfs2_iomap_alloc(inode, iomap, &mp);
1416 release_metapath(&mp);
1421 * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein
1423 * @rd_gh: holder of resource group glock
1424 * @bh: buffer head to sweep
1425 * @start: starting point in bh
1426 * @end: end point in bh
1427 * @meta: true if bh points to metadata (rather than data)
1428 * @btotal: place to keep count of total blocks freed
1430 * We sweep a metadata buffer (provided by the metapath) for blocks we need to
1431 * free, and free them all. However, we do it one rgrp at a time. If this
1432 * block has references to multiple rgrps, we break it into individual
1433 * transactions. This allows other processes to use the rgrps while we're
1434 * focused on a single one, for better concurrency / performance.
1435 * At every transaction boundary, we rewrite the inode into the journal.
1436 * That way the bitmaps are kept consistent with the inode and we can recover
1437 * if we're interrupted by power-outages.
1439 * Returns: 0, or return code if an error occurred.
1440 * *btotal has the total number of blocks freed
1442 static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
1443 struct buffer_head *bh, __be64 *start, __be64 *end,
1444 bool meta, u32 *btotal)
1446 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1447 struct gfs2_rgrpd *rgd;
1448 struct gfs2_trans *tr;
1450 int blks_outside_rgrp;
1451 u64 bn, bstart, isize_blks;
1452 s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */
1454 bool buf_in_tr = false; /* buffer was added to transaction */
1458 if (gfs2_holder_initialized(rd_gh)) {
1459 rgd = gfs2_glock2rgrp(rd_gh->gh_gl);
1460 gfs2_assert_withdraw(sdp,
1461 gfs2_glock_is_locked_by_me(rd_gh->gh_gl));
1463 blks_outside_rgrp = 0;
1467 for (p = start; p < end; p++) {
1470 bn = be64_to_cpu(*p);
1473 if (!rgrp_contains_block(rgd, bn)) {
1474 blks_outside_rgrp++;
1478 rgd = gfs2_blk2rgrpd(sdp, bn, true);
1479 if (unlikely(!rgd)) {
1483 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE,
1484 LM_FLAG_NODE_SCOPE, rd_gh);
1488 /* Must be done with the rgrp glock held: */
1489 if (gfs2_rs_active(&ip->i_res) &&
1490 rgd == ip->i_res.rs_rgd)
1491 gfs2_rs_deltree(&ip->i_res);
1494 /* The size of our transactions will be unknown until we
1495 actually process all the metadata blocks that relate to
1496 the rgrp. So we estimate. We know it can't be more than
1497 the dinode's i_blocks and we don't want to exceed the
1498 journal flush threshold, sd_log_thresh2. */
1499 if (current->journal_info == NULL) {
1500 unsigned int jblocks_rqsted, revokes;
1502 jblocks_rqsted = rgd->rd_length + RES_DINODE +
1504 isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
1505 if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
1507 atomic_read(&sdp->sd_log_thresh2);
1509 jblocks_rqsted += isize_blks;
1510 revokes = jblocks_rqsted;
1512 revokes += end - start;
1513 else if (ip->i_depth)
1514 revokes += sdp->sd_inptrs;
1515 ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes);
1518 down_write(&ip->i_rw_mutex);
1520 /* check if we will exceed the transaction blocks requested */
1521 tr = current->journal_info;
1522 if (tr->tr_num_buf_new + RES_STATFS +
1523 RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) {
1524 /* We set blks_outside_rgrp to ensure the loop will
1525 be repeated for the same rgrp, but with a new
1527 blks_outside_rgrp++;
1528 /* This next part is tricky. If the buffer was added
1529 to the transaction, we've already set some block
1530 pointers to 0, so we better follow through and free
1531 them, or we will introduce corruption (so break).
1532 This may be impossible, or at least rare, but I
1533 decided to cover the case regardless.
1535 If the buffer was not added to the transaction
1536 (this call), doing so would exceed our transaction
1537 size, so we need to end the transaction and start a
1538 new one (so goto). */
1545 gfs2_trans_add_meta(ip->i_gl, bh);
1548 if (bstart + blen == bn) {
1553 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1555 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1561 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1563 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1566 if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
1567 outside the rgrp we just processed,
1568 do it all over again. */
1569 if (current->journal_info) {
1570 struct buffer_head *dibh;
1572 ret = gfs2_meta_inode_buffer(ip, &dibh);
1576 /* Every transaction boundary, we rewrite the dinode
1577 to keep its di_blocks current in case of failure. */
1578 ip->i_inode.i_mtime = ip->i_inode.i_ctime =
1579 current_time(&ip->i_inode);
1580 gfs2_trans_add_meta(ip->i_gl, dibh);
1581 gfs2_dinode_out(ip, dibh->b_data);
1583 up_write(&ip->i_rw_mutex);
1584 gfs2_trans_end(sdp);
1587 gfs2_glock_dq_uninit(rd_gh);
1595 static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h)
1597 if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0])))
1603 * find_nonnull_ptr - find a non-null pointer given a metapath and height
1604 * @sdp: The superblock
1605 * @mp: starting metapath
1606 * @h: desired height to search
1607 * @end_list: See punch_hole().
1608 * @end_aligned: See punch_hole().
1610 * Assumes the metapath is valid (with buffers) out to height h.
1611 * Returns: true if a non-null pointer was found in the metapath buffer
1612 * false if all remaining pointers are NULL in the buffer
1614 static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp,
1616 __u16 *end_list, unsigned int end_aligned)
1618 struct buffer_head *bh = mp->mp_bh[h];
1619 __be64 *first, *ptr, *end;
1621 first = metaptr1(h, mp);
1622 ptr = first + mp->mp_list[h];
1623 end = (__be64 *)(bh->b_data + bh->b_size);
1624 if (end_list && mp_eq_to_hgt(mp, end_list, h)) {
1625 bool keep_end = h < end_aligned;
1626 end = first + end_list[h] + keep_end;
1630 if (*ptr) { /* if we have a non-null pointer */
1631 mp->mp_list[h] = ptr - first;
1633 if (h < GFS2_MAX_META_HEIGHT)
1642 enum dealloc_states {
1643 DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */
1644 DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */
1645 DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */
1646 DEALLOC_DONE = 3, /* process complete */
1650 metapointer_range(struct metapath *mp, int height,
1651 __u16 *start_list, unsigned int start_aligned,
1652 __u16 *end_list, unsigned int end_aligned,
1653 __be64 **start, __be64 **end)
1655 struct buffer_head *bh = mp->mp_bh[height];
1658 first = metaptr1(height, mp);
1660 if (mp_eq_to_hgt(mp, start_list, height)) {
1661 bool keep_start = height < start_aligned;
1662 *start = first + start_list[height] + keep_start;
1664 *end = (__be64 *)(bh->b_data + bh->b_size);
1665 if (end_list && mp_eq_to_hgt(mp, end_list, height)) {
1666 bool keep_end = height < end_aligned;
1667 *end = first + end_list[height] + keep_end;
1671 static inline bool walk_done(struct gfs2_sbd *sdp,
1672 struct metapath *mp, int height,
1673 __u16 *end_list, unsigned int end_aligned)
1678 bool keep_end = height < end_aligned;
1679 if (!mp_eq_to_hgt(mp, end_list, height))
1681 end = end_list[height] + keep_end;
1683 end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs;
1684 return mp->mp_list[height] >= end;
1688 * punch_hole - deallocate blocks in a file
1689 * @ip: inode to truncate
1690 * @offset: the start of the hole
1691 * @length: the size of the hole (or 0 for truncate)
1693 * Punch a hole into a file or truncate a file at a given position. This
1694 * function operates in whole blocks (@offset and @length are rounded
1695 * accordingly); partially filled blocks must be cleared otherwise.
1697 * This function works from the bottom up, and from the right to the left. In
1698 * other words, it strips off the highest layer (data) before stripping any of
1699 * the metadata. Doing it this way is best in case the operation is interrupted
1700 * by power failure, etc. The dinode is rewritten in every transaction to
1701 * guarantee integrity.
1703 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
1705 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1706 u64 maxsize = sdp->sd_heightsize[ip->i_height];
1707 struct metapath mp = {};
1708 struct buffer_head *dibh, *bh;
1709 struct gfs2_holder rd_gh;
1710 unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
1711 u64 lblock = (offset + (1 << bsize_shift) - 1) >> bsize_shift;
1712 __u16 start_list[GFS2_MAX_META_HEIGHT];
1713 __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL;
1714 unsigned int start_aligned, end_aligned;
1715 unsigned int strip_h = ip->i_height - 1;
1718 int mp_h; /* metapath buffers are read in to this height */
1720 __be64 *start, *end;
1722 if (offset >= maxsize) {
1724 * The starting point lies beyond the allocated meta-data;
1725 * there are no blocks do deallocate.
1731 * The start position of the hole is defined by lblock, start_list, and
1732 * start_aligned. The end position of the hole is defined by lend,
1733 * end_list, and end_aligned.
1735 * start_aligned and end_aligned define down to which height the start
1736 * and end positions are aligned to the metadata tree (i.e., the
1737 * position is a multiple of the metadata granularity at the height
1738 * above). This determines at which heights additional meta pointers
1739 * needs to be preserved for the remaining data.
1743 u64 end_offset = offset + length;
1747 * Clip the end at the maximum file size for the given height:
1748 * that's how far the metadata goes; files bigger than that
1749 * will have additional layers of indirection.
1751 if (end_offset > maxsize)
1752 end_offset = maxsize;
1753 lend = end_offset >> bsize_shift;
1758 find_metapath(sdp, lend, &mp, ip->i_height);
1759 end_list = __end_list;
1760 memcpy(end_list, mp.mp_list, sizeof(mp.mp_list));
1762 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1769 find_metapath(sdp, lblock, &mp, ip->i_height);
1770 memcpy(start_list, mp.mp_list, sizeof(start_list));
1772 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1773 if (start_list[mp_h])
1776 start_aligned = mp_h;
1778 ret = gfs2_meta_inode_buffer(ip, &dibh);
1783 ret = lookup_metapath(ip, &mp);
1787 /* issue read-ahead on metadata */
1788 for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
1789 metapointer_range(&mp, mp_h, start_list, start_aligned,
1790 end_list, end_aligned, &start, &end);
1791 gfs2_metapath_ra(ip->i_gl, start, end);
1794 if (mp.mp_aheight == ip->i_height)
1795 state = DEALLOC_MP_FULL; /* We have a complete metapath */
1797 state = DEALLOC_FILL_MP; /* deal with partial metapath */
1799 ret = gfs2_rindex_update(sdp);
1803 ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE);
1806 gfs2_holder_mark_uninitialized(&rd_gh);
1810 while (state != DEALLOC_DONE) {
1812 /* Truncate a full metapath at the given strip height.
1813 * Note that strip_h == mp_h in order to be in this state. */
1814 case DEALLOC_MP_FULL:
1815 bh = mp.mp_bh[mp_h];
1816 gfs2_assert_withdraw(sdp, bh);
1817 if (gfs2_assert_withdraw(sdp,
1818 prev_bnr != bh->b_blocknr)) {
1819 fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u,"
1820 "s_h:%u, mp_h:%u\n",
1821 (unsigned long long)ip->i_no_addr,
1822 prev_bnr, ip->i_height, strip_h, mp_h);
1824 prev_bnr = bh->b_blocknr;
1826 if (gfs2_metatype_check(sdp, bh,
1827 (mp_h ? GFS2_METATYPE_IN :
1828 GFS2_METATYPE_DI))) {
1834 * Below, passing end_aligned as 0 gives us the
1835 * metapointer range excluding the end point: the end
1836 * point is the first metapath we must not deallocate!
1839 metapointer_range(&mp, mp_h, start_list, start_aligned,
1840 end_list, 0 /* end_aligned */,
1842 ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h],
1844 mp_h != ip->i_height - 1,
1847 /* If we hit an error or just swept dinode buffer,
1850 state = DEALLOC_DONE;
1853 state = DEALLOC_MP_LOWER;
1856 /* lower the metapath strip height */
1857 case DEALLOC_MP_LOWER:
1858 /* We're done with the current buffer, so release it,
1859 unless it's the dinode buffer. Then back up to the
1860 previous pointer. */
1862 brelse(mp.mp_bh[mp_h]);
1863 mp.mp_bh[mp_h] = NULL;
1865 /* If we can't get any lower in height, we've stripped
1866 off all we can. Next step is to back up and start
1867 stripping the previous level of metadata. */
1870 memcpy(mp.mp_list, start_list, sizeof(start_list));
1872 state = DEALLOC_FILL_MP;
1875 mp.mp_list[mp_h] = 0;
1876 mp_h--; /* search one metadata height down */
1878 if (walk_done(sdp, &mp, mp_h, end_list, end_aligned))
1880 /* Here we've found a part of the metapath that is not
1881 * allocated. We need to search at that height for the
1882 * next non-null pointer. */
1883 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) {
1884 state = DEALLOC_FILL_MP;
1887 /* No more non-null pointers at this height. Back up
1888 to the previous height and try again. */
1889 break; /* loop around in the same state */
1891 /* Fill the metapath with buffers to the given height. */
1892 case DEALLOC_FILL_MP:
1893 /* Fill the buffers out to the current height. */
1894 ret = fillup_metapath(ip, &mp, mp_h);
1898 /* On the first pass, issue read-ahead on metadata. */
1899 if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) {
1900 unsigned int height = mp.mp_aheight - 1;
1902 /* No read-ahead for data blocks. */
1903 if (mp.mp_aheight - 1 == strip_h)
1906 for (; height >= mp.mp_aheight - ret; height--) {
1907 metapointer_range(&mp, height,
1908 start_list, start_aligned,
1909 end_list, end_aligned,
1911 gfs2_metapath_ra(ip->i_gl, start, end);
1915 /* If buffers found for the entire strip height */
1916 if (mp.mp_aheight - 1 == strip_h) {
1917 state = DEALLOC_MP_FULL;
1920 if (mp.mp_aheight < ip->i_height) /* We have a partial height */
1921 mp_h = mp.mp_aheight - 1;
1923 /* If we find a non-null block pointer, crawl a bit
1924 higher up in the metapath and try again, otherwise
1925 we need to look lower for a new starting point. */
1926 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned))
1929 state = DEALLOC_MP_LOWER;
1935 if (current->journal_info == NULL) {
1936 ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS +
1940 down_write(&ip->i_rw_mutex);
1942 gfs2_statfs_change(sdp, 0, +btotal, 0);
1943 gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
1945 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1946 gfs2_trans_add_meta(ip->i_gl, dibh);
1947 gfs2_dinode_out(ip, dibh->b_data);
1948 up_write(&ip->i_rw_mutex);
1949 gfs2_trans_end(sdp);
1953 if (gfs2_holder_initialized(&rd_gh))
1954 gfs2_glock_dq_uninit(&rd_gh);
1955 if (current->journal_info) {
1956 up_write(&ip->i_rw_mutex);
1957 gfs2_trans_end(sdp);
1960 gfs2_quota_unhold(ip);
1962 release_metapath(&mp);
1966 static int trunc_end(struct gfs2_inode *ip)
1968 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1969 struct buffer_head *dibh;
1972 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1976 down_write(&ip->i_rw_mutex);
1978 error = gfs2_meta_inode_buffer(ip, &dibh);
1982 if (!i_size_read(&ip->i_inode)) {
1984 ip->i_goal = ip->i_no_addr;
1985 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
1986 gfs2_ordered_del_inode(ip);
1988 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1989 ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
1991 gfs2_trans_add_meta(ip->i_gl, dibh);
1992 gfs2_dinode_out(ip, dibh->b_data);
1996 up_write(&ip->i_rw_mutex);
1997 gfs2_trans_end(sdp);
2002 * do_shrink - make a file smaller
2004 * @newsize: the size to make the file
2006 * Called with an exclusive lock on @inode. The @size must
2007 * be equal to or smaller than the current inode size.
2012 static int do_shrink(struct inode *inode, u64 newsize)
2014 struct gfs2_inode *ip = GFS2_I(inode);
2017 error = trunc_start(inode, newsize);
2020 if (gfs2_is_stuffed(ip))
2023 error = punch_hole(ip, newsize, 0);
2025 error = trunc_end(ip);
2030 void gfs2_trim_blocks(struct inode *inode)
2034 ret = do_shrink(inode, inode->i_size);
2039 * do_grow - Touch and update inode size
2041 * @size: The new size
2043 * This function updates the timestamps on the inode and
2044 * may also increase the size of the inode. This function
2045 * must not be called with @size any smaller than the current
2048 * Although it is not strictly required to unstuff files here,
2049 * earlier versions of GFS2 have a bug in the stuffed file reading
2050 * code which will result in a buffer overrun if the size is larger
2051 * than the max stuffed file size. In order to prevent this from
2052 * occurring, such files are unstuffed, but in other cases we can
2053 * just update the inode size directly.
2055 * Returns: 0 on success, or -ve on error
2058 static int do_grow(struct inode *inode, u64 size)
2060 struct gfs2_inode *ip = GFS2_I(inode);
2061 struct gfs2_sbd *sdp = GFS2_SB(inode);
2062 struct gfs2_alloc_parms ap = { .target = 1, };
2063 struct buffer_head *dibh;
2067 if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
2068 error = gfs2_quota_lock_check(ip, &ap);
2072 error = gfs2_inplace_reserve(ip, &ap);
2074 goto do_grow_qunlock;
2078 error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT +
2080 gfs2_is_jdata(ip) ? RES_JDATA : 0) +
2081 (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ?
2084 goto do_grow_release;
2087 error = gfs2_unstuff_dinode(ip);
2092 error = gfs2_meta_inode_buffer(ip, &dibh);
2096 truncate_setsize(inode, size);
2097 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
2098 gfs2_trans_add_meta(ip->i_gl, dibh);
2099 gfs2_dinode_out(ip, dibh->b_data);
2103 gfs2_trans_end(sdp);
2106 gfs2_inplace_release(ip);
2108 gfs2_quota_unlock(ip);
2114 * gfs2_setattr_size - make a file a given size
2116 * @newsize: the size to make the file
2118 * The file size can grow, shrink, or stay the same size. This
2119 * is called holding i_rwsem and an exclusive glock on the inode
2125 int gfs2_setattr_size(struct inode *inode, u64 newsize)
2127 struct gfs2_inode *ip = GFS2_I(inode);
2130 BUG_ON(!S_ISREG(inode->i_mode));
2132 ret = inode_newsize_ok(inode, newsize);
2136 inode_dio_wait(inode);
2138 ret = gfs2_qa_get(ip);
2142 if (newsize >= inode->i_size) {
2143 ret = do_grow(inode, newsize);
2147 ret = do_shrink(inode, newsize);
2154 int gfs2_truncatei_resume(struct gfs2_inode *ip)
2157 error = punch_hole(ip, i_size_read(&ip->i_inode), 0);
2159 error = trunc_end(ip);
2163 int gfs2_file_dealloc(struct gfs2_inode *ip)
2165 return punch_hole(ip, 0, 0);
2169 * gfs2_free_journal_extents - Free cached journal bmap info
2174 void gfs2_free_journal_extents(struct gfs2_jdesc *jd)
2176 struct gfs2_journal_extent *jext;
2178 while(!list_empty(&jd->extent_list)) {
2179 jext = list_first_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2180 list_del(&jext->list);
2186 * gfs2_add_jextent - Add or merge a new extent to extent cache
2187 * @jd: The journal descriptor
2188 * @lblock: The logical block at start of new extent
2189 * @dblock: The physical block at start of new extent
2190 * @blocks: Size of extent in fs blocks
2192 * Returns: 0 on success or -ENOMEM
2195 static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks)
2197 struct gfs2_journal_extent *jext;
2199 if (!list_empty(&jd->extent_list)) {
2200 jext = list_last_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2201 if ((jext->dblock + jext->blocks) == dblock) {
2202 jext->blocks += blocks;
2207 jext = kzalloc(sizeof(struct gfs2_journal_extent), GFP_NOFS);
2210 jext->dblock = dblock;
2211 jext->lblock = lblock;
2212 jext->blocks = blocks;
2213 list_add_tail(&jext->list, &jd->extent_list);
2219 * gfs2_map_journal_extents - Cache journal bmap info
2220 * @sdp: The super block
2221 * @jd: The journal to map
2223 * Create a reusable "extent" mapping from all logical
2224 * blocks to all physical blocks for the given journal. This will save
2225 * us time when writing journal blocks. Most journals will have only one
2226 * extent that maps all their logical blocks. That's because gfs2.mkfs
2227 * arranges the journal blocks sequentially to maximize performance.
2228 * So the extent would map the first block for the entire file length.
2229 * However, gfs2_jadd can happen while file activity is happening, so
2230 * those journals may not be sequential. Less likely is the case where
2231 * the users created their own journals by mounting the metafs and
2232 * laying it out. But it's still possible. These journals might have
2235 * Returns: 0 on success, or error on failure
2238 int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
2242 struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
2243 struct buffer_head bh;
2244 unsigned int shift = sdp->sd_sb.sb_bsize_shift;
2249 start = ktime_get();
2250 lblock_stop = i_size_read(jd->jd_inode) >> shift;
2251 size = (lblock_stop - lblock) << shift;
2253 WARN_ON(!list_empty(&jd->extent_list));
2259 rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0);
2260 if (rc || !buffer_mapped(&bh))
2262 rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift);
2266 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2270 fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid,
2271 jd->nr_extents, ktime_ms_delta(end, start));
2275 fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n",
2277 (unsigned long long)(i_size_read(jd->jd_inode) - size),
2279 fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n",
2280 rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr,
2281 bh.b_state, (unsigned long long)bh.b_size);
2282 gfs2_free_journal_extents(jd);
2287 * gfs2_write_alloc_required - figure out if a write will require an allocation
2288 * @ip: the file being written to
2289 * @offset: the offset to write to
2290 * @len: the number of bytes being written
2292 * Returns: 1 if an alloc is required, 0 otherwise
2295 int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
2298 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2299 struct buffer_head bh;
2301 u64 lblock, lblock_stop, size;
2307 if (gfs2_is_stuffed(ip)) {
2308 if (offset + len > gfs2_max_stuffed_size(ip))
2313 shift = sdp->sd_sb.sb_bsize_shift;
2314 BUG_ON(gfs2_is_dir(ip));
2315 end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift;
2316 lblock = offset >> shift;
2317 lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift;
2318 if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex))
2321 size = (lblock_stop - lblock) << shift;
2325 gfs2_block_map(&ip->i_inode, lblock, &bh, 0);
2326 if (!buffer_mapped(&bh))
2329 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2335 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
2337 struct gfs2_inode *ip = GFS2_I(inode);
2338 struct buffer_head *dibh;
2341 if (offset >= inode->i_size)
2343 if (offset + length > inode->i_size)
2344 length = inode->i_size - offset;
2346 error = gfs2_meta_inode_buffer(ip, &dibh);
2349 gfs2_trans_add_meta(ip->i_gl, dibh);
2350 memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
2356 static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
2359 struct gfs2_sbd *sdp = GFS2_SB(inode);
2360 loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
2364 struct gfs2_trans *tr;
2369 if (chunk > max_chunk)
2372 offs = offset & ~PAGE_MASK;
2373 if (offs && chunk > PAGE_SIZE)
2374 chunk = offs + ((chunk - offs) & PAGE_MASK);
2376 truncate_pagecache_range(inode, offset, chunk);
2380 tr = current->journal_info;
2381 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
2384 gfs2_trans_end(sdp);
2385 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
2392 int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length)
2394 struct inode *inode = file_inode(file);
2395 struct gfs2_inode *ip = GFS2_I(inode);
2396 struct gfs2_sbd *sdp = GFS2_SB(inode);
2397 unsigned int blocksize = i_blocksize(inode);
2401 if (!gfs2_is_stuffed(ip)) {
2402 unsigned int start_off, end_len;
2404 start_off = offset & (blocksize - 1);
2405 end_len = (offset + length) & (blocksize - 1);
2407 unsigned int len = length;
2408 if (length > blocksize - start_off)
2409 len = blocksize - start_off;
2410 error = gfs2_block_zero_range(inode, offset, len);
2413 if (start_off + length < blocksize)
2417 error = gfs2_block_zero_range(inode,
2418 offset + length - end_len, end_len);
2424 start = round_down(offset, blocksize);
2425 end = round_up(offset + length, blocksize) - 1;
2426 error = filemap_write_and_wait_range(inode->i_mapping, start, end);
2430 if (gfs2_is_jdata(ip))
2431 error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
2432 GFS2_JTRUNC_REVOKES);
2434 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2438 if (gfs2_is_stuffed(ip)) {
2439 error = stuffed_zero_range(inode, offset, length);
2444 if (gfs2_is_jdata(ip)) {
2445 BUG_ON(!current->journal_info);
2446 gfs2_journaled_truncate_range(inode, offset, length);
2448 truncate_pagecache_range(inode, offset, offset + length - 1);
2450 file_update_time(file);
2451 mark_inode_dirty(inode);
2453 if (current->journal_info)
2454 gfs2_trans_end(sdp);
2456 if (!gfs2_is_stuffed(ip))
2457 error = punch_hole(ip, offset, length);
2460 if (current->journal_info)
2461 gfs2_trans_end(sdp);
2465 static int gfs2_map_blocks(struct iomap_writepage_ctx *wpc, struct inode *inode,
2470 if (WARN_ON_ONCE(gfs2_is_stuffed(GFS2_I(inode))))
2473 if (offset >= wpc->iomap.offset &&
2474 offset < wpc->iomap.offset + wpc->iomap.length)
2477 memset(&wpc->iomap, 0, sizeof(wpc->iomap));
2478 ret = gfs2_iomap_get(inode, offset, INT_MAX, &wpc->iomap);
2482 const struct iomap_writeback_ops gfs2_writeback_ops = {
2483 .map_blocks = gfs2_map_blocks,