1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
8 #include <linux/mempool.h>
9 /* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
10 #include <linux/blk_types.h>
11 #include <linux/uio.h>
13 #define BIO_MAX_VECS 256U
15 static inline unsigned int bio_max_segs(unsigned int nr_segs)
17 return min(nr_segs, BIO_MAX_VECS);
20 #define bio_prio(bio) (bio)->bi_ioprio
21 #define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio)
23 #define bio_iter_iovec(bio, iter) \
24 bvec_iter_bvec((bio)->bi_io_vec, (iter))
26 #define bio_iter_page(bio, iter) \
27 bvec_iter_page((bio)->bi_io_vec, (iter))
28 #define bio_iter_len(bio, iter) \
29 bvec_iter_len((bio)->bi_io_vec, (iter))
30 #define bio_iter_offset(bio, iter) \
31 bvec_iter_offset((bio)->bi_io_vec, (iter))
33 #define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
34 #define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
35 #define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
37 #define bvec_iter_sectors(iter) ((iter).bi_size >> 9)
38 #define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))
40 #define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter)
41 #define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter)
44 * Return the data direction, READ or WRITE.
46 #define bio_data_dir(bio) \
47 (op_is_write(bio_op(bio)) ? WRITE : READ)
50 * Check whether this bio carries any data or not. A NULL bio is allowed.
52 static inline bool bio_has_data(struct bio *bio)
55 bio->bi_iter.bi_size &&
56 bio_op(bio) != REQ_OP_DISCARD &&
57 bio_op(bio) != REQ_OP_SECURE_ERASE &&
58 bio_op(bio) != REQ_OP_WRITE_ZEROES)
64 static inline bool bio_no_advance_iter(const struct bio *bio)
66 return bio_op(bio) == REQ_OP_DISCARD ||
67 bio_op(bio) == REQ_OP_SECURE_ERASE ||
68 bio_op(bio) == REQ_OP_WRITE_ZEROES;
71 static inline void *bio_data(struct bio *bio)
73 if (bio_has_data(bio))
74 return page_address(bio_page(bio)) + bio_offset(bio);
79 static inline bool bio_next_segment(const struct bio *bio,
80 struct bvec_iter_all *iter)
82 if (iter->idx >= bio->bi_vcnt)
85 bvec_advance(&bio->bi_io_vec[iter->idx], iter);
90 * drivers should _never_ use the all version - the bio may have been split
91 * before it got to the driver and the driver won't own all of it
93 #define bio_for_each_segment_all(bvl, bio, iter) \
94 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
96 static inline void bio_advance_iter(const struct bio *bio,
97 struct bvec_iter *iter, unsigned int bytes)
99 iter->bi_sector += bytes >> 9;
101 if (bio_no_advance_iter(bio))
102 iter->bi_size -= bytes;
104 bvec_iter_advance(bio->bi_io_vec, iter, bytes);
105 /* TODO: It is reasonable to complete bio with error here. */
108 /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */
109 static inline void bio_advance_iter_single(const struct bio *bio,
110 struct bvec_iter *iter,
113 iter->bi_sector += bytes >> 9;
115 if (bio_no_advance_iter(bio))
116 iter->bi_size -= bytes;
118 bvec_iter_advance_single(bio->bi_io_vec, iter, bytes);
121 void __bio_advance(struct bio *, unsigned bytes);
124 * bio_advance - increment/complete a bio by some number of bytes
125 * @bio: bio to advance
126 * @nbytes: number of bytes to complete
128 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
129 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
130 * be updated on the last bvec as well.
132 * @bio will then represent the remaining, uncompleted portion of the io.
134 static inline void bio_advance(struct bio *bio, unsigned int nbytes)
136 if (nbytes == bio->bi_iter.bi_size) {
137 bio->bi_iter.bi_size = 0;
140 __bio_advance(bio, nbytes);
143 #define __bio_for_each_segment(bvl, bio, iter, start) \
144 for (iter = (start); \
146 ((bvl = bio_iter_iovec((bio), (iter))), 1); \
147 bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
149 #define bio_for_each_segment(bvl, bio, iter) \
150 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
152 #define __bio_for_each_bvec(bvl, bio, iter, start) \
153 for (iter = (start); \
155 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
156 bio_advance_iter_single((bio), &(iter), (bvl).bv_len))
158 /* iterate over multi-page bvec */
159 #define bio_for_each_bvec(bvl, bio, iter) \
160 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
163 * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the
164 * same reasons as bio_for_each_segment_all().
166 #define bio_for_each_bvec_all(bvl, bio, i) \
167 for (i = 0, bvl = bio_first_bvec_all(bio); \
168 i < (bio)->bi_vcnt; i++, bvl++)
170 #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
172 static inline unsigned bio_segments(struct bio *bio)
176 struct bvec_iter iter;
179 * We special case discard/write same/write zeroes, because they
180 * interpret bi_size differently:
183 switch (bio_op(bio)) {
185 case REQ_OP_SECURE_ERASE:
186 case REQ_OP_WRITE_ZEROES:
192 bio_for_each_segment(bv, bio, iter)
199 * get a reference to a bio, so it won't disappear. the intended use is
203 * submit_bio(rw, bio);
204 * if (bio->bi_flags ...)
208 * without the bio_get(), it could potentially complete I/O before submit_bio
209 * returns. and then bio would be freed memory when if (bio->bi_flags ...)
212 static inline void bio_get(struct bio *bio)
214 bio->bi_flags |= (1 << BIO_REFFED);
215 smp_mb__before_atomic();
216 atomic_inc(&bio->__bi_cnt);
219 static inline void bio_cnt_set(struct bio *bio, unsigned int count)
222 bio->bi_flags |= (1 << BIO_REFFED);
225 atomic_set(&bio->__bi_cnt, count);
228 static inline bool bio_flagged(struct bio *bio, unsigned int bit)
230 return (bio->bi_flags & (1U << bit)) != 0;
233 static inline void bio_set_flag(struct bio *bio, unsigned int bit)
235 bio->bi_flags |= (1U << bit);
238 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
240 bio->bi_flags &= ~(1U << bit);
243 static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
245 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
246 return bio->bi_io_vec;
249 static inline struct page *bio_first_page_all(struct bio *bio)
251 return bio_first_bvec_all(bio)->bv_page;
254 static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
256 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
257 return &bio->bi_io_vec[bio->bi_vcnt - 1];
261 * struct folio_iter - State for iterating all folios in a bio.
262 * @folio: The current folio we're iterating. NULL after the last folio.
263 * @offset: The byte offset within the current folio.
264 * @length: The number of bytes in this iteration (will not cross folio
271 /* private: for use by the iterator */
276 static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio,
279 struct bio_vec *bvec = bio_first_bvec_all(bio) + i;
281 fi->folio = page_folio(bvec->bv_page);
282 fi->offset = bvec->bv_offset +
283 PAGE_SIZE * (bvec->bv_page - &fi->folio->page);
284 fi->_seg_count = bvec->bv_len;
285 fi->length = min(folio_size(fi->folio) - fi->offset, fi->_seg_count);
289 static inline void bio_next_folio(struct folio_iter *fi, struct bio *bio)
291 fi->_seg_count -= fi->length;
292 if (fi->_seg_count) {
293 fi->folio = folio_next(fi->folio);
295 fi->length = min(folio_size(fi->folio), fi->_seg_count);
296 } else if (fi->_i + 1 < bio->bi_vcnt) {
297 bio_first_folio(fi, bio, fi->_i + 1);
304 * bio_for_each_folio_all - Iterate over each folio in a bio.
305 * @fi: struct folio_iter which is updated for each folio.
306 * @bio: struct bio to iterate over.
308 #define bio_for_each_folio_all(fi, bio) \
309 for (bio_first_folio(&fi, bio, 0); fi.folio; bio_next_folio(&fi, bio))
312 BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
313 BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */
314 BIP_CTRL_NOCHECK = 1 << 2, /* disable HBA integrity checking */
315 BIP_DISK_NOCHECK = 1 << 3, /* disable disk integrity checking */
316 BIP_IP_CHECKSUM = 1 << 4, /* IP checksum */
320 * bio integrity payload
322 struct bio_integrity_payload {
323 struct bio *bip_bio; /* parent bio */
325 struct bvec_iter bip_iter;
327 unsigned short bip_vcnt; /* # of integrity bio_vecs */
328 unsigned short bip_max_vcnt; /* integrity bio_vec slots */
329 unsigned short bip_flags; /* control flags */
331 struct bvec_iter bio_iter; /* for rewinding parent bio */
333 struct work_struct bip_work; /* I/O completion */
335 struct bio_vec *bip_vec;
336 struct bio_vec bip_inline_vecs[];/* embedded bvec array */
339 #if defined(CONFIG_BLK_DEV_INTEGRITY)
341 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
343 if (bio->bi_opf & REQ_INTEGRITY)
344 return bio->bi_integrity;
349 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
351 struct bio_integrity_payload *bip = bio_integrity(bio);
354 return bip->bip_flags & flag;
359 static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
361 return bip->bip_iter.bi_sector;
364 static inline void bip_set_seed(struct bio_integrity_payload *bip,
367 bip->bip_iter.bi_sector = seed;
370 #endif /* CONFIG_BLK_DEV_INTEGRITY */
372 void bio_trim(struct bio *bio, sector_t offset, sector_t size);
373 extern struct bio *bio_split(struct bio *bio, int sectors,
374 gfp_t gfp, struct bio_set *bs);
377 * bio_next_split - get next @sectors from a bio, splitting if necessary
379 * @sectors: number of sectors to split from the front of @bio
381 * @bs: bio set to allocate from
383 * Return: a bio representing the next @sectors of @bio - if the bio is smaller
384 * than @sectors, returns the original bio unchanged.
386 static inline struct bio *bio_next_split(struct bio *bio, int sectors,
387 gfp_t gfp, struct bio_set *bs)
389 if (sectors >= bio_sectors(bio))
392 return bio_split(bio, sectors, gfp, bs);
396 BIOSET_NEED_BVECS = BIT(0),
397 BIOSET_NEED_RESCUER = BIT(1),
398 BIOSET_PERCPU_CACHE = BIT(2),
400 extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
401 extern void bioset_exit(struct bio_set *);
402 extern int biovec_init_pool(mempool_t *pool, int pool_entries);
403 extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
405 struct bio *bio_alloc_bioset(gfp_t gfp, unsigned short nr_iovecs,
407 struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs,
409 struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs);
410 extern void bio_put(struct bio *);
412 extern void __bio_clone_fast(struct bio *, struct bio *);
413 extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
415 extern struct bio_set fs_bio_set;
417 static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned short nr_iovecs)
419 return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
422 void submit_bio(struct bio *bio);
424 extern void bio_endio(struct bio *);
426 static inline void bio_io_error(struct bio *bio)
428 bio->bi_status = BLK_STS_IOERR;
432 static inline void bio_wouldblock_error(struct bio *bio)
434 bio_set_flag(bio, BIO_QUIET);
435 bio->bi_status = BLK_STS_AGAIN;
440 * Calculate number of bvec segments that should be allocated to fit data
441 * pointed by @iter. If @iter is backed by bvec it's going to be reused
442 * instead of allocating a new one.
444 static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
446 if (iov_iter_is_bvec(iter))
448 return iov_iter_npages(iter, max_segs);
451 struct request_queue;
453 extern int submit_bio_wait(struct bio *bio);
454 extern void bio_init(struct bio *bio, struct bio_vec *table,
455 unsigned short max_vecs);
456 extern void bio_uninit(struct bio *);
457 extern void bio_reset(struct bio *);
458 void bio_chain(struct bio *, struct bio *);
460 int bio_add_page(struct bio *, struct page *, unsigned len, unsigned off);
461 bool bio_add_folio(struct bio *, struct folio *, size_t len, size_t off);
462 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
463 unsigned int, unsigned int);
464 int bio_add_zone_append_page(struct bio *bio, struct page *page,
465 unsigned int len, unsigned int offset);
466 void __bio_add_page(struct bio *bio, struct page *page,
467 unsigned int len, unsigned int off);
468 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
469 void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
470 void __bio_release_pages(struct bio *bio, bool mark_dirty);
471 extern void bio_set_pages_dirty(struct bio *bio);
472 extern void bio_check_pages_dirty(struct bio *bio);
474 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
475 struct bio *src, struct bvec_iter *src_iter);
476 extern void bio_copy_data(struct bio *dst, struct bio *src);
477 extern void bio_free_pages(struct bio *bio);
478 void guard_bio_eod(struct bio *bio);
479 void zero_fill_bio(struct bio *bio);
481 static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
483 if (!bio_flagged(bio, BIO_NO_PAGE_REF))
484 __bio_release_pages(bio, mark_dirty);
487 extern const char *bio_devname(struct bio *bio, char *buffer);
489 #define bio_dev(bio) \
490 disk_devt((bio)->bi_bdev->bd_disk)
492 #ifdef CONFIG_BLK_CGROUP
493 void bio_associate_blkg(struct bio *bio);
494 void bio_associate_blkg_from_css(struct bio *bio,
495 struct cgroup_subsys_state *css);
496 void bio_clone_blkg_association(struct bio *dst, struct bio *src);
497 #else /* CONFIG_BLK_CGROUP */
498 static inline void bio_associate_blkg(struct bio *bio) { }
499 static inline void bio_associate_blkg_from_css(struct bio *bio,
500 struct cgroup_subsys_state *css)
502 static inline void bio_clone_blkg_association(struct bio *dst,
504 #endif /* CONFIG_BLK_CGROUP */
506 static inline void bio_set_dev(struct bio *bio, struct block_device *bdev)
508 bio_clear_flag(bio, BIO_REMAPPED);
509 if (bio->bi_bdev != bdev)
510 bio_clear_flag(bio, BIO_THROTTLED);
512 bio_associate_blkg(bio);
515 static inline void bio_copy_dev(struct bio *dst, struct bio *src)
517 bio_clear_flag(dst, BIO_REMAPPED);
518 dst->bi_bdev = src->bi_bdev;
519 bio_clone_blkg_association(dst, src);
523 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
525 * A bio_list anchors a singly-linked list of bios chained through the bi_next
526 * member of the bio. The bio_list also caches the last list member to allow
527 * fast access to the tail.
534 static inline int bio_list_empty(const struct bio_list *bl)
536 return bl->head == NULL;
539 static inline void bio_list_init(struct bio_list *bl)
541 bl->head = bl->tail = NULL;
544 #define BIO_EMPTY_LIST { NULL, NULL }
546 #define bio_list_for_each(bio, bl) \
547 for (bio = (bl)->head; bio; bio = bio->bi_next)
549 static inline unsigned bio_list_size(const struct bio_list *bl)
554 bio_list_for_each(bio, bl)
560 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
565 bl->tail->bi_next = bio;
572 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
574 bio->bi_next = bl->head;
582 static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
588 bl->tail->bi_next = bl2->head;
590 bl->head = bl2->head;
592 bl->tail = bl2->tail;
595 static inline void bio_list_merge_head(struct bio_list *bl,
596 struct bio_list *bl2)
602 bl2->tail->bi_next = bl->head;
604 bl->tail = bl2->tail;
606 bl->head = bl2->head;
609 static inline struct bio *bio_list_peek(struct bio_list *bl)
614 static inline struct bio *bio_list_pop(struct bio_list *bl)
616 struct bio *bio = bl->head;
619 bl->head = bl->head->bi_next;
629 static inline struct bio *bio_list_get(struct bio_list *bl)
631 struct bio *bio = bl->head;
633 bl->head = bl->tail = NULL;
639 * Increment chain count for the bio. Make sure the CHAIN flag update
640 * is visible before the raised count.
642 static inline void bio_inc_remaining(struct bio *bio)
644 bio_set_flag(bio, BIO_CHAIN);
645 smp_mb__before_atomic();
646 atomic_inc(&bio->__bi_remaining);
650 * bio_set is used to allow other portions of the IO system to
651 * allocate their own private memory pools for bio and iovec structures.
652 * These memory pools in turn all allocate from the bio_slab
653 * and the bvec_slabs[].
655 #define BIO_POOL_SIZE 2
658 struct kmem_cache *bio_slab;
659 unsigned int front_pad;
662 * per-cpu bio alloc cache
664 struct bio_alloc_cache __percpu *cache;
668 #if defined(CONFIG_BLK_DEV_INTEGRITY)
669 mempool_t bio_integrity_pool;
670 mempool_t bvec_integrity_pool;
673 unsigned int back_pad;
675 * Deadlock avoidance for stacking block drivers: see comments in
676 * bio_alloc_bioset() for details
678 spinlock_t rescue_lock;
679 struct bio_list rescue_list;
680 struct work_struct rescue_work;
681 struct workqueue_struct *rescue_workqueue;
684 * Hot un-plug notifier for the per-cpu cache, if used
686 struct hlist_node cpuhp_dead;
689 static inline bool bioset_initialized(struct bio_set *bs)
691 return bs->bio_slab != NULL;
694 #if defined(CONFIG_BLK_DEV_INTEGRITY)
696 #define bip_for_each_vec(bvl, bip, iter) \
697 for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
699 #define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
701 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
703 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
704 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
705 extern bool bio_integrity_prep(struct bio *);
706 extern void bio_integrity_advance(struct bio *, unsigned int);
707 extern void bio_integrity_trim(struct bio *);
708 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
709 extern int bioset_integrity_create(struct bio_set *, int);
710 extern void bioset_integrity_free(struct bio_set *);
711 extern void bio_integrity_init(void);
713 #else /* CONFIG_BLK_DEV_INTEGRITY */
715 static inline void *bio_integrity(struct bio *bio)
720 static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
725 static inline void bioset_integrity_free (struct bio_set *bs)
730 static inline bool bio_integrity_prep(struct bio *bio)
735 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
741 static inline void bio_integrity_advance(struct bio *bio,
742 unsigned int bytes_done)
747 static inline void bio_integrity_trim(struct bio *bio)
752 static inline void bio_integrity_init(void)
757 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
762 static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
765 return ERR_PTR(-EINVAL);
768 static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
769 unsigned int len, unsigned int offset)
774 #endif /* CONFIG_BLK_DEV_INTEGRITY */
777 * Mark a bio as polled. Note that for async polled IO, the caller must
778 * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
779 * We cannot block waiting for requests on polled IO, as those completions
780 * must be found by the caller. This is different than IRQ driven IO, where
781 * it's safe to wait for IO to complete.
783 static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
785 bio->bi_opf |= REQ_POLLED;
786 if (!is_sync_kiocb(kiocb))
787 bio->bi_opf |= REQ_NOWAIT;
790 struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp);
792 #endif /* __LINUX_BIO_H */