block: avoid extra iter advance with async iocb

Nobody cares about iov iterators state if we return -EIOCBQUEUED, so as
the we now have __blkdev_direct_IO_async(), which gets pages only once,
we can skip expensive iov_iter_advance(). It's around 1-2% of all CPU
spent.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/a6158edfbfa2ae3bc24aed29a72f035df18fad2f.1635337135.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio.c b/block/bio.c
index ead1f8a..15ab0d6 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -1046,7 +1046,7 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty)
 }
 EXPORT_SYMBOL_GPL(__bio_release_pages);
 
-static void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
+void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
 {
        size_t size = iov_iter_count(iter);
 

diff --git a/block/fops.c b/block/fops.c
index a7b3282..092e507 100644 (file)
--- a/block/fops.c
+++ b/block/fops.c
@@ -352,11 +352,21 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,
        bio->bi_end_io = blkdev_bio_end_io_async;
        bio->bi_ioprio = iocb->ki_ioprio;
 
-       ret = bio_iov_iter_get_pages(bio, iter);
-       if (unlikely(ret)) {
-               bio->bi_status = BLK_STS_IOERR;
-               bio_endio(bio);
-               return ret;
+       if (iov_iter_is_bvec(iter)) {
+               /*
+                * Users don't rely on the iterator being in any particular
+                * state for async I/O returning -EIOCBQUEUED, hence we can
+                * avoid expensive iov_iter_advance(). Bypass
+                * bio_iov_iter_get_pages() and set the bvec directly.
+                */
+               bio_iov_bvec_set(bio, iter);
+       } else {
+               ret = bio_iov_iter_get_pages(bio, iter);
+               if (unlikely(ret)) {
+                       bio->bi_status = BLK_STS_IOERR;
+                       bio_endio(bio);
+                       return ret;
+               }
        }
        dio->size = bio->bi_iter.bi_size;
 

diff --git a/include/linux/bio.h b/include/linux/bio.h
index c88700d..fe6bdfb 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -417,6 +417,7 @@ int bio_add_zone_append_page(struct bio *bio, struct page *page,
 void __bio_add_page(struct bio *bio, struct page *page,
                unsigned int len, unsigned int off);
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
+void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
 void __bio_release_pages(struct bio *bio, bool mark_dirty);
 extern void bio_set_pages_dirty(struct bio *bio);
 extern void bio_check_pages_dirty(struct bio *bio);