block: store bdev->bd_disk->fops->submit_bio state in bdev

We have a long chain of memory dereferencing just to whether or not
this disk has a special submit_bio helper. As that's not necessarily
the common case, add a bd_has_submit_bio state in the bdev to avoid
traversing this memory dependency chain if we don't need to.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bdev.c b/block/bdev.c
index 1795c7d4b99efabc2f071881bbd30b59de923723..850852fe4b78e15b2bb603c7f7c447fc6ab57941 100644 (file)
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -419,6 +419,7 @@ struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
        bdev->bd_inode = inode;
        bdev->bd_queue = disk->queue;
        bdev->bd_stats = alloc_percpu(struct disk_stats);
+       bdev->bd_has_submit_bio = false;
        if (!bdev->bd_stats) {
                iput(inode);
                return NULL;

diff --git a/block/blk-core.c b/block/blk-core.c
index 269765d16cfd9c65732bd6804dda54cbe0009958..06bdb352568c199557f319de5ae25e27139a9097 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -587,14 +587,14 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q,
 
 static void __submit_bio(struct bio *bio)
 {
-       struct gendisk *disk = bio->bi_bdev->bd_disk;
-
        if (unlikely(!blk_crypto_bio_prep(&bio)))
                return;
 
-       if (!disk->fops->submit_bio) {
+       if (!bio->bi_bdev->bd_has_submit_bio) {
                blk_mq_submit_bio(bio);
        } else if (likely(bio_queue_enter(bio) == 0)) {
+               struct gendisk *disk = bio->bi_bdev->bd_disk;
+
                disk->fops->submit_bio(bio);
                blk_queue_exit(disk->queue);
        }
@@ -698,7 +698,7 @@ void submit_bio_noacct_nocheck(struct bio *bio)
         */
        if (current->bio_list)
                bio_list_add(&current->bio_list[0], bio);
-       else if (!bio->bi_bdev->bd_disk->fops->submit_bio)
+       else if (!bio->bi_bdev->bd_has_submit_bio)
                __submit_bio_noacct_mq(bio);
        else
                __submit_bio_noacct(bio);

diff --git a/block/genhd.c b/block/genhd.c
index 02d9cfb9e077aaa5877aad060036191324d2e7a0..cadcb472dfc39385507147d20c40759154e7b2c4 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -420,6 +420,9 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk,
         */
        elevator_init_mq(disk->queue);
 
+       /* Mark bdev as having a submit_bio, if needed */
+       disk->part0->bd_has_submit_bio = disk->fops->submit_bio != NULL;
+
        /*
         * If the driver provides an explicit major number it also must provide
         * the number of minors numbers supported, and those will be used to

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index c1da616fadae11d5bcb9921d88028b6170b2e94b..3e8e9a97de462e06acad28cb070832752aa1793a 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -47,6 +47,7 @@ struct block_device {
        bool                    bd_read_only;   /* read-only policy */
        u8                      bd_partno;
        bool                    bd_write_holder;
+       bool                    bd_has_submit_bio;
        dev_t                   bd_dev;
        atomic_t                bd_openers;
        spinlock_t              bd_size_lock; /* for bd_inode->i_size updates */