block: cache request queue in bdev

There are tons of places where we need to get a request_queue only
having bdev, which turns into bdev->bd_disk->queue. There are probably a
hundred of such places considering inline helpers, and enough of them
are in hot paths.

Cache queue pointer in struct block_device and make use of it in
bdev_get_queue().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/a3bfaecdd28956f03629d0ca5c63ebc096e1c809.1634219547.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bdev.c b/block/bdev.c
index 93b1188..fed8d0c 100644 (file)
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -493,6 +493,7 @@ struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
        spin_lock_init(&bdev->bd_size_lock);
        bdev->bd_partno = partno;
        bdev->bd_inode = inode;
+       bdev->bd_queue = disk->queue;
        bdev->bd_stats = alloc_percpu(struct disk_stats);
        if (!bdev->bd_stats) {
                iput(inode);

diff --git a/block/genhd.c b/block/genhd.c
index ffbdb9b..d148c38 100644 (file)
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1267,6 +1267,9 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
        if (!disk->bdi)
                goto out_free_disk;
 
+       /* bdev_alloc() might need the queue, set before the first call */
+       disk->queue = q;
+
        disk->part0 = bdev_alloc(disk, 0);
        if (!disk->part0)
                goto out_free_bdi;
@@ -1282,7 +1285,6 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id,
        disk_to_dev(disk)->type = &disk_type;
        device_initialize(disk_to_dev(disk));
        inc_diskseq(disk);
-       disk->queue = q;
        q->disk = disk;
        lockdep_init_map(&disk->lockdep_map, "(bio completion)", lkclass, 0);
 #ifdef CONFIG_BLOCK_HOLDER_DEPRECATED

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 72736b4..1e37092 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -38,6 +38,7 @@ struct block_device {
        u8                      bd_partno;
        spinlock_t              bd_size_lock; /* for bd_inode->i_size updates */
        struct gendisk *        bd_disk;
+       struct request_queue *  bd_queue;
 
        /* The counter of freeze processes */
        int                     bd_fsfreeze_count;

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 2a8689e..d5b21fc 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -574,7 +574,7 @@ int iocb_bio_iopoll(struct kiocb *kiocb, unsigned int flags);
 
 static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
 {
-       return bdev->bd_disk->queue;    /* this is never NULL */
+       return bdev->bd_queue;  /* this is never NULL */
 }
 
 /*