blk-mq: avoid to touch q->elevator without any protection

q->elevator is referred in blk_mq_has_sqsched() without any protection,
no .q_usage_counter is held, no queue srcu and rcu read lock is held,
so potential use-after-free may be triggered.

Fix the issue by adding one queue flag for checking if the elevator
uses single queue style dispatch. Meantime the elevator feature flag
of ELEVATOR_F_MQ_AWARE isn't needed any more.

Cc: Jan Kara <jack@suse.cz>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Link: https://lore.kernel.org/r/20220616014401.817001-3-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 0d46cb7..caa55a5 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -7188,6 +7188,9 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e)
        bfq_init_root_group(bfqd->root_group, bfqd);
        bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group);
 
+       /* We dispatch from request queue wide instead of hw queue */
+       blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q);
+
        wbt_disable_default(q);
        return 0;
 

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 9e56a69..eb3c65a 100644 (file)
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -564,6 +564,7 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
        int ret;
 
        if (!e) {
+               blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);
                q->elevator = NULL;
                q->nr_requests = q->tag_set->queue_depth;
                return 0;

diff --git a/block/blk-mq.c b/block/blk-mq.c
index c13d03b..4cdb08a 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2143,20 +2143,6 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 EXPORT_SYMBOL(blk_mq_run_hw_queue);
 
 /*
- * Is the request queue handled by an IO scheduler that does not respect
- * hardware queues when dispatching?
- */
-static bool blk_mq_has_sqsched(struct request_queue *q)
-{
-       struct elevator_queue *e = q->elevator;
-
-       if (e && e->type->ops.dispatch_request &&
-           !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
-               return true;
-       return false;
-}
-
-/*
  * Return prefered queue to dispatch from (if any) for non-mq aware IO
  * scheduler.
  */
@@ -2188,7 +2174,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
        unsigned long i;
 
        sq_hctx = NULL;
-       if (blk_mq_has_sqsched(q))
+       if (blk_queue_sq_sched(q))
                sq_hctx = blk_mq_get_sq_hctx(q);
        queue_for_each_hw_ctx(q, hctx, i) {
                if (blk_mq_hctx_stopped(hctx))
@@ -2216,7 +2202,7 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
        unsigned long i;
 
        sq_hctx = NULL;
-       if (blk_mq_has_sqsched(q))
+       if (blk_queue_sq_sched(q))
                sq_hctx = blk_mq_get_sq_hctx(q);
        queue_for_each_hw_ctx(q, hctx, i) {
                if (blk_mq_hctx_stopped(hctx))

diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index 70ff2a5..8f7c745 100644 (file)
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -421,6 +421,8 @@ static int kyber_init_sched(struct request_queue *q, struct elevator_type *e)
 
        blk_stat_enable_accounting(q);
 
+       blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);
+
        eq->elevator_data = kqd;
        q->elevator = eq;
 
@@ -1033,7 +1035,6 @@ static struct elevator_type kyber_sched = {
 #endif
        .elevator_attrs = kyber_sched_attrs,
        .elevator_name = "kyber",
-       .elevator_features = ELEVATOR_F_MQ_AWARE,
        .elevator_owner = THIS_MODULE,
 };
 

diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 6ed602b..1a9e835 100644 (file)
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -642,6 +642,9 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e)
        spin_lock_init(&dd->lock);
        spin_lock_init(&dd->zone_lock);
 
+       /* We dispatch from request queue wide instead of hw queue */
+       blk_queue_flag_set(QUEUE_FLAG_SQ_SCHED, q);
+
        q->elevator = eq;
        return 0;
 

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 608d577..bb6e3c3 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -575,6 +575,7 @@ struct request_queue {
 #define QUEUE_FLAG_RQ_ALLOC_TIME 27    /* record rq->alloc_time_ns */
 #define QUEUE_FLAG_HCTX_ACTIVE 28      /* at least one blk-mq hctx is active */
 #define QUEUE_FLAG_NOWAIT       29     /* device supports NOWAIT */
+#define QUEUE_FLAG_SQ_SCHED     30     /* single queue style io dispatch */
 
 #define QUEUE_FLAG_MQ_DEFAULT  ((1 << QUEUE_FLAG_IO_STAT) |            \
                                 (1 << QUEUE_FLAG_SAME_COMP) |          \
@@ -616,6 +617,7 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
 #define blk_queue_pm_only(q)   atomic_read(&(q)->pm_only)
 #define blk_queue_registered(q)        test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
 #define blk_queue_nowait(q)    test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
+#define blk_queue_sq_sched(q)  test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
 
 extern void blk_set_pm_only(struct request_queue *q);
 extern void blk_clear_pm_only(struct request_queue *q);
@@ -1006,8 +1008,6 @@ void disk_set_independent_access_ranges(struct gendisk *disk,
  */
 /* Supports zoned block devices sequential write constraint */
 #define ELEVATOR_F_ZBD_SEQ_WRITE       (1U << 0)
-/* Supports scheduling on multiple hardware queues */
-#define ELEVATOR_F_MQ_AWARE            (1U << 1)
 
 extern void blk_queue_required_elevator_features(struct request_queue *q,
                                                 unsigned int features);