blk-mq: pass request queue into get/put budget callback

blk-mq budget is abstract from scsi's device queue depth, and it is
always per-request-queue instead of hctx.

It can be quite absurd to get a budget from one hctx, then dequeue a
request from scheduler queue, and this request may not belong to this
hctx, at least for bfq and deadline.

So fix the mess and always pass request queue to get/put budget
callback.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
Tested-by: Baolin Wang <baolin.wang7@gmail.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Douglas Anderson <dianders@chromium.org>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Cc: Sagi Grimberg <sagi@grimberg.me>
Cc: Baolin Wang <baolin.wang7@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Douglas Anderson <dianders@chromium.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index fdcc2c1..a31e281 100644 (file)
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -108,12 +108,12 @@ static int blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
                        break;
                }
 
-               if (!blk_mq_get_dispatch_budget(hctx))
+               if (!blk_mq_get_dispatch_budget(q))
                        break;
 
                rq = e->type->ops.dispatch_request(hctx);
                if (!rq) {
-                       blk_mq_put_dispatch_budget(hctx);
+                       blk_mq_put_dispatch_budget(q);
                        /*
                         * We're releasing without dispatching. Holding the
                         * budget could have blocked any "hctx"s with the
@@ -173,12 +173,12 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
                if (!sbitmap_any_bit_set(&hctx->ctx_map))
                        break;
 
-               if (!blk_mq_get_dispatch_budget(hctx))
+               if (!blk_mq_get_dispatch_budget(q))
                        break;
 
                rq = blk_mq_dequeue_from_ctx(hctx, ctx);
                if (!rq) {
-                       blk_mq_put_dispatch_budget(hctx);
+                       blk_mq_put_dispatch_budget(q);
                        /*
                         * We're releasing without dispatching. Holding the
                         * budget could have blocked any "hctx"s with the

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 72d3034..8cdc868 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1284,7 +1284,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
                rq = list_first_entry(list, struct request, queuelist);
 
                hctx = rq->mq_hctx;
-               if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) {
+               if (!got_budget && !blk_mq_get_dispatch_budget(q)) {
                        blk_mq_put_driver_tag(rq);
                        no_budget_avail = true;
                        break;
@@ -1299,7 +1299,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
                         * we'll re-run it below.
                         */
                        if (!blk_mq_mark_tag_wait(hctx, rq)) {
-                               blk_mq_put_dispatch_budget(hctx);
+                               blk_mq_put_dispatch_budget(q);
                                /*
                                 * For non-shared tags, the RESTART check
                                 * will suffice.
@@ -1947,11 +1947,11 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
        if (q->elevator && !bypass_insert)
                goto insert;
 
-       if (!blk_mq_get_dispatch_budget(hctx))
+       if (!blk_mq_get_dispatch_budget(q))
                goto insert;
 
        if (!blk_mq_get_driver_tag(rq)) {
-               blk_mq_put_dispatch_budget(hctx);
+               blk_mq_put_dispatch_budget(q);
                goto insert;
        }
 

diff --git a/block/blk-mq.h b/block/blk-mq.h
index c633033..e4af193 100644 (file)
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -179,20 +179,16 @@ unsigned int blk_mq_in_flight(struct request_queue *q, struct hd_struct *part);
 void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part,
                         unsigned int inflight[2]);
 
-static inline void blk_mq_put_dispatch_budget(struct blk_mq_hw_ctx *hctx)
+static inline void blk_mq_put_dispatch_budget(struct request_queue *q)
 {
-       struct request_queue *q = hctx->queue;
-
        if (q->mq_ops->put_budget)
-               q->mq_ops->put_budget(hctx);
+               q->mq_ops->put_budget(q);
 }
 
-static inline bool blk_mq_get_dispatch_budget(struct blk_mq_hw_ctx *hctx)
+static inline bool blk_mq_get_dispatch_budget(struct request_queue *q)
 {
-       struct request_queue *q = hctx->queue;
-
        if (q->mq_ops->get_budget)
-               return q->mq_ops->get_budget(hctx);
+               return q->mq_ops->get_budget(q);
        return true;
 }
 

diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 6ca91d0..534b85e 100644 (file)
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1597,17 +1597,15 @@ static void scsi_mq_done(struct scsi_cmnd *cmd)
        blk_mq_complete_request(cmd->request);
 }
 
-static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
+static void scsi_mq_put_budget(struct request_queue *q)
 {
-       struct request_queue *q = hctx->queue;
        struct scsi_device *sdev = q->queuedata;
 
        atomic_dec(&sdev->device_busy);
 }
 
-static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
+static bool scsi_mq_get_budget(struct request_queue *q)
 {
-       struct request_queue *q = hctx->queue;
        struct scsi_device *sdev = q->queuedata;
 
        return scsi_dev_queue_ready(q, sdev);
@@ -1674,7 +1672,7 @@ out_dec_target_busy:
        if (scsi_target(sdev)->can_queue > 0)
                atomic_dec(&scsi_target(sdev)->target_busy);
 out_put_budget:
-       scsi_mq_put_budget(hctx);
+       scsi_mq_put_budget(q);
        switch (ret) {
        case BLK_STS_OK:
                break;

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 8986e88..faa340b 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -270,8 +270,8 @@ struct blk_mq_queue_data {
 typedef blk_status_t (queue_rq_fn)(struct blk_mq_hw_ctx *,
                const struct blk_mq_queue_data *);
 typedef void (commit_rqs_fn)(struct blk_mq_hw_ctx *);
-typedef bool (get_budget_fn)(struct blk_mq_hw_ctx *);
-typedef void (put_budget_fn)(struct blk_mq_hw_ctx *);
+typedef bool (get_budget_fn)(struct request_queue *);
+typedef void (put_budget_fn)(struct request_queue *);
 typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool);
 typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);
 typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int);