null_blk: poll queue support

There's currently no way to experiment with polled IO with null_blk,
which seems like an oversight. This patch adds support for polled IO.
We keep a list of issued IOs on submit, and then process that list
when mq_ops->poll() is invoked.

A new parameter is added, poll_queues. It defaults to 1 like the
submit queues, meaning we'll have 1 poll queue available.

Fixes-by: Bart Van Assche <bvanassche@acm.org>
Fixes-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
Link: https://lore.kernel.org/r/baca710d-0f2a-16e2-60bd-b105b854e0ae@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c
index e5cbcf5..f4af95c 100644 (file)
--- a/drivers/block/null_blk/main.c
+++ b/drivers/block/null_blk/main.c
@@ -92,6 +92,10 @@ static int g_submit_queues = 1;
 module_param_named(submit_queues, g_submit_queues, int, 0444);
 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
 
+static int g_poll_queues = 1;
+module_param_named(poll_queues, g_poll_queues, int, 0444);
+MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
+
 static int g_home_node = NUMA_NO_NODE;
 module_param_named(home_node, g_home_node, int, 0444);
 MODULE_PARM_DESC(home_node, "Home node for the device");
@@ -347,6 +351,7 @@ static int nullb_apply_submit_queues(struct nullb_device *dev,
 NULLB_DEVICE_ATTR(size, ulong, NULL);
 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
+NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_submit_queues);
 NULLB_DEVICE_ATTR(home_node, uint, NULL);
 NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
 NULLB_DEVICE_ATTR(blocksize, uint, NULL);
@@ -466,6 +471,7 @@ static struct configfs_attribute *nullb_device_attrs[] = {
        &nullb_device_attr_size,
        &nullb_device_attr_completion_nsec,
        &nullb_device_attr_submit_queues,
+       &nullb_device_attr_poll_queues,
        &nullb_device_attr_home_node,
        &nullb_device_attr_queue_mode,
        &nullb_device_attr_blocksize,
@@ -593,6 +599,7 @@ static struct nullb_device *null_alloc_dev(void)
        dev->size = g_gb * 1024;
        dev->completion_nsec = g_completion_nsec;
        dev->submit_queues = g_submit_queues;
+       dev->poll_queues = g_poll_queues;
        dev->home_node = g_home_node;
        dev->queue_mode = g_queue_mode;
        dev->blocksize = g_bs;
@@ -1454,12 +1461,80 @@ static bool should_requeue_request(struct request *rq)
        return false;
 }
 
+static int null_map_queues(struct blk_mq_tag_set *set)
+{
+       struct nullb *nullb = set->driver_data;
+       int i, qoff;
+
+       for (i = 0, qoff = 0; i < set->nr_maps; i++) {
+               struct blk_mq_queue_map *map = &set->map[i];
+
+               switch (i) {
+               case HCTX_TYPE_DEFAULT:
+                       if (nullb)
+                               map->nr_queues = nullb->dev->submit_queues;
+                       else
+                               map->nr_queues = g_submit_queues;
+                       break;
+               case HCTX_TYPE_READ:
+                       map->nr_queues = 0;
+                       continue;
+               case HCTX_TYPE_POLL:
+                       if (nullb)
+                               map->nr_queues = nullb->dev->poll_queues;
+                       else
+                               map->nr_queues = g_poll_queues;
+                       break;
+               }
+               map->queue_offset = qoff;
+               qoff += map->nr_queues;
+               blk_mq_map_queues(map);
+       }
+
+       return 0;
+}
+
+static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
+{
+       struct nullb_queue *nq = hctx->driver_data;
+       LIST_HEAD(list);
+       int nr = 0;
+
+       spin_lock(&nq->poll_lock);
+       list_splice_init(&nq->poll_list, &list);
+       spin_unlock(&nq->poll_lock);
+
+       while (!list_empty(&list)) {
+               struct nullb_cmd *cmd;
+               struct request *req;
+
+               req = list_first_entry(&list, struct request, queuelist);
+               list_del_init(&req->queuelist);
+               cmd = blk_mq_rq_to_pdu(req);
+               cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
+                                               blk_rq_sectors(req));
+               end_cmd(cmd);
+               nr++;
+       }
+
+       return nr;
+}
+
 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
 {
+       struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
        struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
 
        pr_info("rq %p timed out\n", rq);
 
+       if (hctx->type == HCTX_TYPE_POLL) {
+               struct nullb_queue *nq = hctx->driver_data;
+
+               spin_lock(&nq->poll_lock);
+               list_del_init(&rq->queuelist);
+               spin_unlock(&nq->poll_lock);
+       }
+
        /*
         * If the device is marked as blocking (i.e. memory backed or zoned
         * device), the submission path may be blocked waiting for resources
@@ -1480,10 +1555,11 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
        struct nullb_queue *nq = hctx->driver_data;
        sector_t nr_sectors = blk_rq_sectors(bd->rq);
        sector_t sector = blk_rq_pos(bd->rq);
+       const bool is_poll = hctx->type == HCTX_TYPE_POLL;
 
        might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
 
-       if (nq->dev->irqmode == NULL_IRQ_TIMER) {
+       if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
                hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
                cmd->timer.function = null_cmd_timer_expired;
        }
@@ -1507,6 +1583,13 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
                        return BLK_STS_OK;
                }
        }
+
+       if (is_poll) {
+               spin_lock(&nq->poll_lock);
+               list_add_tail(&bd->rq->queuelist, &nq->poll_list);
+               spin_unlock(&nq->poll_lock);
+               return BLK_STS_OK;
+       }
        if (cmd->fake_timeout)
                return BLK_STS_OK;
 
@@ -1542,6 +1625,8 @@ static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
        init_waitqueue_head(&nq->wait);
        nq->queue_depth = nullb->queue_depth;
        nq->dev = nullb->dev;
+       INIT_LIST_HEAD(&nq->poll_list);
+       spin_lock_init(&nq->poll_lock);
 }
 
 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
@@ -1567,6 +1652,8 @@ static const struct blk_mq_ops null_mq_ops = {
        .queue_rq       = null_queue_rq,
        .complete       = null_complete_rq,
        .timeout        = null_timeout_rq,
+       .poll           = null_poll,
+       .map_queues     = null_map_queues,
        .init_hctx      = null_init_hctx,
        .exit_hctx      = null_exit_hctx,
 };
@@ -1663,13 +1750,17 @@ static int setup_commands(struct nullb_queue *nq)
 
 static int setup_queues(struct nullb *nullb)
 {
-       nullb->queues = kcalloc(nr_cpu_ids, sizeof(struct nullb_queue),
+       int nqueues = nr_cpu_ids;
+
+       if (g_poll_queues)
+               nqueues += g_poll_queues;
+
+       nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
                                GFP_KERNEL);
        if (!nullb->queues)
                return -ENOMEM;
 
        nullb->queue_depth = nullb->dev->hw_queue_depth;
-
        return 0;
 }
 
@@ -1721,9 +1812,14 @@ static int null_gendisk_register(struct nullb *nullb)
 
 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
 {
+       int poll_queues;
+
        set->ops = &null_mq_ops;
        set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
                                                g_submit_queues;
+       poll_queues = nullb ? nullb->dev->poll_queues : g_poll_queues;
+       if (poll_queues)
+               set->nr_hw_queues += poll_queues;
        set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
                                                g_hw_queue_depth;
        set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
@@ -1733,7 +1829,11 @@ static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
                set->flags |= BLK_MQ_F_NO_SCHED;
        if (g_shared_tag_bitmap)
                set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
-       set->driver_data = NULL;
+       set->driver_data = nullb;
+       if (g_poll_queues)
+               set->nr_maps = 3;
+       else
+               set->nr_maps = 1;
 
        if ((nullb && nullb->dev->blocking) || g_blocking)
                set->flags |= BLK_MQ_F_BLOCKING;

diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h
index 64bef12..90c3eef 100644 (file)
--- a/drivers/block/null_blk/null_blk.h
+++ b/drivers/block/null_blk/null_blk.h
@@ -32,6 +32,9 @@ struct nullb_queue {
        struct nullb_device *dev;
        unsigned int requeue_selection;
 
+       struct list_head poll_list;
+       spinlock_t poll_lock;
+
        struct nullb_cmd *cmds;
 };
 
@@ -83,6 +86,7 @@ struct nullb_device {
        unsigned int zone_max_open; /* max number of open zones */
        unsigned int zone_max_active; /* max number of active zones */
        unsigned int submit_queues; /* number of submission queues */
+       unsigned int poll_queues; /* number of IOPOLL submission queues */
        unsigned int home_node; /* home node for the device */
        unsigned int queue_mode; /* block interface */
        unsigned int blocksize; /* block size */