block: change plugging to use a singly linked list

Use a singly linked list for the blk_plug. This saves 8 bytes in the
blk_plug struct, and makes for faster list manipulations than doubly
linked lists. As we don't use the doubly linked lists for anything,
singly linked is just fine.

This yields a bump in default (merging enabled) performance from 7.0
to 7.1M IOPS, and ~7.5M IOPS with merging disabled.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index d0c2e11..14d2090 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1550,7 +1550,7 @@ void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)
        if (tsk->plug)
                return;
 
-       INIT_LIST_HEAD(&plug->mq_list);
+       plug->mq_list = NULL;
        plug->cached_rq = NULL;
        plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
        plug->rq_count = 0;
@@ -1640,7 +1640,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
        flush_plug_callbacks(plug, from_schedule);
 
-       if (!list_empty(&plug->mq_list))
+       if (!rq_list_empty(plug->mq_list))
                blk_mq_flush_plug_list(plug, from_schedule);
        if (unlikely(!from_schedule && plug->cached_rq))
                blk_mq_free_plug_rqs(plug);

diff --git a/block/blk-merge.c b/block/blk-merge.c
index c273b58..3e6fa44 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -1090,11 +1090,11 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
        struct request *rq;
 
        plug = blk_mq_plug(q, bio);
-       if (!plug || list_empty(&plug->mq_list))
+       if (!plug || rq_list_empty(plug->mq_list))
                return false;
 
        /* check the previously added entry for a quick merge attempt */
-       rq = list_last_entry(&plug->mq_list, struct request, queuelist);
+       rq = rq_list_peek(&plug->mq_list);
        if (rq->q == q) {
                /*
                 * Only blk-mq multiple hardware queues case checks the rq in

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 8f5c166..7fa3027 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2151,34 +2151,46 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
 
 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
+       struct blk_mq_hw_ctx *this_hctx;
+       struct blk_mq_ctx *this_ctx;
+       unsigned int depth;
        LIST_HEAD(list);
 
-       if (list_empty(&plug->mq_list))
+       if (rq_list_empty(plug->mq_list))
                return;
-       list_splice_init(&plug->mq_list, &list);
        plug->rq_count = 0;
 
+       this_hctx = NULL;
+       this_ctx = NULL;
+       depth = 0;
        do {
-               struct list_head rq_list;
-               struct request *rq, *head_rq = list_entry_rq(list.next);
-               struct list_head *pos = &head_rq->queuelist; /* skip first */
-               struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;
-               struct blk_mq_ctx *this_ctx = head_rq->mq_ctx;
-               unsigned int depth = 1;
-
-               list_for_each_continue(pos, &list) {
-                       rq = list_entry_rq(pos);
-                       BUG_ON(!rq->q);
-                       if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx)
-                               break;
-                       depth++;
+               struct request *rq;
+
+               rq = rq_list_pop(&plug->mq_list);
+
+               if (!this_hctx) {
+                       this_hctx = rq->mq_hctx;
+                       this_ctx = rq->mq_ctx;
+               } else if (this_hctx != rq->mq_hctx || this_ctx != rq->mq_ctx) {
+                       trace_block_unplug(this_hctx->queue, depth,
+                                               !from_schedule);
+                       blk_mq_sched_insert_requests(this_hctx, this_ctx,
+                                               &list, from_schedule);
+                       depth = 0;
+                       this_hctx = rq->mq_hctx;
+                       this_ctx = rq->mq_ctx;
+
                }
 
-               list_cut_before(&rq_list, &list, pos);
-               trace_block_unplug(head_rq->q, depth, !from_schedule);
-               blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
+               list_add(&rq->queuelist, &list);
+               depth++;
+       } while (!rq_list_empty(plug->mq_list));
+
+       if (!list_empty(&list)) {
+               trace_block_unplug(this_hctx->queue, depth, !from_schedule);
+               blk_mq_sched_insert_requests(this_hctx, this_ctx, &list,
                                                from_schedule);
-       } while(!list_empty(&list));
+       }
 }
 
 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
@@ -2358,16 +2370,15 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
 
 static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
 {
-       list_add_tail(&rq->queuelist, &plug->mq_list);
-       plug->rq_count++;
-       if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {
-               struct request *tmp;
+       if (!plug->multiple_queues) {
+               struct request *nxt = rq_list_peek(&plug->mq_list);
 
-               tmp = list_first_entry(&plug->mq_list, struct request,
-                                               queuelist);
-               if (tmp->q != rq->q)
+               if (nxt && nxt->q != rq->q)
                        plug->multiple_queues = true;
        }
+       rq->rq_next = NULL;
+       rq_list_add(&plug->mq_list, rq);
+       plug->rq_count++;
 }
 
 /*
@@ -2479,13 +2490,15 @@ void blk_mq_submit_bio(struct bio *bio)
                unsigned int request_count = plug->rq_count;
                struct request *last = NULL;
 
-               if (!request_count)
+               if (!request_count) {
                        trace_block_plug(q);
-               else
-                       last = list_entry_rq(plug->mq_list.prev);
+               } else if (!blk_queue_nomerges(q)) {
+                       last = rq_list_peek(&plug->mq_list);
+                       if (blk_rq_bytes(last) < BLK_PLUG_FLUSH_SIZE)
+                               last = NULL;
+               }
 
-               if (request_count >= blk_plug_max_rq_count(plug) || (last &&
-                   blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
+               if (request_count >= blk_plug_max_rq_count(plug) || last) {
                        blk_flush_plug_list(plug, false);
                        trace_block_plug(q);
                }
@@ -2505,10 +2518,7 @@ void blk_mq_submit_bio(struct bio *bio)
                 * the plug list is empty, and same_queue_rq is invalid.
                 */
                if (same_queue_rq) {
-                       next_rq = list_last_entry(&plug->mq_list,
-                                                       struct request,
-                                                       queuelist);
-                       list_del_init(&next_rq->queuelist);
+                       next_rq = rq_list_pop(&plug->mq_list);
                        plug->rq_count--;
                }
                blk_add_rq_to_plug(plug, rq);

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index fd9771a..4027112 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -728,7 +728,7 @@ extern void blk_set_queue_dying(struct request_queue *);
  * schedule() where blk_schedule_flush_plug() is called.
  */
 struct blk_plug {
-       struct list_head mq_list; /* blk-mq requests */
+       struct request *mq_list; /* blk-mq requests */
 
        /* if ios_left is > 1, we can batch tag/rq allocations */
        struct request *cached_rq;
@@ -777,8 +777,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk)
        struct blk_plug *plug = tsk->plug;
 
        return plug &&
-                (!list_empty(&plug->mq_list) ||
-                !list_empty(&plug->cb_list));
+                (plug->mq_list || !list_empty(&plug->cb_list));
 }
 
 int blkdev_issue_flush(struct block_device *bdev);