block: handle fast path of bio splitting inline

The fast path is no splitting needed. Separate the handling into a
check part we can inline, and an out-of-line handling path if we do
need to split.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-merge.c b/block/blk-merge.c
index f88d786..ec72723 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -324,6 +324,7 @@ split:
 
 /**
  * __blk_queue_split - split a bio and submit the second half
+ * @q:       [in] request_queue new bio is being queued at
  * @bio:     [in, out] bio to be split
  * @nr_segs: [out] number of segments in the first bio
  *
@@ -334,9 +335,9 @@ split:
  * of the caller to ensure that q->bio_split is only released after processing
  * of the split bio has finished.
  */
-void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)
+void __blk_queue_split(struct request_queue *q, struct bio **bio,
+                      unsigned int *nr_segs)
 {
-       struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue;
        struct bio *split = NULL;
 
        switch (bio_op(*bio)) {
@@ -353,21 +354,6 @@ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)
                                nr_segs);
                break;
        default:
-               /*
-                * All drivers must accept single-segments bios that are <=
-                * PAGE_SIZE.  This is a quick and dirty check that relies on
-                * the fact that bi_io_vec[0] is always valid if a bio has data.
-                * The check might lead to occasional false negatives when bios
-                * are cloned, but compared to the performance impact of cloned
-                * bios themselves the loop below doesn't matter anyway.
-                */
-               if (!q->limits.chunk_sectors &&
-                   (*bio)->bi_vcnt == 1 &&
-                   ((*bio)->bi_io_vec[0].bv_len +
-                    (*bio)->bi_io_vec[0].bv_offset) <= PAGE_SIZE) {
-                       *nr_segs = 1;
-                       break;
-               }
                split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
                break;
        }
@@ -397,9 +383,11 @@ void __blk_queue_split(struct bio **bio, unsigned int *nr_segs)
  */
 void blk_queue_split(struct bio **bio)
 {
+       struct request_queue *q = (*bio)->bi_bdev->bd_disk->queue;
        unsigned int nr_segs;
 
-       __blk_queue_split(bio, &nr_segs);
+       if (blk_may_split(q, *bio))
+               __blk_queue_split(q, bio, &nr_segs);
 }
 EXPORT_SYMBOL(blk_queue_split);
 

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 0860f62..0921908 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2259,11 +2259,12 @@ void blk_mq_submit_bio(struct bio *bio)
        struct request *rq;
        struct blk_plug *plug;
        struct request *same_queue_rq = NULL;
-       unsigned int nr_segs;
+       unsigned int nr_segs = 1;
        blk_status_t ret;
 
        blk_queue_bounce(q, &bio);
-       __blk_queue_split(&bio, &nr_segs);
+       if (blk_may_split(q, bio))
+               __blk_queue_split(q, &bio, &nr_segs);
 
        if (!bio_integrity_prep(bio))
                goto queue_exit;

diff --git a/block/blk.h b/block/blk.h
index fa05d3f..447a2de 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -266,7 +266,32 @@ ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
 ssize_t part_timeout_store(struct device *, struct device_attribute *,
                                const char *, size_t);
 
-void __blk_queue_split(struct bio **bio, unsigned int *nr_segs);
+static inline bool blk_may_split(struct request_queue *q, struct bio *bio)
+{
+       switch (bio_op(bio)) {
+       case REQ_OP_DISCARD:
+       case REQ_OP_SECURE_ERASE:
+       case REQ_OP_WRITE_ZEROES:
+       case REQ_OP_WRITE_SAME:
+               return true; /* non-trivial splitting decisions */
+       default:
+               break;
+       }
+
+       /*
+        * All drivers must accept single-segments bios that are <= PAGE_SIZE.
+        * This is a quick and dirty check that relies on the fact that
+        * bi_io_vec[0] is always valid if a bio has data.  The check might
+        * lead to occasional false negatives when bios are cloned, but compared
+        * to the performance impact of cloned bios themselves the loop below
+        * doesn't matter anyway.
+        */
+       return q->limits.chunk_sectors || bio->bi_vcnt != 1 ||
+               bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
+}
+
+void __blk_queue_split(struct request_queue *q, struct bio **bio,
+                       unsigned int *nr_segs);
 int ll_back_merge_fn(struct request *req, struct bio *bio,
                unsigned int nr_segs);
 bool blk_attempt_req_merge(struct request_queue *q, struct request *rq,