block: move update request helpers into blk-mq.c

For some reason we still have them in blk-core, with the rest of the
request completion being in blk-mq. That causes and out-of-line call
for each completion.

Move them into blk-mq.c instead, where they belong.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-core.c b/block/blk-core.c
index b4094b3..20b6cc0 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -216,7 +216,7 @@ int blk_status_to_errno(blk_status_t status)
 }
 EXPORT_SYMBOL_GPL(blk_status_to_errno);
 
-static void print_req_error(struct request *req, blk_status_t status)
+void blk_print_req_error(struct request *req, blk_status_t status)
 {
        int idx = (__force int)status;
 
@@ -234,33 +234,6 @@ static void print_req_error(struct request *req, blk_status_t status)
                IOPRIO_PRIO_CLASS(req->ioprio));
 }
 
-static void req_bio_endio(struct request *rq, struct bio *bio,
-                         unsigned int nbytes, blk_status_t error)
-{
-       if (error)
-               bio->bi_status = error;
-
-       if (unlikely(rq->rq_flags & RQF_QUIET))
-               bio_set_flag(bio, BIO_QUIET);
-
-       bio_advance(bio, nbytes);
-
-       if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
-               /*
-                * Partial zone append completions cannot be supported as the
-                * BIO fragments may end up not being written sequentially.
-                */
-               if (bio->bi_iter.bi_size)
-                       bio->bi_status = BLK_STS_IOERR;
-               else
-                       bio->bi_iter.bi_sector = rq->__sector;
-       }
-
-       /* don't actually finish bio if it's part of flush sequence */
-       if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
-               bio_endio(bio);
-}
-
 void blk_dump_rq_flags(struct request *rq, char *msg)
 {
        printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
@@ -1311,17 +1284,6 @@ again:
        }
 }
 
-static void blk_account_io_completion(struct request *req, unsigned int bytes)
-{
-       if (req->part && blk_do_io_stat(req)) {
-               const int sgrp = op_stat_group(req_op(req));
-
-               part_stat_lock();
-               part_stat_add(req->part, sectors[sgrp], bytes >> 9);
-               part_stat_unlock();
-       }
-}
-
 void __blk_account_io_done(struct request *req, u64 now)
 {
        const int sgrp = op_stat_group(req_op(req));
@@ -1430,112 +1392,6 @@ void blk_steal_bios(struct bio_list *list, struct request *rq)
 }
 EXPORT_SYMBOL_GPL(blk_steal_bios);
 
-/**
- * blk_update_request - Complete multiple bytes without completing the request
- * @req:      the request being processed
- * @error:    block status code
- * @nr_bytes: number of bytes to complete for @req
- *
- * Description:
- *     Ends I/O on a number of bytes attached to @req, but doesn't complete
- *     the request structure even if @req doesn't have leftover.
- *     If @req has leftover, sets it up for the next range of segments.
- *
- *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
- *     %false return from this function.
- *
- * Note:
- *     The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
- *      except in the consistency check at the end of this function.
- *
- * Return:
- *     %false - this request doesn't have any more data
- *     %true  - this request has more data
- **/
-bool blk_update_request(struct request *req, blk_status_t error,
-               unsigned int nr_bytes)
-{
-       int total_bytes;
-
-       trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
-
-       if (!req->bio)
-               return false;
-
-#ifdef CONFIG_BLK_DEV_INTEGRITY
-       if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
-           error == BLK_STS_OK)
-               req->q->integrity.profile->complete_fn(req, nr_bytes);
-#endif
-
-       if (unlikely(error && !blk_rq_is_passthrough(req) &&
-                    !(req->rq_flags & RQF_QUIET)))
-               print_req_error(req, error);
-
-       blk_account_io_completion(req, nr_bytes);
-
-       total_bytes = 0;
-       while (req->bio) {
-               struct bio *bio = req->bio;
-               unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
-
-               if (bio_bytes == bio->bi_iter.bi_size)
-                       req->bio = bio->bi_next;
-
-               /* Completion has already been traced */
-               bio_clear_flag(bio, BIO_TRACE_COMPLETION);
-               req_bio_endio(req, bio, bio_bytes, error);
-
-               total_bytes += bio_bytes;
-               nr_bytes -= bio_bytes;
-
-               if (!nr_bytes)
-                       break;
-       }
-
-       /*
-        * completely done
-        */
-       if (!req->bio) {
-               /*
-                * Reset counters so that the request stacking driver
-                * can find how many bytes remain in the request
-                * later.
-                */
-               req->__data_len = 0;
-               return false;
-       }
-
-       req->__data_len -= total_bytes;
-
-       /* update sector only for requests with clear definition of sector */
-       if (!blk_rq_is_passthrough(req))
-               req->__sector += total_bytes >> 9;
-
-       /* mixed attributes always follow the first bio */
-       if (req->rq_flags & RQF_MIXED_MERGE) {
-               req->cmd_flags &= ~REQ_FAILFAST_MASK;
-               req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
-       }
-
-       if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
-               /*
-                * If total number of sectors is less than the first segment
-                * size, something has gone terribly wrong.
-                */
-               if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
-                       blk_dump_rq_flags(req, "request botched");
-                       req->__data_len = blk_rq_cur_bytes(req);
-               }
-
-               /* recalculate the number of segments */
-               req->nr_phys_segments = blk_recalc_rq_segments(req);
-       }
-
-       return true;
-}
-EXPORT_SYMBOL_GPL(blk_update_request);
-
 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
 /**
  * rq_flush_dcache_pages - Helper function to flush all pages in a request

diff --git a/block/blk-mq.c b/block/blk-mq.c
index b588782..9cff9e8 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -628,6 +628,150 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug)
        }
 }
 
+static void req_bio_endio(struct request *rq, struct bio *bio,
+                         unsigned int nbytes, blk_status_t error)
+{
+       if (error)
+               bio->bi_status = error;
+
+       if (unlikely(rq->rq_flags & RQF_QUIET))
+               bio_set_flag(bio, BIO_QUIET);
+
+       bio_advance(bio, nbytes);
+
+       if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
+               /*
+                * Partial zone append completions cannot be supported as the
+                * BIO fragments may end up not being written sequentially.
+                */
+               if (bio->bi_iter.bi_size)
+                       bio->bi_status = BLK_STS_IOERR;
+               else
+                       bio->bi_iter.bi_sector = rq->__sector;
+       }
+
+       /* don't actually finish bio if it's part of flush sequence */
+       if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
+               bio_endio(bio);
+}
+
+static void blk_account_io_completion(struct request *req, unsigned int bytes)
+{
+       if (req->part && blk_do_io_stat(req)) {
+               const int sgrp = op_stat_group(req_op(req));
+
+               part_stat_lock();
+               part_stat_add(req->part, sectors[sgrp], bytes >> 9);
+               part_stat_unlock();
+       }
+}
+
+/**
+ * blk_update_request - Complete multiple bytes without completing the request
+ * @req:      the request being processed
+ * @error:    block status code
+ * @nr_bytes: number of bytes to complete for @req
+ *
+ * Description:
+ *     Ends I/O on a number of bytes attached to @req, but doesn't complete
+ *     the request structure even if @req doesn't have leftover.
+ *     If @req has leftover, sets it up for the next range of segments.
+ *
+ *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
+ *     %false return from this function.
+ *
+ * Note:
+ *     The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
+ *      except in the consistency check at the end of this function.
+ *
+ * Return:
+ *     %false - this request doesn't have any more data
+ *     %true  - this request has more data
+ **/
+bool blk_update_request(struct request *req, blk_status_t error,
+               unsigned int nr_bytes)
+{
+       int total_bytes;
+
+       trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
+
+       if (!req->bio)
+               return false;
+
+#ifdef CONFIG_BLK_DEV_INTEGRITY
+       if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
+           error == BLK_STS_OK)
+               req->q->integrity.profile->complete_fn(req, nr_bytes);
+#endif
+
+       if (unlikely(error && !blk_rq_is_passthrough(req) &&
+                    !(req->rq_flags & RQF_QUIET)))
+               blk_print_req_error(req, error);
+
+       blk_account_io_completion(req, nr_bytes);
+
+       total_bytes = 0;
+       while (req->bio) {
+               struct bio *bio = req->bio;
+               unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
+
+               if (bio_bytes == bio->bi_iter.bi_size)
+                       req->bio = bio->bi_next;
+
+               /* Completion has already been traced */
+               bio_clear_flag(bio, BIO_TRACE_COMPLETION);
+               req_bio_endio(req, bio, bio_bytes, error);
+
+               total_bytes += bio_bytes;
+               nr_bytes -= bio_bytes;
+
+               if (!nr_bytes)
+                       break;
+       }
+
+       /*
+        * completely done
+        */
+       if (!req->bio) {
+               /*
+                * Reset counters so that the request stacking driver
+                * can find how many bytes remain in the request
+                * later.
+                */
+               req->__data_len = 0;
+               return false;
+       }
+
+       req->__data_len -= total_bytes;
+
+       /* update sector only for requests with clear definition of sector */
+       if (!blk_rq_is_passthrough(req))
+               req->__sector += total_bytes >> 9;
+
+       /* mixed attributes always follow the first bio */
+       if (req->rq_flags & RQF_MIXED_MERGE) {
+               req->cmd_flags &= ~REQ_FAILFAST_MASK;
+               req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
+       }
+
+       if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
+               /*
+                * If total number of sectors is less than the first segment
+                * size, something has gone terribly wrong.
+                */
+               if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
+                       blk_dump_rq_flags(req, "request botched");
+                       req->__data_len = blk_rq_cur_bytes(req);
+               }
+
+               /* recalculate the number of segments */
+               req->nr_phys_segments = blk_recalc_rq_segments(req);
+       }
+
+       return true;
+}
+EXPORT_SYMBOL_GPL(blk_update_request);
+
 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
 {
        if (blk_mq_need_time_stamp(rq)) {

diff --git a/block/blk.h b/block/blk.h
index 447a2de..e803503 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -215,6 +215,7 @@ static inline void blk_integrity_del(struct gendisk *disk)
 
 unsigned long blk_rq_timeout(unsigned long timeout);
 void blk_add_timer(struct request *req);
+void blk_print_req_error(struct request *req, blk_status_t status);
 
 bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
                unsigned int nr_segs, struct request **same_queue_rq);