block: allow end_io based requests in the completion batch handling

With end_io handlers now being able to potentially pass ownership of
the request upon completion, we can allow requests with end_io handlers
in the batch completion handling.

Reviewed-by: Anuj Gupta <anuj20.g@samsung.com>
Reviewed-by: Keith Busch <kbusch@kernel.org>
Co-developed-by: Stefan Roesch <shr@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index a21631d..8070b6c 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -823,8 +823,10 @@ static void blk_complete_request(struct request *req)
         * can find how many bytes remain in the request
         * later.
         */
-       req->bio = NULL;
-       req->__data_len = 0;
+       if (!req->end_io) {
+               req->bio = NULL;
+               req->__data_len = 0;
+       }
 }
 
 /**
@@ -1055,6 +1057,13 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob)
 
                rq_qos_done(rq->q, rq);
 
+               /*
+                * If end_io handler returns NONE, then it still has
+                * ownership of the request.
+                */
+               if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE)
+                       continue;
+
                WRITE_ONCE(rq->state, MQ_RQ_IDLE);
                if (!req_ref_put_and_test(rq))
                        continue;

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index e6fa49d..50811d0 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -853,8 +853,9 @@ static inline bool blk_mq_add_to_batch(struct request *req,
                                       struct io_comp_batch *iob, int ioerror,
                                       void (*complete)(struct io_comp_batch *))
 {
-       if (!iob || (req->rq_flags & RQF_ELV) || req->end_io || ioerror)
+       if (!iob || (req->rq_flags & RQF_ELV) || ioerror)
                return false;
+
        if (!iob->complete)
                iob->complete = complete;
        else if (iob->complete != complete)