blk-mq: Fix stall due to recursive flush plug

[platform/kernel/linux-starfive.git] / block / blk-merge.c

diff --git a/block/blk-merge.c b/block/blk-merge.c
index ff04e92..cc7f6a4 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -300,6 +300,16 @@ static struct bio *bio_split_rw(struct bio *bio, struct queue_limits *lim,
        *segs = nsegs;
        return NULL;
 split:
+       /*
+        * We can't sanely support splitting for a REQ_NOWAIT bio. End it
+        * with EAGAIN if splitting is required and return an error pointer.
+        */
+       if (bio->bi_opf & REQ_NOWAIT) {
+               bio->bi_status = BLK_STS_AGAIN;
+               bio_endio(bio);
+               return ERR_PTR(-EAGAIN);
+       }
+
        *segs = nsegs;
 
        /*
@@ -348,11 +358,13 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct queue_limits *lim,
        default:
                split = bio_split_rw(bio, lim, nr_segs, bs,
                                get_max_io_size(bio, lim) << SECTOR_SHIFT);
+               if (IS_ERR(split))
+                       return NULL;
                break;
        }
 
        if (split) {
-               /* there isn't chance to merge the splitted bio */
+               /* there isn't chance to merge the split bio */
                split->bi_opf |= REQ_NOMERGE;
 
                blkcg_bio_issue_init(split);
@@ -735,6 +747,33 @@ void blk_rq_set_mixed_merge(struct request *rq)
        rq->rq_flags |= RQF_MIXED_MERGE;
 }
 
+static inline blk_opf_t bio_failfast(const struct bio *bio)
+{
+       if (bio->bi_opf & REQ_RAHEAD)
+               return REQ_FAILFAST_MASK;
+
+       return bio->bi_opf & REQ_FAILFAST_MASK;
+}
+
+/*
+ * After we are marked as MIXED_MERGE, any new RA bio has to be updated
+ * as failfast, and request's failfast has to be updated in case of
+ * front merge.
+ */
+static inline void blk_update_mixed_merge(struct request *req,
+               struct bio *bio, bool front_merge)
+{
+       if (req->rq_flags & RQF_MIXED_MERGE) {
+               if (bio->bi_opf & REQ_RAHEAD)
+                       bio->bi_opf |= REQ_FAILFAST_MASK;
+
+               if (front_merge) {
+                       req->cmd_flags &= ~REQ_FAILFAST_MASK;
+                       req->cmd_flags |= bio->bi_opf & REQ_FAILFAST_MASK;
+               }
+       }
+}
+
 static void blk_account_io_merge_request(struct request *req)
 {
        if (blk_do_io_stat(req)) {
@@ -824,6 +863,8 @@ static struct request *attempt_merge(struct request_queue *q,
        if (!blk_discard_mergable(req))
                elv_merge_requests(q, req, next);
 
+       blk_crypto_rq_put_keyslot(next);
+
        /*
         * 'next' is going away, so update stats accordingly
         */
@@ -932,7 +973,7 @@ enum bio_merge_status {
 static enum bio_merge_status bio_attempt_back_merge(struct request *req,
                struct bio *bio, unsigned int nr_segs)
 {
-       const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK;
+       const blk_opf_t ff = bio_failfast(bio);
 
        if (!ll_back_merge_fn(req, bio, nr_segs))
                return BIO_MERGE_FAILED;
@@ -943,6 +984,8 @@ static enum bio_merge_status bio_attempt_back_merge(struct request *req,
        if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
                blk_rq_set_mixed_merge(req);
 
+       blk_update_mixed_merge(req, bio, false);
+
        req->biotail->bi_next = bio;
        req->biotail = bio;
        req->__data_len += bio->bi_iter.bi_size;
@@ -956,7 +999,7 @@ static enum bio_merge_status bio_attempt_back_merge(struct request *req,
 static enum bio_merge_status bio_attempt_front_merge(struct request *req,
                struct bio *bio, unsigned int nr_segs)
 {
-       const blk_opf_t ff = bio->bi_opf & REQ_FAILFAST_MASK;
+       const blk_opf_t ff = bio_failfast(bio);
 
        if (!ll_front_merge_fn(req, bio, nr_segs))
                return BIO_MERGE_FAILED;
@@ -967,6 +1010,8 @@ static enum bio_merge_status bio_attempt_front_merge(struct request *req,
        if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
                blk_rq_set_mixed_merge(req);
 
+       blk_update_mixed_merge(req, bio, true);
+
        bio->bi_next = req->bio;
        req->bio = bio;