block: fix revalidate performance regression

[platform/kernel/linux-starfive.git] / block / bfq-iosched.c

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 7ea4278..52eb79d 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -386,6 +386,12 @@ static void bfq_put_stable_ref(struct bfq_queue *bfqq);
 
 void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync)
 {
+       struct bfq_queue *old_bfqq = bic->bfqq[is_sync];
+
+       /* Clear bic pointer if bfqq is detached from this bic */
+       if (old_bfqq && old_bfqq->bic == bic)
+               old_bfqq->bic = NULL;
+
        /*
         * If bfqq != NULL, then a non-stable queue merge between
         * bic->bfqq and bfqq is happening here. This causes troubles
@@ -631,6 +637,8 @@ retry:
                                        sched_data->service_tree[i].wsum;
                        }
                }
+               if (!wsum)
+                       continue;
                limit = DIV_ROUND_CLOSEST(limit * entity->weight, wsum);
                if (entity->allocated >= limit) {
                        bfq_log_bfqq(bfqq->bfqd, bfqq,
@@ -3174,7 +3182,7 @@ bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic,
        /*
         * Merge queues (that is, let bic redirect its requests to new_bfqq)
         */
-       bic_set_bfqq(bic, new_bfqq, 1);
+       bic_set_bfqq(bic, new_bfqq, true);
        bfq_mark_bfqq_coop(new_bfqq);
        /*
         * new_bfqq now belongs to at least two bics (it is a shared queue):
@@ -5377,9 +5385,8 @@ static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync)
                unsigned long flags;
 
                spin_lock_irqsave(&bfqd->lock, flags);
-               bfqq->bic = NULL;
-               bfq_exit_bfqq(bfqd, bfqq);
                bic_set_bfqq(bic, NULL, is_sync);
+               bfq_exit_bfqq(bfqd, bfqq);
                spin_unlock_irqrestore(&bfqd->lock, flags);
        }
 }
@@ -5486,9 +5493,11 @@ static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio)
 
        bfqq = bic_to_bfqq(bic, false);
        if (bfqq) {
-               bfq_release_process_ref(bfqd, bfqq);
+               struct bfq_queue *old_bfqq = bfqq;
+
                bfqq = bfq_get_queue(bfqd, bio, false, bic, true);
                bic_set_bfqq(bic, bfqq, false);
+               bfq_release_process_ref(bfqd, old_bfqq);
        }
 
        bfqq = bic_to_bfqq(bic, true);
@@ -6622,7 +6631,7 @@ bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)
                return bfqq;
        }
 
-       bic_set_bfqq(bic, NULL, 1);
+       bic_set_bfqq(bic, NULL, true);
 
        bfq_put_cooperator(bfqq);
 
@@ -6784,6 +6793,12 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
                                bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,
                                                                 true, is_sync,
                                                                 NULL);
+                               if (unlikely(bfqq == &bfqd->oom_bfqq))
+                                       bfqq_already_existing = true;
+                       } else
+                               bfqq_already_existing = true;
+
+                       if (!bfqq_already_existing) {
                                bfqq->waker_bfqq = old_bfqq->waker_bfqq;
                                bfqq->tentative_waker_bfqq = NULL;
 
@@ -6797,8 +6812,7 @@ static struct bfq_queue *bfq_init_rq(struct request *rq)
                                if (bfqq->waker_bfqq)
                                        hlist_add_head(&bfqq->woken_list_node,
                                                       &bfqq->waker_bfqq->woken_list);
-                       } else
-                               bfqq_already_existing = true;
+                       }
                }
        }