block, bfq: split also async bfq_queues on a per-actuator basis

Similarly to sync bfq_queues, also async bfq_queues need to be split
on a per-actuator basis.

Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Signed-off-by: Paolo Valente <paolo.valente@linaro.org>
Signed-off-by: Davide Zini <davidezini2@gmail.com>
Link: https://lore.kernel.org/r/20230103145503.71712-6-paolo.valente@linaro.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 9675a32..0859981 100644 (file)
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -2620,14 +2620,16 @@ static void bfq_bfqq_end_wr(struct bfq_queue *bfqq)
 void bfq_end_wr_async_queues(struct bfq_data *bfqd,
                             struct bfq_group *bfqg)
 {
-       int i, j;
-
-       for (i = 0; i < 2; i++)
-               for (j = 0; j < IOPRIO_NR_LEVELS; j++)
-                       if (bfqg->async_bfqq[i][j])
-                               bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]);
-       if (bfqg->async_idle_bfqq)
-               bfq_bfqq_end_wr(bfqg->async_idle_bfqq);
+       int i, j, k;
+
+       for (k = 0; k < bfqd->num_actuators; k++) {
+               for (i = 0; i < 2; i++)
+                       for (j = 0; j < IOPRIO_NR_LEVELS; j++)
+                               if (bfqg->async_bfqq[i][j][k])
+                                       bfq_bfqq_end_wr(bfqg->async_bfqq[i][j][k]);
+               if (bfqg->async_idle_bfqq[k])
+                       bfq_bfqq_end_wr(bfqg->async_idle_bfqq[k]);
+       }
 }
 
 static void bfq_end_wr(struct bfq_data *bfqd)
@@ -5575,18 +5577,18 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 
 static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd,
                                               struct bfq_group *bfqg,
-                                              int ioprio_class, int ioprio)
+                                              int ioprio_class, int ioprio, int act_idx)
 {
        switch (ioprio_class) {
        case IOPRIO_CLASS_RT:
-               return &bfqg->async_bfqq[0][ioprio];
+               return &bfqg->async_bfqq[0][ioprio][act_idx];
        case IOPRIO_CLASS_NONE:
                ioprio = IOPRIO_BE_NORM;
                fallthrough;
        case IOPRIO_CLASS_BE:
-               return &bfqg->async_bfqq[1][ioprio];
+               return &bfqg->async_bfqq[1][ioprio][act_idx];
        case IOPRIO_CLASS_IDLE:
-               return &bfqg->async_idle_bfqq;
+               return &bfqg->async_idle_bfqq[act_idx];
        default:
                return NULL;
        }
@@ -5759,7 +5761,8 @@ static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd,
        bfqg = bfq_bio_bfqg(bfqd, bio);
        if (!is_sync) {
                async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class,
-                                                 ioprio);
+                                                 ioprio,
+                                                 bfq_actuator_index(bfqd, bio));
                bfqq = *async_bfqq;
                if (bfqq)
                        goto out;
@@ -6982,13 +6985,15 @@ static void __bfq_put_async_bfqq(struct bfq_data *bfqd,
  */
 void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg)
 {
-       int i, j;
+       int i, j, k;
 
-       for (i = 0; i < 2; i++)
-               for (j = 0; j < IOPRIO_NR_LEVELS; j++)
-                       __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]);
+       for (k = 0; k < bfqd->num_actuators; k++) {
+               for (i = 0; i < 2; i++)
+                       for (j = 0; j < IOPRIO_NR_LEVELS; j++)
+                               __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j][k]);
 
-       __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq);
+               __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq[k]);
+       }
 }
 
 /*

diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index 28a884b..ea384d1 100644 (file)
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -980,8 +980,8 @@ struct bfq_group {
 
        struct bfq_data *bfqd;
 
-       struct bfq_queue *async_bfqq[2][IOPRIO_NR_LEVELS];
-       struct bfq_queue *async_idle_bfqq;
+       struct bfq_queue *async_bfqq[2][IOPRIO_NR_LEVELS][BFQ_MAX_ACTUATORS];
+       struct bfq_queue *async_idle_bfqq[BFQ_MAX_ACTUATORS];
 
        struct bfq_entity *my_entity;
 
@@ -998,8 +998,8 @@ struct bfq_group {
        struct bfq_entity entity;
        struct bfq_sched_data sched_data;
 
-       struct bfq_queue *async_bfqq[2][IOPRIO_NR_LEVELS];
-       struct bfq_queue *async_idle_bfqq;
+       struct bfq_queue *async_bfqq[2][IOPRIO_NR_LEVELS][BFQ_MAX_ACTUATORS];
+       struct bfq_queue *async_idle_bfqq[BFQ_MAX_ACTUATORS];
 
        struct rb_root rq_pos_tree;
 };