bfq: Add per-device weight

This adds to BFQ the missing per-device weight interfaces:
blkio.bfq.weight_device on legacy and io.bfq.weight on unified. The
implementation pretty closely resembles what we had in CFQ and the parsing code
is basically reused.

Tests
=====

Using two cgroups and three block devices, having weights setup as:

Cgroup          test1           test2
============================================
default         100             500
sda             500             100
sdb             default         default
sdc             200             200

cgroup v1 runs
--------------

    sda.test1.out:   READ: bw=913MiB/s
    sda.test2.out:   READ: bw=183MiB/s

    sdb.test1.out:   READ: bw=213MiB/s
    sdb.test2.out:   READ: bw=1054MiB/s

    sdc.test1.out:   READ: bw=650MiB/s
    sdc.test2.out:   READ: bw=650MiB/s

cgroup v2 runs
--------------

    sda.test1.out:   READ: bw=915MiB/s
    sda.test2.out:   READ: bw=184MiB/s

    sdb.test1.out:   READ: bw=216MiB/s
    sdb.test2.out:   READ: bw=1069MiB/s

    sdc.test1.out:   READ: bw=621MiB/s
    sdc.test2.out:   READ: bw=622MiB/s

Signed-off-by: Fam Zheng <zhengfeiran@bytedance.com>
Acked-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Paolo Valente <paolo.valente@linaro.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index 102bdfd..86a607c 100644 (file)
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -905,7 +905,7 @@ void bfq_end_wr_async(struct bfq_data *bfqd)
        bfq_end_wr_async_queues(bfqd, bfqd->root_group);
 }
 
-static int bfq_io_show_weight(struct seq_file *sf, void *v)
+static int bfq_io_show_weight_legacy(struct seq_file *sf, void *v)
 {
        struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
        struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
@@ -919,8 +919,32 @@ static int bfq_io_show_weight(struct seq_file *sf, void *v)
        return 0;
 }
 
-static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight)
+static u64 bfqg_prfill_weight_device(struct seq_file *sf,
+                                    struct blkg_policy_data *pd, int off)
+{
+       struct bfq_group *bfqg = pd_to_bfqg(pd);
+
+       if (!bfqg->entity.dev_weight)
+               return 0;
+       return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);
+}
+
+static int bfq_io_show_weight(struct seq_file *sf, void *v)
+{
+       struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+       struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
+
+       seq_printf(sf, "default %u\n", bfqgd->weight);
+       blkcg_print_blkgs(sf, blkcg, bfqg_prfill_weight_device,
+                         &blkcg_policy_bfq, 0, false);
+       return 0;
+}
+
+static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)
 {
+       weight = dev_weight ?: weight;
+
+       bfqg->entity.dev_weight = dev_weight;
        /*
         * Setting the prio_changed flag of the entity
         * to 1 with new_weight == weight would re-set
@@ -968,28 +992,71 @@ static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css,
                struct bfq_group *bfqg = blkg_to_bfqg(blkg);
 
                if (bfqg)
-                       bfq_group_set_weight(bfqg, val);
+                       bfq_group_set_weight(bfqg, val, 0);
        }
        spin_unlock_irq(&blkcg->lock);
 
        return ret;
 }
 
-static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
-                                char *buf, size_t nbytes,
-                                loff_t off)
+static ssize_t bfq_io_set_device_weight(struct kernfs_open_file *of,
+                                       char *buf, size_t nbytes,
+                                       loff_t off)
 {
-       u64 weight;
-       /* First unsigned long found in the file is used */
-       int ret = kstrtoull(strim(buf), 0, &weight);
+       int ret;
+       struct blkg_conf_ctx ctx;
+       struct blkcg *blkcg = css_to_blkcg(of_css(of));
+       struct bfq_group *bfqg;
+       u64 v;
 
+       ret = blkg_conf_prep(blkcg, &blkcg_policy_bfq, buf, &ctx);
        if (ret)
                return ret;
 
-       ret = bfq_io_set_weight_legacy(of_css(of), NULL, weight);
+       if (sscanf(ctx.body, "%llu", &v) == 1) {
+               /* require "default" on dfl */
+               ret = -ERANGE;
+               if (!v)
+                       goto out;
+       } else if (!strcmp(strim(ctx.body), "default")) {
+               v = 0;
+       } else {
+               ret = -EINVAL;
+               goto out;
+       }
+
+       bfqg = blkg_to_bfqg(ctx.blkg);
+
+       ret = -ERANGE;
+       if (!v || (v >= BFQ_MIN_WEIGHT && v <= BFQ_MAX_WEIGHT)) {
+               bfq_group_set_weight(bfqg, bfqg->entity.weight, v);
+               ret = 0;
+       }
+out:
+       blkg_conf_finish(&ctx);
        return ret ?: nbytes;
 }
 
+static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
+                                char *buf, size_t nbytes,
+                                loff_t off)
+{
+       char *endp;
+       int ret;
+       u64 v;
+
+       buf = strim(buf);
+
+       /* "WEIGHT" or "default WEIGHT" sets the default weight */
+       v = simple_strtoull(buf, &endp, 0);
+       if (*endp == '\0' || sscanf(buf, "default %llu", &v) == 1) {
+               ret = bfq_io_set_weight_legacy(of_css(of), NULL, v);
+               return ret ?: nbytes;
+       }
+
+       return bfq_io_set_device_weight(of, buf, nbytes, off);
+}
+
 #ifdef CONFIG_BFQ_CGROUP_DEBUG
 static int bfqg_print_stat(struct seq_file *sf, void *v)
 {
@@ -1146,9 +1213,15 @@ struct cftype bfq_blkcg_legacy_files[] = {
        {
                .name = "bfq.weight",
                .flags = CFTYPE_NOT_ON_ROOT,
-               .seq_show = bfq_io_show_weight,
+               .seq_show = bfq_io_show_weight_legacy,
                .write_u64 = bfq_io_set_weight_legacy,
        },
+       {
+               .name = "bfq.weight_device",
+               .flags = CFTYPE_NOT_ON_ROOT,
+               .seq_show = bfq_io_show_weight,
+               .write = bfq_io_set_weight,
+       },
 
        /* statistics, covers only the tasks in the bfqg */
        {

diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index e80adf8..5d1a519 100644 (file)
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -168,6 +168,9 @@ struct bfq_entity {
        /* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */
        int budget;
 
+       /* device weight, if non-zero, it overrides the default weight of
+        * bfq_group_data */
+       int dev_weight;
        /* weight of the queue */
        int weight;
        /* next weight if a change is in progress */