block: Reduce the amount of memory used for tag sets

Instead of allocating an array of size nr_cpu_ids for set->tags, allocate
an array of size set->nr_hw_queues. This patch improves behavior that was
introduced by commit 868f2f0b7206 ("blk-mq: dynamic h/w context count").

Reallocating tag sets from inside __blk_mq_update_nr_hw_queues() is safe
because:
- All request queues that share the tag sets are frozen before the tag sets
  are reallocated.
- blk_mq_queue_tag_busy_iter() holds q->q_usage_counter while active and
  hence is serialized against __blk_mq_update_nr_hw_queues().

Cc: Keith Busch <keith.busch@intel.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Ming Lei <ming.lei@redhat.com>
Cc: Hannes Reinecke <hare@suse.com>
Cc: Johannes Thumshirn <jthumshirn@suse.de>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq.c b/block/blk-mq.c
index ba09cda49953fee5a92e713f1de1544dd509857f..df41b2d1626151c08b11b04ce3d354a567aed5fe 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2833,19 +2833,6 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
        mutex_unlock(&q->sysfs_lock);
 }
 
-/*
- * Maximum number of hardware queues we support. For single sets, we'll never
- * have more than the CPUs (software queues). For multiple sets, the tag_set
- * user may have set ->nr_hw_queues larger.
- */
-static unsigned int nr_hw_queues(struct blk_mq_tag_set *set)
-{
-       if (set->nr_maps == 1)
-               return nr_cpu_ids;
-
-       return max(set->nr_hw_queues, nr_cpu_ids);
-}
-
 struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
                                                  struct request_queue *q,
                                                  bool elevator_init)
@@ -3012,6 +2999,29 @@ static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
        }
 }
 
+static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
+                                 int cur_nr_hw_queues, int new_nr_hw_queues)
+{
+       struct blk_mq_tags **new_tags;
+
+       if (cur_nr_hw_queues >= new_nr_hw_queues)
+               return 0;
+
+       new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
+                               GFP_KERNEL, set->numa_node);
+       if (!new_tags)
+               return -ENOMEM;
+
+       if (set->tags)
+               memcpy(new_tags, set->tags, cur_nr_hw_queues *
+                      sizeof(*set->tags));
+       kfree(set->tags);
+       set->tags = new_tags;
+       set->nr_hw_queues = new_nr_hw_queues;
+
+       return 0;
+}
+
 /*
  * Alloc a tag set to be associated with one or more request queues.
  * May fail with EINVAL for various error conditions. May adjust the
@@ -3065,9 +3075,7 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
        if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
                set->nr_hw_queues = nr_cpu_ids;
 
-       set->tags = kcalloc_node(nr_hw_queues(set), sizeof(struct blk_mq_tags *),
-                                GFP_KERNEL, set->numa_node);
-       if (!set->tags)
+       if (blk_mq_realloc_tag_set_tags(set, 0, set->nr_hw_queues) < 0)
                return -ENOMEM;
 
        ret = -ENOMEM;
@@ -3108,7 +3116,7 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
 {
        int i, j;
 
-       for (i = 0; i < nr_hw_queues(set); i++)
+       for (i = 0; i < set->nr_hw_queues; i++)
                blk_mq_free_map_and_requests(set, i);
 
        for (j = 0; j < set->nr_maps; j++) {
@@ -3266,6 +3274,10 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
                blk_mq_sysfs_unregister(q);
        }
 
+       if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
+           0)
+               goto reregister;
+
        prev_nr_hw_queues = set->nr_hw_queues;
        set->nr_hw_queues = nr_hw_queues;
        blk_mq_update_queue_map(set);
@@ -3282,6 +3294,7 @@ fallback:
                blk_mq_map_swqueue(q);
        }
 
+reregister:
        list_for_each_entry(q, &set->tag_list, tag_set_list) {
                blk_mq_sysfs_register(q);
                blk_mq_debugfs_register_hctxs(q);