block: move blk_mq_tag_to_rq() inline

This is in the fast path of driver issue or completion, and it's a single
array index operation. Move it inline to avoid a function call for it.

This does mean making struct blk_mq_tags block layer public, but there's
not really much in there.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 78ae2fb..df787b5 100644 (file)
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -4,29 +4,6 @@
 
 struct blk_mq_alloc_data;
 
-/*
- * Tag address space map.
- */
-struct blk_mq_tags {
-       unsigned int nr_tags;
-       unsigned int nr_reserved_tags;
-
-       atomic_t active_queues;
-
-       struct sbitmap_queue bitmap_tags;
-       struct sbitmap_queue breserved_tags;
-
-       struct request **rqs;
-       struct request **static_rqs;
-       struct list_head page_list;
-
-       /*
-        * used to clear request reference in rqs[] before freeing one
-        * request pool
-        */
-       spinlock_t lock;
-};
-
 extern struct blk_mq_tags *blk_mq_init_tags(unsigned int nr_tags,
                                        unsigned int reserved_tags,
                                        int node, int alloc_policy);

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 104019c..8f5c166 100644 (file)
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1120,17 +1120,6 @@ void blk_mq_delay_kick_requeue_list(struct request_queue *q,
 }
 EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
 
-struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
-{
-       if (tag < tags->nr_tags) {
-               prefetch(tags->rqs[tag]);
-               return tags->rqs[tag];
-       }
-
-       return NULL;
-}
-EXPORT_SYMBOL(blk_mq_tag_to_rq);
-
 static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
                               void *priv, bool reserved)
 {

diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 656fe34..6cf35de 100644 (file)
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -7,6 +7,7 @@
 #include <linux/srcu.h>
 #include <linux/lockdep.h>
 #include <linux/scatterlist.h>
+#include <linux/prefetch.h>
 
 struct blk_mq_tags;
 struct blk_flush_queue;
@@ -675,7 +676,40 @@ struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
 struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
                unsigned int op, blk_mq_req_flags_t flags,
                unsigned int hctx_idx);
-struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);
+
+/*
+ * Tag address space map.
+ */
+struct blk_mq_tags {
+       unsigned int nr_tags;
+       unsigned int nr_reserved_tags;
+
+       atomic_t active_queues;
+
+       struct sbitmap_queue bitmap_tags;
+       struct sbitmap_queue breserved_tags;
+
+       struct request **rqs;
+       struct request **static_rqs;
+       struct list_head page_list;
+
+       /*
+        * used to clear request reference in rqs[] before freeing one
+        * request pool
+        */
+       spinlock_t lock;
+};
+
+static inline struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags,
+                                              unsigned int tag)
+{
+       if (tag < tags->nr_tags) {
+               prefetch(tags->rqs[tag]);
+               return tags->rqs[tag];
+       }
+
+       return NULL;
+}
 
 enum {
        BLK_MQ_UNIQUE_TAG_BITS = 16,