return atomic_read(&hctx->nr_active) < depth;
}
-static int __blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
+static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
+ struct sbitmap_queue *bt)
{
- if (!hctx_may_queue(hctx, bt))
+ if (!(data->flags & BLK_MQ_REQ_INTERNAL) &&
+ !hctx_may_queue(data->hctx, bt))
return -1;
return __sbitmap_queue_get(bt);
}
struct sbq_wait_state *ws;
DEFINE_WAIT(wait);
unsigned int tag_offset;
+ bool drop_ctx;
int tag;
if (data->flags & BLK_MQ_REQ_RESERVED) {
tag_offset = tags->nr_reserved_tags;
}
- tag = __blk_mq_get_tag(data->hctx, bt);
+ tag = __blk_mq_get_tag(data, bt);
if (tag != -1)
goto found_tag;
return BLK_MQ_TAG_FAIL;
ws = bt_wait_ptr(bt, data->hctx);
+ drop_ctx = data->ctx == NULL;
do {
prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
- tag = __blk_mq_get_tag(data->hctx, bt);
+ tag = __blk_mq_get_tag(data, bt);
if (tag != -1)
break;
* Retry tag allocation after running the hardware queue,
* as running the queue may also have found completions.
*/
- tag = __blk_mq_get_tag(data->hctx, bt);
+ tag = __blk_mq_get_tag(data, bt);
if (tag != -1)
break;
- blk_mq_put_ctx(data->ctx);
+ if (data->ctx)
+ blk_mq_put_ctx(data->ctx);
io_schedule();
ws = bt_wait_ptr(bt, data->hctx);
} while (1);
+ if (drop_ctx && data->ctx)
+ blk_mq_put_ctx(data->ctx);
+
finish_wait(&ws->wait, &wait);
found_tag:
}
-static unsigned int bt_unused_tags(const struct sbitmap_queue *bt)
-{
- return bt->sb.depth - sbitmap_weight(&bt->sb);
-}
-
static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth,
bool round_robin, int node)
{
(rq->tag & BLK_MQ_UNIQUE_TAG_MASK);
}
EXPORT_SYMBOL(blk_mq_unique_tag);
-
-ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
-{
- char *orig_page = page;
- unsigned int free, res;
-
- if (!tags)
- return 0;
-
- page += sprintf(page, "nr_tags=%u, reserved_tags=%u, "
- "bits_per_word=%u\n",
- tags->nr_tags, tags->nr_reserved_tags,
- 1U << tags->bitmap_tags.sb.shift);
-
- free = bt_unused_tags(&tags->bitmap_tags);
- res = bt_unused_tags(&tags->breserved_tags);
-
- page += sprintf(page, "nr_free=%u, nr_reserved=%u\n", free, res);
- page += sprintf(page, "active_queues=%u\n", atomic_read(&tags->active_queues));
-
- return page - orig_page;
-}