From ff029451496364eef0d342618dfc9972b78f3392 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 11 Jun 2020 08:44:52 +0200 Subject: [PATCH] nvme: use blk_mq_complete_request_remote to avoid an indirect function call Use the new blk_mq_complete_request_remote helper to avoid an indirect function call in the completion fast path. Reviewed-by: Daniel Wagner Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 4 +++- drivers/nvme/host/nvme.h | 7 ++++--- drivers/nvme/host/pci.c | 3 ++- drivers/nvme/host/rdma.c | 4 +++- drivers/nvme/host/tcp.c | 6 ++++-- drivers/nvme/target/loop.c | 3 ++- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index e999a8c..6aa30bb 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -227,6 +227,7 @@ static DECLARE_COMPLETION(nvme_fc_unload_proceed); */ static struct device *fc_udev_device; +static void nvme_fc_complete_rq(struct request *rq); /* *********************** FC-NVME Port Management ************************ */ @@ -2033,7 +2034,8 @@ done: } __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); - nvme_end_request(rq, status, result); + if (!nvme_end_request(rq, status, result)) + nvme_fc_complete_rq(rq); check_error: if (terminate_assoc) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 0346132..61780c3 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -472,7 +472,7 @@ static inline u32 nvme_bytes_to_numd(size_t len) return (len >> 2) - 1; } -static inline void nvme_end_request(struct request *req, __le16 status, +static inline bool nvme_end_request(struct request *req, __le16 status, union nvme_result result) { struct nvme_request *rq = nvme_req(req); @@ -481,8 +481,9 @@ static inline void nvme_end_request(struct request *req, __le16 status, rq->result = result; /* inject error when permitted by fault injection framework */ nvme_should_fail(req); - if (likely(!blk_should_fake_timeout(req->q))) - blk_mq_complete_request(req); + if (unlikely(blk_should_fake_timeout(req->q))) + return true; + return blk_mq_complete_request_remote(req); } static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index e2bacd3..0b3fedc 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -963,7 +963,8 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id); trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); - nvme_end_request(req, cqe->status, cqe->result); + if (!nvme_end_request(req, cqe->status, cqe->result)) + nvme_pci_complete_rq(req); } static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 9c02cf4..97b1054 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -149,6 +149,7 @@ MODULE_PARM_DESC(register_always, static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event); static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void nvme_rdma_complete_rq(struct request *rq); static const struct blk_mq_ops nvme_rdma_mq_ops; static const struct blk_mq_ops nvme_rdma_admin_mq_ops; @@ -1155,7 +1156,8 @@ static void nvme_rdma_end_request(struct nvme_rdma_request *req) if (!refcount_dec_and_test(&req->ref)) return; - nvme_end_request(rq, req->status, req->result); + if (!nvme_end_request(rq, req->status, req->result)) + nvme_rdma_complete_rq(rq); } static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc, diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 3345ec7..157b5b2 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -464,7 +464,8 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, return -EINVAL; } - nvme_end_request(rq, cqe->status, cqe->result); + if (!nvme_end_request(rq, cqe->status, cqe->result)) + nvme_complete_rq(rq); queue->nr_cqe++; return 0; @@ -654,7 +655,8 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status) { union nvme_result res = {}; - nvme_end_request(rq, cpu_to_le16(status << 1), res); + if (!nvme_end_request(rq, cpu_to_le16(status << 1), res)) + nvme_complete_rq(rq); } static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index 0d54e73..1b9c246 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -116,7 +116,8 @@ static void nvme_loop_queue_response(struct nvmet_req *req) return; } - nvme_end_request(rq, cqe->status, cqe->result); + if (!nvme_end_request(rq, cqe->status, cqe->result)) + nvme_loop_complete_rq(rq); } } -- 2.7.4