From 433dc0efd1e041d39a1e993d6879a8c7692b0d2f Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Wed, 15 Dec 2021 15:57:19 +0200 Subject: [PATCH] IB/iser: Don't suppress send completions In order to complete a scsi command and guarantee that the HCA will never perform an access violation when retrying a send operation we must complete a scsi request only when both send and receive completions has arrived. This is a preparation commit that remove the send completions suppression. Next step will be taking care of the local invalidation mechanism and adding a reference counter for commands. Currently, we don't do anything upon getting the send completion and just "consume" it. Link: https://lore.kernel.org/r/20211215135721.3662-5-mgurtovoy@nvidia.com Signed-off-by: Max Gurtovoy Reviewed-by: Israel Rukshin Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/iser/iscsi_iser.h | 7 +------ drivers/infiniband/ulp/iser/iser_initiator.c | 13 +++---------- drivers/infiniband/ulp/iser/iser_verbs.c | 6 ++---- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h index 05a95d5..20af46c 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.h +++ b/drivers/infiniband/ulp/iser/iscsi_iser.h @@ -146,8 +146,6 @@ - ISER_MAX_RX_MISC_PDUS) / \ (1 + ISER_INFLIGHT_DATAOUTS)) -#define ISER_SIGNAL_CMD_COUNT 32 - /* Constant PDU lengths calculations */ #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr)) @@ -364,7 +362,6 @@ struct iser_fr_pool { * @qp: Connection Queue-pair * @cq: Connection completion queue * @cq_size: The number of max outstanding completions - * @sig_count: send work request signal count * @device: reference to iser device * @fr_pool: connection fast registration poool * @pi_support: Indicate device T10-PI support @@ -375,7 +372,6 @@ struct ib_conn { struct ib_qp *qp; struct ib_cq *cq; u32 cq_size; - u8 sig_count; struct iser_device *device; struct iser_fr_pool fr_pool; bool pi_support; @@ -532,8 +528,7 @@ int iser_connect(struct iser_conn *iser_conn, int iser_post_recvl(struct iser_conn *iser_conn); int iser_post_recvm(struct iser_conn *iser_conn, struct iser_rx_desc *rx_desc); -int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, - bool signal); +int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc); int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, struct iser_data_buf *data, diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c index ca22b6d..7788350 100644 --- a/drivers/infiniband/ulp/iser/iser_initiator.c +++ b/drivers/infiniband/ulp/iser/iser_initiator.c @@ -350,11 +350,6 @@ out: return err; } -static inline bool iser_signal_comp(u8 sig_count) -{ - return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0); -} - /** * iser_send_command - send command PDU * @conn: link to matching iscsi connection @@ -371,7 +366,6 @@ int iser_send_command(struct iscsi_conn *conn, struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; struct scsi_cmnd *sc = task->sc; struct iser_tx_desc *tx_desc = &iser_task->desc; - u8 sig_count = ++iser_conn->ib_conn.sig_count; edtl = ntohl(hdr->data_length); @@ -418,8 +412,7 @@ int iser_send_command(struct iscsi_conn *conn, iser_task->status = ISER_TASK_STATUS_STARTED; - err = iser_post_send(&iser_conn->ib_conn, tx_desc, - iser_signal_comp(sig_count)); + err = iser_post_send(&iser_conn->ib_conn, tx_desc); if (!err) return 0; @@ -487,7 +480,7 @@ int iser_send_data_out(struct iscsi_conn *conn, itt, buf_offset, data_seg_len); - err = iser_post_send(&iser_conn->ib_conn, tx_desc, true); + err = iser_post_send(&iser_conn->ib_conn, tx_desc); if (!err) return 0; @@ -550,7 +543,7 @@ int iser_send_control(struct iscsi_conn *conn, goto send_control_error; } - err = iser_post_send(&iser_conn->ib_conn, mdesc, true); + err = iser_post_send(&iser_conn->ib_conn, mdesc); if (!err) return 0; diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index e0d7119..53af7f4 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -871,12 +871,10 @@ int iser_post_recvm(struct iser_conn *iser_conn, struct iser_rx_desc *rx_desc) * iser_post_send - Initiate a Send DTO operation * @ib_conn: connection RDMA resources * @tx_desc: iSER TX descriptor - * @signal: true to send work request as SIGNALED * * Return: 0 on success, -1 on failure */ -int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, - bool signal) +int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc) { struct ib_send_wr *wr = &tx_desc->send_wr; struct ib_send_wr *first_wr; @@ -891,7 +889,7 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, wr->sg_list = tx_desc->tx_sg; wr->num_sge = tx_desc->num_sge; wr->opcode = IB_WR_SEND; - wr->send_flags = signal ? IB_SEND_SIGNALED : 0; + wr->send_flags = IB_SEND_SIGNALED; if (tx_desc->inv_wr.next) first_wr = &tx_desc->inv_wr; -- 2.7.4