2 * Copyright (c) 2016 Chelsio Communications, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/workqueue.h>
10 #include <linux/kthread.h>
11 #include <linux/sched/signal.h>
13 #include <asm/unaligned.h>
15 #include <target/target_core_base.h>
16 #include <target/target_core_fabric.h>
19 struct sge_opaque_hdr {
21 dma_addr_t addr[MAX_SKB_FRAGS + 1];
24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
27 sizeof(struct fw_ofld_tx_data_wr))
29 static struct sk_buff *
30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
32 struct sk_buff *skb = NULL;
35 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
38 skb = alloc_skb_with_frags(hdr_len, len,
44 skb_reserve(skb, TX_HDR_LEN);
45 skb_reset_transport_header(skb);
46 __skb_put(skb, ISCSI_HDR_LEN);
49 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
52 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
54 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
58 skb_reserve(skb, TX_HDR_LEN + iso_len);
59 skb_reset_transport_header(skb);
60 __skb_put(skb, ISCSI_HDR_LEN);
63 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
64 cxgbit_skcb_submode(skb) = submode;
65 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
66 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
72 return __cxgbit_alloc_skb(csk, len, false);
76 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
79 * Returns true if a packet can be sent as an offload WR with immediate
80 * data. We currently use the same limit as for Ethernet packets.
82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
84 int length = skb->len;
86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
87 length += sizeof(struct fw_ofld_tx_data_wr);
89 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
90 length += sizeof(struct cpl_tx_data_iso);
92 #define MAX_IMM_TX_PKT_LEN 256
93 return length <= MAX_IMM_TX_PKT_LEN;
97 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
98 * @n: the number of SGL entries
99 * Calculates the number of flits needed for a scatter/gather list that
100 * can hold the given number of entries.
102 static inline unsigned int cxgbit_sgl_len(unsigned int n)
105 return (3 * n) / 2 + (n & 1) + 2;
109 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
112 * Returns the number of flits needed for the given offload packet.
113 * These packets are already fully constructed and no additional headers
116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
118 unsigned int flits, cnt;
120 if (cxgbit_is_ofld_imm(skb))
121 return DIV_ROUND_UP(skb->len, 8);
122 flits = skb_transport_offset(skb) / 8;
123 cnt = skb_shinfo(skb)->nr_frags;
124 if (skb_tail_pointer(skb) != skb_transport_header(skb))
126 return flits + cxgbit_sgl_len(cnt);
129 #define CXGBIT_ISO_FSLICE 0x1
130 #define CXGBIT_ISO_LSLICE 0x2
132 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
134 struct cpl_tx_data_iso *cpl;
135 unsigned int submode = cxgbit_skcb_submode(skb);
136 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
137 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
139 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl));
141 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
142 CPL_TX_DATA_ISO_FIRST_V(fslice) |
143 CPL_TX_DATA_ISO_LAST_V(lslice) |
144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
145 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
146 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
148 CPL_TX_DATA_ISO_SCSI_V(2));
151 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
152 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
153 cpl->len = htonl(iso_info->len);
154 cpl->reserved2_seglen_offset = htonl(0);
155 cpl->datasn_offset = htonl(0);
156 cpl->buffer_offset = htonl(0);
159 __skb_pull(skb, sizeof(*cpl));
163 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
164 u32 len, u32 credits, u32 compl)
166 struct fw_ofld_tx_data_wr *req;
167 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
168 u32 submode = cxgbit_skcb_submode(skb);
170 u32 hdr_size = sizeof(*req);
171 u32 opcode = FW_OFLD_TX_DATA_WR;
173 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
176 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
177 opcode = FW_ISCSI_TX_DATA_WR;
178 immlen += sizeof(struct cpl_tx_data_iso);
179 hdr_size += sizeof(struct cpl_tx_data_iso);
183 if (cxgbit_is_ofld_imm(skb))
186 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb,
188 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
189 FW_WR_COMPL_V(compl) |
190 FW_WR_IMMDLEN_V(immlen));
191 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
192 FW_WR_LEN16_V(credits));
193 req->plen = htonl(len);
194 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
195 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
197 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
198 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
201 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
206 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
210 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
218 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
219 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
221 if (cxgbit_is_ofld_imm(skb))
222 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
224 credits_needed = DIV_ROUND_UP((8 *
225 cxgbit_calc_tx_flits_ofld(skb)) +
228 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
229 credits_needed += DIV_ROUND_UP(
230 sizeof(struct fw_ofld_tx_data_wr), 16);
232 * Assumes the initial credits is large enough to support
233 * fw_flowc_wr plus largest possible first payload
236 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
237 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
238 csk->wr_cred -= flowclen16;
239 csk->wr_una_cred += flowclen16;
242 if (csk->wr_cred < credits_needed) {
243 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
244 csk, skb->len, skb->data_len,
245 credits_needed, csk->wr_cred);
248 __skb_unlink(skb, &csk->txq);
249 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
250 skb->csum = (__force __wsum)(credits_needed + flowclen16);
251 csk->wr_cred -= credits_needed;
252 csk->wr_una_cred += credits_needed;
254 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
255 csk, skb->len, skb->data_len, credits_needed,
256 csk->wr_cred, csk->wr_una_cred);
258 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
259 len += cxgbit_skcb_tx_extralen(skb);
261 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
262 (!before(csk->write_seq,
263 csk->snd_una + csk->snd_win))) {
265 csk->wr_una_cred = 0;
268 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
272 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
273 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
274 struct cpl_close_con_req *req =
275 (struct cpl_close_con_req *)skb->data;
276 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
277 csk->wr_una_cred = 0;
280 cxgbit_sock_enqueue_wr(csk, skb);
281 t4_set_arp_err_handler(skb, csk,
282 cxgbit_arp_failure_skb_discard);
284 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
285 csk, csk->tid, skb, len);
287 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
291 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
293 spin_lock_bh(&csk->lock);
295 if (before(csk->write_seq, csk->snd_una + csk->snd_win))
296 csk->lock_owner = true;
298 spin_unlock_bh(&csk->lock);
300 return csk->lock_owner;
303 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
305 struct sk_buff_head backlogq;
307 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
309 skb_queue_head_init(&backlogq);
311 spin_lock_bh(&csk->lock);
312 while (skb_queue_len(&csk->backlogq)) {
313 skb_queue_splice_init(&csk->backlogq, &backlogq);
314 spin_unlock_bh(&csk->lock);
316 while ((skb = __skb_dequeue(&backlogq))) {
317 fn = cxgbit_skcb_rx_backlog_fn(skb);
321 spin_lock_bh(&csk->lock);
324 csk->lock_owner = false;
325 spin_unlock_bh(&csk->lock);
328 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
332 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
334 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
335 signal_pending(current))) {
337 __skb_queue_purge(&csk->ppodq);
339 spin_lock_bh(&csk->lock);
340 if (csk->lock_owner) {
341 spin_unlock_bh(&csk->lock);
344 spin_unlock_bh(&csk->lock);
348 csk->write_seq += skb->len +
349 cxgbit_skcb_tx_extralen(skb);
351 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
352 __skb_queue_tail(&csk->txq, skb);
353 cxgbit_push_tx_frames(csk);
356 cxgbit_unlock_sock(csk);
361 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
364 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
365 u32 padding = ((-data_length) & 3);
366 struct scatterlist *sg;
368 unsigned int page_off;
374 * We know each entry in t_data_sg contains a page.
376 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
377 page_off = (data_offset % PAGE_SIZE);
379 while (data_length && (i < nr_frags)) {
380 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
385 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
387 skb->data_len += cur_len;
389 skb->truesize += cur_len;
391 data_length -= cur_len;
401 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
404 skb_fill_page_desc(skb, i, page, 0, padding);
405 skb->data_len += padding;
407 skb->truesize += padding;
414 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
415 struct iscsi_datain_req *dr)
417 struct iscsi_conn *conn = csk->conn;
419 struct iscsi_datain datain;
420 struct cxgbit_iso_info iso_info;
421 u32 data_length = cmd->se_cmd.data_length;
422 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
423 u32 num_pdu, plen, tx_data = 0;
424 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
425 SCF_TRANSPORT_TASK_SENSE);
426 bool set_statsn = false;
429 while (data_length) {
430 num_pdu = (data_length + mrdsl - 1) / mrdsl;
431 if (num_pdu > csk->max_iso_npdu)
432 num_pdu = csk->max_iso_npdu;
434 plen = num_pdu * mrdsl;
435 if (plen > data_length)
438 skb = __cxgbit_alloc_skb(csk, 0, true);
442 memset(skb->data, 0, ISCSI_HDR_LEN);
443 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
444 cxgbit_skcb_submode(skb) |= (csk->submode &
445 CXGBIT_SUBMODE_DCRC);
446 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
447 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
448 ((num_pdu - 1) * ISCSI_HDR_LEN);
450 memset(&datain, 0, sizeof(struct iscsi_datain));
451 memset(&iso_info, 0, sizeof(iso_info));
454 iso_info.flags |= CXGBIT_ISO_FSLICE;
456 if (!(data_length - plen)) {
457 iso_info.flags |= CXGBIT_ISO_LSLICE;
459 datain.flags = ISCSI_FLAG_DATA_STATUS;
460 iscsit_increment_maxcmdsn(cmd, conn->sess);
461 cmd->stat_sn = conn->stat_sn++;
466 iso_info.burst_len = num_pdu * mrdsl;
467 iso_info.mpdu = mrdsl;
468 iso_info.len = ISCSI_HDR_LEN + plen;
470 cxgbit_cpl_tx_data_iso(skb, &iso_info);
472 datain.offset = tx_data;
473 datain.data_sn = cmd->data_sn - 1;
475 iscsit_build_datain_pdu(cmd, conn, &datain,
476 (struct iscsi_data_rsp *)skb->data,
479 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
485 ret = cxgbit_queue_skb(csk, skb);
492 cmd->read_data_done += plen;
493 cmd->data_sn += num_pdu;
496 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
505 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
506 const struct iscsi_datain *datain)
511 skb = cxgbit_alloc_skb(csk, 0);
515 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
517 if (datain->length) {
518 cxgbit_skcb_submode(skb) |= (csk->submode &
519 CXGBIT_SUBMODE_DCRC);
520 cxgbit_skcb_tx_extralen(skb) =
521 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
524 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
530 return cxgbit_queue_skb(csk, skb);
534 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
535 struct iscsi_datain_req *dr,
536 const struct iscsi_datain *datain)
538 struct cxgbit_sock *csk = conn->context;
539 u32 data_length = cmd->se_cmd.data_length;
540 u32 padding = ((-data_length) & 3);
541 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
543 if ((data_length > mrdsl) && (!dr->recovery) &&
544 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
545 atomic_long_add(data_length - datain->length,
546 &conn->sess->tx_data_octets);
547 return cxgbit_tx_datain_iso(csk, cmd, dr);
550 return cxgbit_tx_datain(csk, cmd, datain);
554 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
555 const void *data_buf, u32 data_buf_len)
557 struct cxgbit_sock *csk = conn->context;
559 u32 padding = ((-data_buf_len) & 3);
561 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
565 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
570 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
573 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
574 &pad_bytes, padding);
577 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
578 cxgbit_skcb_submode(skb)];
580 return cxgbit_queue_skb(csk, skb);
584 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
585 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
588 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
590 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
593 int cxgbit_validate_params(struct iscsi_conn *conn)
595 struct cxgbit_sock *csk = conn->context;
596 struct cxgbit_device *cdev = csk->com.cdev;
597 struct iscsi_param *param;
600 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
605 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
608 if (max_xmitdsl > cdev->mdsl) {
609 if (iscsi_change_param_sprintf(
610 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
617 static int cxgbit_set_digest(struct cxgbit_sock *csk)
619 struct iscsi_conn *conn = csk->conn;
620 struct iscsi_param *param;
622 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
624 pr_err("param not found key %s\n", HEADERDIGEST);
628 if (!strcmp(param->value, CRC32C))
629 csk->submode |= CXGBIT_SUBMODE_HCRC;
631 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
634 pr_err("param not found key %s\n", DATADIGEST);
638 if (!strcmp(param->value, CRC32C))
639 csk->submode |= CXGBIT_SUBMODE_DCRC;
641 if (cxgbit_setup_conn_digest(csk)) {
649 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
651 struct iscsi_conn *conn = csk->conn;
652 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
653 struct iscsi_param *param;
655 u32 max_npdu, max_iso_npdu;
657 if (conn->login->leading_connection) {
658 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
661 pr_err("param not found key %s\n", MAXBURSTLENGTH);
665 if (kstrtou32(param->value, 0, &mbl) < 0)
668 mbl = conn->sess->sess_ops->MaxBurstLength;
671 mrdsl = conn_ops->MaxRecvDataSegmentLength;
672 max_npdu = mbl / mrdsl;
674 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD /
675 (ISCSI_HDR_LEN + mrdsl +
676 cxgbit_digest_len[csk->submode]);
678 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
680 if (csk->max_iso_npdu <= 1)
681 csk->max_iso_npdu = 0;
687 * cxgbit_seq_pdu_inorder()
688 * @csk: pointer to cxgbit socket structure
690 * This function checks whether data sequence and data
693 * Return: returns -1 on error, 0 if data sequence and
694 * data pdu are in order, 1 if data sequence or data pdu
697 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
699 struct iscsi_conn *conn = csk->conn;
700 struct iscsi_param *param;
702 if (conn->login->leading_connection) {
703 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
706 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
710 if (strcmp(param->value, YES))
713 param = iscsi_find_param_from_key(DATAPDUINORDER,
716 pr_err("param not found key %s\n", DATAPDUINORDER);
720 if (strcmp(param->value, YES))
724 if (!conn->sess->sess_ops->DataSequenceInOrder)
726 if (!conn->sess->sess_ops->DataPDUInOrder)
733 static int cxgbit_set_params(struct iscsi_conn *conn)
735 struct cxgbit_sock *csk = conn->context;
736 struct cxgbit_device *cdev = csk->com.cdev;
737 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
738 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
739 struct iscsi_param *param;
742 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
743 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
745 if (conn->login->leading_connection) {
746 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
749 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
752 if (kstrtou8(param->value, 0, &erl) < 0)
755 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
761 ret = cxgbit_seq_pdu_inorder(csk);
764 } else if (ret > 0) {
765 if (is_t5(cdev->lldi.adapter_type))
771 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
772 if (cxgbit_set_iso_npdu(csk))
777 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
778 if (cxgbit_setup_conn_pgidx(csk,
779 ppm->tformat.pgsz_idx_dflt))
781 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
786 if (cxgbit_set_digest(csk))
793 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
796 struct cxgbit_sock *csk = conn->context;
799 u8 padding = ((-length) & 3);
801 skb = cxgbit_alloc_skb(csk, length + padding);
804 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
805 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
808 skb_store_bits(skb, ISCSI_HDR_LEN + length,
809 &padding_buf, padding);
811 if (login->login_complete) {
812 if (cxgbit_set_params(conn)) {
817 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
820 if (cxgbit_queue_skb(csk, skb))
823 if ((!login->login_complete) && (!login->login_failed))
824 schedule_delayed_work(&conn->login_work, 0);
830 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
833 struct skb_seq_state st;
835 unsigned int consumed = 0, buf_len;
836 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
838 skb_prepare_seq_read(skb, pdu_cb->doffset,
839 pdu_cb->doffset + pdu_cb->dlen,
843 buf_len = skb_seq_read(consumed, &buf, &st);
845 skb_abort_seq_read(&st);
849 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
854 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
856 struct iscsi_conn *conn = csk->conn;
857 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
858 struct cxgbit_cmd *ccmd;
859 struct iscsi_cmd *cmd;
861 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
863 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
867 ccmd = iscsit_priv_cmd(cmd);
868 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
869 ccmd->setup_ddp = true;
875 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
878 struct iscsi_conn *conn = cmd->conn;
879 struct cxgbit_sock *csk = conn->context;
880 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
882 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
883 pr_err("ImmediateData CRC32C DataDigest error\n");
884 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
885 pr_err("Unable to recover from"
886 " Immediate Data digest failure while"
888 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
889 (unsigned char *)hdr);
890 return IMMEDIATE_DATA_CANNOT_RECOVER;
893 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
894 (unsigned char *)hdr);
895 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
898 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
899 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
900 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
901 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
903 sg_init_table(&ccmd->sg, 1);
904 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
906 get_page(dfrag->page.p);
908 cmd->se_cmd.t_data_sg = &ccmd->sg;
909 cmd->se_cmd.t_data_nents = 1;
911 ccmd->release = true;
913 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
914 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
916 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents);
919 cmd->write_data_done += pdu_cb->dlen;
921 if (cmd->write_data_done == cmd->se_cmd.data_length) {
922 spin_lock_bh(&cmd->istate_lock);
923 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
924 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
925 spin_unlock_bh(&cmd->istate_lock);
928 return IMMEDIATE_DATA_NORMAL_OPERATION;
932 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
935 struct iscsi_conn *conn = cmd->conn;
936 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
938 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
941 goto after_immediate_data;
943 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
944 cmd->first_burst_len);
945 after_immediate_data:
946 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
948 * A PDU/CmdSN carrying Immediate Data passed
949 * DataCRC, check against ExpCmdSN/MaxCmdSN if
950 * Immediate Bit is not set.
952 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
953 (unsigned char *)hdr,
955 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
958 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
959 target_put_sess_cmd(&cmd->se_cmd);
961 } else if (cmd->unsolicited_data) {
962 iscsit_set_unsoliticed_dataout(cmd);
965 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
967 * Immediate Data failed DataCRC and ERL>=1,
968 * silently drop this PDU and let the initiator
969 * plug the CmdSN gap.
971 * FIXME: Send Unsolicited NOPIN with reserved
972 * TTT here to help the initiator figure out
973 * the missing CmdSN, although they should be
974 * intelligent enough to determine the missing
975 * CmdSN and issue a retry to plug the sequence.
977 cmd->i_state = ISTATE_REMOVE;
978 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
979 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
986 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
988 struct iscsi_conn *conn = csk->conn;
989 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
990 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
992 bool dump_payload = false;
994 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
998 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
999 (pdu_cb->nr_dfrags == 1))
1000 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1002 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1006 dump_payload = true;
1011 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
1014 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
1016 struct scatterlist *sg_start;
1017 struct iscsi_conn *conn = csk->conn;
1018 struct iscsi_cmd *cmd = NULL;
1019 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1020 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1021 u32 data_offset = be32_to_cpu(hdr->offset);
1022 u32 data_len = pdu_cb->dlen;
1023 int rc, sg_nents, sg_off;
1024 bool dcrc_err = false;
1026 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1027 u32 offset = be32_to_cpu(hdr->offset);
1029 u32 payload_length = ntoh24(hdr->dlength);
1030 bool success = false;
1032 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1036 ddp_data_len = offset - cmd->write_data_done;
1037 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1039 cmd->write_data_done = offset;
1040 cmd->next_burst_len = ddp_data_len;
1041 cmd->data_sn = be32_to_cpu(hdr->datasn);
1043 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1044 cmd, payload_length, &success);
1050 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1057 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1058 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1059 " DataSN: 0x%08x\n",
1060 hdr->itt, hdr->offset, data_len,
1067 pr_debug("DataOut data_len: %u, "
1068 "write_data_done: %u, data_length: %u\n",
1069 data_len, cmd->write_data_done,
1070 cmd->se_cmd.data_length);
1072 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1073 sg_off = data_offset / PAGE_SIZE;
1074 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1075 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE));
1077 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents);
1082 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1089 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1091 struct iscsi_conn *conn = csk->conn;
1092 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1093 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1094 unsigned char *ping_data = NULL;
1095 u32 payload_length = pdu_cb->dlen;
1098 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1102 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1103 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1104 pr_err("Unable to recover from"
1105 " NOPOUT Ping DataCRC failure while in"
1111 * drop this PDU and let the
1112 * initiator plug the CmdSN gap.
1114 pr_info("Dropping NOPOUT"
1115 " Command CmdSN: 0x%08x due to"
1116 " DataCRC error.\n", hdr->cmdsn);
1123 * Handle NOP-OUT payload for traditional iSCSI sockets
1125 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1126 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1128 pr_err("Unable to allocate memory for"
1129 " NOPOUT ping data.\n");
1134 skb_copy_bits(csk->skb, pdu_cb->doffset,
1135 ping_data, payload_length);
1137 ping_data[payload_length] = '\0';
1139 * Attach ping data to struct iscsi_cmd->buf_ptr.
1141 cmd->buf_ptr = ping_data;
1142 cmd->buf_ptr_size = payload_length;
1144 pr_debug("Got %u bytes of NOPOUT ping"
1145 " data.\n", payload_length);
1146 pr_debug("Ping Data: \"%s\"\n", ping_data);
1149 return iscsit_process_nop_out(conn, cmd, hdr);
1152 iscsit_free_cmd(cmd, false);
1157 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1159 struct iscsi_conn *conn = csk->conn;
1160 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1161 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1162 u32 payload_length = pdu_cb->dlen;
1164 unsigned char *text_in = NULL;
1166 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1170 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1171 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1172 pr_err("Unable to recover from"
1173 " Text Data digest failure while in"
1178 * drop this PDU and let the
1179 * initiator plug the CmdSN gap.
1181 pr_info("Dropping Text"
1182 " Command CmdSN: 0x%08x due to"
1183 " DataCRC error.\n", hdr->cmdsn);
1188 if (payload_length) {
1189 text_in = kzalloc(payload_length, GFP_KERNEL);
1191 pr_err("Unable to allocate text_in of payload_length: %u\n",
1195 skb_copy_bits(csk->skb, pdu_cb->doffset,
1196 text_in, payload_length);
1198 text_in[payload_length - 1] = '\0';
1200 cmd->text_in_ptr = text_in;
1203 return iscsit_process_text_cmd(conn, cmd, hdr);
1206 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1210 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1212 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1213 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1214 struct iscsi_conn *conn = csk->conn;
1215 struct iscsi_cmd *cmd = NULL;
1216 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1220 case ISCSI_OP_SCSI_CMD:
1221 cmd = cxgbit_allocate_cmd(csk);
1225 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1227 case ISCSI_OP_SCSI_DATA_OUT:
1228 ret = cxgbit_handle_iscsi_dataout(csk);
1230 case ISCSI_OP_NOOP_OUT:
1231 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1232 cmd = cxgbit_allocate_cmd(csk);
1237 ret = cxgbit_handle_nop_out(csk, cmd);
1239 case ISCSI_OP_SCSI_TMFUNC:
1240 cmd = cxgbit_allocate_cmd(csk);
1244 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1245 (unsigned char *)hdr);
1248 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1249 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1253 cmd = cxgbit_allocate_cmd(csk);
1258 ret = cxgbit_handle_text_cmd(csk, cmd);
1260 case ISCSI_OP_LOGOUT:
1261 cmd = cxgbit_allocate_cmd(csk);
1265 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1267 wait_for_completion_timeout(&conn->conn_logout_comp,
1268 SECONDS_FOR_LOGOUT_COMP
1271 case ISCSI_OP_SNACK:
1272 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1275 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1283 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1284 (unsigned char *)hdr);
1288 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1290 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1291 struct iscsi_conn *conn = csk->conn;
1292 struct iscsi_hdr *hdr = pdu_cb->hdr;
1295 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1296 atomic_long_inc(&conn->sess->conn_digest_errors);
1300 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1303 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1305 if (conn->sess->sess_ops->SessionType &&
1306 ((!(opcode & ISCSI_OP_TEXT)) ||
1307 (!(opcode & ISCSI_OP_LOGOUT)))) {
1308 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1309 " while in Discovery Session, rejecting.\n", opcode);
1310 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1311 (unsigned char *)hdr);
1315 if (cxgbit_target_rx_opcode(csk) < 0)
1324 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1326 struct iscsi_conn *conn = csk->conn;
1327 struct iscsi_login *login = conn->login;
1328 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1329 struct iscsi_login_req *login_req;
1331 login_req = (struct iscsi_login_req *)login->req;
1332 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1334 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1335 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1336 login_req->flags, login_req->itt, login_req->cmdsn,
1337 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1339 * Setup the initial iscsi_login values from the leading
1340 * login request PDU.
1342 if (login->first_request) {
1343 login_req = (struct iscsi_login_req *)login->req;
1344 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1345 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1347 login->version_min = login_req->min_version;
1348 login->version_max = login_req->max_version;
1349 memcpy(login->isid, login_req->isid, 6);
1350 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1351 login->init_task_tag = login_req->itt;
1352 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1353 login->cid = be16_to_cpu(login_req->cid);
1354 login->tsih = be16_to_cpu(login_req->tsih);
1357 if (iscsi_target_check_login_request(conn, login) < 0)
1360 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1361 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1367 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1369 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1372 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1376 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1377 ret = cxgbit_rx_login_pdu(csk);
1378 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1380 ret = cxgbit_rx_opcode(csk);
1386 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1388 struct skb_shared_info *ssi = skb_shinfo(skb);
1389 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1390 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1393 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1394 skb, skb->head, skb->data, skb->len, skb->data_len,
1396 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1397 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1399 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1400 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1402 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1403 pdu_cb->ddigest, pdu_cb->frags);
1404 for (i = 0; i < ssi->nr_frags; i++)
1405 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1406 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1409 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1411 struct sk_buff *skb = csk->lro_hskb;
1412 struct skb_shared_info *ssi = skb_shinfo(skb);
1415 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1416 for (i = 0; i < ssi->nr_frags; i++)
1417 put_page(skb_frag_page(&ssi->frags[i]));
1420 skb->truesize -= skb->len;
1425 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1427 struct sk_buff *hskb = csk->lro_hskb;
1428 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1429 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1430 struct skb_shared_info *hssi = skb_shinfo(hskb);
1431 struct skb_shared_info *ssi = skb_shinfo(skb);
1432 unsigned int len = 0;
1434 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1435 u8 hfrag_idx = hssi->nr_frags;
1437 hpdu_cb->flags |= pdu_cb->flags;
1438 hpdu_cb->seq = pdu_cb->seq;
1439 hpdu_cb->hdr = pdu_cb->hdr;
1440 hpdu_cb->hlen = pdu_cb->hlen;
1442 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1443 sizeof(skb_frag_t));
1445 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1448 hpdu_cb->hfrag_idx = hfrag_idx;
1450 len = hssi->frags[hfrag_idx].size;
1452 hskb->data_len += len;
1453 hskb->truesize += len;
1456 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1457 u8 dfrag_idx = hssi->nr_frags, i;
1459 hpdu_cb->flags |= pdu_cb->flags;
1460 hpdu_cb->dfrag_idx = dfrag_idx;
1463 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1464 memcpy(&hssi->frags[dfrag_idx],
1465 &ssi->frags[pdu_cb->dfrag_idx + i],
1466 sizeof(skb_frag_t));
1468 get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1470 len += hssi->frags[dfrag_idx].size;
1476 hpdu_cb->dlen = pdu_cb->dlen;
1477 hpdu_cb->doffset = hpdu_cb->hlen;
1478 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1480 hskb->data_len += len;
1481 hskb->truesize += len;
1484 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1485 hpdu_cb->flags |= pdu_cb->flags;
1487 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1488 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1490 hpdu_cb->ddigest = pdu_cb->ddigest;
1491 hpdu_cb->pdulen = pdu_cb->pdulen;
1495 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1497 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1498 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1499 u8 pdu_idx = 0, last_idx = 0;
1502 if (!pdu_cb->complete) {
1503 cxgbit_lro_skb_merge(csk, skb, 0);
1505 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1506 struct sk_buff *hskb = csk->lro_hskb;
1508 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1510 cxgbit_lro_hskb_reset(csk);
1519 if (lro_cb->pdu_idx)
1520 last_idx = lro_cb->pdu_idx - 1;
1522 for (; pdu_idx <= last_idx; pdu_idx++) {
1523 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1528 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1529 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1535 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1537 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1538 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1541 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1542 (pdu_cb->seq != csk->rcv_nxt)) {
1543 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1544 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1545 cxgbit_lro_skb_dump(skb);
1549 csk->rcv_nxt += lro_cb->pdu_totallen;
1551 ret = cxgbit_process_lro_skb(csk, skb);
1553 csk->rx_credits += lro_cb->pdu_totallen;
1555 if (csk->rx_credits >= (csk->rcv_win / 4))
1556 cxgbit_rx_data_ack(csk);
1561 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1563 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1566 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1567 if (is_t5(lldi->adapter_type))
1568 ret = cxgbit_rx_lro_skb(csk, skb);
1570 ret = cxgbit_process_lro_skb(csk, skb);
1577 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1579 spin_lock_bh(&csk->rxq.lock);
1580 if (skb_queue_len(&csk->rxq)) {
1581 skb_queue_splice_init(&csk->rxq, rxq);
1582 spin_unlock_bh(&csk->rxq.lock);
1585 spin_unlock_bh(&csk->rxq.lock);
1589 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1591 struct sk_buff *skb;
1592 struct sk_buff_head rxq;
1594 skb_queue_head_init(&rxq);
1596 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1598 if (signal_pending(current))
1601 while ((skb = __skb_dequeue(&rxq))) {
1602 if (cxgbit_rx_skb(csk, skb))
1608 __skb_queue_purge(&rxq);
1612 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1614 struct cxgbit_sock *csk = conn->context;
1617 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1618 ret = cxgbit_wait_rxq(csk);
1620 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1628 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1630 struct cxgbit_sock *csk = conn->context;
1632 while (!kthread_should_stop()) {
1633 iscsit_thread_check_cpumask(conn, current, 0);
1634 if (cxgbit_wait_rxq(csk))