drivers/scsi/cxgb3i/cxgb3i_offload.c

   1 /*
   2  * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
   3  *
   4  * Copyright (C) 2003-2008 Chelsio Communications.  All rights reserved.
   5  *
   6  * This program is distributed in the hope that it will be useful, but WITHOUT
   7  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   8  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
   9  * release for licensing terms and conditions.
  10  *
  11  * Written by:  Dimitris Michailidis (dm@chelsio.com)
  12  *              Karen Xie (kxie@chelsio.com)
  13  */
  14
  15 #include <linux/if_vlan.h>
  16 #include <linux/slab.h>
  17 #include <linux/version.h>
  18
  19 #include "cxgb3_defs.h"
  20 #include "cxgb3_ctl_defs.h"
  21 #include "firmware_exports.h"
  22 #include "cxgb3i_offload.h"
  23 #include "cxgb3i_pdu.h"
  24 #include "cxgb3i_ddp.h"
  25
  26 #ifdef __DEBUG_C3CN_CONN__
  27 #define c3cn_conn_debug         cxgb3i_log_debug
  28 #else
  29 #define c3cn_conn_debug(fmt...)
  30 #endif
  31
  32 #ifdef __DEBUG_C3CN_TX__
  33 #define c3cn_tx_debug           cxgb3i_log_debug
  34 #else
  35 #define c3cn_tx_debug(fmt...)
  36 #endif
  37
  38 #ifdef __DEBUG_C3CN_RX__
  39 #define c3cn_rx_debug           cxgb3i_log_debug
  40 #else
  41 #define c3cn_rx_debug(fmt...)
  42 #endif
  43
  44 /*
  45  * module parameters releated to offloaded iscsi connection
  46  */
  47 static int cxgb3_rcv_win = 256 * 1024;
  48 module_param(cxgb3_rcv_win, int, 0644);
  49 MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
  50
  51 static int cxgb3_snd_win = 128 * 1024;
  52 module_param(cxgb3_snd_win, int, 0644);
  53 MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=128KB)");
  54
  55 static int cxgb3_rx_credit_thres = 10 * 1024;
  56 module_param(cxgb3_rx_credit_thres, int, 0644);
  57 MODULE_PARM_DESC(rx_credit_thres,
  58                  "RX credits return threshold in bytes (default=10KB)");
  59
  60 static unsigned int cxgb3_max_connect = 8 * 1024;
  61 module_param(cxgb3_max_connect, uint, 0644);
  62 MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
  63
  64 static unsigned int cxgb3_sport_base = 20000;
  65 module_param(cxgb3_sport_base, uint, 0644);
  66 MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
  67
  68 /*
  69  * cxgb3i tcp connection data(per adapter) list
  70  */
  71 static LIST_HEAD(cdata_list);
  72 static DEFINE_RWLOCK(cdata_rwlock);
  73
  74 static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion);
  75 static void c3cn_release_offload_resources(struct s3_conn *c3cn);
  76
  77 /*
  78  * iscsi source port management
  79  *
  80  * Find a free source port in the port allocation map. We use a very simple
  81  * rotor scheme to look for the next free port.
  82  *
  83  * If a source port has been specified make sure that it doesn't collide with
  84  * our normal source port allocation map.  If it's outside the range of our
  85  * allocation/deallocation scheme just let them use it.
  86  *
  87  * If the source port is outside our allocation range, the caller is
  88  * responsible for keeping track of their port usage.
  89  */
  90 static int c3cn_get_port(struct s3_conn *c3cn, struct cxgb3i_sdev_data *cdata)
  91 {
  92         unsigned int start;
  93         int idx;
  94
  95         if (!cdata)
  96                 goto error_out;
  97
  98         if (c3cn->saddr.sin_port) {
  99                 cxgb3i_log_error("connect, sin_port NON-ZERO %u.\n",
 100                                  c3cn->saddr.sin_port);
 101                 return -EADDRINUSE;
 102         }
 103
 104         spin_lock_bh(&cdata->lock);
 105         start = idx = cdata->sport_next;
 106         do {
 107                 if (++idx >= cxgb3_max_connect)
 108                         idx = 0;
 109                 if (!cdata->sport_conn[idx]) {
 110                         c3cn->saddr.sin_port = htons(cxgb3_sport_base + idx);
 111                         cdata->sport_next = idx;
 112                         cdata->sport_conn[idx] = c3cn;
 113                         spin_unlock_bh(&cdata->lock);
 114
 115                         c3cn_conn_debug("%s reserve port %u.\n",
 116                                         cdata->cdev->name,
 117                                         cxgb3_sport_base + idx);
 118                         return 0;
 119                 }
 120         } while (idx != start);
 121         spin_unlock_bh(&cdata->lock);
 122
 123 error_out:
 124         return -EADDRNOTAVAIL;
 125 }
 126
 127 static void c3cn_put_port(struct s3_conn *c3cn)
 128 {
 129         if (!c3cn->cdev)
 130                 return;
 131
 132         if (c3cn->saddr.sin_port) {
 133                 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(c3cn->cdev);
 134                 int idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
 135
 136                 c3cn->saddr.sin_port = 0;
 137                 if (idx < 0 || idx >= cxgb3_max_connect)
 138                         return;
 139                 spin_lock_bh(&cdata->lock);
 140                 cdata->sport_conn[idx] = NULL;
 141                 spin_unlock_bh(&cdata->lock);
 142                 c3cn_conn_debug("%s, release port %u.\n",
 143                                 cdata->cdev->name, cxgb3_sport_base + idx);
 144         }
 145 }
 146
 147 static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 148 {
 149         __set_bit(flag, &c3cn->flags);
 150         c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
 151                         c3cn, flag, c3cn->state, c3cn->flags);
 152 }
 153
 154 static inline void c3cn_clear_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 155 {
 156         __clear_bit(flag, &c3cn->flags);
 157         c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
 158                         c3cn, flag, c3cn->state, c3cn->flags);
 159 }
 160
 161 static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
 162 {
 163         if (c3cn == NULL)
 164                 return 0;
 165         return test_bit(flag, &c3cn->flags);
 166 }
 167
 168 static void c3cn_set_state(struct s3_conn *c3cn, int state)
 169 {
 170         c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn, state);
 171         c3cn->state = state;
 172 }
 173
 174 static inline void c3cn_hold(struct s3_conn *c3cn)
 175 {
 176         atomic_inc(&c3cn->refcnt);
 177 }
 178
 179 static inline void c3cn_put(struct s3_conn *c3cn)
 180 {
 181         if (atomic_dec_and_test(&c3cn->refcnt)) {
 182                 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
 183                                 c3cn, c3cn->state, c3cn->flags);
 184                 kfree(c3cn);
 185         }
 186 }
 187
 188 static void c3cn_closed(struct s3_conn *c3cn)
 189 {
 190         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 191                          c3cn, c3cn->state, c3cn->flags);
 192
 193         c3cn_put_port(c3cn);
 194         c3cn_release_offload_resources(c3cn);
 195         c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
 196         cxgb3i_conn_closing(c3cn);
 197 }
 198
 199 /*
 200  * CPL (Chelsio Protocol Language) defines a message passing interface between
 201  * the host driver and T3 asic.
 202  * The section below implments CPLs that related to iscsi tcp connection
 203  * open/close/abort and data send/receive.
 204  */
 205
 206 /*
 207  * CPL connection active open request: host ->
 208  */
 209 static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
 210 {
 211         int i = 0;
 212
 213         while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
 214                 ++i;
 215         return i;
 216 }
 217
 218 static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
 219 {
 220         unsigned int idx;
 221         struct dst_entry *dst = c3cn->dst_cache;
 222         struct t3cdev *cdev = c3cn->cdev;
 223         const struct t3c_data *td = T3C_DATA(cdev);
 224         u16 advmss = dst_metric(dst, RTAX_ADVMSS);
 225
 226         if (advmss > pmtu - 40)
 227                 advmss = pmtu - 40;
 228         if (advmss < td->mtus[0] - 40)
 229                 advmss = td->mtus[0] - 40;
 230         idx = find_best_mtu(td, advmss + 40);
 231         return idx;
 232 }
 233
 234 static inline int compute_wscale(int win)
 235 {
 236         int wscale = 0;
 237         while (wscale < 14 && (65535<<wscale) < win)
 238                 wscale++;
 239         return wscale;
 240 }
 241
 242 static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
 243 {
 244         int wscale = compute_wscale(cxgb3_rcv_win);
 245         return  V_KEEP_ALIVE(1) |
 246                 F_TCAM_BYPASS |
 247                 V_WND_SCALE(wscale) |
 248                 V_MSS_IDX(c3cn->mss_idx);
 249 }
 250
 251 static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
 252 {
 253         return  V_ULP_MODE(ULP_MODE_ISCSI) |
 254                 V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
 255 }
 256
 257 static void make_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
 258                               unsigned int atid, const struct l2t_entry *e)
 259 {
 260         struct cpl_act_open_req *req;
 261
 262         c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
 263
 264         skb->priority = CPL_PRIORITY_SETUP;
 265         req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
 266         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 267         req->wr.wr_lo = 0;
 268         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
 269         req->local_port = c3cn->saddr.sin_port;
 270         req->peer_port = c3cn->daddr.sin_port;
 271         req->local_ip = c3cn->saddr.sin_addr.s_addr;
 272         req->peer_ip = c3cn->daddr.sin_addr.s_addr;
 273         req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
 274                            V_TX_CHANNEL(e->smt_idx));
 275         req->opt0l = htonl(calc_opt0l(c3cn));
 276         req->params = 0;
 277         req->opt2 = 0;
 278 }
 279
 280 static void fail_act_open(struct s3_conn *c3cn, int errno)
 281 {
 282         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 283                         c3cn, c3cn->state, c3cn->flags);
 284         c3cn->err = errno;
 285         c3cn_closed(c3cn);
 286 }
 287
 288 static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
 289 {
 290         struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
 291
 292         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 293
 294         c3cn_hold(c3cn);
 295         spin_lock_bh(&c3cn->lock);
 296         if (c3cn->state == C3CN_STATE_CONNECTING)
 297                 fail_act_open(c3cn, -EHOSTUNREACH);
 298         spin_unlock_bh(&c3cn->lock);
 299         c3cn_put(c3cn);
 300         __kfree_skb(skb);
 301 }
 302
 303 /*
 304  * CPL connection close request: host ->
 305  *
 306  * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
 307  * the write queue (i.e., after any unsent txt data).
 308  */
 309 static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
 310                        int flags)
 311 {
 312         skb_tcp_seq(skb) = c3cn->write_seq;
 313         skb_flags(skb) = flags;
 314         __skb_queue_tail(&c3cn->write_queue, skb);
 315 }
 316
 317 static void send_close_req(struct s3_conn *c3cn)
 318 {
 319         struct sk_buff *skb = c3cn->cpl_close;
 320         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 321         unsigned int tid = c3cn->tid;
 322
 323         c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
 324                         c3cn, c3cn->state, c3cn->flags);
 325
 326         c3cn->cpl_close = NULL;
 327
 328         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
 329         req->wr.wr_lo = htonl(V_WR_TID(tid));
 330         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 331         req->rsvd = htonl(c3cn->write_seq);
 332
 333         skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
 334         if (c3cn->state != C3CN_STATE_CONNECTING)
 335                 c3cn_push_tx_frames(c3cn, 1);
 336 }
 337
 338 /*
 339  * CPL connection abort request: host ->
 340  *
 341  * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
 342  * for the same connection and also that we do not try to send a message
 343  * after the connection has closed.
 344  */
 345 static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
 346 {
 347         struct cpl_abort_req *req = cplhdr(skb);
 348
 349         c3cn_conn_debug("tdev 0x%p.\n", cdev);
 350
 351         req->cmd = CPL_ABORT_NO_RST;
 352         cxgb3_ofld_send(cdev, skb);
 353 }
 354
 355 static inline void c3cn_purge_write_queue(struct s3_conn *c3cn)
 356 {
 357         struct sk_buff *skb;
 358
 359         while ((skb = __skb_dequeue(&c3cn->write_queue)))
 360                 __kfree_skb(skb);
 361 }
 362
 363 static void send_abort_req(struct s3_conn *c3cn)
 364 {
 365         struct sk_buff *skb = c3cn->cpl_abort_req;
 366         struct cpl_abort_req *req;
 367         unsigned int tid = c3cn->tid;
 368
 369         if (unlikely(c3cn->state == C3CN_STATE_ABORTING) || !skb ||
 370                      !c3cn->cdev)
 371                 return;
 372
 373         c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
 374
 375         c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn);
 376
 377         c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
 378
 379         /* Purge the send queue so we don't send anything after an abort. */
 380         c3cn_purge_write_queue(c3cn);
 381
 382         c3cn->cpl_abort_req = NULL;
 383         req = (struct cpl_abort_req *)skb->head;
 384         memset(req, 0, sizeof(*req));
 385
 386         skb->priority = CPL_PRIORITY_DATA;
 387         set_arp_failure_handler(skb, abort_arp_failure);
 388
 389         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
 390         req->wr.wr_lo = htonl(V_WR_TID(tid));
 391         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
 392         req->rsvd0 = htonl(c3cn->snd_nxt);
 393         req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
 394         req->cmd = CPL_ABORT_SEND_RST;
 395
 396         l2t_send(c3cn->cdev, skb, c3cn->l2t);
 397 }
 398
 399 /*
 400  * CPL connection abort reply: host ->
 401  *
 402  * Send an ABORT_RPL message in response of the ABORT_REQ received.
 403  */
 404 static void send_abort_rpl(struct s3_conn *c3cn, int rst_status)
 405 {
 406         struct sk_buff *skb = c3cn->cpl_abort_rpl;
 407         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 408
 409         c3cn->cpl_abort_rpl = NULL;
 410
 411         skb->priority = CPL_PRIORITY_DATA;
 412         memset(rpl, 0, sizeof(*rpl));
 413         rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
 414         rpl->wr.wr_lo = htonl(V_WR_TID(c3cn->tid));
 415         OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, c3cn->tid));
 416         rpl->cmd = rst_status;
 417
 418         cxgb3_ofld_send(c3cn->cdev, skb);
 419 }
 420
 421 /*
 422  * CPL connection rx data ack: host ->
 423  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 424  * credits sent.
 425  */
 426 static u32 send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack)
 427 {
 428         struct sk_buff *skb;
 429         struct cpl_rx_data_ack *req;
 430
 431         skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
 432         if (!skb)
 433                 return 0;
 434
 435         req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
 436         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 437         req->wr.wr_lo = 0;
 438         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
 439         req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
 440         skb->priority = CPL_PRIORITY_ACK;
 441         cxgb3_ofld_send(c3cn->cdev, skb);
 442         return credits;
 443 }
 444
 445 /*
 446  * CPL connection tx data: host ->
 447  *
 448  * Send iscsi PDU via TX_DATA CPL message. Returns the number of
 449  * credits sent.
 450  * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
 451  * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
 452  */
 453
 454 /*
 455  * For ULP connections HW may inserts digest bytes into the pdu. Those digest
 456  * bytes are not sent by the host but are part of the TCP payload and therefore
 457  * consume TCP sequence space.
 458  */
 459 static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
 460 static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
 461 {
 462         return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
 463 }
 464
 465 static unsigned int wrlen __read_mostly;
 466
 467 /*
 468  * The number of WRs needed for an skb depends on the number of fragments
 469  * in the skb and whether it has any payload in its main body.  This maps the
 470  * length of the gather list represented by an skb into the # of necessary WRs.
 471  * The extra two fragments are for iscsi bhs and payload padding.
 472  */
 473 #define SKB_WR_LIST_SIZE        (MAX_SKB_FRAGS + 2)
 474 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
 475
 476 static void s3_init_wr_tab(unsigned int wr_len)
 477 {
 478         int i;
 479
 480         if (skb_wrs[1])         /* already initialized */
 481                 return;
 482
 483         for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
 484                 int sgl_len = (3 * i) / 2 + (i & 1);
 485
 486                 sgl_len += 3;
 487                 skb_wrs[i] = (sgl_len <= wr_len
 488                               ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
 489         }
 490
 491         wrlen = wr_len * 8;
 492 }
 493
 494 static inline void reset_wr_list(struct s3_conn *c3cn)
 495 {
 496         c3cn->wr_pending_head = c3cn->wr_pending_tail = NULL;
 497 }
 498
 499 /*
 500  * Add a WR to a connections's list of pending WRs.  This is a singly-linked
 501  * list of sk_buffs operating as a FIFO.  The head is kept in wr_pending_head
 502  * and the tail in wr_pending_tail.
 503  */
 504 static inline void enqueue_wr(struct s3_conn *c3cn,
 505                               struct sk_buff *skb)
 506 {
 507         skb_tx_wr_next(skb) = NULL;
 508
 509         /*
 510          * We want to take an extra reference since both us and the driver
 511          * need to free the packet before it's really freed. We know there's
 512          * just one user currently so we use atomic_set rather than skb_get
 513          * to avoid the atomic op.
 514          */
 515         atomic_set(&skb->users, 2);
 516
 517         if (!c3cn->wr_pending_head)
 518                 c3cn->wr_pending_head = skb;
 519         else
 520                 skb_tx_wr_next(c3cn->wr_pending_tail) = skb;
 521         c3cn->wr_pending_tail = skb;
 522 }
 523
 524 static int count_pending_wrs(struct s3_conn *c3cn)
 525 {
 526         int n = 0;
 527         const struct sk_buff *skb = c3cn->wr_pending_head;
 528
 529         while (skb) {
 530                 n += skb->csum;
 531                 skb = skb_tx_wr_next(skb);
 532         }
 533         return n;
 534 }
 535
 536 static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
 537 {
 538         return c3cn->wr_pending_head;
 539 }
 540
 541 static inline void free_wr_skb(struct sk_buff *skb)
 542 {
 543         kfree_skb(skb);
 544 }
 545
 546 static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
 547 {
 548         struct sk_buff *skb = c3cn->wr_pending_head;
 549
 550         if (likely(skb)) {
 551                 /* Don't bother clearing the tail */
 552                 c3cn->wr_pending_head = skb_tx_wr_next(skb);
 553                 skb_tx_wr_next(skb) = NULL;
 554         }
 555         return skb;
 556 }
 557
 558 static void purge_wr_queue(struct s3_conn *c3cn)
 559 {
 560         struct sk_buff *skb;
 561         while ((skb = dequeue_wr(c3cn)) != NULL)
 562                 free_wr_skb(skb);
 563 }
 564
 565 static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
 566                                    int len, int req_completion)
 567 {
 568         struct tx_data_wr *req;
 569
 570         skb_reset_transport_header(skb);
 571         req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
 572         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
 573                         (req_completion ? F_WR_COMPL : 0));
 574         req->wr_lo = htonl(V_WR_TID(c3cn->tid));
 575         req->sndseq = htonl(c3cn->snd_nxt);
 576         /* len includes the length of any HW ULP additions */
 577         req->len = htonl(len);
 578         req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
 579         /* V_TX_ULP_SUBMODE sets both the mode and submode */
 580         req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
 581                            V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
 582
 583         if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
 584                 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
 585                                     V_TX_CPU_IDX(c3cn->qset));
 586                 /* Sendbuffer is in units of 32KB. */
 587                 req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
 588                 c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
 589         }
 590 }
 591
 592 /**
 593  * c3cn_push_tx_frames -- start transmit
 594  * @c3cn: the offloaded connection
 595  * @req_completion: request wr_ack or not
 596  *
 597  * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
 598  * connection's send queue and sends them on to T3.  Must be called with the
 599  * connection's lock held.  Returns the amount of send buffer space that was
 600  * freed as a result of sending queued data to T3.
 601  */
 602 static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
 603 {
 604         kfree_skb(skb);
 605 }
 606
 607 static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
 608 {
 609         int total_size = 0;
 610         struct sk_buff *skb;
 611         struct t3cdev *cdev;
 612         struct cxgb3i_sdev_data *cdata;
 613
 614         if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
 615                      c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
 616                      c3cn->state >= C3CN_STATE_ABORTING)) {
 617                 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
 618                               c3cn, c3cn->state);
 619                 return 0;
 620         }
 621
 622         cdev = c3cn->cdev;
 623         cdata = CXGB3_SDEV_DATA(cdev);
 624
 625         while (c3cn->wr_avail
 626                && (skb = skb_peek(&c3cn->write_queue)) != NULL) {
 627                 int len = skb->len;     /* length before skb_push */
 628                 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
 629                 int wrs_needed = skb_wrs[frags];
 630
 631                 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
 632                         wrs_needed = 1;
 633
 634                 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
 635
 636                 if (c3cn->wr_avail < wrs_needed) {
 637                         c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
 638                                       "wr %d < %u.\n",
 639                                       c3cn, skb->len, skb->data_len, frags,
 640                                       wrs_needed, c3cn->wr_avail);
 641                         break;
 642                 }
 643
 644                 __skb_unlink(skb, &c3cn->write_queue);
 645                 skb->priority = CPL_PRIORITY_DATA;
 646                 skb->csum = wrs_needed; /* remember this until the WR_ACK */
 647                 c3cn->wr_avail -= wrs_needed;
 648                 c3cn->wr_unacked += wrs_needed;
 649                 enqueue_wr(c3cn, skb);
 650
 651                 c3cn_tx_debug("c3cn 0x%p, enqueue, skb len %u/%u, frag %u, "
 652                                 "wr %d, left %u, unack %u.\n",
 653                                 c3cn, skb->len, skb->data_len, frags,
 654                                 wrs_needed, c3cn->wr_avail, c3cn->wr_unacked);
 655
 656
 657                 if (likely(skb_flags(skb) & C3CB_FLAG_NEED_HDR)) {
 658                         if ((req_completion &&
 659                                 c3cn->wr_unacked == wrs_needed) ||
 660                             (skb_flags(skb) & C3CB_FLAG_COMPL) ||
 661                             c3cn->wr_unacked >= c3cn->wr_max / 2) {
 662                                 req_completion = 1;
 663                                 c3cn->wr_unacked = 0;
 664                         }
 665                         len += ulp_extra_len(skb);
 666                         make_tx_data_wr(c3cn, skb, len, req_completion);
 667                         c3cn->snd_nxt += len;
 668                         skb_flags(skb) &= ~C3CB_FLAG_NEED_HDR;
 669                 }
 670
 671                 total_size += skb->truesize;
 672                 set_arp_failure_handler(skb, arp_failure_discard);
 673                 l2t_send(cdev, skb, c3cn->l2t);
 674         }
 675         return total_size;
 676 }
 677
 678 /*
 679  * process_cpl_msg: -> host
 680  * Top-level CPL message processing used by most CPL messages that
 681  * pertain to connections.
 682  */
 683 static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
 684                                               struct sk_buff *),
 685                                    struct s3_conn *c3cn,
 686                                    struct sk_buff *skb)
 687 {
 688         spin_lock_bh(&c3cn->lock);
 689         fn(c3cn, skb);
 690         spin_unlock_bh(&c3cn->lock);
 691 }
 692
 693 /*
 694  * process_cpl_msg_ref: -> host
 695  * Similar to process_cpl_msg() but takes an extra connection reference around
 696  * the call to the handler.  Should be used if the handler may drop a
 697  * connection reference.
 698  */
 699 static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
 700                                                    struct sk_buff *),
 701                                        struct s3_conn *c3cn,
 702                                        struct sk_buff *skb)
 703 {
 704         c3cn_hold(c3cn);
 705         process_cpl_msg(fn, c3cn, skb);
 706         c3cn_put(c3cn);
 707 }
 708
 709 /*
 710  * Process a CPL_ACT_ESTABLISH message: -> host
 711  * Updates connection state from an active establish CPL message.  Runs with
 712  * the connection lock held.
 713  */
 714
 715 static inline void s3_free_atid(struct t3cdev *cdev, unsigned int tid)
 716 {
 717         struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
 718         if (c3cn)
 719                 c3cn_put(c3cn);
 720 }
 721
 722 static void c3cn_established(struct s3_conn *c3cn, u32 snd_isn,
 723                              unsigned int opt)
 724 {
 725         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 726
 727         c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
 728
 729         /*
 730          * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
 731          * pass through opt0.
 732          */
 733         if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
 734                 c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
 735
 736         dst_confirm(c3cn->dst_cache);
 737
 738         smp_mb();
 739
 740         c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
 741 }
 742
 743 static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
 744 {
 745         struct cpl_act_establish *req = cplhdr(skb);
 746         u32 rcv_isn = ntohl(req->rcv_isn);      /* real RCV_ISN + 1 */
 747
 748         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 749                         c3cn, c3cn->state, c3cn->flags);
 750
 751         if (unlikely(c3cn->state != C3CN_STATE_CONNECTING))
 752                 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
 753                                  c3cn->tid, c3cn->state);
 754
 755         c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
 756         c3cn_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 757
 758         __kfree_skb(skb);
 759
 760         if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
 761                 /* upper layer has requested closing */
 762                 send_abort_req(c3cn);
 763         else {
 764                 if (skb_queue_len(&c3cn->write_queue))
 765                         c3cn_push_tx_frames(c3cn, 1);
 766                 cxgb3i_conn_tx_open(c3cn);
 767         }
 768 }
 769
 770 static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
 771                             void *ctx)
 772 {
 773         struct cpl_act_establish *req = cplhdr(skb);
 774         unsigned int tid = GET_TID(req);
 775         unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 776         struct s3_conn *c3cn = ctx;
 777         struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
 778
 779         c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
 780                         tid, c3cn, c3cn->state, c3cn->flags);
 781
 782         c3cn->tid = tid;
 783         c3cn_hold(c3cn);
 784         cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
 785         s3_free_atid(cdev, atid);
 786
 787         c3cn->qset = G_QNUM(ntohl(skb->csum));
 788
 789         process_cpl_msg(process_act_establish, c3cn, skb);
 790         return 0;
 791 }
 792
 793 /*
 794  * Process a CPL_ACT_OPEN_RPL message: -> host
 795  * Handle active open failures.
 796  */
 797 static int act_open_rpl_status_to_errno(int status)
 798 {
 799         switch (status) {
 800         case CPL_ERR_CONN_RESET:
 801                 return -ECONNREFUSED;
 802         case CPL_ERR_ARP_MISS:
 803                 return -EHOSTUNREACH;
 804         case CPL_ERR_CONN_TIMEDOUT:
 805                 return -ETIMEDOUT;
 806         case CPL_ERR_TCAM_FULL:
 807                 return -ENOMEM;
 808         case CPL_ERR_CONN_EXIST:
 809                 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
 810                 return -EADDRINUSE;
 811         default:
 812                 return -EIO;
 813         }
 814 }
 815
 816 static void act_open_retry_timer(unsigned long data)
 817 {
 818         struct sk_buff *skb;
 819         struct s3_conn *c3cn = (struct s3_conn *)data;
 820
 821         c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
 822
 823         spin_lock_bh(&c3cn->lock);
 824         skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
 825         if (!skb)
 826                 fail_act_open(c3cn, -ENOMEM);
 827         else {
 828                 skb->sk = (struct sock *)c3cn;
 829                 set_arp_failure_handler(skb, act_open_req_arp_failure);
 830                 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
 831                 l2t_send(c3cn->cdev, skb, c3cn->l2t);
 832         }
 833         spin_unlock_bh(&c3cn->lock);
 834         c3cn_put(c3cn);
 835 }
 836
 837 static void process_act_open_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
 838 {
 839         struct cpl_act_open_rpl *rpl = cplhdr(skb);
 840
 841         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 842                         c3cn, c3cn->state, c3cn->flags);
 843
 844         if (rpl->status == CPL_ERR_CONN_EXIST &&
 845             c3cn->retry_timer.function != act_open_retry_timer) {
 846                 c3cn->retry_timer.function = act_open_retry_timer;
 847                 if (!mod_timer(&c3cn->retry_timer, jiffies + HZ / 2))
 848                         c3cn_hold(c3cn);
 849         } else
 850                 fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
 851         __kfree_skb(skb);
 852 }
 853
 854 static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 855 {
 856         struct s3_conn *c3cn = ctx;
 857         struct cpl_act_open_rpl *rpl = cplhdr(skb);
 858
 859         c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
 860                         rpl->status, c3cn, c3cn->state, c3cn->flags);
 861
 862         if (rpl->status != CPL_ERR_TCAM_FULL &&
 863             rpl->status != CPL_ERR_CONN_EXIST &&
 864             rpl->status != CPL_ERR_ARP_MISS)
 865                 cxgb3_queue_tid_release(cdev, GET_TID(rpl));
 866
 867         process_cpl_msg_ref(process_act_open_rpl, c3cn, skb);
 868         return 0;
 869 }
 870
 871 /*
 872  * Process PEER_CLOSE CPL messages: -> host
 873  * Handle peer FIN.
 874  */
 875 static void process_peer_close(struct s3_conn *c3cn, struct sk_buff *skb)
 876 {
 877         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 878                         c3cn, c3cn->state, c3cn->flags);
 879
 880         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
 881                 goto out;
 882
 883         switch (c3cn->state) {
 884         case C3CN_STATE_ESTABLISHED:
 885                 c3cn_set_state(c3cn, C3CN_STATE_PASSIVE_CLOSE);
 886                 break;
 887         case C3CN_STATE_ACTIVE_CLOSE:
 888                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
 889                 break;
 890         case C3CN_STATE_CLOSE_WAIT_1:
 891                 c3cn_closed(c3cn);
 892                 break;
 893         case C3CN_STATE_ABORTING:
 894                 break;
 895         default:
 896                 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
 897                                  c3cn->cdev->name, c3cn->tid, c3cn->state);
 898         }
 899
 900         cxgb3i_conn_closing(c3cn);
 901 out:
 902         __kfree_skb(skb);
 903 }
 904
 905 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 906 {
 907         struct s3_conn *c3cn = ctx;
 908
 909         c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
 910                         c3cn, c3cn->state, c3cn->flags);
 911         process_cpl_msg_ref(process_peer_close, c3cn, skb);
 912         return 0;
 913 }
 914
 915 /*
 916  * Process CLOSE_CONN_RPL CPL message: -> host
 917  * Process a peer ACK to our FIN.
 918  */
 919 static void process_close_con_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
 920 {
 921         struct cpl_close_con_rpl *rpl = cplhdr(skb);
 922
 923         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 924                         c3cn, c3cn->state, c3cn->flags);
 925
 926         c3cn->snd_una = ntohl(rpl->snd_nxt) - 1;        /* exclude FIN */
 927
 928         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
 929                 goto out;
 930
 931         switch (c3cn->state) {
 932         case C3CN_STATE_ACTIVE_CLOSE:
 933                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_1);
 934                 break;
 935         case C3CN_STATE_CLOSE_WAIT_1:
 936         case C3CN_STATE_CLOSE_WAIT_2:
 937                 c3cn_closed(c3cn);
 938                 break;
 939         case C3CN_STATE_ABORTING:
 940                 break;
 941         default:
 942                 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
 943                                  c3cn->cdev->name, c3cn->tid, c3cn->state);
 944         }
 945
 946 out:
 947         kfree_skb(skb);
 948 }
 949
 950 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
 951                             void *ctx)
 952 {
 953         struct s3_conn *c3cn = ctx;
 954
 955         c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
 956                          c3cn, c3cn->state, c3cn->flags);
 957
 958         process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
 959         return 0;
 960 }
 961
 962 /*
 963  * Process ABORT_REQ_RSS CPL message: -> host
 964  * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
 965  * request except that we need to reply to it.
 966  */
 967
 968 static int abort_status_to_errno(struct s3_conn *c3cn, int abort_reason,
 969                                  int *need_rst)
 970 {
 971         switch (abort_reason) {
 972         case CPL_ERR_BAD_SYN: /* fall through */
 973         case CPL_ERR_CONN_RESET:
 974                 return c3cn->state > C3CN_STATE_ESTABLISHED ?
 975                         -EPIPE : -ECONNRESET;
 976         case CPL_ERR_XMIT_TIMEDOUT:
 977         case CPL_ERR_PERSIST_TIMEDOUT:
 978         case CPL_ERR_FINWAIT2_TIMEDOUT:
 979         case CPL_ERR_KEEPALIVE_TIMEDOUT:
 980                 return -ETIMEDOUT;
 981         default:
 982                 return -EIO;
 983         }
 984 }
 985
 986 static void process_abort_req(struct s3_conn *c3cn, struct sk_buff *skb)
 987 {
 988         int rst_status = CPL_ABORT_NO_RST;
 989         const struct cpl_abort_req_rss *req = cplhdr(skb);
 990
 991         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
 992                         c3cn, c3cn->state, c3cn->flags);
 993
 994         if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
 995                 c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
 996                 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
 997                 __kfree_skb(skb);
 998                 return;
 999         }
1000
1001         c3cn_clear_flag(c3cn, C3CN_ABORT_REQ_RCVD);
1002         send_abort_rpl(c3cn, rst_status);
1003
1004         if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
1005                 c3cn->err =
1006                     abort_status_to_errno(c3cn, req->status, &rst_status);
1007                 c3cn_closed(c3cn);
1008         }
1009 }
1010
1011 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1012 {
1013         const struct cpl_abort_req_rss *req = cplhdr(skb);
1014         struct s3_conn *c3cn = ctx;
1015
1016         c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
1017                         c3cn, c3cn->state, c3cn->flags);
1018
1019         if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
1020             req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
1021                 __kfree_skb(skb);
1022                 return 0;
1023         }
1024
1025         process_cpl_msg_ref(process_abort_req, c3cn, skb);
1026         return 0;
1027 }
1028
1029 /*
1030  * Process ABORT_RPL_RSS CPL message: -> host
1031  * Process abort replies.  We only process these messages if we anticipate
1032  * them as the coordination between SW and HW in this area is somewhat lacking
1033  * and sometimes we get ABORT_RPLs after we are done with the connection that
1034  * originated the ABORT_REQ.
1035  */
1036 static void process_abort_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
1037 {
1038         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1039                         c3cn, c3cn->state, c3cn->flags);
1040
1041         if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
1042                 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
1043                         c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1044                 else {
1045                         c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1046                         c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_PENDING);
1047                         if (c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD))
1048                                 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1049                                                  c3cn->cdev->name, c3cn->tid);
1050                         c3cn_closed(c3cn);
1051                 }
1052         }
1053         __kfree_skb(skb);
1054 }
1055
1056 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1057 {
1058         struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1059         struct s3_conn *c3cn = ctx;
1060
1061         c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1062                         rpl->status, c3cn, c3cn ? c3cn->state : 0,
1063                         c3cn ? c3cn->flags : 0UL);
1064
1065         /*
1066          * Ignore replies to post-close aborts indicating that the abort was
1067          * requested too late.  These connections are terminated when we get
1068          * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1069          * arrives the TID is either no longer used or it has been recycled.
1070          */
1071         if (rpl->status == CPL_ERR_ABORT_FAILED)
1072                 goto discard;
1073
1074         /*
1075          * Sometimes we've already closed the connection, e.g., a post-close
1076          * abort races with ABORT_REQ_RSS, the latter frees the connection
1077          * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1078          * but FW turns the ABORT_REQ into a regular one and so we get
1079          * ABORT_RPL_RSS with status 0 and no connection.
1080          */
1081         if (!c3cn)
1082                 goto discard;
1083
1084         process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
1085         return 0;
1086
1087 discard:
1088         __kfree_skb(skb);
1089         return 0;
1090 }
1091
1092 /*
1093  * Process RX_ISCSI_HDR CPL message: -> host
1094  * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1095  * follow after the bhs.
1096  */
1097 static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1098 {
1099         struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
1100         struct cpl_iscsi_hdr_norss data_cpl;
1101         struct cpl_rx_data_ddp_norss ddp_cpl;
1102         unsigned int hdr_len, data_len, status;
1103         unsigned int len;
1104         int err;
1105
1106         if (unlikely(c3cn->state >= C3CN_STATE_PASSIVE_CLOSE)) {
1107                 if (c3cn->state != C3CN_STATE_ABORTING)
1108                         send_abort_req(c3cn);
1109                 __kfree_skb(skb);
1110                 return;
1111         }
1112
1113         skb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
1114         skb_flags(skb) = 0;
1115
1116         skb_reset_transport_header(skb);
1117         __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
1118
1119         len = hdr_len = ntohs(hdr_cpl->len);
1120         /* msg coalesce is off or not enough data received */
1121         if (skb->len <= hdr_len) {
1122                 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1123                                  c3cn->cdev->name, c3cn->tid,
1124                                  skb->len, hdr_len);
1125                 goto abort_conn;
1126         }
1127
1128         err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
1129                             sizeof(ddp_cpl));
1130         if (err < 0)
1131                 goto abort_conn;
1132
1133         skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
1134         skb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
1135         skb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
1136         status = ntohl(ddp_cpl.ddp_status);
1137
1138         c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1139                       skb, skb->len, skb_rx_pdulen(skb), status);
1140
1141         if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
1142                 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
1143         if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
1144                 skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
1145         if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
1146                 skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
1147
1148         if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
1149                 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
1150                 if (err < 0)
1151                         goto abort_conn;
1152                 data_len = ntohs(data_cpl.len);
1153                 len += sizeof(data_cpl) + data_len;
1154         } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
1155                 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
1156
1157         c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_rx_pdulen(skb);
1158         __pskb_trim(skb, len);
1159         __skb_queue_tail(&c3cn->receive_queue, skb);
1160         cxgb3i_conn_pdu_ready(c3cn);
1161
1162         return;
1163
1164 abort_conn:
1165         send_abort_req(c3cn);
1166         __kfree_skb(skb);
1167 }
1168
1169 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
1170 {
1171         struct s3_conn *c3cn = ctx;
1172
1173         process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
1174         return 0;
1175 }
1176
1177 /*
1178  * Process TX_DATA_ACK CPL messages: -> host
1179  * Process an acknowledgment of WR completion.  Advance snd_una and send the
1180  * next batch of work requests from the write queue.
1181  */
1182 static void check_wr_invariants(struct s3_conn *c3cn)
1183 {
1184         int pending = count_pending_wrs(c3cn);
1185
1186         if (unlikely(c3cn->wr_avail + pending != c3cn->wr_max))
1187                 cxgb3i_log_error("TID %u: credit imbalance: avail %u, "
1188                                 "pending %u, total should be %u\n",
1189                                 c3cn->tid, c3cn->wr_avail, pending,
1190                                 c3cn->wr_max);
1191 }
1192
1193 static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1194 {
1195         struct cpl_wr_ack *hdr = cplhdr(skb);
1196         unsigned int credits = ntohs(hdr->credits);
1197         u32 snd_una = ntohl(hdr->snd_una);
1198
1199         c3cn_tx_debug("%u WR credits, avail %u, unack %u, TID %u, state %u.\n",
1200                         credits, c3cn->wr_avail, c3cn->wr_unacked,
1201                         c3cn->tid, c3cn->state);
1202
1203         c3cn->wr_avail += credits;
1204         if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
1205                 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
1206
1207         while (credits) {
1208                 struct sk_buff *p = peek_wr(c3cn);
1209
1210                 if (unlikely(!p)) {
1211                         cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1212                                          "nothing pending, state %u\n",
1213                                          credits, c3cn->tid, c3cn->state);
1214                         break;
1215                 }
1216                 if (unlikely(credits < p->csum)) {
1217                         struct tx_data_wr *w = cplhdr(p);
1218                         cxgb3i_log_error("TID %u got %u WR credits need %u, "
1219                                          "len %u, main body %u, frags %u, "
1220                                          "seq # %u, ACK una %u, ACK nxt %u, "
1221                                          "WR_AVAIL %u, WRs pending %u\n",
1222                                          c3cn->tid, credits, p->csum, p->len,
1223                                          p->len - p->data_len,
1224                                          skb_shinfo(p)->nr_frags,
1225                                          ntohl(w->sndseq), snd_una,
1226                                          ntohl(hdr->snd_nxt), c3cn->wr_avail,
1227                                          count_pending_wrs(c3cn) - credits);
1228                         p->csum -= credits;
1229                         break;
1230                 } else {
1231                         dequeue_wr(c3cn);
1232                         credits -= p->csum;
1233                         free_wr_skb(p);
1234                 }
1235         }
1236
1237         check_wr_invariants(c3cn);
1238
1239         if (unlikely(before(snd_una, c3cn->snd_una))) {
1240                 cxgb3i_log_error("TID %u, unexpected sequence # %u in WR_ACK "
1241                                  "snd_una %u\n",
1242                                  c3cn->tid, snd_una, c3cn->snd_una);
1243                 goto out_free;
1244         }
1245
1246         if (c3cn->snd_una != snd_una) {
1247                 c3cn->snd_una = snd_una;
1248                 dst_confirm(c3cn->dst_cache);
1249         }
1250
1251         if (skb_queue_len(&c3cn->write_queue)) {
1252                 if (c3cn_push_tx_frames(c3cn, 0))
1253                         cxgb3i_conn_tx_open(c3cn);
1254         } else
1255                 cxgb3i_conn_tx_open(c3cn);
1256 out_free:
1257         __kfree_skb(skb);
1258 }
1259
1260 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1261 {
1262         struct s3_conn *c3cn = ctx;
1263
1264         process_cpl_msg(process_wr_ack, c3cn, skb);
1265         return 0;
1266 }
1267
1268 /*
1269  * for each connection, pre-allocate skbs needed for close/abort requests. So
1270  * that we can service the request right away.
1271  */
1272 static void c3cn_free_cpl_skbs(struct s3_conn *c3cn)
1273 {
1274         if (c3cn->cpl_close)
1275                 kfree_skb(c3cn->cpl_close);
1276         if (c3cn->cpl_abort_req)
1277                 kfree_skb(c3cn->cpl_abort_req);
1278         if (c3cn->cpl_abort_rpl)
1279                 kfree_skb(c3cn->cpl_abort_rpl);
1280 }
1281
1282 static int c3cn_alloc_cpl_skbs(struct s3_conn *c3cn)
1283 {
1284         c3cn->cpl_close = alloc_skb(sizeof(struct cpl_close_con_req),
1285                                    GFP_KERNEL);
1286         if (!c3cn->cpl_close)
1287                 return -ENOMEM;
1288         skb_put(c3cn->cpl_close, sizeof(struct cpl_close_con_req));
1289
1290         c3cn->cpl_abort_req = alloc_skb(sizeof(struct cpl_abort_req),
1291                                         GFP_KERNEL);
1292         if (!c3cn->cpl_abort_req)
1293                 goto free_cpl_skbs;
1294         skb_put(c3cn->cpl_abort_req, sizeof(struct cpl_abort_req));
1295
1296         c3cn->cpl_abort_rpl = alloc_skb(sizeof(struct cpl_abort_rpl),
1297                                         GFP_KERNEL);
1298         if (!c3cn->cpl_abort_rpl)
1299                 goto free_cpl_skbs;
1300         skb_put(c3cn->cpl_abort_rpl, sizeof(struct cpl_abort_rpl));
1301
1302         return 0;
1303
1304 free_cpl_skbs:
1305         c3cn_free_cpl_skbs(c3cn);
1306         return -ENOMEM;
1307 }
1308
1309 /**
1310  * c3cn_release_offload_resources - release offload resource
1311  * @c3cn: the offloaded iscsi tcp connection.
1312  * Release resources held by an offload connection (TID, L2T entry, etc.)
1313  */
1314 static void c3cn_release_offload_resources(struct s3_conn *c3cn)
1315 {
1316         struct t3cdev *cdev = c3cn->cdev;
1317         unsigned int tid = c3cn->tid;
1318
1319         c3cn->qset = 0;
1320         c3cn_free_cpl_skbs(c3cn);
1321
1322         if (c3cn->wr_avail != c3cn->wr_max) {
1323                 purge_wr_queue(c3cn);
1324                 reset_wr_list(c3cn);
1325         }
1326
1327         if (cdev) {
1328                 if (c3cn->l2t) {
1329                         l2t_release(L2DATA(cdev), c3cn->l2t);
1330                         c3cn->l2t = NULL;
1331                 }
1332                 if (c3cn->state == C3CN_STATE_CONNECTING)
1333                         /* we have ATID */
1334                         s3_free_atid(cdev, tid);
1335                 else {
1336                         /* we have TID */
1337                         cxgb3_remove_tid(cdev, (void *)c3cn, tid);
1338                         c3cn_put(c3cn);
1339                 }
1340         }
1341
1342         c3cn->dst_cache = NULL;
1343         c3cn->cdev = NULL;
1344 }
1345
1346 /**
1347  * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1348  * returns the s3_conn structure allocated.
1349  */
1350 struct s3_conn *cxgb3i_c3cn_create(void)
1351 {
1352         struct s3_conn *c3cn;
1353
1354         c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
1355         if (!c3cn)
1356                 return NULL;
1357
1358         /* pre-allocate close/abort cpl, so we don't need to wait for memory
1359            when close/abort is requested. */
1360         if (c3cn_alloc_cpl_skbs(c3cn) < 0)
1361                 goto free_c3cn;
1362
1363         c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
1364
1365         c3cn->flags = 0;
1366         spin_lock_init(&c3cn->lock);
1367         atomic_set(&c3cn->refcnt, 1);
1368         skb_queue_head_init(&c3cn->receive_queue);
1369         skb_queue_head_init(&c3cn->write_queue);
1370         setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
1371         rwlock_init(&c3cn->callback_lock);
1372
1373         return c3cn;
1374
1375 free_c3cn:
1376         kfree(c3cn);
1377         return NULL;
1378 }
1379
1380 static void c3cn_active_close(struct s3_conn *c3cn)
1381 {
1382         int data_lost;
1383         int close_req = 0;
1384
1385         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1386                          c3cn, c3cn->state, c3cn->flags);
1387
1388         dst_confirm(c3cn->dst_cache);
1389
1390         c3cn_hold(c3cn);
1391         spin_lock_bh(&c3cn->lock);
1392
1393         data_lost = skb_queue_len(&c3cn->receive_queue);
1394         __skb_queue_purge(&c3cn->receive_queue);
1395
1396         switch (c3cn->state) {
1397         case C3CN_STATE_CLOSED:
1398         case C3CN_STATE_ACTIVE_CLOSE:
1399         case C3CN_STATE_CLOSE_WAIT_1:
1400         case C3CN_STATE_CLOSE_WAIT_2:
1401         case C3CN_STATE_ABORTING:
1402                 /* nothing need to be done */
1403                 break;
1404         case C3CN_STATE_CONNECTING:
1405                 /* defer until cpl_act_open_rpl or cpl_act_establish */
1406                 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1407                 break;
1408         case C3CN_STATE_ESTABLISHED:
1409                 close_req = 1;
1410                 c3cn_set_state(c3cn, C3CN_STATE_ACTIVE_CLOSE);
1411                 break;
1412         case C3CN_STATE_PASSIVE_CLOSE:
1413                 close_req = 1;
1414                 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
1415                 break;
1416         }
1417
1418         if (close_req) {
1419                 if (data_lost)
1420                         /* Unread data was tossed, zap the connection. */
1421                         send_abort_req(c3cn);
1422                 else
1423                         send_close_req(c3cn);
1424         }
1425
1426         spin_unlock_bh(&c3cn->lock);
1427         c3cn_put(c3cn);
1428 }
1429
1430 /**
1431  * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1432  *      resource held
1433  * @c3cn: the iscsi tcp connection
1434  */
1435 void cxgb3i_c3cn_release(struct s3_conn *c3cn)
1436 {
1437         c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1438                         c3cn, c3cn->state, c3cn->flags);
1439         if (unlikely(c3cn->state == C3CN_STATE_CONNECTING))
1440                 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1441         else if (likely(c3cn->state != C3CN_STATE_CLOSED))
1442                 c3cn_active_close(c3cn);
1443         c3cn_put(c3cn);
1444 }
1445
1446 static int is_cxgb3_dev(struct net_device *dev)
1447 {
1448         struct cxgb3i_sdev_data *cdata;
1449         struct net_device *ndev = dev;
1450
1451         if (dev->priv_flags & IFF_802_1Q_VLAN)
1452                 ndev = vlan_dev_real_dev(dev);
1453
1454         write_lock(&cdata_rwlock);
1455         list_for_each_entry(cdata, &cdata_list, list) {
1456                 struct adap_ports *ports = &cdata->ports;
1457                 int i;
1458
1459                 for (i = 0; i < ports->nports; i++)
1460                         if (ndev == ports->lldevs[i]) {
1461                                 write_unlock(&cdata_rwlock);
1462                                 return 1;
1463                         }
1464         }
1465         write_unlock(&cdata_rwlock);
1466         return 0;
1467 }
1468
1469 /**
1470  * cxgb3_egress_dev - return the cxgb3 egress device
1471  * @root_dev: the root device anchoring the search
1472  * @c3cn: the connection used to determine egress port in bonding mode
1473  * @context: in bonding mode, indicates a connection set up or failover
1474  *
1475  * Return egress device or NULL if the egress device isn't one of our ports.
1476  */
1477 static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
1478                                            struct s3_conn *c3cn,
1479                                            int context)
1480 {
1481         while (root_dev) {
1482                 if (root_dev->priv_flags & IFF_802_1Q_VLAN)
1483                         root_dev = vlan_dev_real_dev(root_dev);
1484                 else if (is_cxgb3_dev(root_dev))
1485                         return root_dev;
1486                 else
1487                         return NULL;
1488         }
1489         return NULL;
1490 }
1491
1492 static struct rtable *find_route(struct net_device *dev,
1493                                  __be32 saddr, __be32 daddr,
1494                                  __be16 sport, __be16 dport)
1495 {
1496         struct rtable *rt;
1497         struct flowi fl = {
1498                 .oif = dev ? dev->ifindex : 0,
1499                 .nl_u = {
1500                          .ip4_u = {
1501                                    .daddr = daddr,
1502                                    .saddr = saddr,
1503                                    .tos = 0 } },
1504                 .proto = IPPROTO_TCP,
1505                 .uli_u = {
1506                           .ports = {
1507                                     .sport = sport,
1508                                     .dport = dport } } };
1509
1510         if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
1511                 return NULL;
1512         return rt;
1513 }
1514
1515 /*
1516  * Assign offload parameters to some connection fields.
1517  */
1518 static void init_offload_conn(struct s3_conn *c3cn,
1519                               struct t3cdev *cdev,
1520                               struct dst_entry *dst)
1521 {
1522         BUG_ON(c3cn->cdev != cdev);
1523         c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs - 1;
1524         c3cn->wr_unacked = 0;
1525         c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
1526
1527         reset_wr_list(c3cn);
1528 }
1529
1530 static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
1531 {
1532         struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
1533         struct t3cdev *cdev = cdata->cdev;
1534         struct dst_entry *dst = c3cn->dst_cache;
1535         struct sk_buff *skb;
1536
1537         c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1538                         c3cn, c3cn->state, c3cn->flags);
1539         /*
1540          * Initialize connection data.  Note that the flags and ULP mode are
1541          * initialized higher up ...
1542          */
1543         c3cn->dev = dev;
1544         c3cn->cdev = cdev;
1545         c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
1546         if (c3cn->tid < 0)
1547                 goto out_err;
1548
1549         c3cn->qset = 0;
1550         c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
1551         if (!c3cn->l2t)
1552                 goto free_tid;
1553
1554         skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
1555         if (!skb)
1556                 goto free_l2t;
1557
1558         skb->sk = (struct sock *)c3cn;
1559         set_arp_failure_handler(skb, act_open_req_arp_failure);
1560
1561         c3cn_hold(c3cn);
1562
1563         init_offload_conn(c3cn, cdev, dst);
1564         c3cn->err = 0;
1565
1566         make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
1567         l2t_send(cdev, skb, c3cn->l2t);
1568         return 0;
1569
1570 free_l2t:
1571         l2t_release(L2DATA(cdev), c3cn->l2t);
1572 free_tid:
1573         s3_free_atid(cdev, c3cn->tid);
1574         c3cn->tid = 0;
1575 out_err:
1576         return -EINVAL;
1577 }
1578
1579 /**
1580  * cxgb3i_find_dev - find the interface associated with the given address
1581  * @ipaddr: ip address
1582  */
1583 static struct net_device *
1584 cxgb3i_find_dev(struct net_device *dev, __be32 ipaddr)
1585 {
1586         struct flowi fl;
1587         int err;
1588         struct rtable *rt;
1589
1590         memset(&fl, 0, sizeof(fl));
1591         fl.nl_u.ip4_u.daddr = ipaddr;
1592
1593         err = ip_route_output_key(dev ? dev_net(dev) : &init_net, &rt, &fl);
1594         if (!err)
1595                 return (&rt->dst)->dev;
1596
1597         return NULL;
1598 }
1599
1600 /**
1601  * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1602  * @c3cn: the iscsi tcp connection
1603  * @usin: destination address
1604  *
1605  * return 0 if active open request is sent, < 0 otherwise.
1606  */
1607 int cxgb3i_c3cn_connect(struct net_device *dev, struct s3_conn *c3cn,
1608                         struct sockaddr_in *usin)
1609 {
1610         struct rtable *rt;
1611         struct cxgb3i_sdev_data *cdata;
1612         struct t3cdev *cdev;
1613         __be32 sipv4;
1614         struct net_device *dstdev;
1615         int err;
1616
1617         c3cn_conn_debug("c3cn 0x%p, dev 0x%p.\n", c3cn, dev);
1618
1619         if (usin->sin_family != AF_INET)
1620                 return -EAFNOSUPPORT;
1621
1622         c3cn->daddr.sin_port = usin->sin_port;
1623         c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
1624
1625         dstdev = cxgb3i_find_dev(dev, usin->sin_addr.s_addr);
1626         if (!dstdev || !is_cxgb3_dev(dstdev))
1627                 return -ENETUNREACH;
1628
1629         if (dstdev->priv_flags & IFF_802_1Q_VLAN)
1630                 dev = dstdev;
1631
1632         rt = find_route(dev, c3cn->saddr.sin_addr.s_addr,
1633                         c3cn->daddr.sin_addr.s_addr,
1634                         c3cn->saddr.sin_port,
1635                         c3cn->daddr.sin_port);
1636         if (rt == NULL) {
1637                 c3cn_conn_debug("NO route to 0x%x, port %u, dev %s.\n",
1638                                 c3cn->daddr.sin_addr.s_addr,
1639                                 ntohs(c3cn->daddr.sin_port),
1640                                 dev ? dev->name : "any");
1641                 return -ENETUNREACH;
1642         }
1643
1644         if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
1645                 c3cn_conn_debug("multi-cast route to 0x%x, port %u, dev %s.\n",
1646                                 c3cn->daddr.sin_addr.s_addr,
1647                                 ntohs(c3cn->daddr.sin_port),
1648                                 dev ? dev->name : "any");
1649                 ip_rt_put(rt);
1650                 return -ENETUNREACH;
1651         }
1652
1653         if (!c3cn->saddr.sin_addr.s_addr)
1654                 c3cn->saddr.sin_addr.s_addr = rt->rt_src;
1655
1656         /* now commit destination to connection */
1657         c3cn->dst_cache = &rt->dst;
1658
1659         /* try to establish an offloaded connection */
1660         dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
1661         if (dev == NULL) {
1662                 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
1663                 return -ENETUNREACH;
1664         }
1665         cdata = NDEV2CDATA(dev);
1666         cdev = cdata->cdev;
1667
1668         /* get a source port if one hasn't been provided */
1669         err = c3cn_get_port(c3cn, cdata);
1670         if (err)
1671                 return err;
1672
1673         c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1674                         c3cn, ntohs(c3cn->saddr.sin_port));
1675
1676         sipv4 = cxgb3i_get_private_ipv4addr(dev);
1677         if (!sipv4) {
1678                 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
1679                 sipv4 = c3cn->saddr.sin_addr.s_addr;
1680                 cxgb3i_set_private_ipv4addr(dev, sipv4);
1681         } else
1682                 c3cn->saddr.sin_addr.s_addr = sipv4;
1683
1684         c3cn_conn_debug("c3cn 0x%p, %pI4,%u-%pI4,%u SYN_SENT.\n",
1685                         c3cn,
1686                         &c3cn->saddr.sin_addr.s_addr,
1687                         ntohs(c3cn->saddr.sin_port),
1688                         &c3cn->daddr.sin_addr.s_addr,
1689                         ntohs(c3cn->daddr.sin_port));
1690
1691         c3cn_set_state(c3cn, C3CN_STATE_CONNECTING);
1692         if (!initiate_act_open(c3cn, dev))
1693                 return 0;
1694
1695         /*
1696          * If we get here, we don't have an offload connection so simply
1697          * return a failure.
1698          */
1699         err = -ENOTSUPP;
1700
1701         /*
1702          * This trashes the connection and releases the local port,
1703          * if necessary.
1704          */
1705         c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn);
1706         c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
1707         ip_rt_put(rt);
1708         c3cn_put_port(c3cn);
1709         return err;
1710 }
1711
1712 /**
1713  * cxgb3i_c3cn_rx_credits - ack received tcp data.
1714  * @c3cn: iscsi tcp connection
1715  * @copied: # of bytes processed
1716  *
1717  * Called after some received data has been read.  It returns RX credits
1718  * to the HW for the amount of data processed.
1719  */
1720 void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
1721 {
1722         struct t3cdev *cdev;
1723         int must_send;
1724         u32 credits, dack = 0;
1725
1726         if (c3cn->state != C3CN_STATE_ESTABLISHED)
1727                 return;
1728
1729         credits = c3cn->copied_seq - c3cn->rcv_wup;
1730         if (unlikely(!credits))
1731                 return;
1732
1733         cdev = c3cn->cdev;
1734
1735         if (unlikely(cxgb3_rx_credit_thres == 0))
1736                 return;
1737
1738         dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
1739
1740         /*
1741          * For coalescing to work effectively ensure the receive window has
1742          * at least 16KB left.
1743          */
1744         must_send = credits + 16384 >= cxgb3_rcv_win;
1745
1746         if (must_send || credits >= cxgb3_rx_credit_thres)
1747                 c3cn->rcv_wup += send_rx_credits(c3cn, credits, dack);
1748 }
1749
1750 /**
1751  * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1752  * @c3cn: iscsi tcp connection
1753  * @skb: skb contains the iscsi pdu
1754  *
1755  * Add a list of skbs to a connection send queue. The skbs must comply with
1756  * the max size limit of the device and have a headroom of at least
1757  * TX_HEADER_LEN bytes.
1758  * Return # of bytes queued.
1759  */
1760 int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
1761 {
1762         struct sk_buff *next;
1763         int err, copied = 0;
1764
1765         spin_lock_bh(&c3cn->lock);
1766
1767         if (c3cn->state != C3CN_STATE_ESTABLISHED) {
1768                 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1769                               c3cn, c3cn->state);
1770                 err = -EAGAIN;
1771                 goto out_err;
1772         }
1773
1774         if (c3cn->err) {
1775                 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
1776                 err = -EPIPE;
1777                 goto out_err;
1778         }
1779
1780         if (c3cn->write_seq - c3cn->snd_una >= cxgb3_snd_win) {
1781                 c3cn_tx_debug("c3cn 0x%p, snd %u - %u > %u.\n",
1782                                 c3cn, c3cn->write_seq, c3cn->snd_una,
1783                                 cxgb3_snd_win);
1784                 err = -ENOBUFS;
1785                 goto out_err;
1786         }
1787
1788         while (skb) {
1789                 int frags = skb_shinfo(skb)->nr_frags +
1790                                 (skb->len != skb->data_len);
1791
1792                 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
1793                         c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
1794                         err = -EINVAL;
1795                         goto out_err;
1796                 }
1797
1798                 if (frags >= SKB_WR_LIST_SIZE) {
1799                         cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1800                                          c3cn, skb_shinfo(skb)->nr_frags,
1801                                          skb->len, skb->data_len);
1802                         err = -EINVAL;
1803                         goto out_err;
1804                 }
1805
1806                 next = skb->next;
1807                 skb->next = NULL;
1808                 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
1809                 copied += skb->len;
1810                 c3cn->write_seq += skb->len + ulp_extra_len(skb);
1811                 skb = next;
1812         }
1813 done:
1814         if (likely(skb_queue_len(&c3cn->write_queue)))
1815                 c3cn_push_tx_frames(c3cn, 1);
1816         spin_unlock_bh(&c3cn->lock);
1817         return copied;
1818
1819 out_err:
1820         if (copied == 0 && err == -EPIPE)
1821                 copied = c3cn->err ? c3cn->err : -EPIPE;
1822         else
1823                 copied = err;
1824         goto done;
1825 }
1826
1827 static void sdev_data_cleanup(struct cxgb3i_sdev_data *cdata)
1828 {
1829         struct adap_ports *ports = &cdata->ports;
1830         struct s3_conn *c3cn;
1831         int i;
1832
1833         for (i = 0; i < cxgb3_max_connect; i++) {
1834                 if (cdata->sport_conn[i]) {
1835                         c3cn = cdata->sport_conn[i];
1836                         cdata->sport_conn[i] = NULL;
1837
1838                         spin_lock_bh(&c3cn->lock);
1839                         c3cn->cdev = NULL;
1840                         c3cn_set_flag(c3cn, C3CN_OFFLOAD_DOWN);
1841                         c3cn_closed(c3cn);
1842                         spin_unlock_bh(&c3cn->lock);
1843                 }
1844         }
1845
1846         for (i = 0; i < ports->nports; i++)
1847                 NDEV2CDATA(ports->lldevs[i]) = NULL;
1848
1849         cxgb3i_free_big_mem(cdata);
1850 }
1851
1852 void cxgb3i_sdev_cleanup(void)
1853 {
1854         struct cxgb3i_sdev_data *cdata;
1855
1856         write_lock(&cdata_rwlock);
1857         list_for_each_entry(cdata, &cdata_list, list) {
1858                 list_del(&cdata->list);
1859                 sdev_data_cleanup(cdata);
1860         }
1861         write_unlock(&cdata_rwlock);
1862 }
1863
1864 int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
1865 {
1866         cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
1867         cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
1868         cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
1869         cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
1870         cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
1871         cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
1872         cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
1873         cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
1874
1875         if (cxgb3_max_connect > CXGB3I_MAX_CONN)
1876                 cxgb3_max_connect = CXGB3I_MAX_CONN;
1877         return 0;
1878 }
1879
1880 /**
1881  * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1882  * @cdev:       t3cdev adapter
1883  * @client:     cxgb3 driver client
1884  */
1885 void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
1886 {
1887         struct cxgb3i_sdev_data *cdata;
1888         struct ofld_page_info rx_page_info;
1889         unsigned int wr_len;
1890         int mapsize = cxgb3_max_connect * sizeof(struct s3_conn *);
1891         int i;
1892
1893         cdata =  cxgb3i_alloc_big_mem(sizeof(*cdata) + mapsize, GFP_KERNEL);
1894         if (!cdata) {
1895                 cxgb3i_log_warn("t3dev 0x%p, offload up, OOM %d.\n",
1896                                 cdev, mapsize);
1897                 return;
1898         }
1899
1900         if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
1901             cdev->ctl(cdev, GET_PORTS, &cdata->ports) < 0 ||
1902             cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1903                 cxgb3i_log_warn("t3dev 0x%p, offload up, ioctl failed.\n",
1904                                 cdev);
1905                 goto free_cdata;
1906         }
1907
1908         s3_init_wr_tab(wr_len);
1909
1910         spin_lock_init(&cdata->lock);
1911         INIT_LIST_HEAD(&cdata->list);
1912         cdata->cdev = cdev;
1913         cdata->client = client;
1914
1915         for (i = 0; i < cdata->ports.nports; i++)
1916                 NDEV2CDATA(cdata->ports.lldevs[i]) = cdata;
1917
1918         write_lock(&cdata_rwlock);
1919         list_add_tail(&cdata->list, &cdata_list);
1920         write_unlock(&cdata_rwlock);
1921
1922         cxgb3i_log_info("t3dev 0x%p, offload up, added.\n", cdev);
1923         return;
1924
1925 free_cdata:
1926         cxgb3i_free_big_mem(cdata);
1927 }
1928
1929 /**
1930  * cxgb3i_sdev_remove - free the allocated resources for the adapter
1931  * @cdev:       t3cdev adapter
1932  */
1933 void cxgb3i_sdev_remove(struct t3cdev *cdev)
1934 {
1935         struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
1936
1937         cxgb3i_log_info("t3dev 0x%p, offload down, remove.\n", cdev);
1938
1939         write_lock(&cdata_rwlock);
1940         list_del(&cdata->list);
1941         write_unlock(&cdata_rwlock);
1942
1943         sdev_data_cleanup(cdata);
1944 }