drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32 /**
  33  * bnx2x_move_fp - move content of the fastpath structure.
  34  *
  35  * @bp:         driver handle
  36  * @from:       source FP index
  37  * @to:         destination FP index
  38  *
  39  * Makes sure the contents of the bp->fp[to].napi is kept
  40  * intact. This is done by first copying the napi struct from
  41  * the target to the source, and then mem copying the entire
  42  * source onto the target. Update txdata pointers and related
  43  * content.
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  50         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  51         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  52         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  53         int old_max_eth_txqs, new_max_eth_txqs;
  54         int old_txdata_index = 0, new_txdata_index = 0;
  55
  56         /* Copy the NAPI object as it has been already initialized */
  57         from_fp->napi = to_fp->napi;
  58
  59         /* Move bnx2x_fastpath contents */
  60         memcpy(to_fp, from_fp, sizeof(*to_fp));
  61         to_fp->index = to;
  62
  63         /* move sp_objs contents as well, as their indices match fp ones */
  64         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  65
  66         /* move fp_stats contents as well, as their indices match fp ones */
  67         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  68
  69         /* Update txdata pointers in fp and move txdata content accordingly:
  70          * Each fp consumes 'max_cos' txdata structures, so the index should be
  71          * decremented by max_cos x delta.
  72          */
  73
  74         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  75         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  76                                 (bp)->max_cos;
  77         if (from == FCOE_IDX(bp)) {
  78                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  79                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80         }
  81
  82         memcpy(&bp->bnx2x_txq[new_txdata_index],
  83                &bp->bnx2x_txq[old_txdata_index],
  84                sizeof(struct bnx2x_fp_txdata));
  85         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  86 }
  87
  88 /**
  89  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  90  *
  91  * @bp:        driver handle
  92  * @buf:       character buffer to fill with the fw name
  93  * @buf_len:   length of the above buffer
  94  *
  95  */
  96 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
  97 {
  98         if (IS_PF(bp)) {
  99                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 100
 101                 phy_fw_ver[0] = '\0';
 102                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 103                                              phy_fw_ver, PHY_FW_VER_LEN);
 104                 strlcpy(buf, bp->fw_ver, buf_len);
 105                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 106                          "bc %d.%d.%d%s%s",
 107                          (bp->common.bc_ver & 0xff0000) >> 16,
 108                          (bp->common.bc_ver & 0xff00) >> 8,
 109                          (bp->common.bc_ver & 0xff),
 110                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 111         } else {
 112                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 113         }
 114 }
 115
 116 /**
 117  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 118  *
 119  * @bp: driver handle
 120  * @delta:      number of eth queues which were not allocated
 121  */
 122 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 123 {
 124         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 125
 126         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 127          * backward along the array could cause memory to be overriden
 128          */
 129         for (cos = 1; cos < bp->max_cos; cos++) {
 130                 for (i = 0; i < old_eth_num - delta; i++) {
 131                         struct bnx2x_fastpath *fp = &bp->fp[i];
 132                         int new_idx = cos * (old_eth_num - delta) + i;
 133
 134                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 135                                sizeof(struct bnx2x_fp_txdata));
 136                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 137                 }
 138         }
 139 }
 140
 141 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 142
 143 /* free skb in the packet ring at pos idx
 144  * return idx of last bd freed
 145  */
 146 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 147                              u16 idx, unsigned int *pkts_compl,
 148                              unsigned int *bytes_compl)
 149 {
 150         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 151         struct eth_tx_start_bd *tx_start_bd;
 152         struct eth_tx_bd *tx_data_bd;
 153         struct sk_buff *skb = tx_buf->skb;
 154         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 155         int nbd;
 156         u16 split_bd_len = 0;
 157
 158         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 159         prefetch(&skb->end);
 160
 161         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 162            txdata->txq_index, idx, tx_buf, skb);
 163
 164         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 165
 166
 167         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 168 #ifdef BNX2X_STOP_ON_ERROR
 169         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 170                 BNX2X_ERR("BAD nbd!\n");
 171                 bnx2x_panic();
 172         }
 173 #endif
 174         new_cons = nbd + tx_buf->first_bd;
 175
 176         /* Get the next bd */
 177         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 178
 179         /* Skip a parse bd... */
 180         --nbd;
 181         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 182
 183         if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
 184                 /* Skip second parse bd... */
 185                 --nbd;
 186                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 187         }
 188
 189         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
 190         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 191                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 192                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
 193                 --nbd;
 194                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 195         }
 196
 197         /* unmap first bd */
 198         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 199                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
 200                          DMA_TO_DEVICE);
 201
 202         /* now free frags */
 203         while (nbd > 0) {
 204
 205                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 206                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 207                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 208                 if (--nbd)
 209                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 210         }
 211
 212         /* release skb */
 213         WARN_ON(!skb);
 214         if (likely(skb)) {
 215                 (*pkts_compl)++;
 216                 (*bytes_compl) += skb->len;
 217         }
 218
 219         dev_kfree_skb_any(skb);
 220         tx_buf->first_bd = 0;
 221         tx_buf->skb = NULL;
 222
 223         return new_cons;
 224 }
 225
 226 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 227 {
 228         struct netdev_queue *txq;
 229         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 230         unsigned int pkts_compl = 0, bytes_compl = 0;
 231
 232 #ifdef BNX2X_STOP_ON_ERROR
 233         if (unlikely(bp->panic))
 234                 return -1;
 235 #endif
 236
 237         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 238         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 239         sw_cons = txdata->tx_pkt_cons;
 240
 241         while (sw_cons != hw_cons) {
 242                 u16 pkt_cons;
 243
 244                 pkt_cons = TX_BD(sw_cons);
 245
 246                 DP(NETIF_MSG_TX_DONE,
 247                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 248                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 249
 250                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 251                                             &pkts_compl, &bytes_compl);
 252
 253                 sw_cons++;
 254         }
 255
 256         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 257
 258         txdata->tx_pkt_cons = sw_cons;
 259         txdata->tx_bd_cons = bd_cons;
 260
 261         /* Need to make the tx_bd_cons update visible to start_xmit()
 262          * before checking for netif_tx_queue_stopped().  Without the
 263          * memory barrier, there is a small possibility that
 264          * start_xmit() will miss it and cause the queue to be stopped
 265          * forever.
 266          * On the other hand we need an rmb() here to ensure the proper
 267          * ordering of bit testing in the following
 268          * netif_tx_queue_stopped(txq) call.
 269          */
 270         smp_mb();
 271
 272         if (unlikely(netif_tx_queue_stopped(txq))) {
 273                 /* Taking tx_lock() is needed to prevent reenabling the queue
 274                  * while it's empty. This could have happen if rx_action() gets
 275                  * suspended in bnx2x_tx_int() after the condition before
 276                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 277                  *
 278                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 279                  * sends some packets consuming the whole queue again->
 280                  * stops the queue
 281                  */
 282
 283                 __netif_tx_lock(txq, smp_processor_id());
 284
 285                 if ((netif_tx_queue_stopped(txq)) &&
 286                     (bp->state == BNX2X_STATE_OPEN) &&
 287                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 288                         netif_tx_wake_queue(txq);
 289
 290                 __netif_tx_unlock(txq);
 291         }
 292         return 0;
 293 }
 294
 295 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 296                                              u16 idx)
 297 {
 298         u16 last_max = fp->last_max_sge;
 299
 300         if (SUB_S16(idx, last_max) > 0)
 301                 fp->last_max_sge = idx;
 302 }
 303
 304 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 305                                          u16 sge_len,
 306                                          struct eth_end_agg_rx_cqe *cqe)
 307 {
 308         struct bnx2x *bp = fp->bp;
 309         u16 last_max, last_elem, first_elem;
 310         u16 delta = 0;
 311         u16 i;
 312
 313         if (!sge_len)
 314                 return;
 315
 316         /* First mark all used pages */
 317         for (i = 0; i < sge_len; i++)
 318                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 319                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 320
 321         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 322            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 323
 324         /* Here we assume that the last SGE index is the biggest */
 325         prefetch((void *)(fp->sge_mask));
 326         bnx2x_update_last_max_sge(fp,
 327                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 328
 329         last_max = RX_SGE(fp->last_max_sge);
 330         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 331         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 332
 333         /* If ring is not full */
 334         if (last_elem + 1 != first_elem)
 335                 last_elem++;
 336
 337         /* Now update the prod */
 338         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 339                 if (likely(fp->sge_mask[i]))
 340                         break;
 341
 342                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 343                 delta += BIT_VEC64_ELEM_SZ;
 344         }
 345
 346         if (delta > 0) {
 347                 fp->rx_sge_prod += delta;
 348                 /* clear page-end entries */
 349                 bnx2x_clear_sge_mask_next_elems(fp);
 350         }
 351
 352         DP(NETIF_MSG_RX_STATUS,
 353            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 354            fp->last_max_sge, fp->rx_sge_prod);
 355 }
 356
 357 /* Get Toeplitz hash value in the skb using the value from the
 358  * CQE (calculated by HW).
 359  */
 360 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 361                             const struct eth_fast_path_rx_cqe *cqe,
 362                             bool *l4_rxhash)
 363 {
 364         /* Get Toeplitz hash from CQE */
 365         if ((bp->dev->features & NETIF_F_RXHASH) &&
 366             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 367                 enum eth_rss_hash_type htype;
 368
 369                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 370                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 371                              (htype == TCP_IPV6_HASH_TYPE);
 372                 return le32_to_cpu(cqe->rss_hash_result);
 373         }
 374         *l4_rxhash = false;
 375         return 0;
 376 }
 377
 378 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 379                             u16 cons, u16 prod,
 380                             struct eth_fast_path_rx_cqe *cqe)
 381 {
 382         struct bnx2x *bp = fp->bp;
 383         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 384         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 385         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 386         dma_addr_t mapping;
 387         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 388         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 389
 390         /* print error if current state != stop */
 391         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 392                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 393
 394         /* Try to map an empty data buffer from the aggregation info  */
 395         mapping = dma_map_single(&bp->pdev->dev,
 396                                  first_buf->data + NET_SKB_PAD,
 397                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 398         /*
 399          *  ...if it fails - move the skb from the consumer to the producer
 400          *  and set the current aggregation state as ERROR to drop it
 401          *  when TPA_STOP arrives.
 402          */
 403
 404         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 405                 /* Move the BD from the consumer to the producer */
 406                 bnx2x_reuse_rx_data(fp, cons, prod);
 407                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 408                 return;
 409         }
 410
 411         /* move empty data from pool to prod */
 412         prod_rx_buf->data = first_buf->data;
 413         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 414         /* point prod_bd to new data */
 415         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 416         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 417
 418         /* move partial skb from cons to pool (don't unmap yet) */
 419         *first_buf = *cons_rx_buf;
 420
 421         /* mark bin state as START */
 422         tpa_info->parsing_flags =
 423                 le16_to_cpu(cqe->pars_flags.flags);
 424         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 425         tpa_info->tpa_state = BNX2X_TPA_START;
 426         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 427         tpa_info->placement_offset = cqe->placement_offset;
 428         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 429         if (fp->mode == TPA_MODE_GRO) {
 430                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 431                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 432                 tpa_info->gro_size = gro_size;
 433         }
 434
 435 #ifdef BNX2X_STOP_ON_ERROR
 436         fp->tpa_queue_used |= (1 << queue);
 437 #ifdef _ASM_GENERIC_INT_L64_H
 438         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 439 #else
 440         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 441 #endif
 442            fp->tpa_queue_used);
 443 #endif
 444 }
 445
 446 /* Timestamp option length allowed for TPA aggregation:
 447  *
 448  *              nop nop kind length echo val
 449  */
 450 #define TPA_TSTAMP_OPT_LEN      12
 451 /**
 452  * bnx2x_set_gro_params - compute GRO values
 453  *
 454  * @skb:                packet skb
 455  * @parsing_flags:      parsing flags from the START CQE
 456  * @len_on_bd:          total length of the first packet for the
 457  *                      aggregation.
 458  * @pkt_len:            length of all segments
 459  *
 460  * Approximate value of the MSS for this aggregation calculated using
 461  * the first packet of it.
 462  * Compute number of aggregated segments, and gso_type.
 463  */
 464 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 465                                  u16 len_on_bd, unsigned int pkt_len,
 466                                  u16 num_of_coalesced_segs)
 467 {
 468         /* TPA aggregation won't have either IP options or TCP options
 469          * other than timestamp or IPv6 extension headers.
 470          */
 471         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 472
 473         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 474             PRS_FLAG_OVERETH_IPV6) {
 475                 hdrs_len += sizeof(struct ipv6hdr);
 476                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 477         } else {
 478                 hdrs_len += sizeof(struct iphdr);
 479                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 480         }
 481
 482         /* Check if there was a TCP timestamp, if there is it's will
 483          * always be 12 bytes length: nop nop kind length echo val.
 484          *
 485          * Otherwise FW would close the aggregation.
 486          */
 487         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 488                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 489
 490         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 491
 492         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 493          * to skb_shinfo(skb)->gso_segs
 494          */
 495         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 496 }
 497
 498 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 499                               struct bnx2x_fastpath *fp, u16 index)
 500 {
 501         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 502         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 503         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 504         dma_addr_t mapping;
 505
 506         if (unlikely(page == NULL)) {
 507                 BNX2X_ERR("Can't alloc sge\n");
 508                 return -ENOMEM;
 509         }
 510
 511         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 512                                SGE_PAGES, DMA_FROM_DEVICE);
 513         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 514                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 515                 BNX2X_ERR("Can't map sge\n");
 516                 return -ENOMEM;
 517         }
 518
 519         sw_buf->page = page;
 520         dma_unmap_addr_set(sw_buf, mapping, mapping);
 521
 522         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 523         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 524
 525         return 0;
 526 }
 527
 528 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 529                                struct bnx2x_agg_info *tpa_info,
 530                                u16 pages,
 531                                struct sk_buff *skb,
 532                                struct eth_end_agg_rx_cqe *cqe,
 533                                u16 cqe_idx)
 534 {
 535         struct sw_rx_page *rx_pg, old_rx_pg;
 536         u32 i, frag_len, frag_size;
 537         int err, j, frag_id = 0;
 538         u16 len_on_bd = tpa_info->len_on_bd;
 539         u16 full_page = 0, gro_size = 0;
 540
 541         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 542
 543         if (fp->mode == TPA_MODE_GRO) {
 544                 gro_size = tpa_info->gro_size;
 545                 full_page = tpa_info->full_page;
 546         }
 547
 548         /* This is needed in order to enable forwarding support */
 549         if (frag_size)
 550                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 551                                      le16_to_cpu(cqe->pkt_len),
 552                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 553
 554 #ifdef BNX2X_STOP_ON_ERROR
 555         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 556                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 557                           pages, cqe_idx);
 558                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 559                 bnx2x_panic();
 560                 return -EINVAL;
 561         }
 562 #endif
 563
 564         /* Run through the SGL and compose the fragmented skb */
 565         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 566                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 567
 568                 /* FW gives the indices of the SGE as if the ring is an array
 569                    (meaning that "next" element will consume 2 indices) */
 570                 if (fp->mode == TPA_MODE_GRO)
 571                         frag_len = min_t(u32, frag_size, (u32)full_page);
 572                 else /* LRO */
 573                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 574
 575                 rx_pg = &fp->rx_page_ring[sge_idx];
 576                 old_rx_pg = *rx_pg;
 577
 578                 /* If we fail to allocate a substitute page, we simply stop
 579                    where we are and drop the whole packet */
 580                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 581                 if (unlikely(err)) {
 582                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 583                         return err;
 584                 }
 585
 586                 /* Unmap the page as we r going to pass it to the stack */
 587                 dma_unmap_page(&bp->pdev->dev,
 588                                dma_unmap_addr(&old_rx_pg, mapping),
 589                                SGE_PAGES, DMA_FROM_DEVICE);
 590                 /* Add one frag and update the appropriate fields in the skb */
 591                 if (fp->mode == TPA_MODE_LRO)
 592                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 593                 else { /* GRO */
 594                         int rem;
 595                         int offset = 0;
 596                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 597                                 int len = rem > gro_size ? gro_size : rem;
 598                                 skb_fill_page_desc(skb, frag_id++,
 599                                                    old_rx_pg.page, offset, len);
 600                                 if (offset)
 601                                         get_page(old_rx_pg.page);
 602                                 offset += len;
 603                         }
 604                 }
 605
 606                 skb->data_len += frag_len;
 607                 skb->truesize += SGE_PAGES;
 608                 skb->len += frag_len;
 609
 610                 frag_size -= frag_len;
 611         }
 612
 613         return 0;
 614 }
 615
 616 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 617 {
 618         if (fp->rx_frag_size)
 619                 put_page(virt_to_head_page(data));
 620         else
 621                 kfree(data);
 622 }
 623
 624 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 625 {
 626         if (fp->rx_frag_size)
 627                 return netdev_alloc_frag(fp->rx_frag_size);
 628
 629         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 630 }
 631
 632 #ifdef CONFIG_INET
 633 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 634 {
 635         const struct iphdr *iph = ip_hdr(skb);
 636         struct tcphdr *th;
 637
 638         skb_set_transport_header(skb, sizeof(struct iphdr));
 639         th = tcp_hdr(skb);
 640
 641         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 642                                   iph->saddr, iph->daddr, 0);
 643 }
 644
 645 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 646 {
 647         struct ipv6hdr *iph = ipv6_hdr(skb);
 648         struct tcphdr *th;
 649
 650         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 651         th = tcp_hdr(skb);
 652
 653         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 654                                   &iph->saddr, &iph->daddr, 0);
 655 }
 656
 657 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 658                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 659 {
 660         skb_set_network_header(skb, 0);
 661         gro_func(bp, skb);
 662         tcp_gro_complete(skb);
 663 }
 664 #endif
 665
 666 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 667                                struct sk_buff *skb)
 668 {
 669 #ifdef CONFIG_INET
 670         if (skb_shinfo(skb)->gso_size) {
 671                 switch (be16_to_cpu(skb->protocol)) {
 672                 case ETH_P_IP:
 673                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 674                         break;
 675                 case ETH_P_IPV6:
 676                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 677                         break;
 678                 default:
 679                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 680                                   be16_to_cpu(skb->protocol));
 681                 }
 682         }
 683 #endif
 684         skb_record_rx_queue(skb, fp->rx_queue);
 685         napi_gro_receive(&fp->napi, skb);
 686 }
 687
 688 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 689                            struct bnx2x_agg_info *tpa_info,
 690                            u16 pages,
 691                            struct eth_end_agg_rx_cqe *cqe,
 692                            u16 cqe_idx)
 693 {
 694         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 695         u8 pad = tpa_info->placement_offset;
 696         u16 len = tpa_info->len_on_bd;
 697         struct sk_buff *skb = NULL;
 698         u8 *new_data, *data = rx_buf->data;
 699         u8 old_tpa_state = tpa_info->tpa_state;
 700
 701         tpa_info->tpa_state = BNX2X_TPA_STOP;
 702
 703         /* If we there was an error during the handling of the TPA_START -
 704          * drop this aggregation.
 705          */
 706         if (old_tpa_state == BNX2X_TPA_ERROR)
 707                 goto drop;
 708
 709         /* Try to allocate the new data */
 710         new_data = bnx2x_frag_alloc(fp);
 711         /* Unmap skb in the pool anyway, as we are going to change
 712            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 713            fails. */
 714         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 715                          fp->rx_buf_size, DMA_FROM_DEVICE);
 716         if (likely(new_data))
 717                 skb = build_skb(data, fp->rx_frag_size);
 718
 719         if (likely(skb)) {
 720 #ifdef BNX2X_STOP_ON_ERROR
 721                 if (pad + len > fp->rx_buf_size) {
 722                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 723                                   pad, len, fp->rx_buf_size);
 724                         bnx2x_panic();
 725                         return;
 726                 }
 727 #endif
 728
 729                 skb_reserve(skb, pad + NET_SKB_PAD);
 730                 skb_put(skb, len);
 731                 skb->rxhash = tpa_info->rxhash;
 732                 skb->l4_rxhash = tpa_info->l4_rxhash;
 733
 734                 skb->protocol = eth_type_trans(skb, bp->dev);
 735                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 736
 737                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 738                                          skb, cqe, cqe_idx)) {
 739                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 740                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 741                         bnx2x_gro_receive(bp, fp, skb);
 742                 } else {
 743                         DP(NETIF_MSG_RX_STATUS,
 744                            "Failed to allocate new pages - dropping packet!\n");
 745                         dev_kfree_skb_any(skb);
 746                 }
 747
 748
 749                 /* put new data in bin */
 750                 rx_buf->data = new_data;
 751
 752                 return;
 753         }
 754         if (new_data)
 755                 bnx2x_frag_free(fp, new_data);
 756 drop:
 757         /* drop the packet and keep the buffer in the bin */
 758         DP(NETIF_MSG_RX_STATUS,
 759            "Failed to allocate or map a new skb - dropping packet!\n");
 760         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 761 }
 762
 763 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 764                                struct bnx2x_fastpath *fp, u16 index)
 765 {
 766         u8 *data;
 767         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 768         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 769         dma_addr_t mapping;
 770
 771         data = bnx2x_frag_alloc(fp);
 772         if (unlikely(data == NULL))
 773                 return -ENOMEM;
 774
 775         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 776                                  fp->rx_buf_size,
 777                                  DMA_FROM_DEVICE);
 778         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 779                 bnx2x_frag_free(fp, data);
 780                 BNX2X_ERR("Can't map rx data\n");
 781                 return -ENOMEM;
 782         }
 783
 784         rx_buf->data = data;
 785         dma_unmap_addr_set(rx_buf, mapping, mapping);
 786
 787         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 788         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 789
 790         return 0;
 791 }
 792
 793 static
 794 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 795                                  struct bnx2x_fastpath *fp,
 796                                  struct bnx2x_eth_q_stats *qstats)
 797 {
 798         /* Do nothing if no L4 csum validation was done.
 799          * We do not check whether IP csum was validated. For IPv4 we assume
 800          * that if the card got as far as validating the L4 csum, it also
 801          * validated the IP csum. IPv6 has no IP csum.
 802          */
 803         if (cqe->fast_path_cqe.status_flags &
 804             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 805                 return;
 806
 807         /* If L4 validation was done, check if an error was found. */
 808
 809         if (cqe->fast_path_cqe.type_error_flags &
 810             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 811              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 812                 qstats->hw_csum_err++;
 813         else
 814                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 815 }
 816
 817 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 818 {
 819         struct bnx2x *bp = fp->bp;
 820         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 821         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 822         int rx_pkt = 0;
 823
 824 #ifdef BNX2X_STOP_ON_ERROR
 825         if (unlikely(bp->panic))
 826                 return 0;
 827 #endif
 828
 829         /* CQ "next element" is of the size of the regular element,
 830            that's why it's ok here */
 831         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 832         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 833                 hw_comp_cons++;
 834
 835         bd_cons = fp->rx_bd_cons;
 836         bd_prod = fp->rx_bd_prod;
 837         bd_prod_fw = bd_prod;
 838         sw_comp_cons = fp->rx_comp_cons;
 839         sw_comp_prod = fp->rx_comp_prod;
 840
 841         /* Memory barrier necessary as speculative reads of the rx
 842          * buffer can be ahead of the index in the status block
 843          */
 844         rmb();
 845
 846         DP(NETIF_MSG_RX_STATUS,
 847            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 848            fp->index, hw_comp_cons, sw_comp_cons);
 849
 850         while (sw_comp_cons != hw_comp_cons) {
 851                 struct sw_rx_bd *rx_buf = NULL;
 852                 struct sk_buff *skb;
 853                 union eth_rx_cqe *cqe;
 854                 struct eth_fast_path_rx_cqe *cqe_fp;
 855                 u8 cqe_fp_flags;
 856                 enum eth_rx_cqe_type cqe_fp_type;
 857                 u16 len, pad, queue;
 858                 u8 *data;
 859                 bool l4_rxhash;
 860
 861 #ifdef BNX2X_STOP_ON_ERROR
 862                 if (unlikely(bp->panic))
 863                         return 0;
 864 #endif
 865
 866                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 867                 bd_prod = RX_BD(bd_prod);
 868                 bd_cons = RX_BD(bd_cons);
 869
 870                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 871                 cqe_fp = &cqe->fast_path_cqe;
 872                 cqe_fp_flags = cqe_fp->type_error_flags;
 873                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 874
 875                 DP(NETIF_MSG_RX_STATUS,
 876                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 877                    CQE_TYPE(cqe_fp_flags),
 878                    cqe_fp_flags, cqe_fp->status_flags,
 879                    le32_to_cpu(cqe_fp->rss_hash_result),
 880                    le16_to_cpu(cqe_fp->vlan_tag),
 881                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 882
 883                 /* is this a slowpath msg? */
 884                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 885                         bnx2x_sp_event(fp, cqe);
 886                         goto next_cqe;
 887                 }
 888
 889                 rx_buf = &fp->rx_buf_ring[bd_cons];
 890                 data = rx_buf->data;
 891
 892                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 893                         struct bnx2x_agg_info *tpa_info;
 894                         u16 frag_size, pages;
 895 #ifdef BNX2X_STOP_ON_ERROR
 896                         /* sanity check */
 897                         if (fp->disable_tpa &&
 898                             (CQE_TYPE_START(cqe_fp_type) ||
 899                              CQE_TYPE_STOP(cqe_fp_type)))
 900                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 901                                           CQE_TYPE(cqe_fp_type));
 902 #endif
 903
 904                         if (CQE_TYPE_START(cqe_fp_type)) {
 905                                 u16 queue = cqe_fp->queue_index;
 906                                 DP(NETIF_MSG_RX_STATUS,
 907                                    "calling tpa_start on queue %d\n",
 908                                    queue);
 909
 910                                 bnx2x_tpa_start(fp, queue,
 911                                                 bd_cons, bd_prod,
 912                                                 cqe_fp);
 913
 914                                 goto next_rx;
 915
 916                         }
 917                         queue = cqe->end_agg_cqe.queue_index;
 918                         tpa_info = &fp->tpa_info[queue];
 919                         DP(NETIF_MSG_RX_STATUS,
 920                            "calling tpa_stop on queue %d\n",
 921                            queue);
 922
 923                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 924                                     tpa_info->len_on_bd;
 925
 926                         if (fp->mode == TPA_MODE_GRO)
 927                                 pages = (frag_size + tpa_info->full_page - 1) /
 928                                          tpa_info->full_page;
 929                         else
 930                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 931                                         SGE_PAGE_SHIFT;
 932
 933                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 934                                        &cqe->end_agg_cqe, comp_ring_cons);
 935 #ifdef BNX2X_STOP_ON_ERROR
 936                         if (bp->panic)
 937                                 return 0;
 938 #endif
 939
 940                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 941                         goto next_cqe;
 942                 }
 943                 /* non TPA */
 944                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 945                 pad = cqe_fp->placement_offset;
 946                 dma_sync_single_for_cpu(&bp->pdev->dev,
 947                                         dma_unmap_addr(rx_buf, mapping),
 948                                         pad + RX_COPY_THRESH,
 949                                         DMA_FROM_DEVICE);
 950                 pad += NET_SKB_PAD;
 951                 prefetch(data + pad); /* speedup eth_type_trans() */
 952                 /* is this an error packet? */
 953                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 954                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 955                            "ERROR  flags %x  rx packet %u\n",
 956                            cqe_fp_flags, sw_comp_cons);
 957                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 958                         goto reuse_rx;
 959                 }
 960
 961                 /* Since we don't have a jumbo ring
 962                  * copy small packets if mtu > 1500
 963                  */
 964                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 965                     (len <= RX_COPY_THRESH)) {
 966                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 967                         if (skb == NULL) {
 968                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 969                                    "ERROR  packet dropped because of alloc failure\n");
 970                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 971                                 goto reuse_rx;
 972                         }
 973                         memcpy(skb->data, data + pad, len);
 974                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 975                 } else {
 976                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 977                                 dma_unmap_single(&bp->pdev->dev,
 978                                                  dma_unmap_addr(rx_buf, mapping),
 979                                                  fp->rx_buf_size,
 980                                                  DMA_FROM_DEVICE);
 981                                 skb = build_skb(data, fp->rx_frag_size);
 982                                 if (unlikely(!skb)) {
 983                                         bnx2x_frag_free(fp, data);
 984                                         bnx2x_fp_qstats(bp, fp)->
 985                                                         rx_skb_alloc_failed++;
 986                                         goto next_rx;
 987                                 }
 988                                 skb_reserve(skb, pad);
 989                         } else {
 990                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 991                                    "ERROR  packet dropped because of alloc failure\n");
 992                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 993 reuse_rx:
 994                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 995                                 goto next_rx;
 996                         }
 997                 }
 998
 999                 skb_put(skb, len);
1000                 skb->protocol = eth_type_trans(skb, bp->dev);
1001
1002                 /* Set Toeplitz hash for a none-LRO skb */
1003                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
1004                 skb->l4_rxhash = l4_rxhash;
1005
1006                 skb_checksum_none_assert(skb);
1007
1008                 if (bp->dev->features & NETIF_F_RXCSUM)
1009                         bnx2x_csum_validate(skb, cqe, fp,
1010                                             bnx2x_fp_qstats(bp, fp));
1011
1012                 skb_record_rx_queue(skb, fp->rx_queue);
1013
1014                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1015                     PARSING_FLAGS_VLAN)
1016                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1017                                                le16_to_cpu(cqe_fp->vlan_tag));
1018                 napi_gro_receive(&fp->napi, skb);
1019
1020
1021 next_rx:
1022                 rx_buf->data = NULL;
1023
1024                 bd_cons = NEXT_RX_IDX(bd_cons);
1025                 bd_prod = NEXT_RX_IDX(bd_prod);
1026                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1027                 rx_pkt++;
1028 next_cqe:
1029                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1030                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1031
1032                 if (rx_pkt == budget)
1033                         break;
1034         } /* while */
1035
1036         fp->rx_bd_cons = bd_cons;
1037         fp->rx_bd_prod = bd_prod_fw;
1038         fp->rx_comp_cons = sw_comp_cons;
1039         fp->rx_comp_prod = sw_comp_prod;
1040
1041         /* Update producers */
1042         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1043                              fp->rx_sge_prod);
1044
1045         fp->rx_pkt += rx_pkt;
1046         fp->rx_calls++;
1047
1048         return rx_pkt;
1049 }
1050
1051 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1052 {
1053         struct bnx2x_fastpath *fp = fp_cookie;
1054         struct bnx2x *bp = fp->bp;
1055         u8 cos;
1056
1057         DP(NETIF_MSG_INTR,
1058            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1059            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1060
1061         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1062
1063 #ifdef BNX2X_STOP_ON_ERROR
1064         if (unlikely(bp->panic))
1065                 return IRQ_HANDLED;
1066 #endif
1067
1068         /* Handle Rx and Tx according to MSI-X vector */
1069         prefetch(fp->rx_cons_sb);
1070
1071         for_each_cos_in_tx_queue(fp, cos)
1072                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1073
1074         prefetch(&fp->sb_running_index[SM_RX_ID]);
1075         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1076
1077         return IRQ_HANDLED;
1078 }
1079
1080 /* HW Lock for shared dual port PHYs */
1081 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1082 {
1083         mutex_lock(&bp->port.phy_mutex);
1084
1085         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1086 }
1087
1088 void bnx2x_release_phy_lock(struct bnx2x *bp)
1089 {
1090         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1091
1092         mutex_unlock(&bp->port.phy_mutex);
1093 }
1094
1095 /* calculates MF speed according to current linespeed and MF configuration */
1096 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1097 {
1098         u16 line_speed = bp->link_vars.line_speed;
1099         if (IS_MF(bp)) {
1100                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1101                                                    bp->mf_config[BP_VN(bp)]);
1102
1103                 /* Calculate the current MAX line speed limit for the MF
1104                  * devices
1105                  */
1106                 if (IS_MF_SI(bp))
1107                         line_speed = (line_speed * maxCfg) / 100;
1108                 else { /* SD mode */
1109                         u16 vn_max_rate = maxCfg * 100;
1110
1111                         if (vn_max_rate < line_speed)
1112                                 line_speed = vn_max_rate;
1113                 }
1114         }
1115
1116         return line_speed;
1117 }
1118
1119 /**
1120  * bnx2x_fill_report_data - fill link report data to report
1121  *
1122  * @bp:         driver handle
1123  * @data:       link state to update
1124  *
1125  * It uses a none-atomic bit operations because is called under the mutex.
1126  */
1127 static void bnx2x_fill_report_data(struct bnx2x *bp,
1128                                    struct bnx2x_link_report_data *data)
1129 {
1130         u16 line_speed = bnx2x_get_mf_speed(bp);
1131
1132         memset(data, 0, sizeof(*data));
1133
1134         /* Fill the report data: efective line speed */
1135         data->line_speed = line_speed;
1136
1137         /* Link is down */
1138         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1139                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1140                           &data->link_report_flags);
1141
1142         /* Full DUPLEX */
1143         if (bp->link_vars.duplex == DUPLEX_FULL)
1144                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1145
1146         /* Rx Flow Control is ON */
1147         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1148                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1149
1150         /* Tx Flow Control is ON */
1151         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1152                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1153 }
1154
1155 /**
1156  * bnx2x_link_report - report link status to OS.
1157  *
1158  * @bp:         driver handle
1159  *
1160  * Calls the __bnx2x_link_report() under the same locking scheme
1161  * as a link/PHY state managing code to ensure a consistent link
1162  * reporting.
1163  */
1164
1165 void bnx2x_link_report(struct bnx2x *bp)
1166 {
1167         bnx2x_acquire_phy_lock(bp);
1168         __bnx2x_link_report(bp);
1169         bnx2x_release_phy_lock(bp);
1170 }
1171
1172 /**
1173  * __bnx2x_link_report - report link status to OS.
1174  *
1175  * @bp:         driver handle
1176  *
1177  * None atomic inmlementation.
1178  * Should be called under the phy_lock.
1179  */
1180 void __bnx2x_link_report(struct bnx2x *bp)
1181 {
1182         struct bnx2x_link_report_data cur_data;
1183
1184         /* reread mf_cfg */
1185         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1186                 bnx2x_read_mf_cfg(bp);
1187
1188         /* Read the current link report info */
1189         bnx2x_fill_report_data(bp, &cur_data);
1190
1191         /* Don't report link down or exactly the same link status twice */
1192         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1193             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1194                       &bp->last_reported_link.link_report_flags) &&
1195              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1196                       &cur_data.link_report_flags)))
1197                 return;
1198
1199         bp->link_cnt++;
1200
1201         /* We are going to report a new link parameters now -
1202          * remember the current data for the next time.
1203          */
1204         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1205
1206         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1207                      &cur_data.link_report_flags)) {
1208                 netif_carrier_off(bp->dev);
1209                 netdev_err(bp->dev, "NIC Link is Down\n");
1210                 return;
1211         } else {
1212                 const char *duplex;
1213                 const char *flow;
1214
1215                 netif_carrier_on(bp->dev);
1216
1217                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1218                                        &cur_data.link_report_flags))
1219                         duplex = "full";
1220                 else
1221                         duplex = "half";
1222
1223                 /* Handle the FC at the end so that only these flags would be
1224                  * possibly set. This way we may easily check if there is no FC
1225                  * enabled.
1226                  */
1227                 if (cur_data.link_report_flags) {
1228                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1229                                      &cur_data.link_report_flags)) {
1230                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1231                                      &cur_data.link_report_flags))
1232                                         flow = "ON - receive & transmit";
1233                                 else
1234                                         flow = "ON - receive";
1235                         } else {
1236                                 flow = "ON - transmit";
1237                         }
1238                 } else {
1239                         flow = "none";
1240                 }
1241                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1242                             cur_data.line_speed, duplex, flow);
1243         }
1244 }
1245
1246 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1247 {
1248         int i;
1249
1250         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1251                 struct eth_rx_sge *sge;
1252
1253                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1254                 sge->addr_hi =
1255                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1256                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1257
1258                 sge->addr_lo =
1259                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1260                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1261         }
1262 }
1263
1264 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1265                                 struct bnx2x_fastpath *fp, int last)
1266 {
1267         int i;
1268
1269         for (i = 0; i < last; i++) {
1270                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1271                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1272                 u8 *data = first_buf->data;
1273
1274                 if (data == NULL) {
1275                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1276                         continue;
1277                 }
1278                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1279                         dma_unmap_single(&bp->pdev->dev,
1280                                          dma_unmap_addr(first_buf, mapping),
1281                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1282                 bnx2x_frag_free(fp, data);
1283                 first_buf->data = NULL;
1284         }
1285 }
1286
1287 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1288 {
1289         int j;
1290
1291         for_each_rx_queue_cnic(bp, j) {
1292                 struct bnx2x_fastpath *fp = &bp->fp[j];
1293
1294                 fp->rx_bd_cons = 0;
1295
1296                 /* Activate BD ring */
1297                 /* Warning!
1298                  * this will generate an interrupt (to the TSTORM)
1299                  * must only be done after chip is initialized
1300                  */
1301                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1302                                      fp->rx_sge_prod);
1303         }
1304 }
1305
1306 void bnx2x_init_rx_rings(struct bnx2x *bp)
1307 {
1308         int func = BP_FUNC(bp);
1309         u16 ring_prod;
1310         int i, j;
1311
1312         /* Allocate TPA resources */
1313         for_each_eth_queue(bp, j) {
1314                 struct bnx2x_fastpath *fp = &bp->fp[j];
1315
1316                 DP(NETIF_MSG_IFUP,
1317                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1318
1319                 if (!fp->disable_tpa) {
1320                         /* Fill the per-aggregtion pool */
1321                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1322                                 struct bnx2x_agg_info *tpa_info =
1323                                         &fp->tpa_info[i];
1324                                 struct sw_rx_bd *first_buf =
1325                                         &tpa_info->first_buf;
1326
1327                                 first_buf->data = bnx2x_frag_alloc(fp);
1328                                 if (!first_buf->data) {
1329                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1330                                                   j);
1331                                         bnx2x_free_tpa_pool(bp, fp, i);
1332                                         fp->disable_tpa = 1;
1333                                         break;
1334                                 }
1335                                 dma_unmap_addr_set(first_buf, mapping, 0);
1336                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1337                         }
1338
1339                         /* "next page" elements initialization */
1340                         bnx2x_set_next_page_sgl(fp);
1341
1342                         /* set SGEs bit mask */
1343                         bnx2x_init_sge_ring_bit_mask(fp);
1344
1345                         /* Allocate SGEs and initialize the ring elements */
1346                         for (i = 0, ring_prod = 0;
1347                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1348
1349                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1350                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1351                                                   i);
1352                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1353                                                   j);
1354                                         /* Cleanup already allocated elements */
1355                                         bnx2x_free_rx_sge_range(bp, fp,
1356                                                                 ring_prod);
1357                                         bnx2x_free_tpa_pool(bp, fp,
1358                                                             MAX_AGG_QS(bp));
1359                                         fp->disable_tpa = 1;
1360                                         ring_prod = 0;
1361                                         break;
1362                                 }
1363                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1364                         }
1365
1366                         fp->rx_sge_prod = ring_prod;
1367                 }
1368         }
1369
1370         for_each_eth_queue(bp, j) {
1371                 struct bnx2x_fastpath *fp = &bp->fp[j];
1372
1373                 fp->rx_bd_cons = 0;
1374
1375                 /* Activate BD ring */
1376                 /* Warning!
1377                  * this will generate an interrupt (to the TSTORM)
1378                  * must only be done after chip is initialized
1379                  */
1380                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1381                                      fp->rx_sge_prod);
1382
1383                 if (j != 0)
1384                         continue;
1385
1386                 if (CHIP_IS_E1(bp)) {
1387                         REG_WR(bp, BAR_USTRORM_INTMEM +
1388                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1389                                U64_LO(fp->rx_comp_mapping));
1390                         REG_WR(bp, BAR_USTRORM_INTMEM +
1391                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1392                                U64_HI(fp->rx_comp_mapping));
1393                 }
1394         }
1395 }
1396
1397 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1398 {
1399         u8 cos;
1400         struct bnx2x *bp = fp->bp;
1401
1402         for_each_cos_in_tx_queue(fp, cos) {
1403                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1404                 unsigned pkts_compl = 0, bytes_compl = 0;
1405
1406                 u16 sw_prod = txdata->tx_pkt_prod;
1407                 u16 sw_cons = txdata->tx_pkt_cons;
1408
1409                 while (sw_cons != sw_prod) {
1410                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1411                                           &pkts_compl, &bytes_compl);
1412                         sw_cons++;
1413                 }
1414
1415                 netdev_tx_reset_queue(
1416                         netdev_get_tx_queue(bp->dev,
1417                                             txdata->txq_index));
1418         }
1419 }
1420
1421 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1422 {
1423         int i;
1424
1425         for_each_tx_queue_cnic(bp, i) {
1426                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1427         }
1428 }
1429
1430 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1431 {
1432         int i;
1433
1434         for_each_eth_queue(bp, i) {
1435                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1436         }
1437 }
1438
1439 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1440 {
1441         struct bnx2x *bp = fp->bp;
1442         int i;
1443
1444         /* ring wasn't allocated */
1445         if (fp->rx_buf_ring == NULL)
1446                 return;
1447
1448         for (i = 0; i < NUM_RX_BD; i++) {
1449                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1450                 u8 *data = rx_buf->data;
1451
1452                 if (data == NULL)
1453                         continue;
1454                 dma_unmap_single(&bp->pdev->dev,
1455                                  dma_unmap_addr(rx_buf, mapping),
1456                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1457
1458                 rx_buf->data = NULL;
1459                 bnx2x_frag_free(fp, data);
1460         }
1461 }
1462
1463 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1464 {
1465         int j;
1466
1467         for_each_rx_queue_cnic(bp, j) {
1468                 bnx2x_free_rx_bds(&bp->fp[j]);
1469         }
1470 }
1471
1472 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1473 {
1474         int j;
1475
1476         for_each_eth_queue(bp, j) {
1477                 struct bnx2x_fastpath *fp = &bp->fp[j];
1478
1479                 bnx2x_free_rx_bds(fp);
1480
1481                 if (!fp->disable_tpa)
1482                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1483         }
1484 }
1485
1486 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1487 {
1488         bnx2x_free_tx_skbs_cnic(bp);
1489         bnx2x_free_rx_skbs_cnic(bp);
1490 }
1491
1492 void bnx2x_free_skbs(struct bnx2x *bp)
1493 {
1494         bnx2x_free_tx_skbs(bp);
1495         bnx2x_free_rx_skbs(bp);
1496 }
1497
1498 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1499 {
1500         /* load old values */
1501         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1502
1503         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1504                 /* leave all but MAX value */
1505                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1506
1507                 /* set new MAX value */
1508                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1509                                 & FUNC_MF_CFG_MAX_BW_MASK;
1510
1511                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1512         }
1513 }
1514
1515 /**
1516  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1517  *
1518  * @bp:         driver handle
1519  * @nvecs:      number of vectors to be released
1520  */
1521 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1522 {
1523         int i, offset = 0;
1524
1525         if (nvecs == offset)
1526                 return;
1527
1528         /* VFs don't have a default SB */
1529         if (IS_PF(bp)) {
1530                 free_irq(bp->msix_table[offset].vector, bp->dev);
1531                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1532                    bp->msix_table[offset].vector);
1533                 offset++;
1534         }
1535
1536         if (CNIC_SUPPORT(bp)) {
1537                 if (nvecs == offset)
1538                         return;
1539                 offset++;
1540         }
1541
1542         for_each_eth_queue(bp, i) {
1543                 if (nvecs == offset)
1544                         return;
1545                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1546                    i, bp->msix_table[offset].vector);
1547
1548                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1549         }
1550 }
1551
1552 void bnx2x_free_irq(struct bnx2x *bp)
1553 {
1554         if (bp->flags & USING_MSIX_FLAG &&
1555             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1556                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1557
1558                 /* vfs don't have a default status block */
1559                 if (IS_PF(bp))
1560                         nvecs++;
1561
1562                 bnx2x_free_msix_irqs(bp, nvecs);
1563         } else {
1564                 free_irq(bp->dev->irq, bp->dev);
1565         }
1566 }
1567
1568 int bnx2x_enable_msix(struct bnx2x *bp)
1569 {
1570         int msix_vec = 0, i, rc;
1571
1572         /* VFs don't have a default status block */
1573         if (IS_PF(bp)) {
1574                 bp->msix_table[msix_vec].entry = msix_vec;
1575                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1576                                bp->msix_table[0].entry);
1577                 msix_vec++;
1578         }
1579
1580         /* Cnic requires an msix vector for itself */
1581         if (CNIC_SUPPORT(bp)) {
1582                 bp->msix_table[msix_vec].entry = msix_vec;
1583                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1584                                msix_vec, bp->msix_table[msix_vec].entry);
1585                 msix_vec++;
1586         }
1587
1588         /* We need separate vectors for ETH queues only (not FCoE) */
1589         for_each_eth_queue(bp, i) {
1590                 bp->msix_table[msix_vec].entry = msix_vec;
1591                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1592                                msix_vec, msix_vec, i);
1593                 msix_vec++;
1594         }
1595
1596         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1597            msix_vec);
1598
1599         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1600
1601         /*
1602          * reconfigure number of tx/rx queues according to available
1603          * MSI-X vectors
1604          */
1605         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1606                 /* how less vectors we will have? */
1607                 int diff = msix_vec - rc;
1608
1609                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1610
1611                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1612
1613                 if (rc) {
1614                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1615                         goto no_msix;
1616                 }
1617                 /*
1618                  * decrease number of queues by number of unallocated entries
1619                  */
1620                 bp->num_ethernet_queues -= diff;
1621                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1622
1623                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1624                                bp->num_queues);
1625         } else if (rc > 0) {
1626                 /* Get by with single vector */
1627                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1628                 if (rc) {
1629                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1630                                        rc);
1631                         goto no_msix;
1632                 }
1633
1634                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1635                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1636
1637                 BNX2X_DEV_INFO("set number of queues to 1\n");
1638                 bp->num_ethernet_queues = 1;
1639                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1640         } else if (rc < 0) {
1641                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1642                 goto no_msix;
1643         }
1644
1645         bp->flags |= USING_MSIX_FLAG;
1646
1647         return 0;
1648
1649 no_msix:
1650         /* fall to INTx if not enough memory */
1651         if (rc == -ENOMEM)
1652                 bp->flags |= DISABLE_MSI_FLAG;
1653
1654         return rc;
1655 }
1656
1657 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1658 {
1659         int i, rc, offset = 0;
1660
1661         /* no default status block for vf */
1662         if (IS_PF(bp)) {
1663                 rc = request_irq(bp->msix_table[offset++].vector,
1664                                  bnx2x_msix_sp_int, 0,
1665                                  bp->dev->name, bp->dev);
1666                 if (rc) {
1667                         BNX2X_ERR("request sp irq failed\n");
1668                         return -EBUSY;
1669                 }
1670         }
1671
1672         if (CNIC_SUPPORT(bp))
1673                 offset++;
1674
1675         for_each_eth_queue(bp, i) {
1676                 struct bnx2x_fastpath *fp = &bp->fp[i];
1677                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1678                          bp->dev->name, i);
1679
1680                 rc = request_irq(bp->msix_table[offset].vector,
1681                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1682                 if (rc) {
1683                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1684                               bp->msix_table[offset].vector, rc);
1685                         bnx2x_free_msix_irqs(bp, offset);
1686                         return -EBUSY;
1687                 }
1688
1689                 offset++;
1690         }
1691
1692         i = BNX2X_NUM_ETH_QUEUES(bp);
1693         if (IS_PF(bp)) {
1694                 offset = 1 + CNIC_SUPPORT(bp);
1695                 netdev_info(bp->dev,
1696                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1697                             bp->msix_table[0].vector,
1698                             0, bp->msix_table[offset].vector,
1699                             i - 1, bp->msix_table[offset + i - 1].vector);
1700         } else {
1701                 offset = CNIC_SUPPORT(bp);
1702                 netdev_info(bp->dev,
1703                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1704                             0, bp->msix_table[offset].vector,
1705                             i - 1, bp->msix_table[offset + i - 1].vector);
1706         }
1707         return 0;
1708 }
1709
1710 int bnx2x_enable_msi(struct bnx2x *bp)
1711 {
1712         int rc;
1713
1714         rc = pci_enable_msi(bp->pdev);
1715         if (rc) {
1716                 BNX2X_DEV_INFO("MSI is not attainable\n");
1717                 return -1;
1718         }
1719         bp->flags |= USING_MSI_FLAG;
1720
1721         return 0;
1722 }
1723
1724 static int bnx2x_req_irq(struct bnx2x *bp)
1725 {
1726         unsigned long flags;
1727         unsigned int irq;
1728
1729         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1730                 flags = 0;
1731         else
1732                 flags = IRQF_SHARED;
1733
1734         if (bp->flags & USING_MSIX_FLAG)
1735                 irq = bp->msix_table[0].vector;
1736         else
1737                 irq = bp->pdev->irq;
1738
1739         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1740 }
1741
1742 int bnx2x_setup_irqs(struct bnx2x *bp)
1743 {
1744         int rc = 0;
1745         if (bp->flags & USING_MSIX_FLAG &&
1746             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1747                 rc = bnx2x_req_msix_irqs(bp);
1748                 if (rc)
1749                         return rc;
1750         } else {
1751                 rc = bnx2x_req_irq(bp);
1752                 if (rc) {
1753                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1754                         return rc;
1755                 }
1756                 if (bp->flags & USING_MSI_FLAG) {
1757                         bp->dev->irq = bp->pdev->irq;
1758                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1759                                     bp->dev->irq);
1760                 }
1761                 if (bp->flags & USING_MSIX_FLAG) {
1762                         bp->dev->irq = bp->msix_table[0].vector;
1763                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1764                                     bp->dev->irq);
1765                 }
1766         }
1767
1768         return 0;
1769 }
1770
1771 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1772 {
1773         int i;
1774
1775         for_each_rx_queue_cnic(bp, i)
1776                 napi_enable(&bnx2x_fp(bp, i, napi));
1777 }
1778
1779 static void bnx2x_napi_enable(struct bnx2x *bp)
1780 {
1781         int i;
1782
1783         for_each_eth_queue(bp, i)
1784                 napi_enable(&bnx2x_fp(bp, i, napi));
1785 }
1786
1787 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1788 {
1789         int i;
1790
1791         for_each_rx_queue_cnic(bp, i)
1792                 napi_disable(&bnx2x_fp(bp, i, napi));
1793 }
1794
1795 static void bnx2x_napi_disable(struct bnx2x *bp)
1796 {
1797         int i;
1798
1799         for_each_eth_queue(bp, i)
1800                 napi_disable(&bnx2x_fp(bp, i, napi));
1801 }
1802
1803 void bnx2x_netif_start(struct bnx2x *bp)
1804 {
1805         if (netif_running(bp->dev)) {
1806                 bnx2x_napi_enable(bp);
1807                 if (CNIC_LOADED(bp))
1808                         bnx2x_napi_enable_cnic(bp);
1809                 bnx2x_int_enable(bp);
1810                 if (bp->state == BNX2X_STATE_OPEN)
1811                         netif_tx_wake_all_queues(bp->dev);
1812         }
1813 }
1814
1815 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1816 {
1817         bnx2x_int_disable_sync(bp, disable_hw);
1818         bnx2x_napi_disable(bp);
1819         if (CNIC_LOADED(bp))
1820                 bnx2x_napi_disable_cnic(bp);
1821 }
1822
1823 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1824 {
1825         struct bnx2x *bp = netdev_priv(dev);
1826
1827         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1828                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1829                 u16 ether_type = ntohs(hdr->h_proto);
1830
1831                 /* Skip VLAN tag if present */
1832                 if (ether_type == ETH_P_8021Q) {
1833                         struct vlan_ethhdr *vhdr =
1834                                 (struct vlan_ethhdr *)skb->data;
1835
1836                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1837                 }
1838
1839                 /* If ethertype is FCoE or FIP - use FCoE ring */
1840                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1841                         return bnx2x_fcoe_tx(bp, txq_index);
1842         }
1843
1844         /* select a non-FCoE queue */
1845         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1846 }
1847
1848 void bnx2x_set_num_queues(struct bnx2x *bp)
1849 {
1850         /* RSS queues */
1851         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1852
1853         /* override in STORAGE SD modes */
1854         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1855                 bp->num_ethernet_queues = 1;
1856
1857         /* Add special queues */
1858         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1859         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1860
1861         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1862 }
1863
1864 /**
1865  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1866  *
1867  * @bp:         Driver handle
1868  *
1869  * We currently support for at most 16 Tx queues for each CoS thus we will
1870  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1871  * bp->max_cos.
1872  *
1873  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1874  * index after all ETH L2 indices.
1875  *
1876  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1877  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1878  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1879  *
1880  * The proper configuration of skb->queue_mapping is handled by
1881  * bnx2x_select_queue() and __skb_tx_hash().
1882  *
1883  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1884  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1885  */
1886 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1887 {
1888         int rc, tx, rx;
1889
1890         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1891         rx = BNX2X_NUM_ETH_QUEUES(bp);
1892
1893 /* account for fcoe queue */
1894         if (include_cnic && !NO_FCOE(bp)) {
1895                 rx++;
1896                 tx++;
1897         }
1898
1899         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1900         if (rc) {
1901                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1902                 return rc;
1903         }
1904         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1905         if (rc) {
1906                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1907                 return rc;
1908         }
1909
1910         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1911                           tx, rx);
1912
1913         return rc;
1914 }
1915
1916 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1917 {
1918         int i;
1919
1920         for_each_queue(bp, i) {
1921                 struct bnx2x_fastpath *fp = &bp->fp[i];
1922                 u32 mtu;
1923
1924                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1925                 if (IS_FCOE_IDX(i))
1926                         /*
1927                          * Although there are no IP frames expected to arrive to
1928                          * this ring we still want to add an
1929                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1930                          * overrun attack.
1931                          */
1932                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1933                 else
1934                         mtu = bp->dev->mtu;
1935                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1936                                   IP_HEADER_ALIGNMENT_PADDING +
1937                                   ETH_OVREHEAD +
1938                                   mtu +
1939                                   BNX2X_FW_RX_ALIGN_END;
1940                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1941                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1942                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1943                 else
1944                         fp->rx_frag_size = 0;
1945         }
1946 }
1947
1948 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1949 {
1950         int i;
1951         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1952
1953         /* Prepare the initial contents fo the indirection table if RSS is
1954          * enabled
1955          */
1956         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1957                 bp->rss_conf_obj.ind_table[i] =
1958                         bp->fp->cl_id +
1959                         ethtool_rxfh_indir_default(i, num_eth_queues);
1960
1961         /*
1962          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1963          * per-port, so if explicit configuration is needed , do it only
1964          * for a PMF.
1965          *
1966          * For 57712 and newer on the other hand it's a per-function
1967          * configuration.
1968          */
1969         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1970 }
1971
1972 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1973                         bool config_hash)
1974 {
1975         struct bnx2x_config_rss_params params = {NULL};
1976
1977         /* Although RSS is meaningless when there is a single HW queue we
1978          * still need it enabled in order to have HW Rx hash generated.
1979          *
1980          * if (!is_eth_multi(bp))
1981          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1982          */
1983
1984         params.rss_obj = rss_obj;
1985
1986         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1987
1988         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1989
1990         /* RSS configuration */
1991         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1992         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1993         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1994         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1995         if (rss_obj->udp_rss_v4)
1996                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1997         if (rss_obj->udp_rss_v6)
1998                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1999
2000         /* Hash bits */
2001         params.rss_result_mask = MULTI_MASK;
2002
2003         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2004
2005         if (config_hash) {
2006                 /* RSS keys */
2007                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
2008                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2009         }
2010
2011         return bnx2x_config_rss(bp, &params);
2012 }
2013
2014 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2015 {
2016         struct bnx2x_func_state_params func_params = {NULL};
2017
2018         /* Prepare parameters for function state transitions */
2019         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2020
2021         func_params.f_obj = &bp->func_obj;
2022         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2023
2024         func_params.params.hw_init.load_phase = load_code;
2025
2026         return bnx2x_func_state_change(bp, &func_params);
2027 }
2028
2029 /*
2030  * Cleans the object that have internal lists without sending
2031  * ramrods. Should be run when interrutps are disabled.
2032  */
2033 void bnx2x_squeeze_objects(struct bnx2x *bp)
2034 {
2035         int rc;
2036         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2037         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2038         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2039
2040         /***************** Cleanup MACs' object first *************************/
2041
2042         /* Wait for completion of requested */
2043         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2044         /* Perform a dry cleanup */
2045         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2046
2047         /* Clean ETH primary MAC */
2048         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2049         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2050                                  &ramrod_flags);
2051         if (rc != 0)
2052                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2053
2054         /* Cleanup UC list */
2055         vlan_mac_flags = 0;
2056         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2057         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2058                                  &ramrod_flags);
2059         if (rc != 0)
2060                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2061
2062         /***************** Now clean mcast object *****************************/
2063         rparam.mcast_obj = &bp->mcast_obj;
2064         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2065
2066         /* Add a DEL command... */
2067         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2068         if (rc < 0)
2069                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2070                           rc);
2071
2072         /* ...and wait until all pending commands are cleared */
2073         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2074         while (rc != 0) {
2075                 if (rc < 0) {
2076                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2077                                   rc);
2078                         return;
2079                 }
2080
2081                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2082         }
2083 }
2084
2085 #ifndef BNX2X_STOP_ON_ERROR
2086 #define LOAD_ERROR_EXIT(bp, label) \
2087         do { \
2088                 (bp)->state = BNX2X_STATE_ERROR; \
2089                 goto label; \
2090         } while (0)
2091
2092 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2093         do { \
2094                 bp->cnic_loaded = false; \
2095                 goto label; \
2096         } while (0)
2097 #else /*BNX2X_STOP_ON_ERROR*/
2098 #define LOAD_ERROR_EXIT(bp, label) \
2099         do { \
2100                 (bp)->state = BNX2X_STATE_ERROR; \
2101                 (bp)->panic = 1; \
2102                 return -EBUSY; \
2103         } while (0)
2104 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2105         do { \
2106                 bp->cnic_loaded = false; \
2107                 (bp)->panic = 1; \
2108                 return -EBUSY; \
2109         } while (0)
2110 #endif /*BNX2X_STOP_ON_ERROR*/
2111
2112 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2113 {
2114         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2115                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2116         return;
2117 }
2118
2119 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2120 {
2121         int num_groups, vf_headroom = 0;
2122         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2123
2124         /* number of queues for statistics is number of eth queues + FCoE */
2125         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2126
2127         /* Total number of FW statistics requests =
2128          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2129          * and fcoe l2 queue) stats + num of queues (which includes another 1
2130          * for fcoe l2 queue if applicable)
2131          */
2132         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2133
2134         /* vf stats appear in the request list, but their data is allocated by
2135          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2136          * it is used to determine where to place the vf stats queries in the
2137          * request struct
2138          */
2139         if (IS_SRIOV(bp))
2140                 vf_headroom = bnx2x_vf_headroom(bp);
2141
2142         /* Request is built from stats_query_header and an array of
2143          * stats_query_cmd_group each of which contains
2144          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2145          * configured in the stats_query_header.
2146          */
2147         num_groups =
2148                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2149                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2150                  1 : 0));
2151
2152         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2153            bp->fw_stats_num, vf_headroom, num_groups);
2154         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2155                 num_groups * sizeof(struct stats_query_cmd_group);
2156
2157         /* Data for statistics requests + stats_counter
2158          * stats_counter holds per-STORM counters that are incremented
2159          * when STORM has finished with the current request.
2160          * memory for FCoE offloaded statistics are counted anyway,
2161          * even if they will not be sent.
2162          * VF stats are not accounted for here as the data of VF stats is stored
2163          * in memory allocated by the VF, not here.
2164          */
2165         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2166                 sizeof(struct per_pf_stats) +
2167                 sizeof(struct fcoe_statistics_params) +
2168                 sizeof(struct per_queue_stats) * num_queue_stats +
2169                 sizeof(struct stats_counter);
2170
2171         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2172                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2173
2174         /* Set shortcuts */
2175         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2176         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2177         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2178                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2179         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2180                 bp->fw_stats_req_sz;
2181
2182         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x",
2183            U64_HI(bp->fw_stats_req_mapping),
2184            U64_LO(bp->fw_stats_req_mapping));
2185         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x",
2186            U64_HI(bp->fw_stats_data_mapping),
2187            U64_LO(bp->fw_stats_data_mapping));
2188         return 0;
2189
2190 alloc_mem_err:
2191         bnx2x_free_fw_stats_mem(bp);
2192         BNX2X_ERR("Can't allocate FW stats memory\n");
2193         return -ENOMEM;
2194 }
2195
2196 /* send load request to mcp and analyze response */
2197 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2198 {
2199         /* init fw_seq */
2200         bp->fw_seq =
2201                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2202                  DRV_MSG_SEQ_NUMBER_MASK);
2203         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2204
2205         /* Get current FW pulse sequence */
2206         bp->fw_drv_pulse_wr_seq =
2207                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2208                  DRV_PULSE_SEQ_MASK);
2209         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2210
2211         /* load request */
2212         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2213                                         DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2214
2215         /* if mcp fails to respond we must abort */
2216         if (!(*load_code)) {
2217                 BNX2X_ERR("MCP response failure, aborting\n");
2218                 return -EBUSY;
2219         }
2220
2221         /* If mcp refused (e.g. other port is in diagnostic mode) we
2222          * must abort
2223          */
2224         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2225                 BNX2X_ERR("MCP refused load request, aborting\n");
2226                 return -EBUSY;
2227         }
2228         return 0;
2229 }
2230
2231 /* check whether another PF has already loaded FW to chip. In
2232  * virtualized environments a pf from another VM may have already
2233  * initialized the device including loading FW
2234  */
2235 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2236 {
2237         /* is another pf loaded on this engine? */
2238         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2239             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2240                 /* build my FW version dword */
2241                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2242                         (BCM_5710_FW_MINOR_VERSION << 8) +
2243                         (BCM_5710_FW_REVISION_VERSION << 16) +
2244                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2245
2246                 /* read loaded FW from chip */
2247                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2248
2249                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2250                    loaded_fw, my_fw);
2251
2252                 /* abort nic load if version mismatch */
2253                 if (my_fw != loaded_fw) {
2254                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. aborting\n",
2255                                   loaded_fw, my_fw);
2256                         return -EBUSY;
2257                 }
2258         }
2259         return 0;
2260 }
2261
2262 /* returns the "mcp load_code" according to global load_count array */
2263 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2264 {
2265         int path = BP_PATH(bp);
2266
2267         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2268            path, load_count[path][0], load_count[path][1],
2269            load_count[path][2]);
2270         load_count[path][0]++;
2271         load_count[path][1 + port]++;
2272         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2273            path, load_count[path][0], load_count[path][1],
2274            load_count[path][2]);
2275         if (load_count[path][0] == 1)
2276                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2277         else if (load_count[path][1 + port] == 1)
2278                 return FW_MSG_CODE_DRV_LOAD_PORT;
2279         else
2280                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2281 }
2282
2283 /* mark PMF if applicable */
2284 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2285 {
2286         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2287             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2288             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2289                 bp->port.pmf = 1;
2290                 /* We need the barrier to ensure the ordering between the
2291                  * writing to bp->port.pmf here and reading it from the
2292                  * bnx2x_periodic_task().
2293                  */
2294                 smp_mb();
2295         } else {
2296                 bp->port.pmf = 0;
2297         }
2298
2299         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2300 }
2301
2302 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2303 {
2304         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2305              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2306             (bp->common.shmem2_base)) {
2307                 if (SHMEM2_HAS(bp, dcc_support))
2308                         SHMEM2_WR(bp, dcc_support,
2309                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2310                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2311                 if (SHMEM2_HAS(bp, afex_driver_support))
2312                         SHMEM2_WR(bp, afex_driver_support,
2313                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2314         }
2315
2316         /* Set AFEX default VLAN tag to an invalid value */
2317         bp->afex_def_vlan_tag = -1;
2318 }
2319
2320 /**
2321  * bnx2x_bz_fp - zero content of the fastpath structure.
2322  *
2323  * @bp:         driver handle
2324  * @index:      fastpath index to be zeroed
2325  *
2326  * Makes sure the contents of the bp->fp[index].napi is kept
2327  * intact.
2328  */
2329 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2330 {
2331         struct bnx2x_fastpath *fp = &bp->fp[index];
2332
2333         int cos;
2334         struct napi_struct orig_napi = fp->napi;
2335         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2336         /* bzero bnx2x_fastpath contents */
2337         if (fp->tpa_info)
2338                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2339                        sizeof(struct bnx2x_agg_info));
2340         memset(fp, 0, sizeof(*fp));
2341
2342         /* Restore the NAPI object as it has been already initialized */
2343         fp->napi = orig_napi;
2344         fp->tpa_info = orig_tpa_info;
2345         fp->bp = bp;
2346         fp->index = index;
2347         if (IS_ETH_FP(fp))
2348                 fp->max_cos = bp->max_cos;
2349         else
2350                 /* Special queues support only one CoS */
2351                 fp->max_cos = 1;
2352
2353         /* Init txdata pointers */
2354         if (IS_FCOE_FP(fp))
2355                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2356         if (IS_ETH_FP(fp))
2357                 for_each_cos_in_tx_queue(fp, cos)
2358                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2359                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2360
2361         /*
2362          * set the tpa flag for each queue. The tpa flag determines the queue
2363          * minimal size so it must be set prior to queue memory allocation
2364          */
2365         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2366                                   (bp->flags & GRO_ENABLE_FLAG &&
2367                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2368         if (bp->flags & TPA_ENABLE_FLAG)
2369                 fp->mode = TPA_MODE_LRO;
2370         else if (bp->flags & GRO_ENABLE_FLAG)
2371                 fp->mode = TPA_MODE_GRO;
2372
2373         /* We don't want TPA on an FCoE L2 ring */
2374         if (IS_FCOE_FP(fp))
2375                 fp->disable_tpa = 1;
2376 }
2377
2378 int bnx2x_load_cnic(struct bnx2x *bp)
2379 {
2380         int i, rc, port = BP_PORT(bp);
2381
2382         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2383
2384         mutex_init(&bp->cnic_mutex);
2385
2386         if (IS_PF(bp)) {
2387                 rc = bnx2x_alloc_mem_cnic(bp);
2388                 if (rc) {
2389                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2390                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2391                 }
2392         }
2393
2394         rc = bnx2x_alloc_fp_mem_cnic(bp);
2395         if (rc) {
2396                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2397                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2398         }
2399
2400         /* Update the number of queues with the cnic queues */
2401         rc = bnx2x_set_real_num_queues(bp, 1);
2402         if (rc) {
2403                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2404                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2405         }
2406
2407         /* Add all CNIC NAPI objects */
2408         bnx2x_add_all_napi_cnic(bp);
2409         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2410         bnx2x_napi_enable_cnic(bp);
2411
2412         rc = bnx2x_init_hw_func_cnic(bp);
2413         if (rc)
2414                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2415
2416         bnx2x_nic_init_cnic(bp);
2417
2418         if (IS_PF(bp)) {
2419                 /* Enable Timer scan */
2420                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2421
2422                 /* setup cnic queues */
2423                 for_each_cnic_queue(bp, i) {
2424                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2425                         if (rc) {
2426                                 BNX2X_ERR("Queue setup failed\n");
2427                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2428                         }
2429                 }
2430         }
2431
2432         /* Initialize Rx filter. */
2433         netif_addr_lock_bh(bp->dev);
2434         bnx2x_set_rx_mode(bp->dev);
2435         netif_addr_unlock_bh(bp->dev);
2436
2437         /* re-read iscsi info */
2438         bnx2x_get_iscsi_info(bp);
2439         bnx2x_setup_cnic_irq_info(bp);
2440         bnx2x_setup_cnic_info(bp);
2441         bp->cnic_loaded = true;
2442         if (bp->state == BNX2X_STATE_OPEN)
2443                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2444
2445
2446         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2447
2448         return 0;
2449
2450 #ifndef BNX2X_STOP_ON_ERROR
2451 load_error_cnic2:
2452         /* Disable Timer scan */
2453         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2454
2455 load_error_cnic1:
2456         bnx2x_napi_disable_cnic(bp);
2457         /* Update the number of queues without the cnic queues */
2458         rc = bnx2x_set_real_num_queues(bp, 0);
2459         if (rc)
2460                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2461 load_error_cnic0:
2462         BNX2X_ERR("CNIC-related load failed\n");
2463         bnx2x_free_fp_mem_cnic(bp);
2464         bnx2x_free_mem_cnic(bp);
2465         return rc;
2466 #endif /* ! BNX2X_STOP_ON_ERROR */
2467 }
2468
2469 /* must be called with rtnl_lock */
2470 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2471 {
2472         int port = BP_PORT(bp);
2473         int i, rc = 0, load_code = 0;
2474
2475         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2476         DP(NETIF_MSG_IFUP,
2477            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2478
2479 #ifdef BNX2X_STOP_ON_ERROR
2480         if (unlikely(bp->panic)) {
2481                 BNX2X_ERR("Can't load NIC when there is panic\n");
2482                 return -EPERM;
2483         }
2484 #endif
2485
2486         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2487
2488         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2489         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2490                 &bp->last_reported_link.link_report_flags);
2491
2492         if (IS_PF(bp))
2493                 /* must be called before memory allocation and HW init */
2494                 bnx2x_ilt_set_info(bp);
2495
2496         /*
2497          * Zero fastpath structures preserving invariants like napi, which are
2498          * allocated only once, fp index, max_cos, bp pointer.
2499          * Also set fp->disable_tpa and txdata_ptr.
2500          */
2501         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2502         for_each_queue(bp, i)
2503                 bnx2x_bz_fp(bp, i);
2504         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2505                                   bp->num_cnic_queues) *
2506                                   sizeof(struct bnx2x_fp_txdata));
2507
2508         bp->fcoe_init = false;
2509
2510         /* Set the receive queues buffer size */
2511         bnx2x_set_rx_buf_size(bp);
2512
2513         if (IS_PF(bp)) {
2514                 rc = bnx2x_alloc_mem(bp);
2515                 if (rc) {
2516                         BNX2X_ERR("Unable to allocate bp memory\n");
2517                         return rc;
2518                 }
2519         }
2520
2521         /* Allocated memory for FW statistics  */
2522         if (bnx2x_alloc_fw_stats_mem(bp))
2523                 LOAD_ERROR_EXIT(bp, load_error0);
2524
2525         /* need to be done after alloc mem, since it's self adjusting to amount
2526          * of memory available for RSS queues
2527          */
2528         rc = bnx2x_alloc_fp_mem(bp);
2529         if (rc) {
2530                 BNX2X_ERR("Unable to allocate memory for fps\n");
2531                 LOAD_ERROR_EXIT(bp, load_error0);
2532         }
2533
2534         /* request pf to initialize status blocks */
2535         if (IS_VF(bp)) {
2536                 rc = bnx2x_vfpf_init(bp);
2537                 if (rc)
2538                         LOAD_ERROR_EXIT(bp, load_error0);
2539         }
2540
2541         /* As long as bnx2x_alloc_mem() may possibly update
2542          * bp->num_queues, bnx2x_set_real_num_queues() should always
2543          * come after it. At this stage cnic queues are not counted.
2544          */
2545         rc = bnx2x_set_real_num_queues(bp, 0);
2546         if (rc) {
2547                 BNX2X_ERR("Unable to set real_num_queues\n");
2548                 LOAD_ERROR_EXIT(bp, load_error0);
2549         }
2550
2551         /* configure multi cos mappings in kernel.
2552          * this configuration may be overriden by a multi class queue discipline
2553          * or by a dcbx negotiation result.
2554          */
2555         bnx2x_setup_tc(bp->dev, bp->max_cos);
2556
2557         /* Add all NAPI objects */
2558         bnx2x_add_all_napi(bp);
2559         DP(NETIF_MSG_IFUP, "napi added\n");
2560         bnx2x_napi_enable(bp);
2561
2562         if (IS_PF(bp)) {
2563                 /* set pf load just before approaching the MCP */
2564                 bnx2x_set_pf_load(bp);
2565
2566                 /* if mcp exists send load request and analyze response */
2567                 if (!BP_NOMCP(bp)) {
2568                         /* attempt to load pf */
2569                         rc = bnx2x_nic_load_request(bp, &load_code);
2570                         if (rc)
2571                                 LOAD_ERROR_EXIT(bp, load_error1);
2572
2573                         /* what did mcp say? */
2574                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2575                         if (rc) {
2576                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2577                                 LOAD_ERROR_EXIT(bp, load_error2);
2578                         }
2579                 } else {
2580                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2581                 }
2582
2583                 /* mark pmf if applicable */
2584                 bnx2x_nic_load_pmf(bp, load_code);
2585
2586                 /* Init Function state controlling object */
2587                 bnx2x__init_func_obj(bp);
2588
2589                 /* Initialize HW */
2590                 rc = bnx2x_init_hw(bp, load_code);
2591                 if (rc) {
2592                         BNX2X_ERR("HW init failed, aborting\n");
2593                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2594                         LOAD_ERROR_EXIT(bp, load_error2);
2595                 }
2596         }
2597
2598         bnx2x_pre_irq_nic_init(bp);
2599
2600         /* Connect to IRQs */
2601         rc = bnx2x_setup_irqs(bp);
2602         if (rc) {
2603                 BNX2X_ERR("setup irqs failed\n");
2604                 if (IS_PF(bp))
2605                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2606                 LOAD_ERROR_EXIT(bp, load_error2);
2607         }
2608
2609         /* Init per-function objects */
2610         if (IS_PF(bp)) {
2611                 /* Setup NIC internals and enable interrupts */
2612                 bnx2x_post_irq_nic_init(bp, load_code);
2613
2614                 bnx2x_init_bp_objs(bp);
2615                 bnx2x_iov_nic_init(bp);
2616
2617                 /* Set AFEX default VLAN tag to an invalid value */
2618                 bp->afex_def_vlan_tag = -1;
2619                 bnx2x_nic_load_afex_dcc(bp, load_code);
2620                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2621                 rc = bnx2x_func_start(bp);
2622                 if (rc) {
2623                         BNX2X_ERR("Function start failed!\n");
2624                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2625
2626                         LOAD_ERROR_EXIT(bp, load_error3);
2627                 }
2628
2629                 /* Send LOAD_DONE command to MCP */
2630                 if (!BP_NOMCP(bp)) {
2631                         load_code = bnx2x_fw_command(bp,
2632                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2633                         if (!load_code) {
2634                                 BNX2X_ERR("MCP response failure, aborting\n");
2635                                 rc = -EBUSY;
2636                                 LOAD_ERROR_EXIT(bp, load_error3);
2637                         }
2638                 }
2639
2640                 /* initialize FW coalescing state machines in RAM */
2641                 bnx2x_update_coalesce(bp);
2642
2643                 /* setup the leading queue */
2644                 rc = bnx2x_setup_leading(bp);
2645                 if (rc) {
2646                         BNX2X_ERR("Setup leading failed!\n");
2647                         LOAD_ERROR_EXIT(bp, load_error3);
2648                 }
2649
2650                 /* set up the rest of the queues */
2651                 for_each_nondefault_eth_queue(bp, i) {
2652                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2653                         if (rc) {
2654                                 BNX2X_ERR("Queue setup failed\n");
2655                                 LOAD_ERROR_EXIT(bp, load_error3);
2656                         }
2657                 }
2658
2659                 /* setup rss */
2660                 rc = bnx2x_init_rss_pf(bp);
2661                 if (rc) {
2662                         BNX2X_ERR("PF RSS init failed\n");
2663                         LOAD_ERROR_EXIT(bp, load_error3);
2664                 }
2665
2666         } else { /* vf */
2667                 for_each_eth_queue(bp, i) {
2668                         rc = bnx2x_vfpf_setup_q(bp, i);
2669                         if (rc) {
2670                                 BNX2X_ERR("Queue setup failed\n");
2671                                 LOAD_ERROR_EXIT(bp, load_error3);
2672                         }
2673                 }
2674         }
2675
2676         /* Now when Clients are configured we are ready to work */
2677         bp->state = BNX2X_STATE_OPEN;
2678
2679         /* Configure a ucast MAC */
2680         if (IS_PF(bp))
2681                 rc = bnx2x_set_eth_mac(bp, true);
2682         else /* vf */
2683                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2684                                            true);
2685         if (rc) {
2686                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2687                 LOAD_ERROR_EXIT(bp, load_error3);
2688         }
2689
2690         if (IS_PF(bp) && bp->pending_max) {
2691                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2692                 bp->pending_max = 0;
2693         }
2694
2695         if (bp->port.pmf) {
2696                 rc = bnx2x_initial_phy_init(bp, load_mode);
2697                 if (rc)
2698                         LOAD_ERROR_EXIT(bp, load_error3);
2699         }
2700         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2701
2702         /* Start fast path */
2703
2704         /* Initialize Rx filter. */
2705         netif_addr_lock_bh(bp->dev);
2706         bnx2x_set_rx_mode(bp->dev);
2707         netif_addr_unlock_bh(bp->dev);
2708
2709         /* Start the Tx */
2710         switch (load_mode) {
2711         case LOAD_NORMAL:
2712                 /* Tx queue should be only reenabled */
2713                 netif_tx_wake_all_queues(bp->dev);
2714                 break;
2715
2716         case LOAD_OPEN:
2717                 netif_tx_start_all_queues(bp->dev);
2718                 smp_mb__after_clear_bit();
2719                 break;
2720
2721         case LOAD_DIAG:
2722         case LOAD_LOOPBACK_EXT:
2723                 bp->state = BNX2X_STATE_DIAG;
2724                 break;
2725
2726         default:
2727                 break;
2728         }
2729
2730         if (bp->port.pmf)
2731                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2732         else
2733                 bnx2x__link_status_update(bp);
2734
2735         /* start the timer */
2736         mod_timer(&bp->timer, jiffies + bp->current_interval);
2737
2738         if (CNIC_ENABLED(bp))
2739                 bnx2x_load_cnic(bp);
2740
2741         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2742                 /* mark driver is loaded in shmem2 */
2743                 u32 val;
2744                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2745                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2746                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2747                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2748         }
2749
2750         /* Wait for all pending SP commands to complete */
2751         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2752                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2753                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2754                 return -EBUSY;
2755         }
2756
2757         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2758         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2759                 bnx2x_dcbx_init(bp, false);
2760
2761         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2762
2763         return 0;
2764
2765 #ifndef BNX2X_STOP_ON_ERROR
2766 load_error3:
2767         if (IS_PF(bp)) {
2768                 bnx2x_int_disable_sync(bp, 1);
2769
2770                 /* Clean queueable objects */
2771                 bnx2x_squeeze_objects(bp);
2772         }
2773
2774         /* Free SKBs, SGEs, TPA pool and driver internals */
2775         bnx2x_free_skbs(bp);
2776         for_each_rx_queue(bp, i)
2777                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2778
2779         /* Release IRQs */
2780         bnx2x_free_irq(bp);
2781 load_error2:
2782         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2783                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2784                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2785         }
2786
2787         bp->port.pmf = 0;
2788 load_error1:
2789         bnx2x_napi_disable(bp);
2790         bnx2x_del_all_napi(bp);
2791
2792         /* clear pf_load status, as it was already set */
2793         if (IS_PF(bp))
2794                 bnx2x_clear_pf_load(bp);
2795 load_error0:
2796         bnx2x_free_fp_mem(bp);
2797         bnx2x_free_fw_stats_mem(bp);
2798         bnx2x_free_mem(bp);
2799
2800         return rc;
2801 #endif /* ! BNX2X_STOP_ON_ERROR */
2802 }
2803
2804 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2805 {
2806         u8 rc = 0, cos, i;
2807
2808         /* Wait until tx fastpath tasks complete */
2809         for_each_tx_queue(bp, i) {
2810                 struct bnx2x_fastpath *fp = &bp->fp[i];
2811
2812                 for_each_cos_in_tx_queue(fp, cos)
2813                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2814                 if (rc)
2815                         return rc;
2816         }
2817         return 0;
2818 }
2819
2820 /* must be called with rtnl_lock */
2821 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2822 {
2823         int i;
2824         bool global = false;
2825
2826         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2827
2828         /* mark driver is unloaded in shmem2 */
2829         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2830                 u32 val;
2831                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2832                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2833                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2834         }
2835
2836         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2837             (bp->state == BNX2X_STATE_CLOSED ||
2838              bp->state == BNX2X_STATE_ERROR)) {
2839                 /* We can get here if the driver has been unloaded
2840                  * during parity error recovery and is either waiting for a
2841                  * leader to complete or for other functions to unload and
2842                  * then ifdown has been issued. In this case we want to
2843                  * unload and let other functions to complete a recovery
2844                  * process.
2845                  */
2846                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2847                 bp->is_leader = 0;
2848                 bnx2x_release_leader_lock(bp);
2849                 smp_mb();
2850
2851                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2852                 BNX2X_ERR("Can't unload in closed or error state\n");
2853                 return -EINVAL;
2854         }
2855
2856         /* Nothing to do during unload if previous bnx2x_nic_load()
2857          * have not completed succesfully - all resourses are released.
2858          *
2859          * we can get here only after unsuccessful ndo_* callback, during which
2860          * dev->IFF_UP flag is still on.
2861          */
2862         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2863                 return 0;
2864
2865         /* It's important to set the bp->state to the value different from
2866          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2867          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2868          */
2869         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2870         smp_mb();
2871
2872         if (CNIC_LOADED(bp))
2873                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2874
2875         /* Stop Tx */
2876         bnx2x_tx_disable(bp);
2877         netdev_reset_tc(bp->dev);
2878
2879         bp->rx_mode = BNX2X_RX_MODE_NONE;
2880
2881         del_timer_sync(&bp->timer);
2882
2883         if (IS_PF(bp)) {
2884                 /* Set ALWAYS_ALIVE bit in shmem */
2885                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2886                 bnx2x_drv_pulse(bp);
2887                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2888                 bnx2x_save_statistics(bp);
2889         }
2890
2891         /* wait till consumers catch up with producers in all queues */
2892         bnx2x_drain_tx_queues(bp);
2893
2894         /* if VF indicate to PF this function is going down (PF will delete sp
2895          * elements and clear initializations
2896          */
2897         if (IS_VF(bp))
2898                 bnx2x_vfpf_close_vf(bp);
2899         else if (unload_mode != UNLOAD_RECOVERY)
2900                 /* if this is a normal/close unload need to clean up chip*/
2901                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2902         else {
2903                 /* Send the UNLOAD_REQUEST to the MCP */
2904                 bnx2x_send_unload_req(bp, unload_mode);
2905
2906                 /*
2907                  * Prevent transactions to host from the functions on the
2908                  * engine that doesn't reset global blocks in case of global
2909                  * attention once gloabl blocks are reset and gates are opened
2910                  * (the engine which leader will perform the recovery
2911                  * last).
2912                  */
2913                 if (!CHIP_IS_E1x(bp))
2914                         bnx2x_pf_disable(bp);
2915
2916                 /* Disable HW interrupts, NAPI */
2917                 bnx2x_netif_stop(bp, 1);
2918                 /* Delete all NAPI objects */
2919                 bnx2x_del_all_napi(bp);
2920                 if (CNIC_LOADED(bp))
2921                         bnx2x_del_all_napi_cnic(bp);
2922                 /* Release IRQs */
2923                 bnx2x_free_irq(bp);
2924
2925                 /* Report UNLOAD_DONE to MCP */
2926                 bnx2x_send_unload_done(bp, false);
2927         }
2928
2929         /*
2930          * At this stage no more interrupts will arrive so we may safly clean
2931          * the queueable objects here in case they failed to get cleaned so far.
2932          */
2933         if (IS_PF(bp))
2934                 bnx2x_squeeze_objects(bp);
2935
2936         /* There should be no more pending SP commands at this stage */
2937         bp->sp_state = 0;
2938
2939         bp->port.pmf = 0;
2940
2941         /* Free SKBs, SGEs, TPA pool and driver internals */
2942         bnx2x_free_skbs(bp);
2943         if (CNIC_LOADED(bp))
2944                 bnx2x_free_skbs_cnic(bp);
2945         for_each_rx_queue(bp, i)
2946                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2947
2948         bnx2x_free_fp_mem(bp);
2949         if (CNIC_LOADED(bp))
2950                 bnx2x_free_fp_mem_cnic(bp);
2951
2952         if (IS_PF(bp)) {
2953                 if (CNIC_LOADED(bp))
2954                         bnx2x_free_mem_cnic(bp);
2955                 bnx2x_free_mem(bp);
2956         }
2957         bp->state = BNX2X_STATE_CLOSED;
2958         bp->cnic_loaded = false;
2959
2960         /* Check if there are pending parity attentions. If there are - set
2961          * RECOVERY_IN_PROGRESS.
2962          */
2963         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2964                 bnx2x_set_reset_in_progress(bp);
2965
2966                 /* Set RESET_IS_GLOBAL if needed */
2967                 if (global)
2968                         bnx2x_set_reset_global(bp);
2969         }
2970
2971
2972         /* The last driver must disable a "close the gate" if there is no
2973          * parity attention or "process kill" pending.
2974          */
2975         if (IS_PF(bp) &&
2976             !bnx2x_clear_pf_load(bp) &&
2977             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2978                 bnx2x_disable_close_the_gate(bp);
2979
2980         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2981
2982         return 0;
2983 }
2984
2985 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2986 {
2987         u16 pmcsr;
2988
2989         /* If there is no power capability, silently succeed */
2990         if (!bp->pm_cap) {
2991                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2992                 return 0;
2993         }
2994
2995         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2996
2997         switch (state) {
2998         case PCI_D0:
2999                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3000                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3001                                        PCI_PM_CTRL_PME_STATUS));
3002
3003                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3004                         /* delay required during transition out of D3hot */
3005                         msleep(20);
3006                 break;
3007
3008         case PCI_D3hot:
3009                 /* If there are other clients above don't
3010                    shut down the power */
3011                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3012                         return 0;
3013                 /* Don't shut down the power for emulation and FPGA */
3014                 if (CHIP_REV_IS_SLOW(bp))
3015                         return 0;
3016
3017                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3018                 pmcsr |= 3;
3019
3020                 if (bp->wol)
3021                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3022
3023                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3024                                       pmcsr);
3025
3026                 /* No more memory access after this point until
3027                 * device is brought back to D0.
3028                 */
3029                 break;
3030
3031         default:
3032                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3033                 return -EINVAL;
3034         }
3035         return 0;
3036 }
3037
3038 /*
3039  * net_device service functions
3040  */
3041 int bnx2x_poll(struct napi_struct *napi, int budget)
3042 {
3043         int work_done = 0;
3044         u8 cos;
3045         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3046                                                  napi);
3047         struct bnx2x *bp = fp->bp;
3048
3049         while (1) {
3050 #ifdef BNX2X_STOP_ON_ERROR
3051                 if (unlikely(bp->panic)) {
3052                         napi_complete(napi);
3053                         return 0;
3054                 }
3055 #endif
3056
3057                 for_each_cos_in_tx_queue(fp, cos)
3058                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3059                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3060
3061                 if (bnx2x_has_rx_work(fp)) {
3062                         work_done += bnx2x_rx_int(fp, budget - work_done);
3063
3064                         /* must not complete if we consumed full budget */
3065                         if (work_done >= budget)
3066                                 break;
3067                 }
3068
3069                 /* Fall out from the NAPI loop if needed */
3070                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3071
3072                         /* No need to update SB for FCoE L2 ring as long as
3073                          * it's connected to the default SB and the SB
3074                          * has been updated when NAPI was scheduled.
3075                          */
3076                         if (IS_FCOE_FP(fp)) {
3077                                 napi_complete(napi);
3078                                 break;
3079                         }
3080                         bnx2x_update_fpsb_idx(fp);
3081                         /* bnx2x_has_rx_work() reads the status block,
3082                          * thus we need to ensure that status block indices
3083                          * have been actually read (bnx2x_update_fpsb_idx)
3084                          * prior to this check (bnx2x_has_rx_work) so that
3085                          * we won't write the "newer" value of the status block
3086                          * to IGU (if there was a DMA right after
3087                          * bnx2x_has_rx_work and if there is no rmb, the memory
3088                          * reading (bnx2x_update_fpsb_idx) may be postponed
3089                          * to right before bnx2x_ack_sb). In this case there
3090                          * will never be another interrupt until there is
3091                          * another update of the status block, while there
3092                          * is still unhandled work.
3093                          */
3094                         rmb();
3095
3096                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3097                                 napi_complete(napi);
3098                                 /* Re-enable interrupts */
3099                                 DP(NETIF_MSG_RX_STATUS,
3100                                    "Update index to %d\n", fp->fp_hc_idx);
3101                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3102                                              le16_to_cpu(fp->fp_hc_idx),
3103                                              IGU_INT_ENABLE, 1);
3104                                 break;
3105                         }
3106                 }
3107         }
3108
3109         return work_done;
3110 }
3111
3112 /* we split the first BD into headers and data BDs
3113  * to ease the pain of our fellow microcode engineers
3114  * we use one mapping for both BDs
3115  */
3116 static u16 bnx2x_tx_split(struct bnx2x *bp,
3117                           struct bnx2x_fp_txdata *txdata,
3118                           struct sw_tx_bd *tx_buf,
3119                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3120                           u16 bd_prod)
3121 {
3122         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3123         struct eth_tx_bd *d_tx_bd;
3124         dma_addr_t mapping;
3125         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3126
3127         /* first fix first BD */
3128         h_tx_bd->nbytes = cpu_to_le16(hlen);
3129
3130         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3131            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3132
3133         /* now get a new data BD
3134          * (after the pbd) and fill it */
3135         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3136         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3137
3138         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3139                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3140
3141         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3142         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3143         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3144
3145         /* this marks the BD as one that has no individual mapping */
3146         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3147
3148         DP(NETIF_MSG_TX_QUEUED,
3149            "TSO split data size is %d (%x:%x)\n",
3150            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3151
3152         /* update tx_bd */
3153         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3154
3155         return bd_prod;
3156 }
3157
3158 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3159 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3160 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3161 {
3162         __sum16 tsum = (__force __sum16) csum;
3163
3164         if (fix > 0)
3165                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3166                                   csum_partial(t_header - fix, fix, 0)));
3167
3168         else if (fix < 0)
3169                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3170                                   csum_partial(t_header, -fix, 0)));
3171
3172         return bswab16(tsum);
3173 }
3174
3175 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3176 {
3177         u32 rc;
3178         __u8 prot = 0;
3179         __be16 protocol;
3180
3181         if (skb->ip_summed != CHECKSUM_PARTIAL)
3182                 return XMIT_PLAIN;
3183
3184         protocol = vlan_get_protocol(skb);
3185         if (protocol == htons(ETH_P_IPV6)) {
3186                 rc = XMIT_CSUM_V6;
3187                 prot = ipv6_hdr(skb)->nexthdr;
3188         } else {
3189                 rc = XMIT_CSUM_V4;
3190                 prot = ip_hdr(skb)->protocol;
3191         }
3192
3193         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3194                 if (inner_ip_hdr(skb)->version == 6) {
3195                         rc |= XMIT_CSUM_ENC_V6;
3196                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3197                                 rc |= XMIT_CSUM_TCP;
3198                 } else {
3199                         rc |= XMIT_CSUM_ENC_V4;
3200                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3201                                 rc |= XMIT_CSUM_TCP;
3202                 }
3203         }
3204         if (prot == IPPROTO_TCP)
3205                 rc |= XMIT_CSUM_TCP;
3206
3207         if (skb_is_gso_v6(skb)) {
3208                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3209                 if (rc & XMIT_CSUM_ENC)
3210                         rc |= XMIT_GSO_ENC_V6;
3211         } else if (skb_is_gso(skb)) {
3212                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3213                 if (rc & XMIT_CSUM_ENC)
3214                         rc |= XMIT_GSO_ENC_V4;
3215         }
3216
3217         return rc;
3218 }
3219
3220 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3221 /* check if packet requires linearization (packet is too fragmented)
3222    no need to check fragmentation if page size > 8K (there will be no
3223    violation to FW restrictions) */
3224 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3225                              u32 xmit_type)
3226 {
3227         int to_copy = 0;
3228         int hlen = 0;
3229         int first_bd_sz = 0;
3230
3231         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3232         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3233
3234                 if (xmit_type & XMIT_GSO) {
3235                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3236                         /* Check if LSO packet needs to be copied:
3237                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3238                         int wnd_size = MAX_FETCH_BD - 3;
3239                         /* Number of windows to check */
3240                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3241                         int wnd_idx = 0;
3242                         int frag_idx = 0;
3243                         u32 wnd_sum = 0;
3244
3245                         /* Headers length */
3246                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3247                                 tcp_hdrlen(skb);
3248
3249                         /* Amount of data (w/o headers) on linear part of SKB*/
3250                         first_bd_sz = skb_headlen(skb) - hlen;
3251
3252                         wnd_sum  = first_bd_sz;
3253
3254                         /* Calculate the first sum - it's special */
3255                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3256                                 wnd_sum +=
3257                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3258
3259                         /* If there was data on linear skb data - check it */
3260                         if (first_bd_sz > 0) {
3261                                 if (unlikely(wnd_sum < lso_mss)) {
3262                                         to_copy = 1;
3263                                         goto exit_lbl;
3264                                 }
3265
3266                                 wnd_sum -= first_bd_sz;
3267                         }
3268
3269                         /* Others are easier: run through the frag list and
3270                            check all windows */
3271                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3272                                 wnd_sum +=
3273                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3274
3275                                 if (unlikely(wnd_sum < lso_mss)) {
3276                                         to_copy = 1;
3277                                         break;
3278                                 }
3279                                 wnd_sum -=
3280                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3281                         }
3282                 } else {
3283                         /* in non-LSO too fragmented packet should always
3284                            be linearized */
3285                         to_copy = 1;
3286                 }
3287         }
3288
3289 exit_lbl:
3290         if (unlikely(to_copy))
3291                 DP(NETIF_MSG_TX_QUEUED,
3292                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3293                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3294                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3295
3296         return to_copy;
3297 }
3298 #endif
3299
3300 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3301                                  u32 xmit_type)
3302 {
3303         struct ipv6hdr *ipv6;
3304
3305         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3306                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3307                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3308
3309         if (xmit_type & XMIT_GSO_ENC_V6)
3310                 ipv6 = inner_ipv6_hdr(skb);
3311         else if (xmit_type & XMIT_GSO_V6)
3312                 ipv6 = ipv6_hdr(skb);
3313         else
3314                 ipv6 = NULL;
3315
3316         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3317                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3318 }
3319
3320 /**
3321  * bnx2x_set_pbd_gso - update PBD in GSO case.
3322  *
3323  * @skb:        packet skb
3324  * @pbd:        parse BD
3325  * @xmit_type:  xmit flags
3326  */
3327 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3328                               struct eth_tx_parse_bd_e1x *pbd,
3329                               struct eth_tx_start_bd *tx_start_bd,
3330                               u32 xmit_type)
3331 {
3332         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3333         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3334         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3335
3336         if (xmit_type & XMIT_GSO_V4) {
3337                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3338                 pbd->tcp_pseudo_csum =
3339                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3340                                                    ip_hdr(skb)->daddr,
3341                                                    0, IPPROTO_TCP, 0));
3342
3343                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3344                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3345         } else {
3346                 pbd->tcp_pseudo_csum =
3347                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3348                                                  &ipv6_hdr(skb)->daddr,
3349                                                  0, IPPROTO_TCP, 0));
3350         }
3351
3352         pbd->global_data |=
3353                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3354 }
3355
3356 /**
3357  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3358  *
3359  * @bp:                 driver handle
3360  * @skb:                packet skb
3361  * @parsing_data:       data to be updated
3362  * @xmit_type:          xmit flags
3363  *
3364  * 57712/578xx related, when skb has encapsulation
3365  */
3366 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3367                                  u32 *parsing_data, u32 xmit_type)
3368 {
3369         *parsing_data |=
3370                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3371                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3372                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3373
3374         if (xmit_type & XMIT_CSUM_TCP) {
3375                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3376                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3377                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3378
3379                 return skb_inner_transport_header(skb) +
3380                         inner_tcp_hdrlen(skb) - skb->data;
3381         }
3382
3383         /* We support checksum offload for TCP and UDP only.
3384          * No need to pass the UDP header length - it's a constant.
3385          */
3386         return skb_inner_transport_header(skb) +
3387                 sizeof(struct udphdr) - skb->data;
3388 }
3389
3390 /**
3391  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3392  *
3393  * @bp:                 driver handle
3394  * @skb:                packet skb
3395  * @parsing_data:       data to be updated
3396  * @xmit_type:          xmit flags
3397  *
3398  * 57712/578xx related
3399  */
3400 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3401                                 u32 *parsing_data, u32 xmit_type)
3402 {
3403         *parsing_data |=
3404                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3405                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3406                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3407
3408         if (xmit_type & XMIT_CSUM_TCP) {
3409                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3410                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3411                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3412
3413                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3414         }
3415         /* We support checksum offload for TCP and UDP only.
3416          * No need to pass the UDP header length - it's a constant.
3417          */
3418         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3419 }
3420
3421 /* set FW indication according to inner or outer protocols if tunneled */
3422 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3423                                struct eth_tx_start_bd *tx_start_bd,
3424                                u32 xmit_type)
3425 {
3426         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3427
3428         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3429                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3430
3431         if (!(xmit_type & XMIT_CSUM_TCP))
3432                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3433 }
3434
3435 /**
3436  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3437  *
3438  * @bp:         driver handle
3439  * @skb:        packet skb
3440  * @pbd:        parse BD to be updated
3441  * @xmit_type:  xmit flags
3442  */
3443 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3444                              struct eth_tx_parse_bd_e1x *pbd,
3445                              u32 xmit_type)
3446 {
3447         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3448
3449         /* for now NS flag is not used in Linux */
3450         pbd->global_data =
3451                 cpu_to_le16(hlen |
3452                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3453                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3454
3455         pbd->ip_hlen_w = (skb_transport_header(skb) -
3456                         skb_network_header(skb)) >> 1;
3457
3458         hlen += pbd->ip_hlen_w;
3459
3460         /* We support checksum offload for TCP and UDP only */
3461         if (xmit_type & XMIT_CSUM_TCP)
3462                 hlen += tcp_hdrlen(skb) / 2;
3463         else
3464                 hlen += sizeof(struct udphdr) / 2;
3465
3466         pbd->total_hlen_w = cpu_to_le16(hlen);
3467         hlen = hlen*2;
3468
3469         if (xmit_type & XMIT_CSUM_TCP) {
3470                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3471
3472         } else {
3473                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3474
3475                 DP(NETIF_MSG_TX_QUEUED,
3476                    "hlen %d  fix %d  csum before fix %x\n",
3477                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3478
3479                 /* HW bug: fixup the CSUM */
3480                 pbd->tcp_pseudo_csum =
3481                         bnx2x_csum_fix(skb_transport_header(skb),
3482                                        SKB_CS(skb), fix);
3483
3484                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3485                    pbd->tcp_pseudo_csum);
3486         }
3487
3488         return hlen;
3489 }
3490
3491 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3492                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3493                                       struct eth_tx_parse_2nd_bd *pbd2,
3494                                       u16 *global_data,
3495                                       u32 xmit_type)
3496 {
3497         u16 hlen_w = 0;
3498         u8 outerip_off, outerip_len = 0;
3499
3500         /* from outer IP to transport */
3501         hlen_w = (skb_inner_transport_header(skb) -
3502                   skb_network_header(skb)) >> 1;
3503
3504         /* transport len */
3505         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3506
3507         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3508
3509         /* outer IP header info */
3510         if (xmit_type & XMIT_CSUM_V4) {
3511                 struct iphdr *iph = ip_hdr(skb);
3512                 pbd2->fw_ip_csum_wo_len_flags_frag =
3513                         bswab16(csum_fold((~iph->check) -
3514                                           iph->tot_len - iph->frag_off));
3515         } else {
3516                 pbd2->fw_ip_hdr_to_payload_w =
3517                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3518         }
3519
3520         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3521
3522         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3523
3524         if (xmit_type & XMIT_GSO_V4) {
3525                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3526
3527                 pbd_e2->data.tunnel_data.pseudo_csum =
3528                         bswab16(~csum_tcpudp_magic(
3529                                         inner_ip_hdr(skb)->saddr,
3530                                         inner_ip_hdr(skb)->daddr,
3531                                         0, IPPROTO_TCP, 0));
3532
3533                 outerip_len = ip_hdr(skb)->ihl << 1;
3534         } else {
3535                 pbd_e2->data.tunnel_data.pseudo_csum =
3536                         bswab16(~csum_ipv6_magic(
3537                                         &inner_ipv6_hdr(skb)->saddr,
3538                                         &inner_ipv6_hdr(skb)->daddr,
3539                                         0, IPPROTO_TCP, 0));
3540         }
3541
3542         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3543
3544         *global_data |=
3545                 outerip_off |
3546                 (!!(xmit_type & XMIT_CSUM_V6) <<
3547                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3548                 (outerip_len <<
3549                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3550                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3551                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3552
3553         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3554                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3555                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3556         }
3557 }
3558
3559 /* called with netif_tx_lock
3560  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3561  * netif_wake_queue()
3562  */
3563 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3564 {
3565         struct bnx2x *bp = netdev_priv(dev);
3566
3567         struct netdev_queue *txq;
3568         struct bnx2x_fp_txdata *txdata;
3569         struct sw_tx_bd *tx_buf;
3570         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3571         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3572         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3573         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3574         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3575         u32 pbd_e2_parsing_data = 0;
3576         u16 pkt_prod, bd_prod;
3577         int nbd, txq_index;
3578         dma_addr_t mapping;
3579         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3580         int i;
3581         u8 hlen = 0;
3582         __le16 pkt_size = 0;
3583         struct ethhdr *eth;
3584         u8 mac_type = UNICAST_ADDRESS;
3585
3586 #ifdef BNX2X_STOP_ON_ERROR
3587         if (unlikely(bp->panic))
3588                 return NETDEV_TX_BUSY;
3589 #endif
3590
3591         txq_index = skb_get_queue_mapping(skb);
3592         txq = netdev_get_tx_queue(dev, txq_index);
3593
3594         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3595
3596         txdata = &bp->bnx2x_txq[txq_index];
3597
3598         /* enable this debug print to view the transmission queue being used
3599         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3600            txq_index, fp_index, txdata_index); */
3601
3602         /* enable this debug print to view the tranmission details
3603         DP(NETIF_MSG_TX_QUEUED,
3604            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3605            txdata->cid, fp_index, txdata_index, txdata, fp); */
3606
3607         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3608                         skb_shinfo(skb)->nr_frags +
3609                         BDS_PER_TX_PKT +
3610                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3611                 /* Handle special storage cases separately */
3612                 if (txdata->tx_ring_size == 0) {
3613                         struct bnx2x_eth_q_stats *q_stats =
3614                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3615                         q_stats->driver_filtered_tx_pkt++;
3616                         dev_kfree_skb(skb);
3617                         return NETDEV_TX_OK;
3618                 }
3619                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3620                 netif_tx_stop_queue(txq);
3621                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3622
3623                 return NETDEV_TX_BUSY;
3624         }
3625
3626         DP(NETIF_MSG_TX_QUEUED,
3627            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3628            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3629            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3630            skb->len);
3631
3632         eth = (struct ethhdr *)skb->data;
3633
3634         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3635         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3636                 if (is_broadcast_ether_addr(eth->h_dest))
3637                         mac_type = BROADCAST_ADDRESS;
3638                 else
3639                         mac_type = MULTICAST_ADDRESS;
3640         }
3641
3642 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3643         /* First, check if we need to linearize the skb (due to FW
3644            restrictions). No need to check fragmentation if page size > 8K
3645            (there will be no violation to FW restrictions) */
3646         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3647                 /* Statistics of linearization */
3648                 bp->lin_cnt++;
3649                 if (skb_linearize(skb) != 0) {
3650                         DP(NETIF_MSG_TX_QUEUED,
3651                            "SKB linearization failed - silently dropping this SKB\n");
3652                         dev_kfree_skb_any(skb);
3653                         return NETDEV_TX_OK;
3654                 }
3655         }
3656 #endif
3657         /* Map skb linear data for DMA */
3658         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3659                                  skb_headlen(skb), DMA_TO_DEVICE);
3660         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3661                 DP(NETIF_MSG_TX_QUEUED,
3662                    "SKB mapping failed - silently dropping this SKB\n");
3663                 dev_kfree_skb_any(skb);
3664                 return NETDEV_TX_OK;
3665         }
3666         /*
3667         Please read carefully. First we use one BD which we mark as start,
3668         then we have a parsing info BD (used for TSO or xsum),
3669         and only then we have the rest of the TSO BDs.
3670         (don't forget to mark the last one as last,
3671         and to unmap only AFTER you write to the BD ...)
3672         And above all, all pdb sizes are in words - NOT DWORDS!
3673         */
3674
3675         /* get current pkt produced now - advance it just before sending packet
3676          * since mapping of pages may fail and cause packet to be dropped
3677          */
3678         pkt_prod = txdata->tx_pkt_prod;
3679         bd_prod = TX_BD(txdata->tx_bd_prod);
3680
3681         /* get a tx_buf and first BD
3682          * tx_start_bd may be changed during SPLIT,
3683          * but first_bd will always stay first
3684          */
3685         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3686         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3687         first_bd = tx_start_bd;
3688
3689         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3690
3691         /* header nbd: indirectly zero other flags! */
3692         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3693
3694         /* remember the first BD of the packet */
3695         tx_buf->first_bd = txdata->tx_bd_prod;
3696         tx_buf->skb = skb;
3697         tx_buf->flags = 0;
3698
3699         DP(NETIF_MSG_TX_QUEUED,
3700            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3701            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3702
3703         if (vlan_tx_tag_present(skb)) {
3704                 tx_start_bd->vlan_or_ethertype =
3705                     cpu_to_le16(vlan_tx_tag_get(skb));
3706                 tx_start_bd->bd_flags.as_bitfield |=
3707                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3708         } else {
3709                 /* when transmitting in a vf, start bd must hold the ethertype
3710                  * for fw to enforce it
3711                  */
3712                 if (IS_VF(bp))
3713                         tx_start_bd->vlan_or_ethertype =
3714                                 cpu_to_le16(ntohs(eth->h_proto));
3715                 else
3716                         /* used by FW for packet accounting */
3717                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3718         }
3719
3720         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3721
3722         /* turn on parsing and get a BD */
3723         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3724
3725         if (xmit_type & XMIT_CSUM)
3726                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3727
3728         if (!CHIP_IS_E1x(bp)) {
3729                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3730                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3731
3732                 if (xmit_type & XMIT_CSUM_ENC) {
3733                         u16 global_data = 0;
3734
3735                         /* Set PBD in enc checksum offload case */
3736                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3737                                                       &pbd_e2_parsing_data,
3738                                                       xmit_type);
3739
3740                         /* turn on 2nd parsing and get a BD */
3741                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3742
3743                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3744
3745                         memset(pbd2, 0, sizeof(*pbd2));
3746
3747                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3748                                 (skb_inner_network_header(skb) -
3749                                  skb->data) >> 1;
3750
3751                         if (xmit_type & XMIT_GSO_ENC)
3752                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3753                                                           &global_data,
3754                                                           xmit_type);
3755
3756                         pbd2->global_data = cpu_to_le16(global_data);
3757
3758                         /* add addition parse BD indication to start BD */
3759                         SET_FLAG(tx_start_bd->general_data,
3760                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3761                         /* set encapsulation flag in start BD */
3762                         SET_FLAG(tx_start_bd->general_data,
3763                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3764
3765                         tx_buf->flags |= BNX2X_HAS_SECOND_PBD;
3766
3767                         nbd++;
3768                 } else if (xmit_type & XMIT_CSUM) {
3769                         /* Set PBD in checksum offload case w/o encapsulation */
3770                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3771                                                      &pbd_e2_parsing_data,
3772                                                      xmit_type);
3773                 }
3774
3775                 /* Add the macs to the parsing BD this is a vf */
3776                 if (IS_VF(bp)) {
3777                         /* override GRE parameters in BD */
3778                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3779                                               &pbd_e2->data.mac_addr.src_mid,
3780                                               &pbd_e2->data.mac_addr.src_lo,
3781                                               eth->h_source);
3782
3783                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3784                                               &pbd_e2->data.mac_addr.dst_mid,
3785                                               &pbd_e2->data.mac_addr.dst_lo,
3786                                               eth->h_dest);
3787                 }
3788
3789                 SET_FLAG(pbd_e2_parsing_data,
3790                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3791         } else {
3792                 u16 global_data = 0;
3793                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3794                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3795                 /* Set PBD in checksum offload case */
3796                 if (xmit_type & XMIT_CSUM)
3797                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3798
3799                 SET_FLAG(global_data,
3800                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3801                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3802         }
3803
3804         /* Setup the data pointer of the first BD of the packet */
3805         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3806         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3807         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3808         pkt_size = tx_start_bd->nbytes;
3809
3810         DP(NETIF_MSG_TX_QUEUED,
3811            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3812            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3813            le16_to_cpu(tx_start_bd->nbytes),
3814            tx_start_bd->bd_flags.as_bitfield,
3815            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3816
3817         if (xmit_type & XMIT_GSO) {
3818
3819                 DP(NETIF_MSG_TX_QUEUED,
3820                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3821                    skb->len, hlen, skb_headlen(skb),
3822                    skb_shinfo(skb)->gso_size);
3823
3824                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3825
3826                 if (unlikely(skb_headlen(skb) > hlen)) {
3827                         nbd++;
3828                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3829                                                  &tx_start_bd, hlen,
3830                                                  bd_prod);
3831                 }
3832                 if (!CHIP_IS_E1x(bp))
3833                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3834                                              xmit_type);
3835                 else
3836                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3837         }
3838
3839         /* Set the PBD's parsing_data field if not zero
3840          * (for the chips newer than 57711).
3841          */
3842         if (pbd_e2_parsing_data)
3843                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3844
3845         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3846
3847         /* Handle fragmented skb */
3848         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3849                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3850
3851                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3852                                            skb_frag_size(frag), DMA_TO_DEVICE);
3853                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3854                         unsigned int pkts_compl = 0, bytes_compl = 0;
3855
3856                         DP(NETIF_MSG_TX_QUEUED,
3857                            "Unable to map page - dropping packet...\n");
3858
3859                         /* we need unmap all buffers already mapped
3860                          * for this SKB;
3861                          * first_bd->nbd need to be properly updated
3862                          * before call to bnx2x_free_tx_pkt
3863                          */
3864                         first_bd->nbd = cpu_to_le16(nbd);
3865                         bnx2x_free_tx_pkt(bp, txdata,
3866                                           TX_BD(txdata->tx_pkt_prod),
3867                                           &pkts_compl, &bytes_compl);
3868                         return NETDEV_TX_OK;
3869                 }
3870
3871                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3872                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3873                 if (total_pkt_bd == NULL)
3874                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3875
3876                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3877                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3878                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3879                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3880                 nbd++;
3881
3882                 DP(NETIF_MSG_TX_QUEUED,
3883                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3884                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3885                    le16_to_cpu(tx_data_bd->nbytes));
3886         }
3887
3888         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3889
3890         /* update with actual num BDs */
3891         first_bd->nbd = cpu_to_le16(nbd);
3892
3893         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3894
3895         /* now send a tx doorbell, counting the next BD
3896          * if the packet contains or ends with it
3897          */
3898         if (TX_BD_POFF(bd_prod) < nbd)
3899                 nbd++;
3900
3901         /* total_pkt_bytes should be set on the first data BD if
3902          * it's not an LSO packet and there is more than one
3903          * data BD. In this case pkt_size is limited by an MTU value.
3904          * However we prefer to set it for an LSO packet (while we don't
3905          * have to) in order to save some CPU cycles in a none-LSO
3906          * case, when we much more care about them.
3907          */
3908         if (total_pkt_bd != NULL)
3909                 total_pkt_bd->total_pkt_bytes = pkt_size;
3910
3911         if (pbd_e1x)
3912                 DP(NETIF_MSG_TX_QUEUED,
3913                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3914                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3915                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3916                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3917                     le16_to_cpu(pbd_e1x->total_hlen_w));
3918         if (pbd_e2)
3919                 DP(NETIF_MSG_TX_QUEUED,
3920                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3921                    pbd_e2,
3922                    pbd_e2->data.mac_addr.dst_hi,
3923                    pbd_e2->data.mac_addr.dst_mid,
3924                    pbd_e2->data.mac_addr.dst_lo,
3925                    pbd_e2->data.mac_addr.src_hi,
3926                    pbd_e2->data.mac_addr.src_mid,
3927                    pbd_e2->data.mac_addr.src_lo,
3928                    pbd_e2->parsing_data);
3929         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3930
3931         netdev_tx_sent_queue(txq, skb->len);
3932
3933         skb_tx_timestamp(skb);
3934
3935         txdata->tx_pkt_prod++;
3936         /*
3937          * Make sure that the BD data is updated before updating the producer
3938          * since FW might read the BD right after the producer is updated.
3939          * This is only applicable for weak-ordered memory model archs such
3940          * as IA-64. The following barrier is also mandatory since FW will
3941          * assumes packets must have BDs.
3942          */
3943         wmb();
3944
3945         txdata->tx_db.data.prod += nbd;
3946         barrier();
3947
3948         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3949
3950         mmiowb();
3951
3952         txdata->tx_bd_prod += nbd;
3953
3954         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3955                 netif_tx_stop_queue(txq);
3956
3957                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3958                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3959                  * fp->bd_tx_cons */
3960                 smp_mb();
3961
3962                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3963                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3964                         netif_tx_wake_queue(txq);
3965         }
3966         txdata->tx_pkt++;
3967
3968         return NETDEV_TX_OK;
3969 }
3970
3971 /**
3972  * bnx2x_setup_tc - routine to configure net_device for multi tc
3973  *
3974  * @netdev: net device to configure
3975  * @tc: number of traffic classes to enable
3976  *
3977  * callback connected to the ndo_setup_tc function pointer
3978  */
3979 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3980 {
3981         int cos, prio, count, offset;
3982         struct bnx2x *bp = netdev_priv(dev);
3983
3984         /* setup tc must be called under rtnl lock */
3985         ASSERT_RTNL();
3986
3987         /* no traffic classes requested. aborting */
3988         if (!num_tc) {
3989                 netdev_reset_tc(dev);
3990                 return 0;
3991         }
3992
3993         /* requested to support too many traffic classes */
3994         if (num_tc > bp->max_cos) {
3995                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3996                           num_tc, bp->max_cos);
3997                 return -EINVAL;
3998         }
3999
4000         /* declare amount of supported traffic classes */
4001         if (netdev_set_num_tc(dev, num_tc)) {
4002                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4003                 return -EINVAL;
4004         }
4005
4006         /* configure priority to traffic class mapping */
4007         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4008                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4009                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4010                    "mapping priority %d to tc %d\n",
4011                    prio, bp->prio_to_cos[prio]);
4012         }
4013
4014
4015         /* Use this configuration to diffrentiate tc0 from other COSes
4016            This can be used for ets or pfc, and save the effort of setting
4017            up a multio class queue disc or negotiating DCBX with a switch
4018         netdev_set_prio_tc_map(dev, 0, 0);
4019         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4020         for (prio = 1; prio < 16; prio++) {
4021                 netdev_set_prio_tc_map(dev, prio, 1);
4022                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4023         } */
4024
4025         /* configure traffic class to transmission queue mapping */
4026         for (cos = 0; cos < bp->max_cos; cos++) {
4027                 count = BNX2X_NUM_ETH_QUEUES(bp);
4028                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4029                 netdev_set_tc_queue(dev, cos, count, offset);
4030                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4031                    "mapping tc %d to offset %d count %d\n",
4032                    cos, offset, count);
4033         }
4034
4035         return 0;
4036 }
4037
4038 /* called with rtnl_lock */
4039 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4040 {
4041         struct sockaddr *addr = p;
4042         struct bnx2x *bp = netdev_priv(dev);
4043         int rc = 0;
4044
4045         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4046                 BNX2X_ERR("Requested MAC address is not valid\n");
4047                 return -EINVAL;
4048         }
4049
4050         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4051             !is_zero_ether_addr(addr->sa_data)) {
4052                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4053                 return -EINVAL;
4054         }
4055
4056         if (netif_running(dev))  {
4057                 rc = bnx2x_set_eth_mac(bp, false);
4058                 if (rc)
4059                         return rc;
4060         }
4061
4062         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4063
4064         if (netif_running(dev))
4065                 rc = bnx2x_set_eth_mac(bp, true);
4066
4067         return rc;
4068 }
4069
4070 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4071 {
4072         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4073         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4074         u8 cos;
4075
4076         /* Common */
4077
4078         if (IS_FCOE_IDX(fp_index)) {
4079                 memset(sb, 0, sizeof(union host_hc_status_block));
4080                 fp->status_blk_mapping = 0;
4081         } else {
4082                 /* status blocks */
4083                 if (!CHIP_IS_E1x(bp))
4084                         BNX2X_PCI_FREE(sb->e2_sb,
4085                                        bnx2x_fp(bp, fp_index,
4086                                                 status_blk_mapping),
4087                                        sizeof(struct host_hc_status_block_e2));
4088                 else
4089                         BNX2X_PCI_FREE(sb->e1x_sb,
4090                                        bnx2x_fp(bp, fp_index,
4091                                                 status_blk_mapping),
4092                                        sizeof(struct host_hc_status_block_e1x));
4093         }
4094
4095         /* Rx */
4096         if (!skip_rx_queue(bp, fp_index)) {
4097                 bnx2x_free_rx_bds(fp);
4098
4099                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4100                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4101                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4102                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4103                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4104
4105                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4106                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4107                                sizeof(struct eth_fast_path_rx_cqe) *
4108                                NUM_RCQ_BD);
4109
4110                 /* SGE ring */
4111                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4112                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4113                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4114                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4115         }
4116
4117         /* Tx */
4118         if (!skip_tx_queue(bp, fp_index)) {
4119                 /* fastpath tx rings: tx_buf tx_desc */
4120                 for_each_cos_in_tx_queue(fp, cos) {
4121                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4122
4123                         DP(NETIF_MSG_IFDOWN,
4124                            "freeing tx memory of fp %d cos %d cid %d\n",
4125                            fp_index, cos, txdata->cid);
4126
4127                         BNX2X_FREE(txdata->tx_buf_ring);
4128                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4129                                 txdata->tx_desc_mapping,
4130                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4131                 }
4132         }
4133         /* end of fastpath */
4134 }
4135
4136 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4137 {
4138         int i;
4139         for_each_cnic_queue(bp, i)
4140                 bnx2x_free_fp_mem_at(bp, i);
4141 }
4142
4143 void bnx2x_free_fp_mem(struct bnx2x *bp)
4144 {
4145         int i;
4146         for_each_eth_queue(bp, i)
4147                 bnx2x_free_fp_mem_at(bp, i);
4148 }
4149
4150 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4151 {
4152         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4153         if (!CHIP_IS_E1x(bp)) {
4154                 bnx2x_fp(bp, index, sb_index_values) =
4155                         (__le16 *)status_blk.e2_sb->sb.index_values;
4156                 bnx2x_fp(bp, index, sb_running_index) =
4157                         (__le16 *)status_blk.e2_sb->sb.running_index;
4158         } else {
4159                 bnx2x_fp(bp, index, sb_index_values) =
4160                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4161                 bnx2x_fp(bp, index, sb_running_index) =
4162                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4163         }
4164 }
4165
4166 /* Returns the number of actually allocated BDs */
4167 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4168                               int rx_ring_size)
4169 {
4170         struct bnx2x *bp = fp->bp;
4171         u16 ring_prod, cqe_ring_prod;
4172         int i, failure_cnt = 0;
4173
4174         fp->rx_comp_cons = 0;
4175         cqe_ring_prod = ring_prod = 0;
4176
4177         /* This routine is called only during fo init so
4178          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4179          */
4180         for (i = 0; i < rx_ring_size; i++) {
4181                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4182                         failure_cnt++;
4183                         continue;
4184                 }
4185                 ring_prod = NEXT_RX_IDX(ring_prod);
4186                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4187                 WARN_ON(ring_prod <= (i - failure_cnt));
4188         }
4189
4190         if (failure_cnt)
4191                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4192                           i - failure_cnt, fp->index);
4193
4194         fp->rx_bd_prod = ring_prod;
4195         /* Limit the CQE producer by the CQE ring size */
4196         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4197                                cqe_ring_prod);
4198         fp->rx_pkt = fp->rx_calls = 0;
4199
4200         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4201
4202         return i - failure_cnt;
4203 }
4204
4205 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4206 {
4207         int i;
4208
4209         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4210                 struct eth_rx_cqe_next_page *nextpg;
4211
4212                 nextpg = (struct eth_rx_cqe_next_page *)
4213                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4214                 nextpg->addr_hi =
4215                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4216                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4217                 nextpg->addr_lo =
4218                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4219                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4220         }
4221 }
4222
4223 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4224 {
4225         union host_hc_status_block *sb;
4226         struct bnx2x_fastpath *fp = &bp->fp[index];
4227         int ring_size = 0;
4228         u8 cos;
4229         int rx_ring_size = 0;
4230
4231         if (!bp->rx_ring_size &&
4232             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4233                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4234                 bp->rx_ring_size = rx_ring_size;
4235         } else if (!bp->rx_ring_size) {
4236                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4237
4238                 if (CHIP_IS_E3(bp)) {
4239                         u32 cfg = SHMEM_RD(bp,
4240                                            dev_info.port_hw_config[BP_PORT(bp)].
4241                                            default_cfg);
4242
4243                         /* Decrease ring size for 1G functions */
4244                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4245                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4246                                 rx_ring_size /= 10;
4247                 }
4248
4249                 /* allocate at least number of buffers required by FW */
4250                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4251                                      MIN_RX_SIZE_TPA, rx_ring_size);
4252
4253                 bp->rx_ring_size = rx_ring_size;
4254         } else /* if rx_ring_size specified - use it */
4255                 rx_ring_size = bp->rx_ring_size;
4256
4257         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4258
4259         /* Common */
4260         sb = &bnx2x_fp(bp, index, status_blk);
4261
4262         if (!IS_FCOE_IDX(index)) {
4263                 /* status blocks */
4264                 if (!CHIP_IS_E1x(bp))
4265                         BNX2X_PCI_ALLOC(sb->e2_sb,
4266                                 &bnx2x_fp(bp, index, status_blk_mapping),
4267                                 sizeof(struct host_hc_status_block_e2));
4268                 else
4269                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4270                                 &bnx2x_fp(bp, index, status_blk_mapping),
4271                             sizeof(struct host_hc_status_block_e1x));
4272         }
4273
4274         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4275          * set shortcuts for it.
4276          */
4277         if (!IS_FCOE_IDX(index))
4278                 set_sb_shortcuts(bp, index);
4279
4280         /* Tx */
4281         if (!skip_tx_queue(bp, index)) {
4282                 /* fastpath tx rings: tx_buf tx_desc */
4283                 for_each_cos_in_tx_queue(fp, cos) {
4284                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4285
4286                         DP(NETIF_MSG_IFUP,
4287                            "allocating tx memory of fp %d cos %d\n",
4288                            index, cos);
4289
4290                         BNX2X_ALLOC(txdata->tx_buf_ring,
4291                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4292                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4293                                 &txdata->tx_desc_mapping,
4294                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4295                 }
4296         }
4297
4298         /* Rx */
4299         if (!skip_rx_queue(bp, index)) {
4300                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4301                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4302                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4303                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4304                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4305                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4306
4307                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4308                                 &bnx2x_fp(bp, index, rx_comp_mapping),
4309                                 sizeof(struct eth_fast_path_rx_cqe) *
4310                                 NUM_RCQ_BD);
4311
4312                 /* SGE ring */
4313                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4314                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4315                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4316                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4317                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4318                 /* RX BD ring */
4319                 bnx2x_set_next_page_rx_bd(fp);
4320
4321                 /* CQ ring */
4322                 bnx2x_set_next_page_rx_cq(fp);
4323
4324                 /* BDs */
4325                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4326                 if (ring_size < rx_ring_size)
4327                         goto alloc_mem_err;
4328         }
4329
4330         return 0;
4331
4332 /* handles low memory cases */
4333 alloc_mem_err:
4334         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4335                                                 index, ring_size);
4336         /* FW will drop all packets if queue is not big enough,
4337          * In these cases we disable the queue
4338          * Min size is different for OOO, TPA and non-TPA queues
4339          */
4340         if (ring_size < (fp->disable_tpa ?
4341                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4342                         /* release memory allocated for this queue */
4343                         bnx2x_free_fp_mem_at(bp, index);
4344                         return -ENOMEM;
4345         }
4346         return 0;
4347 }
4348
4349 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4350 {
4351         if (!NO_FCOE(bp))
4352                 /* FCoE */
4353                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4354                         /* we will fail load process instead of mark
4355                          * NO_FCOE_FLAG
4356                          */
4357                         return -ENOMEM;
4358
4359         return 0;
4360 }
4361
4362 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4363 {
4364         int i;
4365
4366         /* 1. Allocate FP for leading - fatal if error
4367          * 2. Allocate RSS - fix number of queues if error
4368          */
4369
4370         /* leading */
4371         if (bnx2x_alloc_fp_mem_at(bp, 0))
4372                 return -ENOMEM;
4373
4374         /* RSS */
4375         for_each_nondefault_eth_queue(bp, i)
4376                 if (bnx2x_alloc_fp_mem_at(bp, i))
4377                         break;
4378
4379         /* handle memory failures */
4380         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4381                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4382
4383                 WARN_ON(delta < 0);
4384                 bnx2x_shrink_eth_fp(bp, delta);
4385                 if (CNIC_SUPPORT(bp))
4386                         /* move non eth FPs next to last eth FP
4387                          * must be done in that order
4388                          * FCOE_IDX < FWD_IDX < OOO_IDX
4389                          */
4390
4391                         /* move FCoE fp even NO_FCOE_FLAG is on */
4392                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4393                 bp->num_ethernet_queues -= delta;
4394                 bp->num_queues = bp->num_ethernet_queues +
4395                                  bp->num_cnic_queues;
4396                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4397                           bp->num_queues + delta, bp->num_queues);
4398         }
4399
4400         return 0;
4401 }
4402
4403 void bnx2x_free_mem_bp(struct bnx2x *bp)
4404 {
4405         int i;
4406
4407         for (i = 0; i < bp->fp_array_size; i++)
4408                 kfree(bp->fp[i].tpa_info);
4409         kfree(bp->fp);
4410         kfree(bp->sp_objs);
4411         kfree(bp->fp_stats);
4412         kfree(bp->bnx2x_txq);
4413         kfree(bp->msix_table);
4414         kfree(bp->ilt);
4415 }
4416
4417 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4418 {
4419         struct bnx2x_fastpath *fp;
4420         struct msix_entry *tbl;
4421         struct bnx2x_ilt *ilt;
4422         int msix_table_size = 0;
4423         int fp_array_size, txq_array_size;
4424         int i;
4425
4426         /*
4427          * The biggest MSI-X table we might need is as a maximum number of fast
4428          * path IGU SBs plus default SB (for PF only).
4429          */
4430         msix_table_size = bp->igu_sb_cnt;
4431         if (IS_PF(bp))
4432                 msix_table_size++;
4433         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4434
4435         /* fp array: RSS plus CNIC related L2 queues */
4436         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4437         bp->fp_array_size = fp_array_size;
4438         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4439
4440         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4441         if (!fp)
4442                 goto alloc_err;
4443         for (i = 0; i < bp->fp_array_size; i++) {
4444                 fp[i].tpa_info =
4445                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4446                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4447                 if (!(fp[i].tpa_info))
4448                         goto alloc_err;
4449         }
4450
4451         bp->fp = fp;
4452
4453         /* allocate sp objs */
4454         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4455                               GFP_KERNEL);
4456         if (!bp->sp_objs)
4457                 goto alloc_err;
4458
4459         /* allocate fp_stats */
4460         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4461                                GFP_KERNEL);
4462         if (!bp->fp_stats)
4463                 goto alloc_err;
4464
4465         /* Allocate memory for the transmission queues array */
4466         txq_array_size =
4467                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4468         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4469
4470         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4471                                 GFP_KERNEL);
4472         if (!bp->bnx2x_txq)
4473                 goto alloc_err;
4474
4475         /* msix table */
4476         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4477         if (!tbl)
4478                 goto alloc_err;
4479         bp->msix_table = tbl;
4480
4481         /* ilt */
4482         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4483         if (!ilt)
4484                 goto alloc_err;
4485         bp->ilt = ilt;
4486
4487         return 0;
4488 alloc_err:
4489         bnx2x_free_mem_bp(bp);
4490         return -ENOMEM;
4491
4492 }
4493
4494 int bnx2x_reload_if_running(struct net_device *dev)
4495 {
4496         struct bnx2x *bp = netdev_priv(dev);
4497
4498         if (unlikely(!netif_running(dev)))
4499                 return 0;
4500
4501         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4502         return bnx2x_nic_load(bp, LOAD_NORMAL);
4503 }
4504
4505 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4506 {
4507         u32 sel_phy_idx = 0;
4508         if (bp->link_params.num_phys <= 1)
4509                 return INT_PHY;
4510
4511         if (bp->link_vars.link_up) {
4512                 sel_phy_idx = EXT_PHY1;
4513                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4514                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4515                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4516                         sel_phy_idx = EXT_PHY2;
4517         } else {
4518
4519                 switch (bnx2x_phy_selection(&bp->link_params)) {
4520                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4521                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4522                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4523                        sel_phy_idx = EXT_PHY1;
4524                        break;
4525                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4526                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4527                        sel_phy_idx = EXT_PHY2;
4528                        break;
4529                 }
4530         }
4531
4532         return sel_phy_idx;
4533
4534 }
4535 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4536 {
4537         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4538         /*
4539          * The selected activated PHY is always after swapping (in case PHY
4540          * swapping is enabled). So when swapping is enabled, we need to reverse
4541          * the configuration
4542          */
4543
4544         if (bp->link_params.multi_phy_config &
4545             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4546                 if (sel_phy_idx == EXT_PHY1)
4547                         sel_phy_idx = EXT_PHY2;
4548                 else if (sel_phy_idx == EXT_PHY2)
4549                         sel_phy_idx = EXT_PHY1;
4550         }
4551         return LINK_CONFIG_IDX(sel_phy_idx);
4552 }
4553
4554 #ifdef NETDEV_FCOE_WWNN
4555 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4556 {
4557         struct bnx2x *bp = netdev_priv(dev);
4558         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4559
4560         switch (type) {
4561         case NETDEV_FCOE_WWNN:
4562                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4563                                 cp->fcoe_wwn_node_name_lo);
4564                 break;
4565         case NETDEV_FCOE_WWPN:
4566                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4567                                 cp->fcoe_wwn_port_name_lo);
4568                 break;
4569         default:
4570                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4571                 return -EINVAL;
4572         }
4573
4574         return 0;
4575 }
4576 #endif
4577
4578 /* called with rtnl_lock */
4579 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4580 {
4581         struct bnx2x *bp = netdev_priv(dev);
4582
4583         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4584                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4585                 return -EAGAIN;
4586         }
4587
4588         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4589             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4590                 BNX2X_ERR("Can't support requested MTU size\n");
4591                 return -EINVAL;
4592         }
4593
4594         /* This does not race with packet allocation
4595          * because the actual alloc size is
4596          * only updated as part of load
4597          */
4598         dev->mtu = new_mtu;
4599
4600         return bnx2x_reload_if_running(dev);
4601 }
4602
4603 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4604                                      netdev_features_t features)
4605 {
4606         struct bnx2x *bp = netdev_priv(dev);
4607
4608         /* TPA requires Rx CSUM offloading */
4609         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4610                 features &= ~NETIF_F_LRO;
4611                 features &= ~NETIF_F_GRO;
4612         }
4613
4614         return features;
4615 }
4616
4617 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4618 {
4619         struct bnx2x *bp = netdev_priv(dev);
4620         u32 flags = bp->flags;
4621         bool bnx2x_reload = false;
4622
4623         if (features & NETIF_F_LRO)
4624                 flags |= TPA_ENABLE_FLAG;
4625         else
4626                 flags &= ~TPA_ENABLE_FLAG;
4627
4628         if (features & NETIF_F_GRO)
4629                 flags |= GRO_ENABLE_FLAG;
4630         else
4631                 flags &= ~GRO_ENABLE_FLAG;
4632
4633         if (features & NETIF_F_LOOPBACK) {
4634                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4635                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4636                         bnx2x_reload = true;
4637                 }
4638         } else {
4639                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4640                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4641                         bnx2x_reload = true;
4642                 }
4643         }
4644
4645         if (flags ^ bp->flags) {
4646                 bp->flags = flags;
4647                 bnx2x_reload = true;
4648         }
4649
4650         if (bnx2x_reload) {
4651                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4652                         return bnx2x_reload_if_running(dev);
4653                 /* else: bnx2x_nic_load() will be called at end of recovery */
4654         }
4655
4656         return 0;
4657 }
4658
4659 void bnx2x_tx_timeout(struct net_device *dev)
4660 {
4661         struct bnx2x *bp = netdev_priv(dev);
4662
4663 #ifdef BNX2X_STOP_ON_ERROR
4664         if (!bp->panic)
4665                 bnx2x_panic();
4666 #endif
4667
4668         smp_mb__before_clear_bit();
4669         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4670         smp_mb__after_clear_bit();
4671
4672         /* This allows the netif to be shutdown gracefully before resetting */
4673         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4674 }
4675
4676 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4677 {
4678         struct net_device *dev = pci_get_drvdata(pdev);
4679         struct bnx2x *bp;
4680
4681         if (!dev) {
4682                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4683                 return -ENODEV;
4684         }
4685         bp = netdev_priv(dev);
4686
4687         rtnl_lock();
4688
4689         pci_save_state(pdev);
4690
4691         if (!netif_running(dev)) {
4692                 rtnl_unlock();
4693                 return 0;
4694         }
4695
4696         netif_device_detach(dev);
4697
4698         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4699
4700         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4701
4702         rtnl_unlock();
4703
4704         return 0;
4705 }
4706
4707 int bnx2x_resume(struct pci_dev *pdev)
4708 {
4709         struct net_device *dev = pci_get_drvdata(pdev);
4710         struct bnx2x *bp;
4711         int rc;
4712
4713         if (!dev) {
4714                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4715                 return -ENODEV;
4716         }
4717         bp = netdev_priv(dev);
4718
4719         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4720                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4721                 return -EAGAIN;
4722         }
4723
4724         rtnl_lock();
4725
4726         pci_restore_state(pdev);
4727
4728         if (!netif_running(dev)) {
4729                 rtnl_unlock();
4730                 return 0;
4731         }
4732
4733         bnx2x_set_power_state(bp, PCI_D0);
4734         netif_device_attach(dev);
4735
4736         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4737
4738         rtnl_unlock();
4739
4740         return rc;
4741 }
4742
4743
4744 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4745                               u32 cid)
4746 {
4747         /* ustorm cxt validation */
4748         cxt->ustorm_ag_context.cdu_usage =
4749                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4750                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4751         /* xcontext validation */
4752         cxt->xstorm_ag_context.cdu_reserved =
4753                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4754                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4755 }
4756
4757 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4758                                     u8 fw_sb_id, u8 sb_index,
4759                                     u8 ticks)
4760 {
4761
4762         u32 addr = BAR_CSTRORM_INTMEM +
4763                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4764         REG_WR8(bp, addr, ticks);
4765         DP(NETIF_MSG_IFUP,
4766            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4767            port, fw_sb_id, sb_index, ticks);
4768 }
4769
4770 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4771                                     u16 fw_sb_id, u8 sb_index,
4772                                     u8 disable)
4773 {
4774         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4775         u32 addr = BAR_CSTRORM_INTMEM +
4776                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4777         u8 flags = REG_RD8(bp, addr);
4778         /* clear and set */
4779         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4780         flags |= enable_flag;
4781         REG_WR8(bp, addr, flags);
4782         DP(NETIF_MSG_IFUP,
4783            "port %x fw_sb_id %d sb_index %d disable %d\n",
4784            port, fw_sb_id, sb_index, disable);
4785 }
4786
4787 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4788                                     u8 sb_index, u8 disable, u16 usec)
4789 {
4790         int port = BP_PORT(bp);
4791         u8 ticks = usec / BNX2X_BTR;
4792
4793         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4794
4795         disable = disable ? 1 : (usec ? 0 : 1);
4796         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4797 }