drivers/net/ethernet/broadcom/bnxt/bnxt.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2014-2016 Broadcom Corporation
   4  * Copyright (c) 2016-2018 Broadcom Limited
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  */
  10
  11 #include <linux/module.h>
  12
  13 #include <linux/stringify.h>
  14 #include <linux/kernel.h>
  15 #include <linux/timer.h>
  16 #include <linux/errno.h>
  17 #include <linux/ioport.h>
  18 #include <linux/slab.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/pci.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/etherdevice.h>
  24 #include <linux/skbuff.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/bitops.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/delay.h>
  30 #include <asm/byteorder.h>
  31 #include <asm/page.h>
  32 #include <linux/time.h>
  33 #include <linux/mii.h>
  34 #include <linux/if.h>
  35 #include <linux/if_vlan.h>
  36 #include <linux/if_bridge.h>
  37 #include <linux/rtc.h>
  38 #include <linux/bpf.h>
  39 #include <net/ip.h>
  40 #include <net/tcp.h>
  41 #include <net/udp.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <net/udp_tunnel.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/prefetch.h>
  47 #include <linux/cache.h>
  48 #include <linux/log2.h>
  49 #include <linux/aer.h>
  50 #include <linux/bitmap.h>
  51 #include <linux/cpu_rmap.h>
  52 #include <linux/cpumask.h>
  53 #include <net/pkt_cls.h>
  54 #include <linux/hwmon.h>
  55 #include <linux/hwmon-sysfs.h>
  56
  57 #include "bnxt_hsi.h"
  58 #include "bnxt.h"
  59 #include "bnxt_ulp.h"
  60 #include "bnxt_sriov.h"
  61 #include "bnxt_ethtool.h"
  62 #include "bnxt_dcb.h"
  63 #include "bnxt_xdp.h"
  64 #include "bnxt_vfr.h"
  65 #include "bnxt_tc.h"
  66 #include "bnxt_devlink.h"
  67 #include "bnxt_debugfs.h"
  68
  69 #define BNXT_TX_TIMEOUT         (5 * HZ)
  70
  71 static const char version[] =
  72         "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
  73
  74 MODULE_LICENSE("GPL");
  75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
  76 MODULE_VERSION(DRV_MODULE_VERSION);
  77
  78 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
  79 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
  80 #define BNXT_RX_COPY_THRESH 256
  81
  82 #define BNXT_TX_PUSH_THRESH 164
  83
  84 enum board_idx {
  85         BCM57301,
  86         BCM57302,
  87         BCM57304,
  88         BCM57417_NPAR,
  89         BCM58700,
  90         BCM57311,
  91         BCM57312,
  92         BCM57402,
  93         BCM57404,
  94         BCM57406,
  95         BCM57402_NPAR,
  96         BCM57407,
  97         BCM57412,
  98         BCM57414,
  99         BCM57416,
 100         BCM57417,
 101         BCM57412_NPAR,
 102         BCM57314,
 103         BCM57417_SFP,
 104         BCM57416_SFP,
 105         BCM57404_NPAR,
 106         BCM57406_NPAR,
 107         BCM57407_SFP,
 108         BCM57407_NPAR,
 109         BCM57414_NPAR,
 110         BCM57416_NPAR,
 111         BCM57452,
 112         BCM57454,
 113         BCM5745x_NPAR,
 114         BCM57508,
 115         BCM58802,
 116         BCM58804,
 117         BCM58808,
 118         NETXTREME_E_VF,
 119         NETXTREME_C_VF,
 120         NETXTREME_S_VF,
 121 };
 122
 123 /* indexed by enum above */
 124 static const struct {
 125         char *name;
 126 } board_info[] = {
 127         [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
 128         [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
 129         [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 130         [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
 131         [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
 132         [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
 133         [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
 134         [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
 135         [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
 136         [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
 137         [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
 138         [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
 139         [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
 140         [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
 141         [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
 142         [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
 143         [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
 144         [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
 145         [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
 146         [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
 147         [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
 148         [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
 149         [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
 150         [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
 151         [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
 152         [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 153         [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 154         [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 155         [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 156         [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
 157         [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 158         [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 159         [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 160         [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
 161         [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
 162         [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
 163 };
 164
 165 static const struct pci_device_id bnxt_pci_tbl[] = {
 166         { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
 167         { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 168         { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 169         { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 170         { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
 171         { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
 172         { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
 173         { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
 174         { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
 175         { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
 176         { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
 177         { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
 178         { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
 179         { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
 180         { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
 181         { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
 182         { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
 183         { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
 184         { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
 185         { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
 186         { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
 187         { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
 188         { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
 189         { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
 190         { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
 191         { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
 192         { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
 193         { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
 194         { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
 195         { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
 196         { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
 197         { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
 198         { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
 199         { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
 200         { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
 201         { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
 202         { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
 203         { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
 204 #ifdef CONFIG_BNXT_SRIOV
 205         { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
 206         { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
 207         { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
 208         { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
 209         { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
 210         { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
 211         { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
 212         { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
 213         { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
 214 #endif
 215         { 0 }
 216 };
 217
 218 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 219
 220 static const u16 bnxt_vf_req_snif[] = {
 221         HWRM_FUNC_CFG,
 222         HWRM_FUNC_VF_CFG,
 223         HWRM_PORT_PHY_QCFG,
 224         HWRM_CFA_L2_FILTER_ALLOC,
 225 };
 226
 227 static const u16 bnxt_async_events_arr[] = {
 228         ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
 229         ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
 230         ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
 231         ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
 232         ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
 233 };
 234
 235 static struct workqueue_struct *bnxt_pf_wq;
 236
 237 static bool bnxt_vf_pciid(enum board_idx idx)
 238 {
 239         return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
 240                 idx == NETXTREME_S_VF);
 241 }
 242
 243 #define DB_CP_REARM_FLAGS       (DB_KEY_CP | DB_IDX_VALID)
 244 #define DB_CP_FLAGS             (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
 245 #define DB_CP_IRQ_DIS_FLAGS     (DB_KEY_CP | DB_IRQ_DIS)
 246
 247 #define BNXT_CP_DB_IRQ_DIS(db)                                          \
 248                 writel(DB_CP_IRQ_DIS_FLAGS, db)
 249
 250 #define BNXT_DB_CQ(db, idx)                                             \
 251         writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
 252
 253 #define BNXT_DB_NQ_P5(db, idx)                                          \
 254         writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
 255
 256 #define BNXT_DB_CQ_ARM(db, idx)                                         \
 257         writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
 258
 259 #define BNXT_DB_NQ_ARM_P5(db, idx)                                      \
 260         writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
 261
 262 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 263 {
 264         if (bp->flags & BNXT_FLAG_CHIP_P5)
 265                 BNXT_DB_NQ_P5(db, idx);
 266         else
 267                 BNXT_DB_CQ(db, idx);
 268 }
 269
 270 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 271 {
 272         if (bp->flags & BNXT_FLAG_CHIP_P5)
 273                 BNXT_DB_NQ_ARM_P5(db, idx);
 274         else
 275                 BNXT_DB_CQ_ARM(db, idx);
 276 }
 277
 278 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
 279 {
 280         if (bp->flags & BNXT_FLAG_CHIP_P5)
 281                 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
 282                        db->doorbell);
 283         else
 284                 BNXT_DB_CQ(db, idx);
 285 }
 286
 287 const u16 bnxt_lhint_arr[] = {
 288         TX_BD_FLAGS_LHINT_512_AND_SMALLER,
 289         TX_BD_FLAGS_LHINT_512_TO_1023,
 290         TX_BD_FLAGS_LHINT_1024_TO_2047,
 291         TX_BD_FLAGS_LHINT_1024_TO_2047,
 292         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 293         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 294         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 295         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 296         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 297         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 298         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 299         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 300         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 301         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 302         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 303         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 304         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 305         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 306         TX_BD_FLAGS_LHINT_2048_AND_LARGER,
 307 };
 308
 309 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 310 {
 311         struct metadata_dst *md_dst = skb_metadata_dst(skb);
 312
 313         if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
 314                 return 0;
 315
 316         return md_dst->u.port_info.port_id;
 317 }
 318
 319 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 320 {
 321         struct bnxt *bp = netdev_priv(dev);
 322         struct tx_bd *txbd;
 323         struct tx_bd_ext *txbd1;
 324         struct netdev_queue *txq;
 325         int i;
 326         dma_addr_t mapping;
 327         unsigned int length, pad = 0;
 328         u32 len, free_size, vlan_tag_flags, cfa_action, flags;
 329         u16 prod, last_frag;
 330         struct pci_dev *pdev = bp->pdev;
 331         struct bnxt_tx_ring_info *txr;
 332         struct bnxt_sw_tx_bd *tx_buf;
 333
 334         i = skb_get_queue_mapping(skb);
 335         if (unlikely(i >= bp->tx_nr_rings)) {
 336                 dev_kfree_skb_any(skb);
 337                 return NETDEV_TX_OK;
 338         }
 339
 340         txq = netdev_get_tx_queue(dev, i);
 341         txr = &bp->tx_ring[bp->tx_ring_map[i]];
 342         prod = txr->tx_prod;
 343
 344         free_size = bnxt_tx_avail(bp, txr);
 345         if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
 346                 netif_tx_stop_queue(txq);
 347                 return NETDEV_TX_BUSY;
 348         }
 349
 350         length = skb->len;
 351         len = skb_headlen(skb);
 352         last_frag = skb_shinfo(skb)->nr_frags;
 353
 354         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 355
 356         txbd->tx_bd_opaque = prod;
 357
 358         tx_buf = &txr->tx_buf_ring[prod];
 359         tx_buf->skb = skb;
 360         tx_buf->nr_frags = last_frag;
 361
 362         vlan_tag_flags = 0;
 363         cfa_action = bnxt_xmit_get_cfa_action(skb);
 364         if (skb_vlan_tag_present(skb)) {
 365                 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
 366                                  skb_vlan_tag_get(skb);
 367                 /* Currently supports 8021Q, 8021AD vlan offloads
 368                  * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
 369                  */
 370                 if (skb->vlan_proto == htons(ETH_P_8021Q))
 371                         vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
 372         }
 373
 374         if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
 375                 struct tx_push_buffer *tx_push_buf = txr->tx_push;
 376                 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
 377                 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
 378                 void __iomem *db = txr->tx_db.doorbell;
 379                 void *pdata = tx_push_buf->data;
 380                 u64 *end;
 381                 int j, push_len;
 382
 383                 /* Set COAL_NOW to be ready quickly for the next push */
 384                 tx_push->tx_bd_len_flags_type =
 385                         cpu_to_le32((length << TX_BD_LEN_SHIFT) |
 386                                         TX_BD_TYPE_LONG_TX_BD |
 387                                         TX_BD_FLAGS_LHINT_512_AND_SMALLER |
 388                                         TX_BD_FLAGS_COAL_NOW |
 389                                         TX_BD_FLAGS_PACKET_END |
 390                                         (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
 391
 392                 if (skb->ip_summed == CHECKSUM_PARTIAL)
 393                         tx_push1->tx_bd_hsize_lflags =
 394                                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 395                 else
 396                         tx_push1->tx_bd_hsize_lflags = 0;
 397
 398                 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 399                 tx_push1->tx_bd_cfa_action =
 400                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 401
 402                 end = pdata + length;
 403                 end = PTR_ALIGN(end, 8) - 1;
 404                 *end = 0;
 405
 406                 skb_copy_from_linear_data(skb, pdata, len);
 407                 pdata += len;
 408                 for (j = 0; j < last_frag; j++) {
 409                         skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
 410                         void *fptr;
 411
 412                         fptr = skb_frag_address_safe(frag);
 413                         if (!fptr)
 414                                 goto normal_tx;
 415
 416                         memcpy(pdata, fptr, skb_frag_size(frag));
 417                         pdata += skb_frag_size(frag);
 418                 }
 419
 420                 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 421                 txbd->tx_bd_haddr = txr->data_mapping;
 422                 prod = NEXT_TX(prod);
 423                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 424                 memcpy(txbd, tx_push1, sizeof(*txbd));
 425                 prod = NEXT_TX(prod);
 426                 tx_push->doorbell =
 427                         cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
 428                 txr->tx_prod = prod;
 429
 430                 tx_buf->is_push = 1;
 431                 netdev_tx_sent_queue(txq, skb->len);
 432                 wmb();  /* Sync is_push and byte queue before pushing data */
 433
 434                 push_len = (length + sizeof(*tx_push) + 7) / 8;
 435                 if (push_len > 16) {
 436                         __iowrite64_copy(db, tx_push_buf, 16);
 437                         __iowrite32_copy(db + 4, tx_push_buf + 1,
 438                                          (push_len - 16) << 1);
 439                 } else {
 440                         __iowrite64_copy(db, tx_push_buf, push_len);
 441                 }
 442
 443                 goto tx_done;
 444         }
 445
 446 normal_tx:
 447         if (length < BNXT_MIN_PKT_SIZE) {
 448                 pad = BNXT_MIN_PKT_SIZE - length;
 449                 if (skb_pad(skb, pad)) {
 450                         /* SKB already freed. */
 451                         tx_buf->skb = NULL;
 452                         return NETDEV_TX_OK;
 453                 }
 454                 length = BNXT_MIN_PKT_SIZE;
 455         }
 456
 457         mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
 458
 459         if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
 460                 dev_kfree_skb_any(skb);
 461                 tx_buf->skb = NULL;
 462                 return NETDEV_TX_OK;
 463         }
 464
 465         dma_unmap_addr_set(tx_buf, mapping, mapping);
 466         flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
 467                 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 468
 469         txbd->tx_bd_haddr = cpu_to_le64(mapping);
 470
 471         prod = NEXT_TX(prod);
 472         txbd1 = (struct tx_bd_ext *)
 473                 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 474
 475         txbd1->tx_bd_hsize_lflags = 0;
 476         if (skb_is_gso(skb)) {
 477                 u32 hdr_len;
 478
 479                 if (skb->encapsulation)
 480                         hdr_len = skb_inner_network_offset(skb) +
 481                                 skb_inner_network_header_len(skb) +
 482                                 inner_tcp_hdrlen(skb);
 483                 else
 484                         hdr_len = skb_transport_offset(skb) +
 485                                 tcp_hdrlen(skb);
 486
 487                 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
 488                                         TX_BD_FLAGS_T_IPID |
 489                                         (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
 490                 length = skb_shinfo(skb)->gso_size;
 491                 txbd1->tx_bd_mss = cpu_to_le32(length);
 492                 length += hdr_len;
 493         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 494                 txbd1->tx_bd_hsize_lflags =
 495                         cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
 496                 txbd1->tx_bd_mss = 0;
 497         }
 498
 499         length >>= 9;
 500         flags |= bnxt_lhint_arr[length];
 501         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 502
 503         txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 504         txbd1->tx_bd_cfa_action =
 505                         cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
 506         for (i = 0; i < last_frag; i++) {
 507                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 508
 509                 prod = NEXT_TX(prod);
 510                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 511
 512                 len = skb_frag_size(frag);
 513                 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
 514                                            DMA_TO_DEVICE);
 515
 516                 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
 517                         goto tx_dma_error;
 518
 519                 tx_buf = &txr->tx_buf_ring[prod];
 520                 dma_unmap_addr_set(tx_buf, mapping, mapping);
 521
 522                 txbd->tx_bd_haddr = cpu_to_le64(mapping);
 523
 524                 flags = len << TX_BD_LEN_SHIFT;
 525                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
 526         }
 527
 528         flags &= ~TX_BD_LEN;
 529         txbd->tx_bd_len_flags_type =
 530                 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
 531                             TX_BD_FLAGS_PACKET_END);
 532
 533         netdev_tx_sent_queue(txq, skb->len);
 534
 535         /* Sync BD data before updating doorbell */
 536         wmb();
 537
 538         prod = NEXT_TX(prod);
 539         txr->tx_prod = prod;
 540
 541         if (!skb->xmit_more || netif_xmit_stopped(txq))
 542                 bnxt_db_write(bp, &txr->tx_db, prod);
 543
 544 tx_done:
 545
 546         mmiowb();
 547
 548         if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
 549                 if (skb->xmit_more && !tx_buf->is_push)
 550                         bnxt_db_write(bp, &txr->tx_db, prod);
 551
 552                 netif_tx_stop_queue(txq);
 553
 554                 /* netif_tx_stop_queue() must be done before checking
 555                  * tx index in bnxt_tx_avail() below, because in
 556                  * bnxt_tx_int(), we update tx index before checking for
 557                  * netif_tx_queue_stopped().
 558                  */
 559                 smp_mb();
 560                 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
 561                         netif_tx_wake_queue(txq);
 562         }
 563         return NETDEV_TX_OK;
 564
 565 tx_dma_error:
 566         last_frag = i;
 567
 568         /* start back at beginning and unmap skb */
 569         prod = txr->tx_prod;
 570         tx_buf = &txr->tx_buf_ring[prod];
 571         tx_buf->skb = NULL;
 572         dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 573                          skb_headlen(skb), PCI_DMA_TODEVICE);
 574         prod = NEXT_TX(prod);
 575
 576         /* unmap remaining mapped pages */
 577         for (i = 0; i < last_frag; i++) {
 578                 prod = NEXT_TX(prod);
 579                 tx_buf = &txr->tx_buf_ring[prod];
 580                 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 581                                skb_frag_size(&skb_shinfo(skb)->frags[i]),
 582                                PCI_DMA_TODEVICE);
 583         }
 584
 585         dev_kfree_skb_any(skb);
 586         return NETDEV_TX_OK;
 587 }
 588
 589 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 590 {
 591         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 592         struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
 593         u16 cons = txr->tx_cons;
 594         struct pci_dev *pdev = bp->pdev;
 595         int i;
 596         unsigned int tx_bytes = 0;
 597
 598         for (i = 0; i < nr_pkts; i++) {
 599                 struct bnxt_sw_tx_bd *tx_buf;
 600                 struct sk_buff *skb;
 601                 int j, last;
 602
 603                 tx_buf = &txr->tx_buf_ring[cons];
 604                 cons = NEXT_TX(cons);
 605                 skb = tx_buf->skb;
 606                 tx_buf->skb = NULL;
 607
 608                 if (tx_buf->is_push) {
 609                         tx_buf->is_push = 0;
 610                         goto next_tx_int;
 611                 }
 612
 613                 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
 614                                  skb_headlen(skb), PCI_DMA_TODEVICE);
 615                 last = tx_buf->nr_frags;
 616
 617                 for (j = 0; j < last; j++) {
 618                         cons = NEXT_TX(cons);
 619                         tx_buf = &txr->tx_buf_ring[cons];
 620                         dma_unmap_page(
 621                                 &pdev->dev,
 622                                 dma_unmap_addr(tx_buf, mapping),
 623                                 skb_frag_size(&skb_shinfo(skb)->frags[j]),
 624                                 PCI_DMA_TODEVICE);
 625                 }
 626
 627 next_tx_int:
 628                 cons = NEXT_TX(cons);
 629
 630                 tx_bytes += skb->len;
 631                 dev_kfree_skb_any(skb);
 632         }
 633
 634         netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
 635         txr->tx_cons = cons;
 636
 637         /* Need to make the tx_cons update visible to bnxt_start_xmit()
 638          * before checking for netif_tx_queue_stopped().  Without the
 639          * memory barrier, there is a small possibility that bnxt_start_xmit()
 640          * will miss it and cause the queue to be stopped forever.
 641          */
 642         smp_mb();
 643
 644         if (unlikely(netif_tx_queue_stopped(txq)) &&
 645             (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
 646                 __netif_tx_lock(txq, smp_processor_id());
 647                 if (netif_tx_queue_stopped(txq) &&
 648                     bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
 649                     txr->dev_state != BNXT_DEV_STATE_CLOSING)
 650                         netif_tx_wake_queue(txq);
 651                 __netif_tx_unlock(txq);
 652         }
 653 }
 654
 655 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 656                                          gfp_t gfp)
 657 {
 658         struct device *dev = &bp->pdev->dev;
 659         struct page *page;
 660
 661         page = alloc_page(gfp);
 662         if (!page)
 663                 return NULL;
 664
 665         *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
 666                                       DMA_ATTR_WEAK_ORDERING);
 667         if (dma_mapping_error(dev, *mapping)) {
 668                 __free_page(page);
 669                 return NULL;
 670         }
 671         *mapping += bp->rx_dma_offset;
 672         return page;
 673 }
 674
 675 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
 676                                        gfp_t gfp)
 677 {
 678         u8 *data;
 679         struct pci_dev *pdev = bp->pdev;
 680
 681         data = kmalloc(bp->rx_buf_size, gfp);
 682         if (!data)
 683                 return NULL;
 684
 685         *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
 686                                         bp->rx_buf_use_size, bp->rx_dir,
 687                                         DMA_ATTR_WEAK_ORDERING);
 688
 689         if (dma_mapping_error(&pdev->dev, *mapping)) {
 690                 kfree(data);
 691                 data = NULL;
 692         }
 693         return data;
 694 }
 695
 696 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 697                        u16 prod, gfp_t gfp)
 698 {
 699         struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 700         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
 701         dma_addr_t mapping;
 702
 703         if (BNXT_RX_PAGE_MODE(bp)) {
 704                 struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp);
 705
 706                 if (!page)
 707                         return -ENOMEM;
 708
 709                 rx_buf->data = page;
 710                 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
 711         } else {
 712                 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
 713
 714                 if (!data)
 715                         return -ENOMEM;
 716
 717                 rx_buf->data = data;
 718                 rx_buf->data_ptr = data + bp->rx_offset;
 719         }
 720         rx_buf->mapping = mapping;
 721
 722         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 723         return 0;
 724 }
 725
 726 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
 727 {
 728         u16 prod = rxr->rx_prod;
 729         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
 730         struct rx_bd *cons_bd, *prod_bd;
 731
 732         prod_rx_buf = &rxr->rx_buf_ring[prod];
 733         cons_rx_buf = &rxr->rx_buf_ring[cons];
 734
 735         prod_rx_buf->data = data;
 736         prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
 737
 738         prod_rx_buf->mapping = cons_rx_buf->mapping;
 739
 740         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 741         cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
 742
 743         prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
 744 }
 745
 746 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
 747 {
 748         u16 next, max = rxr->rx_agg_bmap_size;
 749
 750         next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
 751         if (next >= max)
 752                 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
 753         return next;
 754 }
 755
 756 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
 757                                      struct bnxt_rx_ring_info *rxr,
 758                                      u16 prod, gfp_t gfp)
 759 {
 760         struct rx_bd *rxbd =
 761                 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 762         struct bnxt_sw_rx_agg_bd *rx_agg_buf;
 763         struct pci_dev *pdev = bp->pdev;
 764         struct page *page;
 765         dma_addr_t mapping;
 766         u16 sw_prod = rxr->rx_sw_agg_prod;
 767         unsigned int offset = 0;
 768
 769         if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
 770                 page = rxr->rx_page;
 771                 if (!page) {
 772                         page = alloc_page(gfp);
 773                         if (!page)
 774                                 return -ENOMEM;
 775                         rxr->rx_page = page;
 776                         rxr->rx_page_offset = 0;
 777                 }
 778                 offset = rxr->rx_page_offset;
 779                 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
 780                 if (rxr->rx_page_offset == PAGE_SIZE)
 781                         rxr->rx_page = NULL;
 782                 else
 783                         get_page(page);
 784         } else {
 785                 page = alloc_page(gfp);
 786                 if (!page)
 787                         return -ENOMEM;
 788         }
 789
 790         mapping = dma_map_page_attrs(&pdev->dev, page, offset,
 791                                      BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
 792                                      DMA_ATTR_WEAK_ORDERING);
 793         if (dma_mapping_error(&pdev->dev, mapping)) {
 794                 __free_page(page);
 795                 return -EIO;
 796         }
 797
 798         if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 799                 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 800
 801         __set_bit(sw_prod, rxr->rx_agg_bmap);
 802         rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
 803         rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
 804
 805         rx_agg_buf->page = page;
 806         rx_agg_buf->offset = offset;
 807         rx_agg_buf->mapping = mapping;
 808         rxbd->rx_bd_haddr = cpu_to_le64(mapping);
 809         rxbd->rx_bd_opaque = sw_prod;
 810         return 0;
 811 }
 812
 813 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 cp_cons,
 814                                    u32 agg_bufs)
 815 {
 816         struct bnxt_napi *bnapi = cpr->bnapi;
 817         struct bnxt *bp = bnapi->bp;
 818         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 819         u16 prod = rxr->rx_agg_prod;
 820         u16 sw_prod = rxr->rx_sw_agg_prod;
 821         u32 i;
 822
 823         for (i = 0; i < agg_bufs; i++) {
 824                 u16 cons;
 825                 struct rx_agg_cmp *agg;
 826                 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
 827                 struct rx_bd *prod_bd;
 828                 struct page *page;
 829
 830                 agg = (struct rx_agg_cmp *)
 831                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 832                 cons = agg->rx_agg_cmp_opaque;
 833                 __clear_bit(cons, rxr->rx_agg_bmap);
 834
 835                 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
 836                         sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
 837
 838                 __set_bit(sw_prod, rxr->rx_agg_bmap);
 839                 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
 840                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 841
 842                 /* It is possible for sw_prod to be equal to cons, so
 843                  * set cons_rx_buf->page to NULL first.
 844                  */
 845                 page = cons_rx_buf->page;
 846                 cons_rx_buf->page = NULL;
 847                 prod_rx_buf->page = page;
 848                 prod_rx_buf->offset = cons_rx_buf->offset;
 849
 850                 prod_rx_buf->mapping = cons_rx_buf->mapping;
 851
 852                 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
 853
 854                 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
 855                 prod_bd->rx_bd_opaque = sw_prod;
 856
 857                 prod = NEXT_RX_AGG(prod);
 858                 sw_prod = NEXT_RX_AGG(sw_prod);
 859                 cp_cons = NEXT_CMP(cp_cons);
 860         }
 861         rxr->rx_agg_prod = prod;
 862         rxr->rx_sw_agg_prod = sw_prod;
 863 }
 864
 865 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
 866                                         struct bnxt_rx_ring_info *rxr,
 867                                         u16 cons, void *data, u8 *data_ptr,
 868                                         dma_addr_t dma_addr,
 869                                         unsigned int offset_and_len)
 870 {
 871         unsigned int payload = offset_and_len >> 16;
 872         unsigned int len = offset_and_len & 0xffff;
 873         struct skb_frag_struct *frag;
 874         struct page *page = data;
 875         u16 prod = rxr->rx_prod;
 876         struct sk_buff *skb;
 877         int off, err;
 878
 879         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 880         if (unlikely(err)) {
 881                 bnxt_reuse_rx_data(rxr, cons, data);
 882                 return NULL;
 883         }
 884         dma_addr -= bp->rx_dma_offset;
 885         dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
 886                              DMA_ATTR_WEAK_ORDERING);
 887
 888         if (unlikely(!payload))
 889                 payload = eth_get_headlen(data_ptr, len);
 890
 891         skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
 892         if (!skb) {
 893                 __free_page(page);
 894                 return NULL;
 895         }
 896
 897         off = (void *)data_ptr - page_address(page);
 898         skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
 899         memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
 900                payload + NET_IP_ALIGN);
 901
 902         frag = &skb_shinfo(skb)->frags[0];
 903         skb_frag_size_sub(frag, payload);
 904         frag->page_offset += payload;
 905         skb->data_len -= payload;
 906         skb->tail += payload;
 907
 908         return skb;
 909 }
 910
 911 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
 912                                    struct bnxt_rx_ring_info *rxr, u16 cons,
 913                                    void *data, u8 *data_ptr,
 914                                    dma_addr_t dma_addr,
 915                                    unsigned int offset_and_len)
 916 {
 917         u16 prod = rxr->rx_prod;
 918         struct sk_buff *skb;
 919         int err;
 920
 921         err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
 922         if (unlikely(err)) {
 923                 bnxt_reuse_rx_data(rxr, cons, data);
 924                 return NULL;
 925         }
 926
 927         skb = build_skb(data, 0);
 928         dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
 929                                bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
 930         if (!skb) {
 931                 kfree(data);
 932                 return NULL;
 933         }
 934
 935         skb_reserve(skb, bp->rx_offset);
 936         skb_put(skb, offset_and_len & 0xffff);
 937         return skb;
 938 }
 939
 940 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
 941                                      struct bnxt_cp_ring_info *cpr,
 942                                      struct sk_buff *skb, u16 cp_cons,
 943                                      u32 agg_bufs)
 944 {
 945         struct bnxt_napi *bnapi = cpr->bnapi;
 946         struct pci_dev *pdev = bp->pdev;
 947         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 948         u16 prod = rxr->rx_agg_prod;
 949         u32 i;
 950
 951         for (i = 0; i < agg_bufs; i++) {
 952                 u16 cons, frag_len;
 953                 struct rx_agg_cmp *agg;
 954                 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
 955                 struct page *page;
 956                 dma_addr_t mapping;
 957
 958                 agg = (struct rx_agg_cmp *)
 959                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
 960                 cons = agg->rx_agg_cmp_opaque;
 961                 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
 962                             RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
 963
 964                 cons_rx_buf = &rxr->rx_agg_ring[cons];
 965                 skb_fill_page_desc(skb, i, cons_rx_buf->page,
 966                                    cons_rx_buf->offset, frag_len);
 967                 __clear_bit(cons, rxr->rx_agg_bmap);
 968
 969                 /* It is possible for bnxt_alloc_rx_page() to allocate
 970                  * a sw_prod index that equals the cons index, so we
 971                  * need to clear the cons entry now.
 972                  */
 973                 mapping = cons_rx_buf->mapping;
 974                 page = cons_rx_buf->page;
 975                 cons_rx_buf->page = NULL;
 976
 977                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
 978                         struct skb_shared_info *shinfo;
 979                         unsigned int nr_frags;
 980
 981                         shinfo = skb_shinfo(skb);
 982                         nr_frags = --shinfo->nr_frags;
 983                         __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
 984
 985                         dev_kfree_skb(skb);
 986
 987                         cons_rx_buf->page = page;
 988
 989                         /* Update prod since possibly some pages have been
 990                          * allocated already.
 991                          */
 992                         rxr->rx_agg_prod = prod;
 993                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs - i);
 994                         return NULL;
 995                 }
 996
 997                 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
 998                                      PCI_DMA_FROMDEVICE,
 999                                      DMA_ATTR_WEAK_ORDERING);
1000
1001                 skb->data_len += frag_len;
1002                 skb->len += frag_len;
1003                 skb->truesize += PAGE_SIZE;
1004
1005                 prod = NEXT_RX_AGG(prod);
1006                 cp_cons = NEXT_CMP(cp_cons);
1007         }
1008         rxr->rx_agg_prod = prod;
1009         return skb;
1010 }
1011
1012 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1013                                u8 agg_bufs, u32 *raw_cons)
1014 {
1015         u16 last;
1016         struct rx_agg_cmp *agg;
1017
1018         *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1019         last = RING_CMP(*raw_cons);
1020         agg = (struct rx_agg_cmp *)
1021                 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1022         return RX_AGG_CMP_VALID(agg, *raw_cons);
1023 }
1024
1025 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1026                                             unsigned int len,
1027                                             dma_addr_t mapping)
1028 {
1029         struct bnxt *bp = bnapi->bp;
1030         struct pci_dev *pdev = bp->pdev;
1031         struct sk_buff *skb;
1032
1033         skb = napi_alloc_skb(&bnapi->napi, len);
1034         if (!skb)
1035                 return NULL;
1036
1037         dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1038                                 bp->rx_dir);
1039
1040         memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1041                len + NET_IP_ALIGN);
1042
1043         dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1044                                    bp->rx_dir);
1045
1046         skb_put(skb, len);
1047         return skb;
1048 }
1049
1050 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1051                            u32 *raw_cons, void *cmp)
1052 {
1053         struct rx_cmp *rxcmp = cmp;
1054         u32 tmp_raw_cons = *raw_cons;
1055         u8 cmp_type, agg_bufs = 0;
1056
1057         cmp_type = RX_CMP_TYPE(rxcmp);
1058
1059         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1060                 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1061                             RX_CMP_AGG_BUFS) >>
1062                            RX_CMP_AGG_BUFS_SHIFT;
1063         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1064                 struct rx_tpa_end_cmp *tpa_end = cmp;
1065
1066                 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1067                             RX_TPA_END_CMP_AGG_BUFS) >>
1068                            RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1069         }
1070
1071         if (agg_bufs) {
1072                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1073                         return -EBUSY;
1074         }
1075         *raw_cons = tmp_raw_cons;
1076         return 0;
1077 }
1078
1079 static void bnxt_queue_sp_work(struct bnxt *bp)
1080 {
1081         if (BNXT_PF(bp))
1082                 queue_work(bnxt_pf_wq, &bp->sp_task);
1083         else
1084                 schedule_work(&bp->sp_task);
1085 }
1086
1087 static void bnxt_cancel_sp_work(struct bnxt *bp)
1088 {
1089         if (BNXT_PF(bp))
1090                 flush_workqueue(bnxt_pf_wq);
1091         else
1092                 cancel_work_sync(&bp->sp_task);
1093 }
1094
1095 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1096 {
1097         if (!rxr->bnapi->in_reset) {
1098                 rxr->bnapi->in_reset = true;
1099                 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1100                 bnxt_queue_sp_work(bp);
1101         }
1102         rxr->rx_next_cons = 0xffff;
1103 }
1104
1105 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1106                            struct rx_tpa_start_cmp *tpa_start,
1107                            struct rx_tpa_start_cmp_ext *tpa_start1)
1108 {
1109         u8 agg_id = TPA_START_AGG_ID(tpa_start);
1110         u16 cons, prod;
1111         struct bnxt_tpa_info *tpa_info;
1112         struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1113         struct rx_bd *prod_bd;
1114         dma_addr_t mapping;
1115
1116         cons = tpa_start->rx_tpa_start_cmp_opaque;
1117         prod = rxr->rx_prod;
1118         cons_rx_buf = &rxr->rx_buf_ring[cons];
1119         prod_rx_buf = &rxr->rx_buf_ring[prod];
1120         tpa_info = &rxr->rx_tpa[agg_id];
1121
1122         if (unlikely(cons != rxr->rx_next_cons)) {
1123                 bnxt_sched_reset(bp, rxr);
1124                 return;
1125         }
1126         /* Store cfa_code in tpa_info to use in tpa_end
1127          * completion processing.
1128          */
1129         tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1130         prod_rx_buf->data = tpa_info->data;
1131         prod_rx_buf->data_ptr = tpa_info->data_ptr;
1132
1133         mapping = tpa_info->mapping;
1134         prod_rx_buf->mapping = mapping;
1135
1136         prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1137
1138         prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1139
1140         tpa_info->data = cons_rx_buf->data;
1141         tpa_info->data_ptr = cons_rx_buf->data_ptr;
1142         cons_rx_buf->data = NULL;
1143         tpa_info->mapping = cons_rx_buf->mapping;
1144
1145         tpa_info->len =
1146                 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1147                                 RX_TPA_START_CMP_LEN_SHIFT;
1148         if (likely(TPA_START_HASH_VALID(tpa_start))) {
1149                 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1150
1151                 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1152                 tpa_info->gso_type = SKB_GSO_TCPV4;
1153                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1154                 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1155                         tpa_info->gso_type = SKB_GSO_TCPV6;
1156                 tpa_info->rss_hash =
1157                         le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1158         } else {
1159                 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1160                 tpa_info->gso_type = 0;
1161                 if (netif_msg_rx_err(bp))
1162                         netdev_warn(bp->dev, "TPA packet without valid hash\n");
1163         }
1164         tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1165         tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1166         tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1167
1168         rxr->rx_prod = NEXT_RX(prod);
1169         cons = NEXT_RX(cons);
1170         rxr->rx_next_cons = NEXT_RX(cons);
1171         cons_rx_buf = &rxr->rx_buf_ring[cons];
1172
1173         bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1174         rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1175         cons_rx_buf->data = NULL;
1176 }
1177
1178 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 cp_cons,
1179                            u32 agg_bufs)
1180 {
1181         if (agg_bufs)
1182                 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs);
1183 }
1184
1185 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1186                                            int payload_off, int tcp_ts,
1187                                            struct sk_buff *skb)
1188 {
1189 #ifdef CONFIG_INET
1190         struct tcphdr *th;
1191         int len, nw_off;
1192         u16 outer_ip_off, inner_ip_off, inner_mac_off;
1193         u32 hdr_info = tpa_info->hdr_info;
1194         bool loopback = false;
1195
1196         inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1197         inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1198         outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1199
1200         /* If the packet is an internal loopback packet, the offsets will
1201          * have an extra 4 bytes.
1202          */
1203         if (inner_mac_off == 4) {
1204                 loopback = true;
1205         } else if (inner_mac_off > 4) {
1206                 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1207                                             ETH_HLEN - 2));
1208
1209                 /* We only support inner iPv4/ipv6.  If we don't see the
1210                  * correct protocol ID, it must be a loopback packet where
1211                  * the offsets are off by 4.
1212                  */
1213                 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1214                         loopback = true;
1215         }
1216         if (loopback) {
1217                 /* internal loopback packet, subtract all offsets by 4 */
1218                 inner_ip_off -= 4;
1219                 inner_mac_off -= 4;
1220                 outer_ip_off -= 4;
1221         }
1222
1223         nw_off = inner_ip_off - ETH_HLEN;
1224         skb_set_network_header(skb, nw_off);
1225         if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1226                 struct ipv6hdr *iph = ipv6_hdr(skb);
1227
1228                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1229                 len = skb->len - skb_transport_offset(skb);
1230                 th = tcp_hdr(skb);
1231                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1232         } else {
1233                 struct iphdr *iph = ip_hdr(skb);
1234
1235                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1236                 len = skb->len - skb_transport_offset(skb);
1237                 th = tcp_hdr(skb);
1238                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1239         }
1240
1241         if (inner_mac_off) { /* tunnel */
1242                 struct udphdr *uh = NULL;
1243                 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1244                                             ETH_HLEN - 2));
1245
1246                 if (proto == htons(ETH_P_IP)) {
1247                         struct iphdr *iph = (struct iphdr *)skb->data;
1248
1249                         if (iph->protocol == IPPROTO_UDP)
1250                                 uh = (struct udphdr *)(iph + 1);
1251                 } else {
1252                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1253
1254                         if (iph->nexthdr == IPPROTO_UDP)
1255                                 uh = (struct udphdr *)(iph + 1);
1256                 }
1257                 if (uh) {
1258                         if (uh->check)
1259                                 skb_shinfo(skb)->gso_type |=
1260                                         SKB_GSO_UDP_TUNNEL_CSUM;
1261                         else
1262                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1263                 }
1264         }
1265 #endif
1266         return skb;
1267 }
1268
1269 #define BNXT_IPV4_HDR_SIZE      (sizeof(struct iphdr) + sizeof(struct tcphdr))
1270 #define BNXT_IPV6_HDR_SIZE      (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1271
1272 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1273                                            int payload_off, int tcp_ts,
1274                                            struct sk_buff *skb)
1275 {
1276 #ifdef CONFIG_INET
1277         struct tcphdr *th;
1278         int len, nw_off, tcp_opt_len = 0;
1279
1280         if (tcp_ts)
1281                 tcp_opt_len = 12;
1282
1283         if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1284                 struct iphdr *iph;
1285
1286                 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1287                          ETH_HLEN;
1288                 skb_set_network_header(skb, nw_off);
1289                 iph = ip_hdr(skb);
1290                 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1291                 len = skb->len - skb_transport_offset(skb);
1292                 th = tcp_hdr(skb);
1293                 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1294         } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1295                 struct ipv6hdr *iph;
1296
1297                 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1298                          ETH_HLEN;
1299                 skb_set_network_header(skb, nw_off);
1300                 iph = ipv6_hdr(skb);
1301                 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1302                 len = skb->len - skb_transport_offset(skb);
1303                 th = tcp_hdr(skb);
1304                 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1305         } else {
1306                 dev_kfree_skb_any(skb);
1307                 return NULL;
1308         }
1309
1310         if (nw_off) { /* tunnel */
1311                 struct udphdr *uh = NULL;
1312
1313                 if (skb->protocol == htons(ETH_P_IP)) {
1314                         struct iphdr *iph = (struct iphdr *)skb->data;
1315
1316                         if (iph->protocol == IPPROTO_UDP)
1317                                 uh = (struct udphdr *)(iph + 1);
1318                 } else {
1319                         struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1320
1321                         if (iph->nexthdr == IPPROTO_UDP)
1322                                 uh = (struct udphdr *)(iph + 1);
1323                 }
1324                 if (uh) {
1325                         if (uh->check)
1326                                 skb_shinfo(skb)->gso_type |=
1327                                         SKB_GSO_UDP_TUNNEL_CSUM;
1328                         else
1329                                 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1330                 }
1331         }
1332 #endif
1333         return skb;
1334 }
1335
1336 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1337                                            struct bnxt_tpa_info *tpa_info,
1338                                            struct rx_tpa_end_cmp *tpa_end,
1339                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1340                                            struct sk_buff *skb)
1341 {
1342 #ifdef CONFIG_INET
1343         int payload_off;
1344         u16 segs;
1345
1346         segs = TPA_END_TPA_SEGS(tpa_end);
1347         if (segs == 1)
1348                 return skb;
1349
1350         NAPI_GRO_CB(skb)->count = segs;
1351         skb_shinfo(skb)->gso_size =
1352                 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1353         skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1354         payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1355                        RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
1356                       RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
1357         skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1358         if (likely(skb))
1359                 tcp_gro_complete(skb);
1360 #endif
1361         return skb;
1362 }
1363
1364 /* Given the cfa_code of a received packet determine which
1365  * netdev (vf-rep or PF) the packet is destined to.
1366  */
1367 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1368 {
1369         struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1370
1371         /* if vf-rep dev is NULL, the must belongs to the PF */
1372         return dev ? dev : bp->dev;
1373 }
1374
1375 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1376                                            struct bnxt_cp_ring_info *cpr,
1377                                            u32 *raw_cons,
1378                                            struct rx_tpa_end_cmp *tpa_end,
1379                                            struct rx_tpa_end_cmp_ext *tpa_end1,
1380                                            u8 *event)
1381 {
1382         struct bnxt_napi *bnapi = cpr->bnapi;
1383         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1384         u8 agg_id = TPA_END_AGG_ID(tpa_end);
1385         u8 *data_ptr, agg_bufs;
1386         u16 cp_cons = RING_CMP(*raw_cons);
1387         unsigned int len;
1388         struct bnxt_tpa_info *tpa_info;
1389         dma_addr_t mapping;
1390         struct sk_buff *skb;
1391         void *data;
1392
1393         if (unlikely(bnapi->in_reset)) {
1394                 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1395
1396                 if (rc < 0)
1397                         return ERR_PTR(-EBUSY);
1398                 return NULL;
1399         }
1400
1401         tpa_info = &rxr->rx_tpa[agg_id];
1402         data = tpa_info->data;
1403         data_ptr = tpa_info->data_ptr;
1404         prefetch(data_ptr);
1405         len = tpa_info->len;
1406         mapping = tpa_info->mapping;
1407
1408         agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1409                     RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1410
1411         if (agg_bufs) {
1412                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1413                         return ERR_PTR(-EBUSY);
1414
1415                 *event |= BNXT_AGG_EVENT;
1416                 cp_cons = NEXT_CMP(cp_cons);
1417         }
1418
1419         if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1420                 bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1421                 if (agg_bufs > MAX_SKB_FRAGS)
1422                         netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1423                                     agg_bufs, (int)MAX_SKB_FRAGS);
1424                 return NULL;
1425         }
1426
1427         if (len <= bp->rx_copy_thresh) {
1428                 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1429                 if (!skb) {
1430                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1431                         return NULL;
1432                 }
1433         } else {
1434                 u8 *new_data;
1435                 dma_addr_t new_mapping;
1436
1437                 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1438                 if (!new_data) {
1439                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1440                         return NULL;
1441                 }
1442
1443                 tpa_info->data = new_data;
1444                 tpa_info->data_ptr = new_data + bp->rx_offset;
1445                 tpa_info->mapping = new_mapping;
1446
1447                 skb = build_skb(data, 0);
1448                 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1449                                        bp->rx_buf_use_size, bp->rx_dir,
1450                                        DMA_ATTR_WEAK_ORDERING);
1451
1452                 if (!skb) {
1453                         kfree(data);
1454                         bnxt_abort_tpa(cpr, cp_cons, agg_bufs);
1455                         return NULL;
1456                 }
1457                 skb_reserve(skb, bp->rx_offset);
1458                 skb_put(skb, len);
1459         }
1460
1461         if (agg_bufs) {
1462                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs);
1463                 if (!skb) {
1464                         /* Page reuse already handled by bnxt_rx_pages(). */
1465                         return NULL;
1466                 }
1467         }
1468
1469         skb->protocol =
1470                 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1471
1472         if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1473                 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1474
1475         if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1476             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1477                 u16 vlan_proto = tpa_info->metadata >>
1478                         RX_CMP_FLAGS2_METADATA_TPID_SFT;
1479                 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1480
1481                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1482         }
1483
1484         skb_checksum_none_assert(skb);
1485         if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1486                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1487                 skb->csum_level =
1488                         (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1489         }
1490
1491         if (TPA_END_GRO(tpa_end))
1492                 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1493
1494         return skb;
1495 }
1496
1497 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1498                              struct sk_buff *skb)
1499 {
1500         if (skb->dev != bp->dev) {
1501                 /* this packet belongs to a vf-rep */
1502                 bnxt_vf_rep_rx(bp, skb);
1503                 return;
1504         }
1505         skb_record_rx_queue(skb, bnapi->index);
1506         napi_gro_receive(&bnapi->napi, skb);
1507 }
1508
1509 /* returns the following:
1510  * 1       - 1 packet successfully received
1511  * 0       - successful TPA_START, packet not completed yet
1512  * -EBUSY  - completion ring does not have all the agg buffers yet
1513  * -ENOMEM - packet aborted due to out of memory
1514  * -EIO    - packet aborted due to hw error indicated in BD
1515  */
1516 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1517                        u32 *raw_cons, u8 *event)
1518 {
1519         struct bnxt_napi *bnapi = cpr->bnapi;
1520         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1521         struct net_device *dev = bp->dev;
1522         struct rx_cmp *rxcmp;
1523         struct rx_cmp_ext *rxcmp1;
1524         u32 tmp_raw_cons = *raw_cons;
1525         u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1526         struct bnxt_sw_rx_bd *rx_buf;
1527         unsigned int len;
1528         u8 *data_ptr, agg_bufs, cmp_type;
1529         dma_addr_t dma_addr;
1530         struct sk_buff *skb;
1531         void *data;
1532         int rc = 0;
1533         u32 misc;
1534
1535         rxcmp = (struct rx_cmp *)
1536                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1537
1538         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1539         cp_cons = RING_CMP(tmp_raw_cons);
1540         rxcmp1 = (struct rx_cmp_ext *)
1541                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1542
1543         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1544                 return -EBUSY;
1545
1546         cmp_type = RX_CMP_TYPE(rxcmp);
1547
1548         prod = rxr->rx_prod;
1549
1550         if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1551                 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1552                                (struct rx_tpa_start_cmp_ext *)rxcmp1);
1553
1554                 *event |= BNXT_RX_EVENT;
1555                 goto next_rx_no_prod_no_len;
1556
1557         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1558                 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1559                                    (struct rx_tpa_end_cmp *)rxcmp,
1560                                    (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1561
1562                 if (IS_ERR(skb))
1563                         return -EBUSY;
1564
1565                 rc = -ENOMEM;
1566                 if (likely(skb)) {
1567                         bnxt_deliver_skb(bp, bnapi, skb);
1568                         rc = 1;
1569                 }
1570                 *event |= BNXT_RX_EVENT;
1571                 goto next_rx_no_prod_no_len;
1572         }
1573
1574         cons = rxcmp->rx_cmp_opaque;
1575         rx_buf = &rxr->rx_buf_ring[cons];
1576         data = rx_buf->data;
1577         data_ptr = rx_buf->data_ptr;
1578         if (unlikely(cons != rxr->rx_next_cons)) {
1579                 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1580
1581                 bnxt_sched_reset(bp, rxr);
1582                 return rc1;
1583         }
1584         prefetch(data_ptr);
1585
1586         misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1587         agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1588
1589         if (agg_bufs) {
1590                 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1591                         return -EBUSY;
1592
1593                 cp_cons = NEXT_CMP(cp_cons);
1594                 *event |= BNXT_AGG_EVENT;
1595         }
1596         *event |= BNXT_RX_EVENT;
1597
1598         rx_buf->data = NULL;
1599         if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1600                 bnxt_reuse_rx_data(rxr, cons, data);
1601                 if (agg_bufs)
1602                         bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs);
1603
1604                 rc = -EIO;
1605                 goto next_rx;
1606         }
1607
1608         len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1609         dma_addr = rx_buf->mapping;
1610
1611         if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1612                 rc = 1;
1613                 goto next_rx;
1614         }
1615
1616         if (len <= bp->rx_copy_thresh) {
1617                 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1618                 bnxt_reuse_rx_data(rxr, cons, data);
1619                 if (!skb) {
1620                         rc = -ENOMEM;
1621                         goto next_rx;
1622                 }
1623         } else {
1624                 u32 payload;
1625
1626                 if (rx_buf->data_ptr == data_ptr)
1627                         payload = misc & RX_CMP_PAYLOAD_OFFSET;
1628                 else
1629                         payload = 0;
1630                 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1631                                       payload | len);
1632                 if (!skb) {
1633                         rc = -ENOMEM;
1634                         goto next_rx;
1635                 }
1636         }
1637
1638         if (agg_bufs) {
1639                 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs);
1640                 if (!skb) {
1641                         rc = -ENOMEM;
1642                         goto next_rx;
1643                 }
1644         }
1645
1646         if (RX_CMP_HASH_VALID(rxcmp)) {
1647                 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1648                 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1649
1650                 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1651                 if (hash_type != 1 && hash_type != 3)
1652                         type = PKT_HASH_TYPE_L3;
1653                 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1654         }
1655
1656         cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1657         skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1658
1659         if ((rxcmp1->rx_cmp_flags2 &
1660              cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1661             (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1662                 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1663                 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1664                 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1665
1666                 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1667         }
1668
1669         skb_checksum_none_assert(skb);
1670         if (RX_CMP_L4_CS_OK(rxcmp1)) {
1671                 if (dev->features & NETIF_F_RXCSUM) {
1672                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1673                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1674                 }
1675         } else {
1676                 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1677                         if (dev->features & NETIF_F_RXCSUM)
1678                                 bnapi->cp_ring.rx_l4_csum_errors++;
1679                 }
1680         }
1681
1682         bnxt_deliver_skb(bp, bnapi, skb);
1683         rc = 1;
1684
1685 next_rx:
1686         rxr->rx_prod = NEXT_RX(prod);
1687         rxr->rx_next_cons = NEXT_RX(cons);
1688
1689         cpr->rx_packets += 1;
1690         cpr->rx_bytes += len;
1691
1692 next_rx_no_prod_no_len:
1693         *raw_cons = tmp_raw_cons;
1694
1695         return rc;
1696 }
1697
1698 /* In netpoll mode, if we are using a combined completion ring, we need to
1699  * discard the rx packets and recycle the buffers.
1700  */
1701 static int bnxt_force_rx_discard(struct bnxt *bp,
1702                                  struct bnxt_cp_ring_info *cpr,
1703                                  u32 *raw_cons, u8 *event)
1704 {
1705         u32 tmp_raw_cons = *raw_cons;
1706         struct rx_cmp_ext *rxcmp1;
1707         struct rx_cmp *rxcmp;
1708         u16 cp_cons;
1709         u8 cmp_type;
1710
1711         cp_cons = RING_CMP(tmp_raw_cons);
1712         rxcmp = (struct rx_cmp *)
1713                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1714
1715         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1716         cp_cons = RING_CMP(tmp_raw_cons);
1717         rxcmp1 = (struct rx_cmp_ext *)
1718                         &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1719
1720         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1721                 return -EBUSY;
1722
1723         cmp_type = RX_CMP_TYPE(rxcmp);
1724         if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1725                 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1726                         cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1727         } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1728                 struct rx_tpa_end_cmp_ext *tpa_end1;
1729
1730                 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1731                 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1732                         cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1733         }
1734         return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1735 }
1736
1737 #define BNXT_GET_EVENT_PORT(data)       \
1738         ((data) &                       \
1739          ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1740
1741 static int bnxt_async_event_process(struct bnxt *bp,
1742                                     struct hwrm_async_event_cmpl *cmpl)
1743 {
1744         u16 event_id = le16_to_cpu(cmpl->event_id);
1745
1746         /* TODO CHIMP_FW: Define event id's for link change, error etc */
1747         switch (event_id) {
1748         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1749                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1750                 struct bnxt_link_info *link_info = &bp->link_info;
1751
1752                 if (BNXT_VF(bp))
1753                         goto async_event_process_exit;
1754
1755                 /* print unsupported speed warning in forced speed mode only */
1756                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1757                     (data1 & 0x20000)) {
1758                         u16 fw_speed = link_info->force_link_speed;
1759                         u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1760
1761                         if (speed != SPEED_UNKNOWN)
1762                                 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1763                                             speed);
1764                 }
1765                 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1766         }
1767         /* fall through */
1768         case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1769                 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1770                 break;
1771         case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1772                 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1773                 break;
1774         case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1775                 u32 data1 = le32_to_cpu(cmpl->event_data1);
1776                 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1777
1778                 if (BNXT_VF(bp))
1779                         break;
1780
1781                 if (bp->pf.port_id != port_id)
1782                         break;
1783
1784                 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1785                 break;
1786         }
1787         case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1788                 if (BNXT_PF(bp))
1789                         goto async_event_process_exit;
1790                 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1791                 break;
1792         default:
1793                 goto async_event_process_exit;
1794         }
1795         bnxt_queue_sp_work(bp);
1796 async_event_process_exit:
1797         bnxt_ulp_async_events(bp, cmpl);
1798         return 0;
1799 }
1800
1801 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1802 {
1803         u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1804         struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1805         struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1806                                 (struct hwrm_fwd_req_cmpl *)txcmp;
1807
1808         switch (cmpl_type) {
1809         case CMPL_BASE_TYPE_HWRM_DONE:
1810                 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1811                 if (seq_id == bp->hwrm_intr_seq_id)
1812                         bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1813                 else
1814                         netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1815                 break;
1816
1817         case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1818                 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1819
1820                 if ((vf_id < bp->pf.first_vf_id) ||
1821                     (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1822                         netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1823                                    vf_id);
1824                         return -EINVAL;
1825                 }
1826
1827                 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1828                 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1829                 bnxt_queue_sp_work(bp);
1830                 break;
1831
1832         case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1833                 bnxt_async_event_process(bp,
1834                                          (struct hwrm_async_event_cmpl *)txcmp);
1835
1836         default:
1837                 break;
1838         }
1839
1840         return 0;
1841 }
1842
1843 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1844 {
1845         struct bnxt_napi *bnapi = dev_instance;
1846         struct bnxt *bp = bnapi->bp;
1847         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1848         u32 cons = RING_CMP(cpr->cp_raw_cons);
1849
1850         cpr->event_ctr++;
1851         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1852         napi_schedule(&bnapi->napi);
1853         return IRQ_HANDLED;
1854 }
1855
1856 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1857 {
1858         u32 raw_cons = cpr->cp_raw_cons;
1859         u16 cons = RING_CMP(raw_cons);
1860         struct tx_cmp *txcmp;
1861
1862         txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1863
1864         return TX_CMP_VALID(txcmp, raw_cons);
1865 }
1866
1867 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1868 {
1869         struct bnxt_napi *bnapi = dev_instance;
1870         struct bnxt *bp = bnapi->bp;
1871         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1872         u32 cons = RING_CMP(cpr->cp_raw_cons);
1873         u32 int_status;
1874
1875         prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1876
1877         if (!bnxt_has_work(bp, cpr)) {
1878                 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1879                 /* return if erroneous interrupt */
1880                 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1881                         return IRQ_NONE;
1882         }
1883
1884         /* disable ring IRQ */
1885         BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
1886
1887         /* Return here if interrupt is shared and is disabled. */
1888         if (unlikely(atomic_read(&bp->intr_sem) != 0))
1889                 return IRQ_HANDLED;
1890
1891         napi_schedule(&bnapi->napi);
1892         return IRQ_HANDLED;
1893 }
1894
1895 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1896                             int budget)
1897 {
1898         struct bnxt_napi *bnapi = cpr->bnapi;
1899         u32 raw_cons = cpr->cp_raw_cons;
1900         u32 cons;
1901         int tx_pkts = 0;
1902         int rx_pkts = 0;
1903         u8 event = 0;
1904         struct tx_cmp *txcmp;
1905
1906         cpr->has_more_work = 0;
1907         while (1) {
1908                 int rc;
1909
1910                 cons = RING_CMP(raw_cons);
1911                 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1912
1913                 if (!TX_CMP_VALID(txcmp, raw_cons))
1914                         break;
1915
1916                 /* The valid test of the entry must be done first before
1917                  * reading any further.
1918                  */
1919                 dma_rmb();
1920                 cpr->had_work_done = 1;
1921                 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1922                         tx_pkts++;
1923                         /* return full budget so NAPI will complete. */
1924                         if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
1925                                 rx_pkts = budget;
1926                                 raw_cons = NEXT_RAW_CMP(raw_cons);
1927                                 if (budget)
1928                                         cpr->has_more_work = 1;
1929                                 break;
1930                         }
1931                 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1932                         if (likely(budget))
1933                                 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
1934                         else
1935                                 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
1936                                                            &event);
1937                         if (likely(rc >= 0))
1938                                 rx_pkts += rc;
1939                         /* Increment rx_pkts when rc is -ENOMEM to count towards
1940                          * the NAPI budget.  Otherwise, we may potentially loop
1941                          * here forever if we consistently cannot allocate
1942                          * buffers.
1943                          */
1944                         else if (rc == -ENOMEM && budget)
1945                                 rx_pkts++;
1946                         else if (rc == -EBUSY)  /* partial completion */
1947                                 break;
1948                 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1949                                      CMPL_BASE_TYPE_HWRM_DONE) ||
1950                                     (TX_CMP_TYPE(txcmp) ==
1951                                      CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1952                                     (TX_CMP_TYPE(txcmp) ==
1953                                      CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1954                         bnxt_hwrm_handler(bp, txcmp);
1955                 }
1956                 raw_cons = NEXT_RAW_CMP(raw_cons);
1957
1958                 if (rx_pkts && rx_pkts == budget) {
1959                         cpr->has_more_work = 1;
1960                         break;
1961                 }
1962         }
1963
1964         if (event & BNXT_TX_EVENT) {
1965                 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
1966                 u16 prod = txr->tx_prod;
1967
1968                 /* Sync BD data before updating doorbell */
1969                 wmb();
1970
1971                 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
1972         }
1973
1974         cpr->cp_raw_cons = raw_cons;
1975         bnapi->tx_pkts += tx_pkts;
1976         bnapi->events |= event;
1977         return rx_pkts;
1978 }
1979
1980 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
1981 {
1982         if (bnapi->tx_pkts) {
1983                 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
1984                 bnapi->tx_pkts = 0;
1985         }
1986
1987         if (bnapi->events & BNXT_RX_EVENT) {
1988                 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1989
1990                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
1991                 if (bnapi->events & BNXT_AGG_EVENT)
1992                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
1993         }
1994         bnapi->events = 0;
1995 }
1996
1997 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1998                           int budget)
1999 {
2000         struct bnxt_napi *bnapi = cpr->bnapi;
2001         int rx_pkts;
2002
2003         rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2004
2005         /* ACK completion ring before freeing tx ring and producing new
2006          * buffers in rx/agg rings to prevent overflowing the completion
2007          * ring.
2008          */
2009         bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2010
2011         __bnxt_poll_work_done(bp, bnapi);
2012         return rx_pkts;
2013 }
2014
2015 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2016 {
2017         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2018         struct bnxt *bp = bnapi->bp;
2019         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2020         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2021         struct tx_cmp *txcmp;
2022         struct rx_cmp_ext *rxcmp1;
2023         u32 cp_cons, tmp_raw_cons;
2024         u32 raw_cons = cpr->cp_raw_cons;
2025         u32 rx_pkts = 0;
2026         u8 event = 0;
2027
2028         while (1) {
2029                 int rc;
2030
2031                 cp_cons = RING_CMP(raw_cons);
2032                 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2033
2034                 if (!TX_CMP_VALID(txcmp, raw_cons))
2035                         break;
2036
2037                 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2038                         tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2039                         cp_cons = RING_CMP(tmp_raw_cons);
2040                         rxcmp1 = (struct rx_cmp_ext *)
2041                           &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2042
2043                         if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2044                                 break;
2045
2046                         /* force an error to recycle the buffer */
2047                         rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2048                                 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2049
2050                         rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2051                         if (likely(rc == -EIO) && budget)
2052                                 rx_pkts++;
2053                         else if (rc == -EBUSY)  /* partial completion */
2054                                 break;
2055                 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2056                                     CMPL_BASE_TYPE_HWRM_DONE)) {
2057                         bnxt_hwrm_handler(bp, txcmp);
2058                 } else {
2059                         netdev_err(bp->dev,
2060                                    "Invalid completion received on special ring\n");
2061                 }
2062                 raw_cons = NEXT_RAW_CMP(raw_cons);
2063
2064                 if (rx_pkts == budget)
2065                         break;
2066         }
2067
2068         cpr->cp_raw_cons = raw_cons;
2069         BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2070         bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2071
2072         if (event & BNXT_AGG_EVENT)
2073                 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2074
2075         if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2076                 napi_complete_done(napi, rx_pkts);
2077                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2078         }
2079         return rx_pkts;
2080 }
2081
2082 static int bnxt_poll(struct napi_struct *napi, int budget)
2083 {
2084         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2085         struct bnxt *bp = bnapi->bp;
2086         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2087         int work_done = 0;
2088
2089         while (1) {
2090                 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2091
2092                 if (work_done >= budget) {
2093                         if (!budget)
2094                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2095                         break;
2096                 }
2097
2098                 if (!bnxt_has_work(bp, cpr)) {
2099                         if (napi_complete_done(napi, work_done))
2100                                 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2101                         break;
2102                 }
2103         }
2104         if (bp->flags & BNXT_FLAG_DIM) {
2105                 struct net_dim_sample dim_sample;
2106
2107                 net_dim_sample(cpr->event_ctr,
2108                                cpr->rx_packets,
2109                                cpr->rx_bytes,
2110                                &dim_sample);
2111                 net_dim(&cpr->dim, dim_sample);
2112         }
2113         mmiowb();
2114         return work_done;
2115 }
2116
2117 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2118 {
2119         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2120         int i, work_done = 0;
2121
2122         for (i = 0; i < 2; i++) {
2123                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2124
2125                 if (cpr2) {
2126                         work_done += __bnxt_poll_work(bp, cpr2,
2127                                                       budget - work_done);
2128                         cpr->has_more_work |= cpr2->has_more_work;
2129                 }
2130         }
2131         return work_done;
2132 }
2133
2134 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2135                                  u64 dbr_type, bool all)
2136 {
2137         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2138         int i;
2139
2140         for (i = 0; i < 2; i++) {
2141                 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2142                 struct bnxt_db_info *db;
2143
2144                 if (cpr2 && (all || cpr2->had_work_done)) {
2145                         db = &cpr2->cp_db;
2146                         writeq(db->db_key64 | dbr_type |
2147                                RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2148                         cpr2->had_work_done = 0;
2149                 }
2150         }
2151         __bnxt_poll_work_done(bp, bnapi);
2152 }
2153
2154 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2155 {
2156         struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2157         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2158         u32 raw_cons = cpr->cp_raw_cons;
2159         struct bnxt *bp = bnapi->bp;
2160         struct nqe_cn *nqcmp;
2161         int work_done = 0;
2162         u32 cons;
2163
2164         if (cpr->has_more_work) {
2165                 cpr->has_more_work = 0;
2166                 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2167                 if (cpr->has_more_work) {
2168                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2169                         return work_done;
2170                 }
2171                 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2172                 if (napi_complete_done(napi, work_done))
2173                         BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2174                 return work_done;
2175         }
2176         while (1) {
2177                 cons = RING_CMP(raw_cons);
2178                 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2179
2180                 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2181                         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2182                                              false);
2183                         cpr->cp_raw_cons = raw_cons;
2184                         if (napi_complete_done(napi, work_done))
2185                                 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2186                                                   cpr->cp_raw_cons);
2187                         return work_done;
2188                 }
2189
2190                 /* The valid test of the entry must be done first before
2191                  * reading any further.
2192                  */
2193                 dma_rmb();
2194
2195                 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2196                         u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2197                         struct bnxt_cp_ring_info *cpr2;
2198
2199                         cpr2 = cpr->cp_ring_arr[idx];
2200                         work_done += __bnxt_poll_work(bp, cpr2,
2201                                                       budget - work_done);
2202                         cpr->has_more_work = cpr2->has_more_work;
2203                 } else {
2204                         bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2205                 }
2206                 raw_cons = NEXT_RAW_CMP(raw_cons);
2207                 if (cpr->has_more_work)
2208                         break;
2209         }
2210         __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2211         cpr->cp_raw_cons = raw_cons;
2212         return work_done;
2213 }
2214
2215 static void bnxt_free_tx_skbs(struct bnxt *bp)
2216 {
2217         int i, max_idx;
2218         struct pci_dev *pdev = bp->pdev;
2219
2220         if (!bp->tx_ring)
2221                 return;
2222
2223         max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2224         for (i = 0; i < bp->tx_nr_rings; i++) {
2225                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2226                 int j;
2227
2228                 for (j = 0; j < max_idx;) {
2229                         struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2230                         struct sk_buff *skb = tx_buf->skb;
2231                         int k, last;
2232
2233                         if (!skb) {
2234                                 j++;
2235                                 continue;
2236                         }
2237
2238                         tx_buf->skb = NULL;
2239
2240                         if (tx_buf->is_push) {
2241                                 dev_kfree_skb(skb);
2242                                 j += 2;
2243                                 continue;
2244                         }
2245
2246                         dma_unmap_single(&pdev->dev,
2247                                          dma_unmap_addr(tx_buf, mapping),
2248                                          skb_headlen(skb),
2249                                          PCI_DMA_TODEVICE);
2250
2251                         last = tx_buf->nr_frags;
2252                         j += 2;
2253                         for (k = 0; k < last; k++, j++) {
2254                                 int ring_idx = j & bp->tx_ring_mask;
2255                                 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2256
2257                                 tx_buf = &txr->tx_buf_ring[ring_idx];
2258                                 dma_unmap_page(
2259                                         &pdev->dev,
2260                                         dma_unmap_addr(tx_buf, mapping),
2261                                         skb_frag_size(frag), PCI_DMA_TODEVICE);
2262                         }
2263                         dev_kfree_skb(skb);
2264                 }
2265                 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2266         }
2267 }
2268
2269 static void bnxt_free_rx_skbs(struct bnxt *bp)
2270 {
2271         int i, max_idx, max_agg_idx;
2272         struct pci_dev *pdev = bp->pdev;
2273
2274         if (!bp->rx_ring)
2275                 return;
2276
2277         max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2278         max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2279         for (i = 0; i < bp->rx_nr_rings; i++) {
2280                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2281                 int j;
2282
2283                 if (rxr->rx_tpa) {
2284                         for (j = 0; j < MAX_TPA; j++) {
2285                                 struct bnxt_tpa_info *tpa_info =
2286                                                         &rxr->rx_tpa[j];
2287                                 u8 *data = tpa_info->data;
2288
2289                                 if (!data)
2290                                         continue;
2291
2292                                 dma_unmap_single_attrs(&pdev->dev,
2293                                                        tpa_info->mapping,
2294                                                        bp->rx_buf_use_size,
2295                                                        bp->rx_dir,
2296                                                        DMA_ATTR_WEAK_ORDERING);
2297
2298                                 tpa_info->data = NULL;
2299
2300                                 kfree(data);
2301                         }
2302                 }
2303
2304                 for (j = 0; j < max_idx; j++) {
2305                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2306                         dma_addr_t mapping = rx_buf->mapping;
2307                         void *data = rx_buf->data;
2308
2309                         if (!data)
2310                                 continue;
2311
2312                         rx_buf->data = NULL;
2313
2314                         if (BNXT_RX_PAGE_MODE(bp)) {
2315                                 mapping -= bp->rx_dma_offset;
2316                                 dma_unmap_page_attrs(&pdev->dev, mapping,
2317                                                      PAGE_SIZE, bp->rx_dir,
2318                                                      DMA_ATTR_WEAK_ORDERING);
2319                                 __free_page(data);
2320                         } else {
2321                                 dma_unmap_single_attrs(&pdev->dev, mapping,
2322                                                        bp->rx_buf_use_size,
2323                                                        bp->rx_dir,
2324                                                        DMA_ATTR_WEAK_ORDERING);
2325                                 kfree(data);
2326                         }
2327                 }
2328
2329                 for (j = 0; j < max_agg_idx; j++) {
2330                         struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2331                                 &rxr->rx_agg_ring[j];
2332                         struct page *page = rx_agg_buf->page;
2333
2334                         if (!page)
2335                                 continue;
2336
2337                         dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2338                                              BNXT_RX_PAGE_SIZE,
2339                                              PCI_DMA_FROMDEVICE,
2340                                              DMA_ATTR_WEAK_ORDERING);
2341
2342                         rx_agg_buf->page = NULL;
2343                         __clear_bit(j, rxr->rx_agg_bmap);
2344
2345                         __free_page(page);
2346                 }
2347                 if (rxr->rx_page) {
2348                         __free_page(rxr->rx_page);
2349                         rxr->rx_page = NULL;
2350                 }
2351         }
2352 }
2353
2354 static void bnxt_free_skbs(struct bnxt *bp)
2355 {
2356         bnxt_free_tx_skbs(bp);
2357         bnxt_free_rx_skbs(bp);
2358 }
2359
2360 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2361 {
2362         struct pci_dev *pdev = bp->pdev;
2363         int i;
2364
2365         for (i = 0; i < rmem->nr_pages; i++) {
2366                 if (!rmem->pg_arr[i])
2367                         continue;
2368
2369                 dma_free_coherent(&pdev->dev, rmem->page_size,
2370                                   rmem->pg_arr[i], rmem->dma_arr[i]);
2371
2372                 rmem->pg_arr[i] = NULL;
2373         }
2374         if (rmem->pg_tbl) {
2375                 dma_free_coherent(&pdev->dev, rmem->nr_pages * 8,
2376                                   rmem->pg_tbl, rmem->pg_tbl_map);
2377                 rmem->pg_tbl = NULL;
2378         }
2379         if (rmem->vmem_size && *rmem->vmem) {
2380                 vfree(*rmem->vmem);
2381                 *rmem->vmem = NULL;
2382         }
2383 }
2384
2385 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2386 {
2387         struct pci_dev *pdev = bp->pdev;
2388         u64 valid_bit = 0;
2389         int i;
2390
2391         if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2392                 valid_bit = PTU_PTE_VALID;
2393         if (rmem->nr_pages > 1) {
2394                 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev,
2395                                                   rmem->nr_pages * 8,
2396                                                   &rmem->pg_tbl_map,
2397                                                   GFP_KERNEL);
2398                 if (!rmem->pg_tbl)
2399                         return -ENOMEM;
2400         }
2401
2402         for (i = 0; i < rmem->nr_pages; i++) {
2403                 u64 extra_bits = valid_bit;
2404
2405                 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2406                                                      rmem->page_size,
2407                                                      &rmem->dma_arr[i],
2408                                                      GFP_KERNEL);
2409                 if (!rmem->pg_arr[i])
2410                         return -ENOMEM;
2411
2412                 if (rmem->nr_pages > 1) {
2413                         if (i == rmem->nr_pages - 2 &&
2414                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2415                                 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2416                         else if (i == rmem->nr_pages - 1 &&
2417                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2418                                 extra_bits |= PTU_PTE_LAST;
2419                         rmem->pg_tbl[i] =
2420                                 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2421                 }
2422         }
2423
2424         if (rmem->vmem_size) {
2425                 *rmem->vmem = vzalloc(rmem->vmem_size);
2426                 if (!(*rmem->vmem))
2427                         return -ENOMEM;
2428         }
2429         return 0;
2430 }
2431
2432 static void bnxt_free_rx_rings(struct bnxt *bp)
2433 {
2434         int i;
2435
2436         if (!bp->rx_ring)
2437                 return;
2438
2439         for (i = 0; i < bp->rx_nr_rings; i++) {
2440                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2441                 struct bnxt_ring_struct *ring;
2442
2443                 if (rxr->xdp_prog)
2444                         bpf_prog_put(rxr->xdp_prog);
2445
2446                 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2447                         xdp_rxq_info_unreg(&rxr->xdp_rxq);
2448
2449                 kfree(rxr->rx_tpa);
2450                 rxr->rx_tpa = NULL;
2451
2452                 kfree(rxr->rx_agg_bmap);
2453                 rxr->rx_agg_bmap = NULL;
2454
2455                 ring = &rxr->rx_ring_struct;
2456                 bnxt_free_ring(bp, &ring->ring_mem);
2457
2458                 ring = &rxr->rx_agg_ring_struct;
2459                 bnxt_free_ring(bp, &ring->ring_mem);
2460         }
2461 }
2462
2463 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2464 {
2465         int i, rc, agg_rings = 0, tpa_rings = 0;
2466
2467         if (!bp->rx_ring)
2468                 return -ENOMEM;
2469
2470         if (bp->flags & BNXT_FLAG_AGG_RINGS)
2471                 agg_rings = 1;
2472
2473         if (bp->flags & BNXT_FLAG_TPA)
2474                 tpa_rings = 1;
2475
2476         for (i = 0; i < bp->rx_nr_rings; i++) {
2477                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2478                 struct bnxt_ring_struct *ring;
2479
2480                 ring = &rxr->rx_ring_struct;
2481
2482                 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2483                 if (rc < 0)
2484                         return rc;
2485
2486                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2487                 if (rc)
2488                         return rc;
2489
2490                 ring->grp_idx = i;
2491                 if (agg_rings) {
2492                         u16 mem_size;
2493
2494                         ring = &rxr->rx_agg_ring_struct;
2495                         rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2496                         if (rc)
2497                                 return rc;
2498
2499                         ring->grp_idx = i;
2500                         rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2501                         mem_size = rxr->rx_agg_bmap_size / 8;
2502                         rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2503                         if (!rxr->rx_agg_bmap)
2504                                 return -ENOMEM;
2505
2506                         if (tpa_rings) {
2507                                 rxr->rx_tpa = kcalloc(MAX_TPA,
2508                                                 sizeof(struct bnxt_tpa_info),
2509                                                 GFP_KERNEL);
2510                                 if (!rxr->rx_tpa)
2511                                         return -ENOMEM;
2512                         }
2513                 }
2514         }
2515         return 0;
2516 }
2517
2518 static void bnxt_free_tx_rings(struct bnxt *bp)
2519 {
2520         int i;
2521         struct pci_dev *pdev = bp->pdev;
2522
2523         if (!bp->tx_ring)
2524                 return;
2525
2526         for (i = 0; i < bp->tx_nr_rings; i++) {
2527                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2528                 struct bnxt_ring_struct *ring;
2529
2530                 if (txr->tx_push) {
2531                         dma_free_coherent(&pdev->dev, bp->tx_push_size,
2532                                           txr->tx_push, txr->tx_push_mapping);
2533                         txr->tx_push = NULL;
2534                 }
2535
2536                 ring = &txr->tx_ring_struct;
2537
2538                 bnxt_free_ring(bp, &ring->ring_mem);
2539         }
2540 }
2541
2542 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2543 {
2544         int i, j, rc;
2545         struct pci_dev *pdev = bp->pdev;
2546
2547         bp->tx_push_size = 0;
2548         if (bp->tx_push_thresh) {
2549                 int push_size;
2550
2551                 push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2552                                         bp->tx_push_thresh);
2553
2554                 if (push_size > 256) {
2555                         push_size = 0;
2556                         bp->tx_push_thresh = 0;
2557                 }
2558
2559                 bp->tx_push_size = push_size;
2560         }
2561
2562         for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2563                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2564                 struct bnxt_ring_struct *ring;
2565                 u8 qidx;
2566
2567                 ring = &txr->tx_ring_struct;
2568
2569                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2570                 if (rc)
2571                         return rc;
2572
2573                 ring->grp_idx = txr->bnapi->index;
2574                 if (bp->tx_push_size) {
2575                         dma_addr_t mapping;
2576
2577                         /* One pre-allocated DMA buffer to backup
2578                          * TX push operation
2579                          */
2580                         txr->tx_push = dma_alloc_coherent(&pdev->dev,
2581                                                 bp->tx_push_size,
2582                                                 &txr->tx_push_mapping,
2583                                                 GFP_KERNEL);
2584
2585                         if (!txr->tx_push)
2586                                 return -ENOMEM;
2587
2588                         mapping = txr->tx_push_mapping +
2589                                 sizeof(struct tx_push_bd);
2590                         txr->data_mapping = cpu_to_le64(mapping);
2591
2592                         memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
2593                 }
2594                 qidx = bp->tc_to_qidx[j];
2595                 ring->queue_id = bp->q_info[qidx].queue_id;
2596                 if (i < bp->tx_nr_rings_xdp)
2597                         continue;
2598                 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2599                         j++;
2600         }
2601         return 0;
2602 }
2603
2604 static void bnxt_free_cp_rings(struct bnxt *bp)
2605 {
2606         int i;
2607
2608         if (!bp->bnapi)
2609                 return;
2610
2611         for (i = 0; i < bp->cp_nr_rings; i++) {
2612                 struct bnxt_napi *bnapi = bp->bnapi[i];
2613                 struct bnxt_cp_ring_info *cpr;
2614                 struct bnxt_ring_struct *ring;
2615                 int j;
2616
2617                 if (!bnapi)
2618                         continue;
2619
2620                 cpr = &bnapi->cp_ring;
2621                 ring = &cpr->cp_ring_struct;
2622
2623                 bnxt_free_ring(bp, &ring->ring_mem);
2624
2625                 for (j = 0; j < 2; j++) {
2626                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2627
2628                         if (cpr2) {
2629                                 ring = &cpr2->cp_ring_struct;
2630                                 bnxt_free_ring(bp, &ring->ring_mem);
2631                                 kfree(cpr2);
2632                                 cpr->cp_ring_arr[j] = NULL;
2633                         }
2634                 }
2635         }
2636 }
2637
2638 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
2639 {
2640         struct bnxt_ring_mem_info *rmem;
2641         struct bnxt_ring_struct *ring;
2642         struct bnxt_cp_ring_info *cpr;
2643         int rc;
2644
2645         cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
2646         if (!cpr)
2647                 return NULL;
2648
2649         ring = &cpr->cp_ring_struct;
2650         rmem = &ring->ring_mem;
2651         rmem->nr_pages = bp->cp_nr_pages;
2652         rmem->page_size = HW_CMPD_RING_SIZE;
2653         rmem->pg_arr = (void **)cpr->cp_desc_ring;
2654         rmem->dma_arr = cpr->cp_desc_mapping;
2655         rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
2656         rc = bnxt_alloc_ring(bp, rmem);
2657         if (rc) {
2658                 bnxt_free_ring(bp, rmem);
2659                 kfree(cpr);
2660                 cpr = NULL;
2661         }
2662         return cpr;
2663 }
2664
2665 static int bnxt_alloc_cp_rings(struct bnxt *bp)
2666 {
2667         bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
2668         int i, rc, ulp_base_vec, ulp_msix;
2669
2670         ulp_msix = bnxt_get_ulp_msix_num(bp);
2671         ulp_base_vec = bnxt_get_ulp_msix_base(bp);
2672         for (i = 0; i < bp->cp_nr_rings; i++) {
2673                 struct bnxt_napi *bnapi = bp->bnapi[i];
2674                 struct bnxt_cp_ring_info *cpr;
2675                 struct bnxt_ring_struct *ring;
2676
2677                 if (!bnapi)
2678                         continue;
2679
2680                 cpr = &bnapi->cp_ring;
2681                 cpr->bnapi = bnapi;
2682                 ring = &cpr->cp_ring_struct;
2683
2684                 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2685                 if (rc)
2686                         return rc;
2687
2688                 if (ulp_msix && i >= ulp_base_vec)
2689                         ring->map_idx = i + ulp_msix;
2690                 else
2691                         ring->map_idx = i;
2692
2693                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2694                         continue;
2695
2696                 if (i < bp->rx_nr_rings) {
2697                         struct bnxt_cp_ring_info *cpr2 =
2698                                 bnxt_alloc_cp_sub_ring(bp);
2699
2700                         cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
2701                         if (!cpr2)
2702                                 return -ENOMEM;
2703                         cpr2->bnapi = bnapi;
2704                 }
2705                 if ((sh && i < bp->tx_nr_rings) ||
2706                     (!sh && i >= bp->rx_nr_rings)) {
2707                         struct bnxt_cp_ring_info *cpr2 =
2708                                 bnxt_alloc_cp_sub_ring(bp);
2709
2710                         cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
2711                         if (!cpr2)
2712                                 return -ENOMEM;
2713                         cpr2->bnapi = bnapi;
2714                 }
2715         }
2716         return 0;
2717 }
2718
2719 static void bnxt_init_ring_struct(struct bnxt *bp)
2720 {
2721         int i;
2722
2723         for (i = 0; i < bp->cp_nr_rings; i++) {
2724                 struct bnxt_napi *bnapi = bp->bnapi[i];
2725                 struct bnxt_ring_mem_info *rmem;
2726                 struct bnxt_cp_ring_info *cpr;
2727                 struct bnxt_rx_ring_info *rxr;
2728                 struct bnxt_tx_ring_info *txr;
2729                 struct bnxt_ring_struct *ring;
2730
2731                 if (!bnapi)
2732                         continue;
2733
2734                 cpr = &bnapi->cp_ring;
2735                 ring = &cpr->cp_ring_struct;
2736                 rmem = &ring->ring_mem;
2737                 rmem->nr_pages = bp->cp_nr_pages;
2738                 rmem->page_size = HW_CMPD_RING_SIZE;
2739                 rmem->pg_arr = (void **)cpr->cp_desc_ring;
2740                 rmem->dma_arr = cpr->cp_desc_mapping;
2741                 rmem->vmem_size = 0;
2742
2743                 rxr = bnapi->rx_ring;
2744                 if (!rxr)
2745                         goto skip_rx;
2746
2747                 ring = &rxr->rx_ring_struct;
2748                 rmem = &ring->ring_mem;
2749                 rmem->nr_pages = bp->rx_nr_pages;
2750                 rmem->page_size = HW_RXBD_RING_SIZE;
2751                 rmem->pg_arr = (void **)rxr->rx_desc_ring;
2752                 rmem->dma_arr = rxr->rx_desc_mapping;
2753                 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2754                 rmem->vmem = (void **)&rxr->rx_buf_ring;
2755
2756                 ring = &rxr->rx_agg_ring_struct;
2757                 rmem = &ring->ring_mem;
2758                 rmem->nr_pages = bp->rx_agg_nr_pages;
2759                 rmem->page_size = HW_RXBD_RING_SIZE;
2760                 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
2761                 rmem->dma_arr = rxr->rx_agg_desc_mapping;
2762                 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2763                 rmem->vmem = (void **)&rxr->rx_agg_ring;
2764
2765 skip_rx:
2766                 txr = bnapi->tx_ring;
2767                 if (!txr)
2768                         continue;
2769
2770                 ring = &txr->tx_ring_struct;
2771                 rmem = &ring->ring_mem;
2772                 rmem->nr_pages = bp->tx_nr_pages;
2773                 rmem->page_size = HW_RXBD_RING_SIZE;
2774                 rmem->pg_arr = (void **)txr->tx_desc_ring;
2775                 rmem->dma_arr = txr->tx_desc_mapping;
2776                 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2777                 rmem->vmem = (void **)&txr->tx_buf_ring;
2778         }
2779 }
2780
2781 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2782 {
2783         int i;
2784         u32 prod;
2785         struct rx_bd **rx_buf_ring;
2786
2787         rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
2788         for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
2789                 int j;
2790                 struct rx_bd *rxbd;
2791
2792                 rxbd = rx_buf_ring[i];
2793                 if (!rxbd)
2794                         continue;
2795
2796                 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2797                         rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2798                         rxbd->rx_bd_opaque = prod;
2799                 }
2800         }
2801 }
2802
2803 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2804 {
2805         struct net_device *dev = bp->dev;
2806         struct bnxt_rx_ring_info *rxr;
2807         struct bnxt_ring_struct *ring;
2808         u32 prod, type;
2809         int i;
2810
2811         type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2812                 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2813
2814         if (NET_IP_ALIGN == 2)
2815                 type |= RX_BD_FLAGS_SOP;
2816
2817         rxr = &bp->rx_ring[ring_nr];
2818         ring = &rxr->rx_ring_struct;
2819         bnxt_init_rxbd_pages(ring, type);
2820
2821         if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
2822                 rxr->xdp_prog = bpf_prog_add(bp->xdp_prog, 1);
2823                 if (IS_ERR(rxr->xdp_prog)) {
2824                         int rc = PTR_ERR(rxr->xdp_prog);
2825
2826                         rxr->xdp_prog = NULL;
2827                         return rc;
2828                 }
2829         }
2830         prod = rxr->rx_prod;
2831         for (i = 0; i < bp->rx_ring_size; i++) {
2832                 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2833                         netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2834                                     ring_nr, i, bp->rx_ring_size);
2835                         break;
2836                 }
2837                 prod = NEXT_RX(prod);
2838         }
2839         rxr->rx_prod = prod;
2840         ring->fw_ring_id = INVALID_HW_RING_ID;
2841
2842         ring = &rxr->rx_agg_ring_struct;
2843         ring->fw_ring_id = INVALID_HW_RING_ID;
2844
2845         if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2846                 return 0;
2847
2848         type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2849                 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2850
2851         bnxt_init_rxbd_pages(ring, type);
2852
2853         prod = rxr->rx_agg_prod;
2854         for (i = 0; i < bp->rx_agg_ring_size; i++) {
2855                 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2856                         netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2857                                     ring_nr, i, bp->rx_ring_size);
2858                         break;
2859                 }
2860                 prod = NEXT_RX_AGG(prod);
2861         }
2862         rxr->rx_agg_prod = prod;
2863
2864         if (bp->flags & BNXT_FLAG_TPA) {
2865                 if (rxr->rx_tpa) {
2866                         u8 *data;
2867                         dma_addr_t mapping;
2868
2869                         for (i = 0; i < MAX_TPA; i++) {
2870                                 data = __bnxt_alloc_rx_data(bp, &mapping,
2871                                                             GFP_KERNEL);
2872                                 if (!data)
2873                                         return -ENOMEM;
2874
2875                                 rxr->rx_tpa[i].data = data;
2876                                 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
2877                                 rxr->rx_tpa[i].mapping = mapping;
2878                         }
2879                 } else {
2880                         netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2881                         return -ENOMEM;
2882                 }
2883         }
2884
2885         return 0;
2886 }
2887
2888 static void bnxt_init_cp_rings(struct bnxt *bp)
2889 {
2890         int i, j;
2891
2892         for (i = 0; i < bp->cp_nr_rings; i++) {
2893                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
2894                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
2895
2896                 ring->fw_ring_id = INVALID_HW_RING_ID;
2897                 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
2898                 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
2899                 for (j = 0; j < 2; j++) {
2900                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2901
2902                         if (!cpr2)
2903                                 continue;
2904
2905                         ring = &cpr2->cp_ring_struct;
2906                         ring->fw_ring_id = INVALID_HW_RING_ID;
2907                         cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
2908                         cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
2909                 }
2910         }
2911 }
2912
2913 static int bnxt_init_rx_rings(struct bnxt *bp)
2914 {
2915         int i, rc = 0;
2916
2917         if (BNXT_RX_PAGE_MODE(bp)) {
2918                 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
2919                 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
2920         } else {
2921                 bp->rx_offset = BNXT_RX_OFFSET;
2922                 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
2923         }
2924
2925         for (i = 0; i < bp->rx_nr_rings; i++) {
2926                 rc = bnxt_init_one_rx_ring(bp, i);
2927                 if (rc)
2928                         break;
2929         }
2930
2931         return rc;
2932 }
2933
2934 static int bnxt_init_tx_rings(struct bnxt *bp)
2935 {
2936         u16 i;
2937
2938         bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2939                                    MAX_SKB_FRAGS + 1);
2940
2941         for (i = 0; i < bp->tx_nr_rings; i++) {
2942                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2943                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2944
2945                 ring->fw_ring_id = INVALID_HW_RING_ID;
2946         }
2947
2948         return 0;
2949 }
2950
2951 static void bnxt_free_ring_grps(struct bnxt *bp)
2952 {
2953         kfree(bp->grp_info);
2954         bp->grp_info = NULL;
2955 }
2956
2957 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2958 {
2959         int i;
2960
2961         if (irq_re_init) {
2962                 bp->grp_info = kcalloc(bp->cp_nr_rings,
2963                                        sizeof(struct bnxt_ring_grp_info),
2964                                        GFP_KERNEL);
2965                 if (!bp->grp_info)
2966                         return -ENOMEM;
2967         }
2968         for (i = 0; i < bp->cp_nr_rings; i++) {
2969                 if (irq_re_init)
2970                         bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2971                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2972                 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2973                 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2974                 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2975         }
2976         return 0;
2977 }
2978
2979 static void bnxt_free_vnics(struct bnxt *bp)
2980 {
2981         kfree(bp->vnic_info);
2982         bp->vnic_info = NULL;
2983         bp->nr_vnics = 0;
2984 }
2985
2986 static int bnxt_alloc_vnics(struct bnxt *bp)
2987 {
2988         int num_vnics = 1;
2989
2990 #ifdef CONFIG_RFS_ACCEL
2991         if (bp->flags & BNXT_FLAG_RFS)
2992                 num_vnics += bp->rx_nr_rings;
2993 #endif
2994
2995         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
2996                 num_vnics++;
2997
2998         bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2999                                 GFP_KERNEL);
3000         if (!bp->vnic_info)
3001                 return -ENOMEM;
3002
3003         bp->nr_vnics = num_vnics;
3004         return 0;
3005 }
3006
3007 static void bnxt_init_vnics(struct bnxt *bp)
3008 {
3009         int i;
3010
3011         for (i = 0; i < bp->nr_vnics; i++) {
3012                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3013                 int j;
3014
3015                 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3016                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3017                         vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3018
3019                 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3020
3021                 if (bp->vnic_info[i].rss_hash_key) {
3022                         if (i == 0)
3023                                 prandom_bytes(vnic->rss_hash_key,
3024                                               HW_HASH_KEY_SIZE);
3025                         else
3026                                 memcpy(vnic->rss_hash_key,
3027                                        bp->vnic_info[0].rss_hash_key,
3028                                        HW_HASH_KEY_SIZE);
3029                 }
3030         }
3031 }
3032
3033 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3034 {
3035         int pages;
3036
3037         pages = ring_size / desc_per_pg;
3038
3039         if (!pages)
3040                 return 1;
3041
3042         pages++;
3043
3044         while (pages & (pages - 1))
3045                 pages++;
3046
3047         return pages;
3048 }
3049
3050 void bnxt_set_tpa_flags(struct bnxt *bp)
3051 {
3052         bp->flags &= ~BNXT_FLAG_TPA;
3053         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3054                 return;
3055         if (bp->dev->features & NETIF_F_LRO)
3056                 bp->flags |= BNXT_FLAG_LRO;
3057         else if (bp->dev->features & NETIF_F_GRO_HW)
3058                 bp->flags |= BNXT_FLAG_GRO;
3059 }
3060
3061 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3062  * be set on entry.
3063  */
3064 void bnxt_set_ring_params(struct bnxt *bp)
3065 {
3066         u32 ring_size, rx_size, rx_space;
3067         u32 agg_factor = 0, agg_ring_size = 0;
3068
3069         /* 8 for CRC and VLAN */
3070         rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3071
3072         rx_space = rx_size + NET_SKB_PAD +
3073                 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3074
3075         bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3076         ring_size = bp->rx_ring_size;
3077         bp->rx_agg_ring_size = 0;
3078         bp->rx_agg_nr_pages = 0;
3079
3080         if (bp->flags & BNXT_FLAG_TPA)
3081                 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3082
3083         bp->flags &= ~BNXT_FLAG_JUMBO;
3084         if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3085                 u32 jumbo_factor;
3086
3087                 bp->flags |= BNXT_FLAG_JUMBO;
3088                 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3089                 if (jumbo_factor > agg_factor)
3090                         agg_factor = jumbo_factor;
3091         }
3092         agg_ring_size = ring_size * agg_factor;
3093
3094         if (agg_ring_size) {
3095                 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3096                                                         RX_DESC_CNT);
3097                 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3098                         u32 tmp = agg_ring_size;
3099
3100                         bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3101                         agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3102                         netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3103                                     tmp, agg_ring_size);
3104                 }
3105                 bp->rx_agg_ring_size = agg_ring_size;
3106                 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3107                 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3108                 rx_space = rx_size + NET_SKB_PAD +
3109                         SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3110         }
3111
3112         bp->rx_buf_use_size = rx_size;
3113         bp->rx_buf_size = rx_space;
3114
3115         bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3116         bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3117
3118         ring_size = bp->tx_ring_size;
3119         bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3120         bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3121
3122         ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3123         bp->cp_ring_size = ring_size;
3124
3125         bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3126         if (bp->cp_nr_pages > MAX_CP_PAGES) {
3127                 bp->cp_nr_pages = MAX_CP_PAGES;
3128                 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3129                 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3130                             ring_size, bp->cp_ring_size);
3131         }
3132         bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3133         bp->cp_ring_mask = bp->cp_bit - 1;
3134 }
3135
3136 /* Changing allocation mode of RX rings.
3137  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3138  */
3139 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3140 {
3141         if (page_mode) {
3142                 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3143                         return -EOPNOTSUPP;
3144                 bp->dev->max_mtu =
3145                         min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3146                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3147                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3148                 bp->rx_dir = DMA_BIDIRECTIONAL;
3149                 bp->rx_skb_func = bnxt_rx_page_skb;
3150                 /* Disable LRO or GRO_HW */
3151                 netdev_update_features(bp->dev);
3152         } else {
3153                 bp->dev->max_mtu = bp->max_mtu;
3154                 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3155                 bp->rx_dir = DMA_FROM_DEVICE;
3156                 bp->rx_skb_func = bnxt_rx_skb;
3157         }
3158         return 0;
3159 }
3160
3161 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3162 {
3163         int i;
3164         struct bnxt_vnic_info *vnic;
3165         struct pci_dev *pdev = bp->pdev;
3166
3167         if (!bp->vnic_info)
3168                 return;
3169
3170         for (i = 0; i < bp->nr_vnics; i++) {
3171                 vnic = &bp->vnic_info[i];
3172
3173                 kfree(vnic->fw_grp_ids);
3174                 vnic->fw_grp_ids = NULL;
3175
3176                 kfree(vnic->uc_list);
3177                 vnic->uc_list = NULL;
3178
3179                 if (vnic->mc_list) {
3180                         dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3181                                           vnic->mc_list, vnic->mc_list_mapping);
3182                         vnic->mc_list = NULL;
3183                 }
3184
3185                 if (vnic->rss_table) {
3186                         dma_free_coherent(&pdev->dev, PAGE_SIZE,
3187                                           vnic->rss_table,
3188                                           vnic->rss_table_dma_addr);
3189                         vnic->rss_table = NULL;
3190                 }
3191
3192                 vnic->rss_hash_key = NULL;
3193                 vnic->flags = 0;
3194         }
3195 }
3196
3197 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3198 {
3199         int i, rc = 0, size;
3200         struct bnxt_vnic_info *vnic;
3201         struct pci_dev *pdev = bp->pdev;
3202         int max_rings;
3203
3204         for (i = 0; i < bp->nr_vnics; i++) {
3205                 vnic = &bp->vnic_info[i];
3206
3207                 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3208                         int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3209
3210                         if (mem_size > 0) {
3211                                 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3212                                 if (!vnic->uc_list) {
3213                                         rc = -ENOMEM;
3214                                         goto out;
3215                                 }
3216                         }
3217                 }
3218
3219                 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3220                         vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3221                         vnic->mc_list =
3222                                 dma_alloc_coherent(&pdev->dev,
3223                                                    vnic->mc_list_size,
3224                                                    &vnic->mc_list_mapping,
3225                                                    GFP_KERNEL);
3226                         if (!vnic->mc_list) {
3227                                 rc = -ENOMEM;
3228                                 goto out;
3229                         }
3230                 }
3231
3232                 if (bp->flags & BNXT_FLAG_CHIP_P5)
3233                         goto vnic_skip_grps;
3234
3235                 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3236                         max_rings = bp->rx_nr_rings;
3237                 else
3238                         max_rings = 1;
3239
3240                 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3241                 if (!vnic->fw_grp_ids) {
3242                         rc = -ENOMEM;
3243                         goto out;
3244                 }
3245 vnic_skip_grps:
3246                 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3247                     !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3248                         continue;
3249
3250                 /* Allocate rss table and hash key */
3251                 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3252                                                      &vnic->rss_table_dma_addr,
3253                                                      GFP_KERNEL);
3254                 if (!vnic->rss_table) {
3255                         rc = -ENOMEM;
3256                         goto out;
3257                 }
3258
3259                 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3260
3261                 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3262                 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3263         }
3264         return 0;
3265
3266 out:
3267         return rc;
3268 }
3269
3270 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3271 {
3272         struct pci_dev *pdev = bp->pdev;
3273
3274         if (bp->hwrm_cmd_resp_addr) {
3275                 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3276                                   bp->hwrm_cmd_resp_dma_addr);
3277                 bp->hwrm_cmd_resp_addr = NULL;
3278         }
3279 }
3280
3281 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3282 {
3283         struct pci_dev *pdev = bp->pdev;
3284
3285         bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3286                                                    &bp->hwrm_cmd_resp_dma_addr,
3287                                                    GFP_KERNEL);
3288         if (!bp->hwrm_cmd_resp_addr)
3289                 return -ENOMEM;
3290
3291         return 0;
3292 }
3293
3294 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3295 {
3296         if (bp->hwrm_short_cmd_req_addr) {
3297                 struct pci_dev *pdev = bp->pdev;
3298
3299                 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3300                                   bp->hwrm_short_cmd_req_addr,
3301                                   bp->hwrm_short_cmd_req_dma_addr);
3302                 bp->hwrm_short_cmd_req_addr = NULL;
3303         }
3304 }
3305
3306 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3307 {
3308         struct pci_dev *pdev = bp->pdev;
3309
3310         bp->hwrm_short_cmd_req_addr =
3311                 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3312                                    &bp->hwrm_short_cmd_req_dma_addr,
3313                                    GFP_KERNEL);
3314         if (!bp->hwrm_short_cmd_req_addr)
3315                 return -ENOMEM;
3316
3317         return 0;
3318 }
3319
3320 static void bnxt_free_stats(struct bnxt *bp)
3321 {
3322         u32 size, i;
3323         struct pci_dev *pdev = bp->pdev;
3324
3325         bp->flags &= ~BNXT_FLAG_PORT_STATS;
3326         bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3327
3328         if (bp->hw_rx_port_stats) {
3329                 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3330                                   bp->hw_rx_port_stats,
3331                                   bp->hw_rx_port_stats_map);
3332                 bp->hw_rx_port_stats = NULL;
3333         }
3334
3335         if (bp->hw_tx_port_stats_ext) {
3336                 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3337                                   bp->hw_tx_port_stats_ext,
3338                                   bp->hw_tx_port_stats_ext_map);
3339                 bp->hw_tx_port_stats_ext = NULL;
3340         }
3341
3342         if (bp->hw_rx_port_stats_ext) {
3343                 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3344                                   bp->hw_rx_port_stats_ext,
3345                                   bp->hw_rx_port_stats_ext_map);
3346                 bp->hw_rx_port_stats_ext = NULL;
3347         }
3348
3349         if (!bp->bnapi)
3350                 return;
3351
3352         size = sizeof(struct ctx_hw_stats);
3353
3354         for (i = 0; i < bp->cp_nr_rings; i++) {
3355                 struct bnxt_napi *bnapi = bp->bnapi[i];
3356                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3357
3358                 if (cpr->hw_stats) {
3359                         dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3360                                           cpr->hw_stats_map);
3361                         cpr->hw_stats = NULL;
3362                 }
3363         }
3364 }
3365
3366 static int bnxt_alloc_stats(struct bnxt *bp)
3367 {
3368         u32 size, i;
3369         struct pci_dev *pdev = bp->pdev;
3370
3371         size = sizeof(struct ctx_hw_stats);
3372
3373         for (i = 0; i < bp->cp_nr_rings; i++) {
3374                 struct bnxt_napi *bnapi = bp->bnapi[i];
3375                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3376
3377                 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3378                                                    &cpr->hw_stats_map,
3379                                                    GFP_KERNEL);
3380                 if (!cpr->hw_stats)
3381                         return -ENOMEM;
3382
3383                 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3384         }
3385
3386         if (BNXT_PF(bp) && bp->chip_num != CHIP_NUM_58700) {
3387                 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3388                                          sizeof(struct tx_port_stats) + 1024;
3389
3390                 bp->hw_rx_port_stats =
3391                         dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3392                                            &bp->hw_rx_port_stats_map,
3393                                            GFP_KERNEL);
3394                 if (!bp->hw_rx_port_stats)
3395                         return -ENOMEM;
3396
3397                 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
3398                                        512;
3399                 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3400                                            sizeof(struct rx_port_stats) + 512;
3401                 bp->flags |= BNXT_FLAG_PORT_STATS;
3402
3403                 /* Display extended statistics only if FW supports it */
3404                 if (bp->hwrm_spec_code < 0x10804 ||
3405                     bp->hwrm_spec_code == 0x10900)
3406                         return 0;
3407
3408                 bp->hw_rx_port_stats_ext =
3409                         dma_zalloc_coherent(&pdev->dev,
3410                                             sizeof(struct rx_port_stats_ext),
3411                                             &bp->hw_rx_port_stats_ext_map,
3412                                             GFP_KERNEL);
3413                 if (!bp->hw_rx_port_stats_ext)
3414                         return 0;
3415
3416                 if (bp->hwrm_spec_code >= 0x10902) {
3417                         bp->hw_tx_port_stats_ext =
3418                                 dma_zalloc_coherent(&pdev->dev,
3419                                             sizeof(struct tx_port_stats_ext),
3420                                             &bp->hw_tx_port_stats_ext_map,
3421                                             GFP_KERNEL);
3422                 }
3423                 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3424         }
3425         return 0;
3426 }
3427
3428 static void bnxt_clear_ring_indices(struct bnxt *bp)
3429 {
3430         int i;
3431
3432         if (!bp->bnapi)
3433                 return;
3434
3435         for (i = 0; i < bp->cp_nr_rings; i++) {
3436                 struct bnxt_napi *bnapi = bp->bnapi[i];
3437                 struct bnxt_cp_ring_info *cpr;
3438                 struct bnxt_rx_ring_info *rxr;
3439                 struct bnxt_tx_ring_info *txr;
3440
3441                 if (!bnapi)
3442                         continue;
3443
3444                 cpr = &bnapi->cp_ring;
3445                 cpr->cp_raw_cons = 0;
3446
3447                 txr = bnapi->tx_ring;
3448                 if (txr) {
3449                         txr->tx_prod = 0;
3450                         txr->tx_cons = 0;
3451                 }
3452
3453                 rxr = bnapi->rx_ring;
3454                 if (rxr) {
3455                         rxr->rx_prod = 0;
3456                         rxr->rx_agg_prod = 0;
3457                         rxr->rx_sw_agg_prod = 0;
3458                         rxr->rx_next_cons = 0;
3459                 }
3460         }
3461 }
3462
3463 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3464 {
3465 #ifdef CONFIG_RFS_ACCEL
3466         int i;
3467
3468         /* Under rtnl_lock and all our NAPIs have been disabled.  It's
3469          * safe to delete the hash table.
3470          */
3471         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3472                 struct hlist_head *head;
3473                 struct hlist_node *tmp;
3474                 struct bnxt_ntuple_filter *fltr;
3475
3476                 head = &bp->ntp_fltr_hash_tbl[i];
3477                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3478                         hlist_del(&fltr->hash);
3479                         kfree(fltr);
3480                 }
3481         }
3482         if (irq_reinit) {
3483                 kfree(bp->ntp_fltr_bmap);
3484                 bp->ntp_fltr_bmap = NULL;
3485         }
3486         bp->ntp_fltr_count = 0;
3487 #endif
3488 }
3489
3490 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3491 {
3492 #ifdef CONFIG_RFS_ACCEL
3493         int i, rc = 0;
3494
3495         if (!(bp->flags & BNXT_FLAG_RFS))
3496                 return 0;
3497
3498         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3499                 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3500
3501         bp->ntp_fltr_count = 0;
3502         bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3503                                     sizeof(long),
3504                                     GFP_KERNEL);
3505
3506         if (!bp->ntp_fltr_bmap)
3507                 rc = -ENOMEM;
3508
3509         return rc;
3510 #else
3511         return 0;
3512 #endif
3513 }
3514
3515 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3516 {
3517         bnxt_free_vnic_attributes(bp);
3518         bnxt_free_tx_rings(bp);
3519         bnxt_free_rx_rings(bp);
3520         bnxt_free_cp_rings(bp);
3521         bnxt_free_ntp_fltrs(bp, irq_re_init);
3522         if (irq_re_init) {
3523                 bnxt_free_stats(bp);
3524                 bnxt_free_ring_grps(bp);
3525                 bnxt_free_vnics(bp);
3526                 kfree(bp->tx_ring_map);
3527                 bp->tx_ring_map = NULL;
3528                 kfree(bp->tx_ring);
3529                 bp->tx_ring = NULL;
3530                 kfree(bp->rx_ring);
3531                 bp->rx_ring = NULL;
3532                 kfree(bp->bnapi);
3533                 bp->bnapi = NULL;
3534         } else {
3535                 bnxt_clear_ring_indices(bp);
3536         }
3537 }
3538
3539 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3540 {
3541         int i, j, rc, size, arr_size;
3542         void *bnapi;
3543
3544         if (irq_re_init) {
3545                 /* Allocate bnapi mem pointer array and mem block for
3546                  * all queues
3547                  */
3548                 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3549                                 bp->cp_nr_rings);
3550                 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3551                 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3552                 if (!bnapi)
3553                         return -ENOMEM;
3554
3555                 bp->bnapi = bnapi;
3556                 bnapi += arr_size;
3557                 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3558                         bp->bnapi[i] = bnapi;
3559                         bp->bnapi[i]->index = i;
3560                         bp->bnapi[i]->bp = bp;
3561                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3562                                 struct bnxt_cp_ring_info *cpr =
3563                                         &bp->bnapi[i]->cp_ring;
3564
3565                                 cpr->cp_ring_struct.ring_mem.flags =
3566                                         BNXT_RMEM_RING_PTE_FLAG;
3567                         }
3568                 }
3569
3570                 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3571                                       sizeof(struct bnxt_rx_ring_info),
3572                                       GFP_KERNEL);
3573                 if (!bp->rx_ring)
3574                         return -ENOMEM;
3575
3576                 for (i = 0; i < bp->rx_nr_rings; i++) {
3577                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3578
3579                         if (bp->flags & BNXT_FLAG_CHIP_P5) {
3580                                 rxr->rx_ring_struct.ring_mem.flags =
3581                                         BNXT_RMEM_RING_PTE_FLAG;
3582                                 rxr->rx_agg_ring_struct.ring_mem.flags =
3583                                         BNXT_RMEM_RING_PTE_FLAG;
3584                         }
3585                         rxr->bnapi = bp->bnapi[i];
3586                         bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
3587                 }
3588
3589                 bp->tx_ring = kcalloc(bp->tx_nr_rings,
3590                                       sizeof(struct bnxt_tx_ring_info),
3591                                       GFP_KERNEL);
3592                 if (!bp->tx_ring)
3593                         return -ENOMEM;
3594
3595                 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
3596                                           GFP_KERNEL);
3597
3598                 if (!bp->tx_ring_map)
3599                         return -ENOMEM;
3600
3601                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
3602                         j = 0;
3603                 else
3604                         j = bp->rx_nr_rings;
3605
3606                 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
3607                         struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3608
3609                         if (bp->flags & BNXT_FLAG_CHIP_P5)
3610                                 txr->tx_ring_struct.ring_mem.flags =
3611                                         BNXT_RMEM_RING_PTE_FLAG;
3612                         txr->bnapi = bp->bnapi[j];
3613                         bp->bnapi[j]->tx_ring = txr;
3614                         bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
3615                         if (i >= bp->tx_nr_rings_xdp) {
3616                                 txr->txq_index = i - bp->tx_nr_rings_xdp;
3617                                 bp->bnapi[j]->tx_int = bnxt_tx_int;
3618                         } else {
3619                                 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
3620                                 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
3621                         }
3622                 }
3623
3624                 rc = bnxt_alloc_stats(bp);
3625                 if (rc)
3626                         goto alloc_mem_err;
3627
3628                 rc = bnxt_alloc_ntp_fltrs(bp);
3629                 if (rc)
3630                         goto alloc_mem_err;
3631
3632                 rc = bnxt_alloc_vnics(bp);
3633                 if (rc)
3634                         goto alloc_mem_err;
3635         }
3636
3637         bnxt_init_ring_struct(bp);
3638
3639         rc = bnxt_alloc_rx_rings(bp);
3640         if (rc)
3641                 goto alloc_mem_err;
3642
3643         rc = bnxt_alloc_tx_rings(bp);
3644         if (rc)
3645                 goto alloc_mem_err;
3646
3647         rc = bnxt_alloc_cp_rings(bp);
3648         if (rc)
3649                 goto alloc_mem_err;
3650
3651         bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
3652                                   BNXT_VNIC_UCAST_FLAG;
3653         rc = bnxt_alloc_vnic_attributes(bp);
3654         if (rc)
3655                 goto alloc_mem_err;
3656         return 0;
3657
3658 alloc_mem_err:
3659         bnxt_free_mem(bp, true);
3660         return rc;
3661 }
3662
3663 static void bnxt_disable_int(struct bnxt *bp)
3664 {
3665         int i;
3666
3667         if (!bp->bnapi)
3668                 return;
3669
3670         for (i = 0; i < bp->cp_nr_rings; i++) {
3671                 struct bnxt_napi *bnapi = bp->bnapi[i];
3672                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3673                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3674
3675                 if (ring->fw_ring_id != INVALID_HW_RING_ID)
3676                         bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
3677         }
3678 }
3679
3680 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
3681 {
3682         struct bnxt_napi *bnapi = bp->bnapi[n];
3683         struct bnxt_cp_ring_info *cpr;
3684
3685         cpr = &bnapi->cp_ring;
3686         return cpr->cp_ring_struct.map_idx;
3687 }
3688
3689 static void bnxt_disable_int_sync(struct bnxt *bp)
3690 {
3691         int i;
3692
3693         atomic_inc(&bp->intr_sem);
3694
3695         bnxt_disable_int(bp);
3696         for (i = 0; i < bp->cp_nr_rings; i++) {
3697                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
3698
3699                 synchronize_irq(bp->irq_tbl[map_idx].vector);
3700         }
3701 }
3702
3703 static void bnxt_enable_int(struct bnxt *bp)
3704 {
3705         int i;
3706
3707         atomic_set(&bp->intr_sem, 0);
3708         for (i = 0; i < bp->cp_nr_rings; i++) {
3709                 struct bnxt_napi *bnapi = bp->bnapi[i];
3710                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3711
3712                 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
3713         }
3714 }
3715
3716 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
3717                             u16 cmpl_ring, u16 target_id)
3718 {
3719         struct input *req = request;
3720
3721         req->req_type = cpu_to_le16(req_type);
3722         req->cmpl_ring = cpu_to_le16(cmpl_ring);
3723         req->target_id = cpu_to_le16(target_id);
3724         req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
3725 }
3726
3727 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
3728                                  int timeout, bool silent)
3729 {
3730         int i, intr_process, rc, tmo_count;
3731         struct input *req = msg;
3732         u32 *data = msg;
3733         __le32 *resp_len;
3734         u8 *valid;
3735         u16 cp_ring_id, len = 0;
3736         struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
3737         u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
3738         struct hwrm_short_input short_input = {0};
3739
3740         req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
3741         memset(resp, 0, PAGE_SIZE);
3742         cp_ring_id = le16_to_cpu(req->cmpl_ring);
3743         intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
3744
3745         if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
3746                 if (msg_len > bp->hwrm_max_ext_req_len ||
3747                     !bp->hwrm_short_cmd_req_addr)
3748                         return -EINVAL;
3749         }
3750
3751         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
3752             msg_len > BNXT_HWRM_MAX_REQ_LEN) {
3753                 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
3754                 u16 max_msg_len;
3755
3756                 /* Set boundary for maximum extended request length for short
3757                  * cmd format. If passed up from device use the max supported
3758                  * internal req length.
3759                  */
3760                 max_msg_len = bp->hwrm_max_ext_req_len;
3761
3762                 memcpy(short_cmd_req, req, msg_len);
3763                 if (msg_len < max_msg_len)
3764                         memset(short_cmd_req + msg_len, 0,
3765                                max_msg_len - msg_len);
3766
3767                 short_input.req_type = req->req_type;
3768                 short_input.signature =
3769                                 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
3770                 short_input.size = cpu_to_le16(msg_len);
3771                 short_input.req_addr =
3772                         cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
3773
3774                 data = (u32 *)&short_input;
3775                 msg_len = sizeof(short_input);
3776
3777                 /* Sync memory write before updating doorbell */
3778                 wmb();
3779
3780                 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
3781         }
3782
3783         /* Write request msg to hwrm channel */
3784         __iowrite32_copy(bp->bar0, data, msg_len / 4);
3785
3786         for (i = msg_len; i < max_req_len; i += 4)
3787                 writel(0, bp->bar0 + i);
3788
3789         /* currently supports only one outstanding message */
3790         if (intr_process)
3791                 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
3792
3793         /* Ring channel doorbell */
3794         writel(1, bp->bar0 + 0x100);
3795
3796         if (!timeout)
3797                 timeout = DFLT_HWRM_CMD_TIMEOUT;
3798         /* convert timeout to usec */
3799         timeout *= 1000;
3800
3801         i = 0;
3802         /* Short timeout for the first few iterations:
3803          * number of loops = number of loops for short timeout +
3804          * number of loops for standard timeout.
3805          */
3806         tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
3807         timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
3808         tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
3809         resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
3810         if (intr_process) {
3811                 /* Wait until hwrm response cmpl interrupt is processed */
3812                 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
3813                        i++ < tmo_count) {
3814                         /* on first few passes, just barely sleep */
3815                         if (i < HWRM_SHORT_TIMEOUT_COUNTER)
3816                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
3817                                              HWRM_SHORT_MAX_TIMEOUT);
3818                         else
3819                                 usleep_range(HWRM_MIN_TIMEOUT,
3820                                              HWRM_MAX_TIMEOUT);
3821                 }
3822
3823                 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
3824                         netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
3825                                    le16_to_cpu(req->req_type));
3826                         return -1;
3827                 }
3828                 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3829                       HWRM_RESP_LEN_SFT;
3830                 valid = bp->hwrm_cmd_resp_addr + len - 1;
3831         } else {
3832                 int j;
3833
3834                 /* Check if response len is updated */
3835                 for (i = 0; i < tmo_count; i++) {
3836                         len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
3837                               HWRM_RESP_LEN_SFT;
3838                         if (len)
3839                                 break;
3840                         /* on first few passes, just barely sleep */
3841                         if (i < DFLT_HWRM_CMD_TIMEOUT)
3842                                 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
3843                                              HWRM_SHORT_MAX_TIMEOUT);
3844                         else
3845                                 usleep_range(HWRM_MIN_TIMEOUT,
3846                                              HWRM_MAX_TIMEOUT);
3847                 }
3848
3849                 if (i >= tmo_count) {
3850                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
3851                                    HWRM_TOTAL_TIMEOUT(i),
3852                                    le16_to_cpu(req->req_type),
3853                                    le16_to_cpu(req->seq_id), len);
3854                         return -1;
3855                 }
3856
3857                 /* Last byte of resp contains valid bit */
3858                 valid = bp->hwrm_cmd_resp_addr + len - 1;
3859                 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
3860                         /* make sure we read from updated DMA memory */
3861                         dma_rmb();
3862                         if (*valid)
3863                                 break;
3864                         udelay(1);
3865                 }
3866
3867                 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
3868                         netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
3869                                    HWRM_TOTAL_TIMEOUT(i),
3870                                    le16_to_cpu(req->req_type),
3871                                    le16_to_cpu(req->seq_id), len, *valid);
3872                         return -1;
3873                 }
3874         }
3875
3876         /* Zero valid bit for compatibility.  Valid bit in an older spec
3877          * may become a new field in a newer spec.  We must make sure that
3878          * a new field not implemented by old spec will read zero.
3879          */
3880         *valid = 0;
3881         rc = le16_to_cpu(resp->error_code);
3882         if (rc && !silent)
3883                 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
3884                            le16_to_cpu(resp->req_type),
3885                            le16_to_cpu(resp->seq_id), rc);
3886         return rc;
3887 }
3888
3889 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3890 {
3891         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
3892 }
3893
3894 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3895                               int timeout)
3896 {
3897         return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3898 }
3899
3900 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
3901 {
3902         int rc;
3903
3904         mutex_lock(&bp->hwrm_cmd_lock);
3905         rc = _hwrm_send_message(bp, msg, msg_len, timeout);
3906         mutex_unlock(&bp->hwrm_cmd_lock);
3907         return rc;
3908 }
3909
3910 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
3911                              int timeout)
3912 {
3913         int rc;
3914
3915         mutex_lock(&bp->hwrm_cmd_lock);
3916         rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
3917         mutex_unlock(&bp->hwrm_cmd_lock);
3918         return rc;
3919 }
3920
3921 int bnxt_hwrm_func_rgtr_async_events(struct bnxt *bp, unsigned long *bmap,
3922                                      int bmap_size)
3923 {
3924         struct hwrm_func_drv_rgtr_input req = {0};
3925         DECLARE_BITMAP(async_events_bmap, 256);
3926         u32 *events = (u32 *)async_events_bmap;
3927         int i;
3928
3929         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3930
3931         req.enables =
3932                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
3933
3934         memset(async_events_bmap, 0, sizeof(async_events_bmap));
3935         for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
3936                 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
3937
3938         if (bmap && bmap_size) {
3939                 for (i = 0; i < bmap_size; i++) {
3940                         if (test_bit(i, bmap))
3941                                 __set_bit(i, async_events_bmap);
3942                 }
3943         }
3944
3945         for (i = 0; i < 8; i++)
3946                 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
3947
3948         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3949 }
3950
3951 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
3952 {
3953         struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
3954         struct hwrm_func_drv_rgtr_input req = {0};
3955         int rc;
3956
3957         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
3958
3959         req.enables =
3960                 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
3961                             FUNC_DRV_RGTR_REQ_ENABLES_VER);
3962
3963         req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
3964         req.flags = cpu_to_le32(FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE);
3965         req.ver_maj_8b = DRV_VER_MAJ;
3966         req.ver_min_8b = DRV_VER_MIN;
3967         req.ver_upd_8b = DRV_VER_UPD;
3968         req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
3969         req.ver_min = cpu_to_le16(DRV_VER_MIN);
3970         req.ver_upd = cpu_to_le16(DRV_VER_UPD);
3971
3972         if (BNXT_PF(bp)) {
3973                 u32 data[8];
3974                 int i;
3975
3976                 memset(data, 0, sizeof(data));
3977                 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
3978                         u16 cmd = bnxt_vf_req_snif[i];
3979                         unsigned int bit, idx;
3980
3981                         idx = cmd / 32;
3982                         bit = cmd % 32;
3983                         data[idx] |= 1 << bit;
3984                 }
3985
3986                 for (i = 0; i < 8; i++)
3987                         req.vf_req_fwd[i] = cpu_to_le32(data[i]);
3988
3989                 req.enables |=
3990                         cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
3991         }
3992
3993         mutex_lock(&bp->hwrm_cmd_lock);
3994         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3995         if (rc)
3996                 rc = -EIO;
3997         else if (resp->flags &
3998                  cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
3999                 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4000         mutex_unlock(&bp->hwrm_cmd_lock);
4001         return rc;
4002 }
4003
4004 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4005 {
4006         struct hwrm_func_drv_unrgtr_input req = {0};
4007
4008         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4009         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4010 }
4011
4012 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4013 {
4014         u32 rc = 0;
4015         struct hwrm_tunnel_dst_port_free_input req = {0};
4016
4017         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4018         req.tunnel_type = tunnel_type;
4019
4020         switch (tunnel_type) {
4021         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4022                 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4023                 break;
4024         case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4025                 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4026                 break;
4027         default:
4028                 break;
4029         }
4030
4031         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4032         if (rc)
4033                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4034                            rc);
4035         return rc;
4036 }
4037
4038 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4039                                            u8 tunnel_type)
4040 {
4041         u32 rc = 0;
4042         struct hwrm_tunnel_dst_port_alloc_input req = {0};
4043         struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4044
4045         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4046
4047         req.tunnel_type = tunnel_type;
4048         req.tunnel_dst_port_val = port;
4049
4050         mutex_lock(&bp->hwrm_cmd_lock);
4051         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4052         if (rc) {
4053                 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4054                            rc);
4055                 goto err_out;
4056         }
4057
4058         switch (tunnel_type) {
4059         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4060                 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
4061                 break;
4062         case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4063                 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
4064                 break;
4065         default:
4066                 break;
4067         }
4068
4069 err_out:
4070         mutex_unlock(&bp->hwrm_cmd_lock);
4071         return rc;
4072 }
4073
4074 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4075 {
4076         struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4077         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4078
4079         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4080         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4081
4082         req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4083         req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4084         req.mask = cpu_to_le32(vnic->rx_mask);
4085         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4086 }
4087
4088 #ifdef CONFIG_RFS_ACCEL
4089 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4090                                             struct bnxt_ntuple_filter *fltr)
4091 {
4092         struct hwrm_cfa_ntuple_filter_free_input req = {0};
4093
4094         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4095         req.ntuple_filter_id = fltr->filter_id;
4096         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4097 }
4098
4099 #define BNXT_NTP_FLTR_FLAGS                                     \
4100         (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |     \
4101          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |        \
4102          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |      \
4103          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |      \
4104          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |       \
4105          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |  \
4106          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |       \
4107          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |  \
4108          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |      \
4109          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |         \
4110          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |    \
4111          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |         \
4112          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |    \
4113          CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4114
4115 #define BNXT_NTP_TUNNEL_FLTR_FLAG                               \
4116                 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4117
4118 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4119                                              struct bnxt_ntuple_filter *fltr)
4120 {
4121         int rc = 0;
4122         struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4123         struct hwrm_cfa_ntuple_filter_alloc_output *resp =
4124                 bp->hwrm_cmd_resp_addr;
4125         struct flow_keys *keys = &fltr->fkeys;
4126         struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
4127
4128         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4129         req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4130
4131         req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4132
4133         req.ethertype = htons(ETH_P_IP);
4134         memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4135         req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4136         req.ip_protocol = keys->basic.ip_proto;
4137
4138         if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4139                 int i;
4140
4141                 req.ethertype = htons(ETH_P_IPV6);
4142                 req.ip_addr_type =
4143                         CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4144                 *(struct in6_addr *)&req.src_ipaddr[0] =
4145                         keys->addrs.v6addrs.src;
4146                 *(struct in6_addr *)&req.dst_ipaddr[0] =
4147                         keys->addrs.v6addrs.dst;
4148                 for (i = 0; i < 4; i++) {
4149                         req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4150                         req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4151                 }
4152         } else {
4153                 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4154                 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4155                 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4156                 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4157         }
4158         if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4159                 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4160                 req.tunnel_type =
4161                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4162         }
4163
4164         req.src_port = keys->ports.src;
4165         req.src_port_mask = cpu_to_be16(0xffff);
4166         req.dst_port = keys->ports.dst;
4167         req.dst_port_mask = cpu_to_be16(0xffff);
4168
4169         req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4170         mutex_lock(&bp->hwrm_cmd_lock);
4171         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4172         if (!rc)
4173                 fltr->filter_id = resp->ntuple_filter_id;
4174         mutex_unlock(&bp->hwrm_cmd_lock);
4175         return rc;
4176 }
4177 #endif
4178
4179 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4180                                      u8 *mac_addr)
4181 {
4182         u32 rc = 0;
4183         struct hwrm_cfa_l2_filter_alloc_input req = {0};
4184         struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4185
4186         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4187         req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4188         if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4189                 req.flags |=
4190                         cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4191         req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4192         req.enables =
4193                 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4194                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4195                             CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4196         memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4197         req.l2_addr_mask[0] = 0xff;
4198         req.l2_addr_mask[1] = 0xff;
4199         req.l2_addr_mask[2] = 0xff;
4200         req.l2_addr_mask[3] = 0xff;
4201         req.l2_addr_mask[4] = 0xff;
4202         req.l2_addr_mask[5] = 0xff;
4203
4204         mutex_lock(&bp->hwrm_cmd_lock);
4205         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4206         if (!rc)
4207                 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4208                                                         resp->l2_filter_id;
4209         mutex_unlock(&bp->hwrm_cmd_lock);
4210         return rc;
4211 }
4212
4213 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4214 {
4215         u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4216         int rc = 0;
4217
4218         /* Any associated ntuple filters will also be cleared by firmware. */
4219         mutex_lock(&bp->hwrm_cmd_lock);
4220         for (i = 0; i < num_of_vnics; i++) {
4221                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4222
4223                 for (j = 0; j < vnic->uc_filter_count; j++) {
4224                         struct hwrm_cfa_l2_filter_free_input req = {0};
4225
4226                         bnxt_hwrm_cmd_hdr_init(bp, &req,
4227                                                HWRM_CFA_L2_FILTER_FREE, -1, -1);
4228
4229                         req.l2_filter_id = vnic->fw_l2_filter_id[j];
4230
4231                         rc = _hwrm_send_message(bp, &req, sizeof(req),
4232                                                 HWRM_CMD_TIMEOUT);
4233                 }
4234                 vnic->uc_filter_count = 0;
4235         }
4236         mutex_unlock(&bp->hwrm_cmd_lock);
4237
4238         return rc;
4239 }
4240
4241 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4242 {
4243         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4244         struct hwrm_vnic_tpa_cfg_input req = {0};
4245
4246         if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4247                 return 0;
4248
4249         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4250
4251         if (tpa_flags) {
4252                 u16 mss = bp->dev->mtu - 40;
4253                 u32 nsegs, n, segs = 0, flags;
4254
4255                 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4256                         VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4257                         VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4258                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4259                         VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4260                 if (tpa_flags & BNXT_FLAG_GRO)
4261                         flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4262
4263                 req.flags = cpu_to_le32(flags);
4264
4265                 req.enables =
4266                         cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4267                                     VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4268                                     VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4269
4270                 /* Number of segs are log2 units, and first packet is not
4271                  * included as part of this units.
4272                  */
4273                 if (mss <= BNXT_RX_PAGE_SIZE) {
4274                         n = BNXT_RX_PAGE_SIZE / mss;
4275                         nsegs = (MAX_SKB_FRAGS - 1) * n;
4276                 } else {
4277                         n = mss / BNXT_RX_PAGE_SIZE;
4278                         if (mss & (BNXT_RX_PAGE_SIZE - 1))
4279                                 n++;
4280                         nsegs = (MAX_SKB_FRAGS - n) / n;
4281                 }
4282
4283                 segs = ilog2(nsegs);
4284                 req.max_agg_segs = cpu_to_le16(segs);
4285                 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
4286
4287                 req.min_agg_len = cpu_to_le32(512);
4288         }
4289         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4290
4291         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4292 }
4293
4294 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4295 {
4296         struct bnxt_ring_grp_info *grp_info;
4297
4298         grp_info = &bp->grp_info[ring->grp_idx];
4299         return grp_info->cp_fw_ring_id;
4300 }
4301
4302 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4303 {
4304         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4305                 struct bnxt_napi *bnapi = rxr->bnapi;
4306                 struct bnxt_cp_ring_info *cpr;
4307
4308                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4309                 return cpr->cp_ring_struct.fw_ring_id;
4310         } else {
4311                 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4312         }
4313 }
4314
4315 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4316 {
4317         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4318                 struct bnxt_napi *bnapi = txr->bnapi;
4319                 struct bnxt_cp_ring_info *cpr;
4320
4321                 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4322                 return cpr->cp_ring_struct.fw_ring_id;
4323         } else {
4324                 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4325         }
4326 }
4327
4328 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4329 {
4330         u32 i, j, max_rings;
4331         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4332         struct hwrm_vnic_rss_cfg_input req = {0};
4333
4334         if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4335             vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
4336                 return 0;
4337
4338         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4339         if (set_rss) {
4340                 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4341                 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4342                 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4343                         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4344                                 max_rings = bp->rx_nr_rings - 1;
4345                         else
4346                                 max_rings = bp->rx_nr_rings;
4347                 } else {
4348                         max_rings = 1;
4349                 }
4350
4351                 /* Fill the RSS indirection table with ring group ids */
4352                 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4353                         if (j == max_rings)
4354                                 j = 0;
4355                         vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4356                 }
4357
4358                 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4359                 req.hash_key_tbl_addr =
4360                         cpu_to_le64(vnic->rss_hash_key_dma_addr);
4361         }
4362         req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4363         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4364 }
4365
4366 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4367 {
4368         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4369         u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4370         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4371         struct hwrm_vnic_rss_cfg_input req = {0};
4372
4373         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4374         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4375         if (!set_rss) {
4376                 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4377                 return 0;
4378         }
4379         req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4380         req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4381         req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4382         req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4383         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4384         for (i = 0, k = 0; i < nr_ctxs; i++) {
4385                 __le16 *ring_tbl = vnic->rss_table;
4386                 int rc;
4387
4388                 req.ring_table_pair_index = i;
4389                 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4390                 for (j = 0; j < 64; j++) {
4391                         u16 ring_id;
4392
4393                         ring_id = rxr->rx_ring_struct.fw_ring_id;
4394                         *ring_tbl++ = cpu_to_le16(ring_id);
4395                         ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4396                         *ring_tbl++ = cpu_to_le16(ring_id);
4397                         rxr++;
4398                         k++;
4399                         if (k == max_rings) {
4400                                 k = 0;
4401                                 rxr = &bp->rx_ring[0];
4402                         }
4403                 }
4404                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4405                 if (rc)
4406                         return -EIO;
4407         }
4408         return 0;
4409 }
4410
4411 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4412 {
4413         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4414         struct hwrm_vnic_plcmodes_cfg_input req = {0};
4415
4416         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4417         req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4418                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4419                                 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4420         req.enables =
4421                 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4422                             VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4423         /* thresholds not implemented in firmware yet */
4424         req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4425         req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4426         req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4427         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4428 }
4429
4430 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4431                                         u16 ctx_idx)
4432 {
4433         struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4434
4435         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4436         req.rss_cos_lb_ctx_id =
4437                 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4438
4439         hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4440         bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4441 }
4442
4443 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4444 {
4445         int i, j;
4446
4447         for (i = 0; i < bp->nr_vnics; i++) {
4448                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4449
4450                 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4451                         if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4452                                 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4453                 }
4454         }
4455         bp->rsscos_nr_ctxs = 0;
4456 }
4457
4458 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4459 {
4460         int rc;
4461         struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4462         struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4463                                                 bp->hwrm_cmd_resp_addr;
4464
4465         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4466                                -1);
4467
4468         mutex_lock(&bp->hwrm_cmd_lock);
4469         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4470         if (!rc)
4471                 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4472                         le16_to_cpu(resp->rss_cos_lb_ctx_id);
4473         mutex_unlock(&bp->hwrm_cmd_lock);
4474
4475         return rc;
4476 }
4477
4478 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4479 {
4480         if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4481                 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4482         return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4483 }
4484
4485 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
4486 {
4487         unsigned int ring = 0, grp_idx;
4488         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4489         struct hwrm_vnic_cfg_input req = {0};
4490         u16 def_vlan = 0;
4491
4492         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
4493
4494         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4495                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4496
4497                 req.default_rx_ring_id =
4498                         cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
4499                 req.default_cmpl_ring_id =
4500                         cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
4501                 req.enables =
4502                         cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
4503                                     VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
4504                 goto vnic_mru;
4505         }
4506         req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
4507         /* Only RSS support for now TBD: COS & LB */
4508         if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
4509                 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4510                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4511                                            VNIC_CFG_REQ_ENABLES_MRU);
4512         } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
4513                 req.rss_rule =
4514                         cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
4515                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
4516                                            VNIC_CFG_REQ_ENABLES_MRU);
4517                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
4518         } else {
4519                 req.rss_rule = cpu_to_le16(0xffff);
4520         }
4521
4522         if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
4523             (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
4524                 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
4525                 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
4526         } else {
4527                 req.cos_rule = cpu_to_le16(0xffff);
4528         }
4529
4530         if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4531                 ring = 0;
4532         else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
4533                 ring = vnic_id - 1;
4534         else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
4535                 ring = bp->rx_nr_rings - 1;
4536
4537         grp_idx = bp->rx_ring[ring].bnapi->index;
4538         req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
4539         req.lb_rule = cpu_to_le16(0xffff);
4540 vnic_mru:
4541         req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
4542                               VLAN_HLEN);
4543
4544         req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4545 #ifdef CONFIG_BNXT_SRIOV
4546         if (BNXT_VF(bp))
4547                 def_vlan = bp->vf.vlan;
4548 #endif
4549         if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
4550                 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
4551         if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
4552                 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
4553
4554         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4555 }
4556
4557 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
4558 {
4559         u32 rc = 0;
4560
4561         if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
4562                 struct hwrm_vnic_free_input req = {0};
4563
4564                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
4565                 req.vnic_id =
4566                         cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
4567
4568                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4569                 if (rc)
4570                         return rc;
4571                 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
4572         }
4573         return rc;
4574 }
4575
4576 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
4577 {
4578         u16 i;
4579
4580         for (i = 0; i < bp->nr_vnics; i++)
4581                 bnxt_hwrm_vnic_free_one(bp, i);
4582 }
4583
4584 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
4585                                 unsigned int start_rx_ring_idx,
4586                                 unsigned int nr_rings)
4587 {
4588         int rc = 0;
4589         unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
4590         struct hwrm_vnic_alloc_input req = {0};
4591         struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4592         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4593
4594         if (bp->flags & BNXT_FLAG_CHIP_P5)
4595                 goto vnic_no_ring_grps;
4596
4597         /* map ring groups to this vnic */
4598         for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
4599                 grp_idx = bp->rx_ring[i].bnapi->index;
4600                 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
4601                         netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
4602                                    j, nr_rings);
4603                         break;
4604                 }
4605                 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
4606         }
4607
4608 vnic_no_ring_grps:
4609         for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
4610                 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
4611         if (vnic_id == 0)
4612                 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
4613
4614         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
4615
4616         mutex_lock(&bp->hwrm_cmd_lock);
4617         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4618         if (!rc)
4619                 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
4620         mutex_unlock(&bp->hwrm_cmd_lock);
4621         return rc;
4622 }
4623
4624 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
4625 {
4626         struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4627         struct hwrm_vnic_qcaps_input req = {0};
4628         int rc;
4629
4630         if (bp->hwrm_spec_code < 0x10600)
4631                 return 0;
4632
4633         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
4634         mutex_lock(&bp->hwrm_cmd_lock);
4635         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4636         if (!rc) {
4637                 u32 flags = le32_to_cpu(resp->flags);
4638
4639                 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
4640                     (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
4641                         bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
4642                 if (flags &
4643                     VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
4644                         bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
4645         }
4646         mutex_unlock(&bp->hwrm_cmd_lock);
4647         return rc;
4648 }
4649
4650 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
4651 {
4652         u16 i;
4653         u32 rc = 0;
4654
4655         if (bp->flags & BNXT_FLAG_CHIP_P5)
4656                 return 0;
4657
4658         mutex_lock(&bp->hwrm_cmd_lock);
4659         for (i = 0; i < bp->rx_nr_rings; i++) {
4660                 struct hwrm_ring_grp_alloc_input req = {0};
4661                 struct hwrm_ring_grp_alloc_output *resp =
4662                                         bp->hwrm_cmd_resp_addr;
4663                 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
4664
4665                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
4666
4667                 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
4668                 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
4669                 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
4670                 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
4671
4672                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4673                                         HWRM_CMD_TIMEOUT);
4674                 if (rc)
4675                         break;
4676
4677                 bp->grp_info[grp_idx].fw_grp_id =
4678                         le32_to_cpu(resp->ring_group_id);
4679         }
4680         mutex_unlock(&bp->hwrm_cmd_lock);
4681         return rc;
4682 }
4683
4684 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
4685 {
4686         u16 i;
4687         u32 rc = 0;
4688         struct hwrm_ring_grp_free_input req = {0};
4689
4690         if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
4691                 return 0;
4692
4693         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
4694
4695         mutex_lock(&bp->hwrm_cmd_lock);
4696         for (i = 0; i < bp->cp_nr_rings; i++) {
4697                 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
4698                         continue;
4699                 req.ring_group_id =
4700                         cpu_to_le32(bp->grp_info[i].fw_grp_id);
4701
4702                 rc = _hwrm_send_message(bp, &req, sizeof(req),
4703                                         HWRM_CMD_TIMEOUT);
4704                 if (rc)
4705                         break;
4706                 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4707         }
4708         mutex_unlock(&bp->hwrm_cmd_lock);
4709         return rc;
4710 }
4711
4712 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
4713                                     struct bnxt_ring_struct *ring,
4714                                     u32 ring_type, u32 map_index)
4715 {
4716         int rc = 0, err = 0;
4717         struct hwrm_ring_alloc_input req = {0};
4718         struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4719         struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
4720         struct bnxt_ring_grp_info *grp_info;
4721         u16 ring_id;
4722
4723         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
4724
4725         req.enables = 0;
4726         if (rmem->nr_pages > 1) {
4727                 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
4728                 /* Page size is in log2 units */
4729                 req.page_size = BNXT_PAGE_SHIFT;
4730                 req.page_tbl_depth = 1;
4731         } else {
4732                 req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
4733         }
4734         req.fbo = 0;
4735         /* Association of ring index with doorbell index and MSIX number */
4736         req.logical_id = cpu_to_le16(map_index);
4737
4738         switch (ring_type) {
4739         case HWRM_RING_ALLOC_TX: {
4740                 struct bnxt_tx_ring_info *txr;
4741
4742                 txr = container_of(ring, struct bnxt_tx_ring_info,
4743                                    tx_ring_struct);
4744                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
4745                 /* Association of transmit ring with completion ring */
4746                 grp_info = &bp->grp_info[ring->grp_idx];
4747                 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
4748                 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
4749                 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4750                 req.queue_id = cpu_to_le16(ring->queue_id);
4751                 break;
4752         }
4753         case HWRM_RING_ALLOC_RX:
4754                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4755                 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
4756                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4757                         u16 flags = 0;
4758
4759                         /* Association of rx ring with stats context */
4760                         grp_info = &bp->grp_info[ring->grp_idx];
4761                         req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
4762                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4763                         req.enables |= cpu_to_le32(
4764                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
4765                         if (NET_IP_ALIGN == 2)
4766                                 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
4767                         req.flags = cpu_to_le16(flags);
4768                 }
4769                 break;
4770         case HWRM_RING_ALLOC_AGG:
4771                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4772                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
4773                         /* Association of agg ring with rx ring */
4774                         grp_info = &bp->grp_info[ring->grp_idx];
4775                         req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
4776                         req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
4777                         req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
4778                         req.enables |= cpu_to_le32(
4779                                 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
4780                                 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
4781                 } else {
4782                         req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
4783                 }
4784                 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
4785                 break;
4786         case HWRM_RING_ALLOC_CMPL:
4787                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
4788                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4789                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4790                         /* Association of cp ring with nq */
4791                         grp_info = &bp->grp_info[map_index];
4792                         req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
4793                         req.cq_handle = cpu_to_le64(ring->handle);
4794                         req.enables |= cpu_to_le32(
4795                                 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
4796                 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
4797                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4798                 }
4799                 break;
4800         case HWRM_RING_ALLOC_NQ:
4801                 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
4802                 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
4803                 if (bp->flags & BNXT_FLAG_USING_MSIX)
4804                         req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
4805                 break;
4806         default:
4807                 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
4808                            ring_type);
4809                 return -1;
4810         }
4811
4812         mutex_lock(&bp->hwrm_cmd_lock);
4813         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4814         err = le16_to_cpu(resp->error_code);
4815         ring_id = le16_to_cpu(resp->ring_id);
4816         mutex_unlock(&bp->hwrm_cmd_lock);
4817
4818         if (rc || err) {
4819                 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
4820                            ring_type, rc, err);
4821                 return -EIO;
4822         }
4823         ring->fw_ring_id = ring_id;
4824         return rc;
4825 }
4826
4827 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
4828 {
4829         int rc;
4830
4831         if (BNXT_PF(bp)) {
4832                 struct hwrm_func_cfg_input req = {0};
4833
4834                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
4835                 req.fid = cpu_to_le16(0xffff);
4836                 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4837                 req.async_event_cr = cpu_to_le16(idx);
4838                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4839         } else {
4840                 struct hwrm_func_vf_cfg_input req = {0};
4841
4842                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
4843                 req.enables =
4844                         cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
4845                 req.async_event_cr = cpu_to_le16(idx);
4846                 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4847         }
4848         return rc;
4849 }
4850
4851 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
4852                         u32 map_idx, u32 xid)
4853 {
4854         if (bp->flags & BNXT_FLAG_CHIP_P5) {
4855                 if (BNXT_PF(bp))
4856                         db->doorbell = bp->bar1 + 0x10000;
4857                 else
4858                         db->doorbell = bp->bar1 + 0x4000;
4859                 switch (ring_type) {
4860                 case HWRM_RING_ALLOC_TX:
4861                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
4862                         break;
4863                 case HWRM_RING_ALLOC_RX:
4864                 case HWRM_RING_ALLOC_AGG:
4865                         db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
4866                         break;
4867                 case HWRM_RING_ALLOC_CMPL:
4868                         db->db_key64 = DBR_PATH_L2;
4869                         break;
4870                 case HWRM_RING_ALLOC_NQ:
4871                         db->db_key64 = DBR_PATH_L2;
4872                         break;
4873                 }
4874                 db->db_key64 |= (u64)xid << DBR_XID_SFT;
4875         } else {
4876                 db->doorbell = bp->bar1 + map_idx * 0x80;
4877                 switch (ring_type) {
4878                 case HWRM_RING_ALLOC_TX:
4879                         db->db_key32 = DB_KEY_TX;
4880                         break;
4881                 case HWRM_RING_ALLOC_RX:
4882                 case HWRM_RING_ALLOC_AGG:
4883                         db->db_key32 = DB_KEY_RX;
4884                         break;
4885                 case HWRM_RING_ALLOC_CMPL:
4886                         db->db_key32 = DB_KEY_CP;
4887                         break;
4888                 }
4889         }
4890 }
4891
4892 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
4893 {
4894         int i, rc = 0;
4895         u32 type;
4896
4897         if (bp->flags & BNXT_FLAG_CHIP_P5)
4898                 type = HWRM_RING_ALLOC_NQ;
4899         else
4900                 type = HWRM_RING_ALLOC_CMPL;
4901         for (i = 0; i < bp->cp_nr_rings; i++) {
4902                 struct bnxt_napi *bnapi = bp->bnapi[i];
4903                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4904                 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4905                 u32 map_idx = ring->map_idx;
4906
4907                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
4908                 if (rc)
4909                         goto err_out;
4910                 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
4911                 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4912                 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
4913
4914                 if (!i) {
4915                         rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
4916                         if (rc)
4917                                 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
4918                 }
4919         }
4920
4921         type = HWRM_RING_ALLOC_TX;
4922         for (i = 0; i < bp->tx_nr_rings; i++) {
4923                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4924                 struct bnxt_ring_struct *ring;
4925                 u32 map_idx;
4926
4927                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4928                         struct bnxt_napi *bnapi = txr->bnapi;
4929                         struct bnxt_cp_ring_info *cpr, *cpr2;
4930                         u32 type2 = HWRM_RING_ALLOC_CMPL;
4931
4932                         cpr = &bnapi->cp_ring;
4933                         cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
4934                         ring = &cpr2->cp_ring_struct;
4935                         ring->handle = BNXT_TX_HDL;
4936                         map_idx = bnapi->index;
4937                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
4938                         if (rc)
4939                                 goto err_out;
4940                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
4941                                     ring->fw_ring_id);
4942                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
4943                 }
4944                 ring = &txr->tx_ring_struct;
4945                 map_idx = i;
4946                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
4947                 if (rc)
4948                         goto err_out;
4949                 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
4950         }
4951
4952         type = HWRM_RING_ALLOC_RX;
4953         for (i = 0; i < bp->rx_nr_rings; i++) {
4954                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4955                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
4956                 struct bnxt_napi *bnapi = rxr->bnapi;
4957                 u32 map_idx = bnapi->index;
4958
4959                 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
4960                 if (rc)
4961                         goto err_out;
4962                 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
4963                 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
4964                 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
4965                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4966                         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4967                         u32 type2 = HWRM_RING_ALLOC_CMPL;
4968                         struct bnxt_cp_ring_info *cpr2;
4969
4970                         cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
4971                         ring = &cpr2->cp_ring_struct;
4972                         ring->handle = BNXT_RX_HDL;
4973                         rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
4974                         if (rc)
4975                                 goto err_out;
4976                         bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
4977                                     ring->fw_ring_id);
4978                         bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
4979                 }
4980         }
4981
4982         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4983                 type = HWRM_RING_ALLOC_AGG;
4984                 for (i = 0; i < bp->rx_nr_rings; i++) {
4985                         struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4986                         struct bnxt_ring_struct *ring =
4987                                                 &rxr->rx_agg_ring_struct;
4988                         u32 grp_idx = ring->grp_idx;
4989                         u32 map_idx = grp_idx + bp->rx_nr_rings;
4990
4991                         rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
4992                         if (rc)
4993                                 goto err_out;
4994
4995                         bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
4996                                     ring->fw_ring_id);
4997                         bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
4998                         bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
4999                 }
5000         }
5001 err_out:
5002         return rc;
5003 }
5004
5005 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5006                                    struct bnxt_ring_struct *ring,
5007                                    u32 ring_type, int cmpl_ring_id)
5008 {
5009         int rc;
5010         struct hwrm_ring_free_input req = {0};
5011         struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5012         u16 error_code;
5013
5014         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5015         req.ring_type = ring_type;
5016         req.ring_id = cpu_to_le16(ring->fw_ring_id);
5017
5018         mutex_lock(&bp->hwrm_cmd_lock);
5019         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5020         error_code = le16_to_cpu(resp->error_code);
5021         mutex_unlock(&bp->hwrm_cmd_lock);
5022
5023         if (rc || error_code) {
5024                 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5025                            ring_type, rc, error_code);
5026                 return -EIO;
5027         }
5028         return 0;
5029 }
5030
5031 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5032 {
5033         u32 type;
5034         int i;
5035
5036         if (!bp->bnapi)
5037                 return;
5038
5039         for (i = 0; i < bp->tx_nr_rings; i++) {
5040                 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5041                 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5042                 u32 cmpl_ring_id;
5043
5044                 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5045                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5046                         hwrm_ring_free_send_msg(bp, ring,
5047                                                 RING_FREE_REQ_RING_TYPE_TX,
5048                                                 close_path ? cmpl_ring_id :
5049                                                 INVALID_HW_RING_ID);
5050                         ring->fw_ring_id = INVALID_HW_RING_ID;
5051                 }
5052         }
5053
5054         for (i = 0; i < bp->rx_nr_rings; i++) {
5055                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5056                 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5057                 u32 grp_idx = rxr->bnapi->index;
5058                 u32 cmpl_ring_id;
5059
5060                 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5061                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5062                         hwrm_ring_free_send_msg(bp, ring,
5063                                                 RING_FREE_REQ_RING_TYPE_RX,
5064                                                 close_path ? cmpl_ring_id :
5065                                                 INVALID_HW_RING_ID);
5066                         ring->fw_ring_id = INVALID_HW_RING_ID;
5067                         bp->grp_info[grp_idx].rx_fw_ring_id =
5068                                 INVALID_HW_RING_ID;
5069                 }
5070         }
5071
5072         if (bp->flags & BNXT_FLAG_CHIP_P5)
5073                 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5074         else
5075                 type = RING_FREE_REQ_RING_TYPE_RX;
5076         for (i = 0; i < bp->rx_nr_rings; i++) {
5077                 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5078                 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5079                 u32 grp_idx = rxr->bnapi->index;
5080                 u32 cmpl_ring_id;
5081
5082                 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5083                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5084                         hwrm_ring_free_send_msg(bp, ring, type,
5085                                                 close_path ? cmpl_ring_id :
5086                                                 INVALID_HW_RING_ID);
5087                         ring->fw_ring_id = INVALID_HW_RING_ID;
5088                         bp->grp_info[grp_idx].agg_fw_ring_id =
5089                                 INVALID_HW_RING_ID;
5090                 }
5091         }
5092
5093         /* The completion rings are about to be freed.  After that the
5094          * IRQ doorbell will not work anymore.  So we need to disable
5095          * IRQ here.
5096          */
5097         bnxt_disable_int_sync(bp);
5098
5099         if (bp->flags & BNXT_FLAG_CHIP_P5)
5100                 type = RING_FREE_REQ_RING_TYPE_NQ;
5101         else
5102                 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5103         for (i = 0; i < bp->cp_nr_rings; i++) {
5104                 struct bnxt_napi *bnapi = bp->bnapi[i];
5105                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5106                 struct bnxt_ring_struct *ring;
5107                 int j;
5108
5109                 for (j = 0; j < 2; j++) {
5110                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5111
5112                         if (cpr2) {
5113                                 ring = &cpr2->cp_ring_struct;
5114                                 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5115                                         continue;
5116                                 hwrm_ring_free_send_msg(bp, ring,
5117                                         RING_FREE_REQ_RING_TYPE_L2_CMPL,
5118                                         INVALID_HW_RING_ID);
5119                                 ring->fw_ring_id = INVALID_HW_RING_ID;
5120                         }
5121                 }
5122                 ring = &cpr->cp_ring_struct;
5123                 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5124                         hwrm_ring_free_send_msg(bp, ring, type,
5125                                                 INVALID_HW_RING_ID);
5126                         ring->fw_ring_id = INVALID_HW_RING_ID;
5127                         bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5128                 }
5129         }
5130 }
5131
5132 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5133                            bool shared);
5134
5135 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5136 {
5137         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5138         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5139         struct hwrm_func_qcfg_input req = {0};
5140         int rc;
5141
5142         if (bp->hwrm_spec_code < 0x10601)
5143                 return 0;
5144
5145         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5146         req.fid = cpu_to_le16(0xffff);
5147         mutex_lock(&bp->hwrm_cmd_lock);
5148         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5149         if (rc) {
5150                 mutex_unlock(&bp->hwrm_cmd_lock);
5151                 return -EIO;
5152         }
5153
5154         hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5155         if (BNXT_NEW_RM(bp)) {
5156                 u16 cp, stats;
5157
5158                 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5159                 hw_resc->resv_hw_ring_grps =
5160                         le32_to_cpu(resp->alloc_hw_ring_grps);
5161                 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5162                 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5163                 stats = le16_to_cpu(resp->alloc_stat_ctx);
5164                 cp = min_t(u16, cp, stats);
5165                 hw_resc->resv_irqs = cp;
5166                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5167                         int rx = hw_resc->resv_rx_rings;
5168                         int tx = hw_resc->resv_tx_rings;
5169
5170                         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5171                                 rx >>= 1;
5172                         if (cp < (rx + tx)) {
5173                                 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5174                                 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5175                                         rx <<= 1;
5176                                 hw_resc->resv_rx_rings = rx;
5177                                 hw_resc->resv_tx_rings = tx;
5178                         }
5179                         hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5180                         hw_resc->resv_hw_ring_grps = rx;
5181                 }
5182                 hw_resc->resv_cp_rings = cp;
5183         }
5184         mutex_unlock(&bp->hwrm_cmd_lock);
5185         return 0;
5186 }
5187
5188 /* Caller must hold bp->hwrm_cmd_lock */
5189 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5190 {
5191         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5192         struct hwrm_func_qcfg_input req = {0};
5193         int rc;
5194
5195         if (bp->hwrm_spec_code < 0x10601)
5196                 return 0;
5197
5198         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5199         req.fid = cpu_to_le16(fid);
5200         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5201         if (!rc)
5202                 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5203
5204         return rc;
5205 }
5206
5207 static bool bnxt_rfs_supported(struct bnxt *bp);
5208
5209 static void
5210 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5211                              int tx_rings, int rx_rings, int ring_grps,
5212                              int cp_rings, int vnics)
5213 {
5214         u32 enables = 0;
5215
5216         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5217         req->fid = cpu_to_le16(0xffff);
5218         enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5219         req->num_tx_rings = cpu_to_le16(tx_rings);
5220         if (BNXT_NEW_RM(bp)) {
5221                 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5222                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5223                         enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5224                         enables |= tx_rings + ring_grps ?
5225                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5226                                    FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5227                         enables |= rx_rings ?
5228                                 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5229                 } else {
5230                         enables |= cp_rings ?
5231                                    FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5232                                    FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5233                         enables |= ring_grps ?
5234                                    FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5235                                    FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5236                 }
5237                 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5238
5239                 req->num_rx_rings = cpu_to_le16(rx_rings);
5240                 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5241                         req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5242                         req->num_msix = cpu_to_le16(cp_rings);
5243                         req->num_rsscos_ctxs =
5244                                 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5245                 } else {
5246                         req->num_cmpl_rings = cpu_to_le16(cp_rings);
5247                         req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5248                         req->num_rsscos_ctxs = cpu_to_le16(1);
5249                         if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5250                             bnxt_rfs_supported(bp))
5251                                 req->num_rsscos_ctxs =
5252                                         cpu_to_le16(ring_grps + 1);
5253                 }
5254                 req->num_stat_ctxs = req->num_cmpl_rings;
5255                 req->num_vnics = cpu_to_le16(vnics);
5256         }
5257         req->enables = cpu_to_le32(enables);
5258 }
5259
5260 static void
5261 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5262                              struct hwrm_func_vf_cfg_input *req, int tx_rings,
5263                              int rx_rings, int ring_grps, int cp_rings,
5264                              int vnics)
5265 {
5266         u32 enables = 0;
5267
5268         bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5269         enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5270         enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5271                               FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5272         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5273                 enables |= tx_rings + ring_grps ?
5274                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5275                            FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5276         } else {
5277                 enables |= cp_rings ?
5278                            FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
5279                            FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5280                 enables |= ring_grps ?
5281                            FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5282         }
5283         enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5284         enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5285
5286         req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
5287         req->num_tx_rings = cpu_to_le16(tx_rings);
5288         req->num_rx_rings = cpu_to_le16(rx_rings);
5289         if (bp->flags & BNXT_FLAG_CHIP_P5) {
5290                 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5291                 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5292         } else {
5293                 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5294                 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5295                 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5296         }
5297         req->num_stat_ctxs = req->num_cmpl_rings;
5298         req->num_vnics = cpu_to_le16(vnics);
5299
5300         req->enables = cpu_to_le32(enables);
5301 }
5302
5303 static int
5304 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5305                            int ring_grps, int cp_rings, int vnics)
5306 {
5307         struct hwrm_func_cfg_input req = {0};
5308         int rc;
5309
5310         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5311                                      cp_rings, vnics);
5312         if (!req.enables)
5313                 return 0;
5314
5315         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5316         if (rc)
5317                 return -ENOMEM;
5318
5319         if (bp->hwrm_spec_code < 0x10601)
5320                 bp->hw_resc.resv_tx_rings = tx_rings;
5321
5322         rc = bnxt_hwrm_get_rings(bp);
5323         return rc;
5324 }
5325
5326 static int
5327 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5328                            int ring_grps, int cp_rings, int vnics)
5329 {
5330         struct hwrm_func_vf_cfg_input req = {0};
5331         int rc;
5332
5333         if (!BNXT_NEW_RM(bp)) {
5334                 bp->hw_resc.resv_tx_rings = tx_rings;
5335                 return 0;
5336         }
5337
5338         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5339                                      cp_rings, vnics);
5340         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5341         if (rc)
5342                 return -ENOMEM;
5343
5344         rc = bnxt_hwrm_get_rings(bp);
5345         return rc;
5346 }
5347
5348 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
5349                                    int cp, int vnic)
5350 {
5351         if (BNXT_PF(bp))
5352                 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic);
5353         else
5354                 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic);
5355 }
5356
5357 static int bnxt_nq_rings_in_use(struct bnxt *bp)
5358 {
5359         int cp = bp->cp_nr_rings;
5360         int ulp_msix, ulp_base;
5361
5362         ulp_msix = bnxt_get_ulp_msix_num(bp);
5363         if (ulp_msix) {
5364                 ulp_base = bnxt_get_ulp_msix_base(bp);
5365                 cp += ulp_msix;
5366                 if ((ulp_base + ulp_msix) > cp)
5367                         cp = ulp_base + ulp_msix;
5368         }
5369         return cp;
5370 }
5371
5372 static int bnxt_cp_rings_in_use(struct bnxt *bp)
5373 {
5374         int cp;
5375
5376         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5377                 return bnxt_nq_rings_in_use(bp);
5378
5379         cp = bp->tx_nr_rings + bp->rx_nr_rings;
5380         return cp;
5381 }
5382
5383 static bool bnxt_need_reserve_rings(struct bnxt *bp)
5384 {
5385         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5386         int cp = bnxt_cp_rings_in_use(bp);
5387         int nq = bnxt_nq_rings_in_use(bp);
5388         int rx = bp->rx_nr_rings;
5389         int vnic = 1, grp = rx;
5390
5391         if (bp->hwrm_spec_code < 0x10601)
5392                 return false;
5393
5394         if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5395                 return true;
5396
5397         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5398                 vnic = rx + 1;
5399         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5400                 rx <<= 1;
5401         if (BNXT_NEW_RM(bp) &&
5402             (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
5403              hw_resc->resv_irqs < nq || hw_resc->resv_vnics != vnic ||
5404              (hw_resc->resv_hw_ring_grps != grp &&
5405               !(bp->flags & BNXT_FLAG_CHIP_P5))))
5406                 return true;
5407         return false;
5408 }
5409
5410 static int __bnxt_reserve_rings(struct bnxt *bp)
5411 {
5412         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5413         int cp = bnxt_nq_rings_in_use(bp);
5414         int tx = bp->tx_nr_rings;
5415         int rx = bp->rx_nr_rings;
5416         int grp, rx_rings, rc;
5417         bool sh = false;
5418         int vnic = 1;
5419
5420         if (!bnxt_need_reserve_rings(bp))
5421                 return 0;
5422
5423         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5424                 sh = true;
5425         if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5426                 vnic = rx + 1;
5427         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5428                 rx <<= 1;
5429         grp = bp->rx_nr_rings;
5430
5431         rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic);
5432         if (rc)
5433                 return rc;
5434
5435         tx = hw_resc->resv_tx_rings;
5436         if (BNXT_NEW_RM(bp)) {
5437                 rx = hw_resc->resv_rx_rings;
5438                 cp = hw_resc->resv_irqs;
5439                 grp = hw_resc->resv_hw_ring_grps;
5440                 vnic = hw_resc->resv_vnics;
5441         }
5442
5443         rx_rings = rx;
5444         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5445                 if (rx >= 2) {
5446                         rx_rings = rx >> 1;
5447                 } else {
5448                         if (netif_running(bp->dev))
5449                                 return -ENOMEM;
5450
5451                         bp->flags &= ~BNXT_FLAG_AGG_RINGS;
5452                         bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
5453                         bp->dev->hw_features &= ~NETIF_F_LRO;
5454                         bp->dev->features &= ~NETIF_F_LRO;
5455                         bnxt_set_ring_params(bp);
5456                 }
5457         }
5458         rx_rings = min_t(int, rx_rings, grp);
5459         rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
5460         if (bp->flags & BNXT_FLAG_AGG_RINGS)
5461                 rx = rx_rings << 1;
5462         cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
5463         bp->tx_nr_rings = tx;
5464         bp->rx_nr_rings = rx_rings;
5465         bp->cp_nr_rings = cp;
5466
5467         if (!tx || !rx || !cp || !grp || !vnic)
5468                 return -ENOMEM;
5469
5470         return rc;
5471 }
5472
5473 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5474                                     int ring_grps, int cp_rings, int vnics)
5475 {
5476         struct hwrm_func_vf_cfg_input req = {0};
5477         u32 flags;
5478         int rc;
5479
5480         if (!BNXT_NEW_RM(bp))
5481                 return 0;
5482
5483         __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5484                                      cp_rings, vnics);
5485         flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
5486                 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5487                 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5488                 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5489                 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
5490                 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5491         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5492                 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5493
5494         req.flags = cpu_to_le32(flags);
5495         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5496         if (rc)
5497                 return -ENOMEM;
5498         return 0;
5499 }
5500
5501 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5502                                     int ring_grps, int cp_rings, int vnics)
5503 {
5504         struct hwrm_func_cfg_input req = {0};
5505         u32 flags;
5506         int rc;
5507
5508         __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5509                                      cp_rings, vnics);
5510         flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
5511         if (BNXT_NEW_RM(bp)) {
5512                 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
5513                          FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
5514                          FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
5515                          FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
5516                 if (bp->flags & BNXT_FLAG_CHIP_P5)
5517                         flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
5518                 else
5519                         flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
5520         }
5521
5522         req.flags = cpu_to_le32(flags);
5523         rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5524         if (rc)
5525                 return -ENOMEM;
5526         return 0;
5527 }
5528
5529 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5530                                  int ring_grps, int cp_rings, int vnics)
5531 {
5532         if (bp->hwrm_spec_code < 0x10801)
5533                 return 0;
5534
5535         if (BNXT_PF(bp))
5536                 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
5537                                                 ring_grps, cp_rings, vnics);
5538
5539         return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
5540                                         cp_rings, vnics);
5541 }
5542
5543 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
5544 {
5545         struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5546         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5547         struct hwrm_ring_aggint_qcaps_input req = {0};
5548         int rc;
5549
5550         coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
5551         coal_cap->num_cmpl_dma_aggr_max = 63;
5552         coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
5553         coal_cap->cmpl_aggr_dma_tmr_max = 65535;
5554         coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
5555         coal_cap->int_lat_tmr_min_max = 65535;
5556         coal_cap->int_lat_tmr_max_max = 65535;
5557         coal_cap->num_cmpl_aggr_int_max = 65535;
5558         coal_cap->timer_units = 80;
5559
5560         if (bp->hwrm_spec_code < 0x10902)
5561                 return;
5562
5563         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
5564         mutex_lock(&bp->hwrm_cmd_lock);
5565         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5566         if (!rc) {
5567                 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
5568                 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
5569                 coal_cap->num_cmpl_dma_aggr_max =
5570                         le16_to_cpu(resp->num_cmpl_dma_aggr_max);
5571                 coal_cap->num_cmpl_dma_aggr_during_int_max =
5572                         le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
5573                 coal_cap->cmpl_aggr_dma_tmr_max =
5574                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
5575                 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
5576                         le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
5577                 coal_cap->int_lat_tmr_min_max =
5578                         le16_to_cpu(resp->int_lat_tmr_min_max);
5579                 coal_cap->int_lat_tmr_max_max =
5580                         le16_to_cpu(resp->int_lat_tmr_max_max);
5581                 coal_cap->num_cmpl_aggr_int_max =
5582                         le16_to_cpu(resp->num_cmpl_aggr_int_max);
5583                 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
5584         }
5585         mutex_unlock(&bp->hwrm_cmd_lock);
5586 }
5587
5588 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
5589 {
5590         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5591
5592         return usec * 1000 / coal_cap->timer_units;
5593 }
5594
5595 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
5596         struct bnxt_coal *hw_coal,
5597         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
5598 {
5599         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5600         u32 cmpl_params = coal_cap->cmpl_params;
5601         u16 val, tmr, max, flags = 0;
5602
5603         max = hw_coal->bufs_per_record * 128;
5604         if (hw_coal->budget)
5605                 max = hw_coal->bufs_per_record * hw_coal->budget;
5606         max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
5607
5608         val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
5609         req->num_cmpl_aggr_int = cpu_to_le16(val);
5610
5611         val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
5612         req->num_cmpl_dma_aggr = cpu_to_le16(val);
5613
5614         val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
5615                       coal_cap->num_cmpl_dma_aggr_during_int_max);
5616         req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
5617
5618         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
5619         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
5620         req->int_lat_tmr_max = cpu_to_le16(tmr);
5621
5622         /* min timer set to 1/2 of interrupt timer */
5623         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
5624                 val = tmr / 2;
5625                 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
5626                 req->int_lat_tmr_min = cpu_to_le16(val);
5627                 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5628         }
5629
5630         /* buf timer set to 1/4 of interrupt timer */
5631         val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
5632         req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
5633
5634         if (cmpl_params &
5635             RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
5636                 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
5637                 val = clamp_t(u16, tmr, 1,
5638                               coal_cap->cmpl_aggr_dma_tmr_during_int_max);
5639                 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(tmr);
5640                 req->enables |=
5641                         cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
5642         }
5643
5644         if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
5645                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
5646         if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
5647             hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
5648                 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
5649         req->flags = cpu_to_le16(flags);
5650         req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
5651 }
5652
5653 /* Caller holds bp->hwrm_cmd_lock */
5654 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
5655                                    struct bnxt_coal *hw_coal)
5656 {
5657         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
5658         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5659         struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
5660         u32 nq_params = coal_cap->nq_params;
5661         u16 tmr;
5662
5663         if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
5664                 return 0;
5665
5666         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
5667                                -1, -1);
5668         req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
5669         req.flags =
5670                 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
5671
5672         tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
5673         tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
5674         req.int_lat_tmr_min = cpu_to_le16(tmr);
5675         req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
5676         return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5677 }
5678
5679 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
5680 {
5681         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
5682         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5683         struct bnxt_coal coal;
5684
5685         /* Tick values in micro seconds.
5686          * 1 coal_buf x bufs_per_record = 1 completion record.
5687          */
5688         memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
5689
5690         coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
5691         coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
5692
5693         if (!bnapi->rx_ring)
5694                 return -ENODEV;
5695
5696         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
5697                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5698
5699         bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
5700
5701         req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
5702
5703         return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
5704                                  HWRM_CMD_TIMEOUT);
5705 }
5706
5707 int bnxt_hwrm_set_coal(struct bnxt *bp)
5708 {
5709         int i, rc = 0;
5710         struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
5711                                                            req_tx = {0}, *req;
5712
5713         bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
5714                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5715         bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
5716                                HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
5717
5718         bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
5719         bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
5720
5721         mutex_lock(&bp->hwrm_cmd_lock);
5722         for (i = 0; i < bp->cp_nr_rings; i++) {
5723                 struct bnxt_napi *bnapi = bp->bnapi[i];
5724                 struct bnxt_coal *hw_coal;
5725                 u16 ring_id;
5726
5727                 req = &req_rx;
5728                 if (!bnapi->rx_ring) {
5729                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
5730                         req = &req_tx;
5731                 } else {
5732                         ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
5733                 }
5734                 req->ring_id = cpu_to_le16(ring_id);
5735
5736                 rc = _hwrm_send_message(bp, req, sizeof(*req),
5737                                         HWRM_CMD_TIMEOUT);
5738                 if (rc)
5739                         break;
5740
5741                 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5742                         continue;
5743
5744                 if (bnapi->rx_ring && bnapi->tx_ring) {
5745                         req = &req_tx;
5746                         ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
5747                         req->ring_id = cpu_to_le16(ring_id);
5748                         rc = _hwrm_send_message(bp, req, sizeof(*req),
5749                                                 HWRM_CMD_TIMEOUT);
5750                         if (rc)
5751                                 break;
5752                 }
5753                 if (bnapi->rx_ring)
5754                         hw_coal = &bp->rx_coal;
5755                 else
5756                         hw_coal = &bp->tx_coal;
5757                 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
5758         }
5759         mutex_unlock(&bp->hwrm_cmd_lock);
5760         return rc;
5761 }
5762
5763 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
5764 {
5765         int rc = 0, i;
5766         struct hwrm_stat_ctx_free_input req = {0};
5767
5768         if (!bp->bnapi)
5769                 return 0;
5770
5771         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5772                 return 0;
5773
5774         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
5775
5776         mutex_lock(&bp->hwrm_cmd_lock);
5777         for (i = 0; i < bp->cp_nr_rings; i++) {
5778                 struct bnxt_napi *bnapi = bp->bnapi[i];
5779                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5780
5781                 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
5782                         req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
5783
5784                         rc = _hwrm_send_message(bp, &req, sizeof(req),
5785                                                 HWRM_CMD_TIMEOUT);
5786                         if (rc)
5787                                 break;
5788
5789                         cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
5790                 }
5791         }
5792         mutex_unlock(&bp->hwrm_cmd_lock);
5793         return rc;
5794 }
5795
5796 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
5797 {
5798         int rc = 0, i;
5799         struct hwrm_stat_ctx_alloc_input req = {0};
5800         struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5801
5802         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5803                 return 0;
5804
5805         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
5806
5807         req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
5808
5809         mutex_lock(&bp->hwrm_cmd_lock);
5810         for (i = 0; i < bp->cp_nr_rings; i++) {
5811                 struct bnxt_napi *bnapi = bp->bnapi[i];
5812                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5813
5814                 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
5815
5816                 rc = _hwrm_send_message(bp, &req, sizeof(req),
5817                                         HWRM_CMD_TIMEOUT);
5818                 if (rc)
5819                         break;
5820
5821                 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
5822
5823                 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
5824         }
5825         mutex_unlock(&bp->hwrm_cmd_lock);
5826         return rc;
5827 }
5828
5829 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
5830 {
5831         struct hwrm_func_qcfg_input req = {0};
5832         struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5833         u16 flags;
5834         int rc;
5835
5836         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5837         req.fid = cpu_to_le16(0xffff);
5838         mutex_lock(&bp->hwrm_cmd_lock);
5839         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5840         if (rc)
5841                 goto func_qcfg_exit;
5842
5843 #ifdef CONFIG_BNXT_SRIOV
5844         if (BNXT_VF(bp)) {
5845                 struct bnxt_vf_info *vf = &bp->vf;
5846
5847                 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
5848         }
5849 #endif
5850         flags = le16_to_cpu(resp->flags);
5851         if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
5852                      FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
5853                 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
5854                 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
5855                         bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
5856         }
5857         if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
5858                 bp->flags |= BNXT_FLAG_MULTI_HOST;
5859
5860         switch (resp->port_partition_type) {
5861         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
5862         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
5863         case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
5864                 bp->port_partition_type = resp->port_partition_type;
5865                 break;
5866         }
5867         if (bp->hwrm_spec_code < 0x10707 ||
5868             resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
5869                 bp->br_mode = BRIDGE_MODE_VEB;
5870         else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
5871                 bp->br_mode = BRIDGE_MODE_VEPA;
5872         else
5873                 bp->br_mode = BRIDGE_MODE_UNDEF;
5874
5875         bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
5876         if (!bp->max_mtu)
5877                 bp->max_mtu = BNXT_MAX_MTU;
5878
5879 func_qcfg_exit:
5880         mutex_unlock(&bp->hwrm_cmd_lock);
5881         return rc;
5882 }
5883
5884 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
5885 {
5886         struct hwrm_func_backing_store_qcaps_input req = {0};
5887         struct hwrm_func_backing_store_qcaps_output *resp =
5888                 bp->hwrm_cmd_resp_addr;
5889         int rc;
5890
5891         if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
5892                 return 0;
5893
5894         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
5895         mutex_lock(&bp->hwrm_cmd_lock);
5896         rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5897         if (!rc) {
5898                 struct bnxt_ctx_pg_info *ctx_pg;
5899                 struct bnxt_ctx_mem_info *ctx;
5900                 int i;
5901
5902                 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
5903                 if (!ctx) {
5904                         rc = -ENOMEM;
5905                         goto ctx_err;
5906                 }
5907                 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
5908                 if (!ctx_pg) {
5909                         kfree(ctx);
5910                         rc = -ENOMEM;
5911                         goto ctx_err;
5912                 }
5913                 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
5914                         ctx->tqm_mem[i] = ctx_pg;
5915
5916                 bp->ctx = ctx;
5917                 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
5918                 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
5919                 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
5920                 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
5921                 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
5922                 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
5923                 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
5924                 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
5925                 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
5926                 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
5927                 ctx->vnic_max_vnic_entries =
5928                         le16_to_cpu(resp->vnic_max_vnic_entries);
5929                 ctx->vnic_max_ring_table_entries =
5930                         le16_to_cpu(resp->vnic_max_ring_table_entries);
5931                 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
5932                 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
5933                 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
5934                 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
5935                 ctx->tqm_min_entries_per_ring =
5936                         le32_to_cpu(resp->tqm_min_entries_per_ring);
5937                 ctx->tqm_max_entries_per_ring =
5938                         le32_to_cpu(resp->tqm_max_entries_per_ring);
5939                 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
5940                 if (!ctx->tqm_entries_multiple)
5941                         ctx->tqm_entries_multiple = 1;
5942                 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
5943                 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
5944                 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
5945                 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
5946         } else {
5947                 rc = 0;
5948         }
5949 ctx_err:
5950         mutex_unlock(&bp->hwrm_cmd_lock);
5951         return rc;
5952 }
5953
5954 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
5955                                   __le64 *pg_dir)
5956 {
5957         u8 pg_size = 0;
5958
5959         if (BNXT_PAGE_SHIFT == 13)
5960                 pg_size = 1 << 4;
5961         else if (BNXT_PAGE_SIZE == 16)
5962                 pg_size = 2 << 4;
5963
5964         *pg_attr = pg_size;
5965         if (rmem->nr_pages > 1) {
5966                 *pg_attr |= 1;
5967                 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
5968         } else {
5969                 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
5970         }
5971 }
5972
5973 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES                 \
5974         (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |                \
5975          FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |               \
5976          FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |                \
5977          FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |              \
5978          FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
5979
5980 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
5981 {
5982         struct hwrm_func_backing_store_cfg_input req = {0};
5983         struct bnxt_ctx_mem_info *ctx = bp->ctx;
5984         struct bnxt_ctx_pg_info *ctx_pg;
5985         __le32 *num_entries;
5986         __le64 *pg_dir;
5987         u8 *pg_attr;
5988         int i, rc;
5989         u32 ena;
5990
5991         if (!ctx)
5992                 return 0;
5993
5994         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
5995         req.enables = cpu_to_le32(enables);
5996
5997         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
5998                 ctx_pg = &ctx->qp_mem;
5999                 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6000                 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6001                 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6002                 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6003                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6004                                       &req.qpc_pg_size_qpc_lvl,
6005                                       &req.qpc_page_dir);
6006         }
6007         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6008                 ctx_pg = &ctx->srq_mem;
6009                 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6010                 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6011                 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6012                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6013                                       &req.srq_pg_size_srq_lvl,
6014                                       &req.srq_page_dir);
6015         }
6016         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6017                 ctx_pg = &ctx->cq_mem;
6018                 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6019                 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6020                 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6021                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6022                                       &req.cq_page_dir);
6023         }
6024         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6025                 ctx_pg = &ctx->vnic_mem;
6026                 req.vnic_num_vnic_entries =
6027                         cpu_to_le16(ctx->vnic_max_vnic_entries);
6028                 req.vnic_num_ring_table_entries =
6029                         cpu_to_le16(ctx->vnic_max_ring_table_entries);
6030                 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6031                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6032                                       &req.vnic_pg_size_vnic_lvl,
6033                                       &req.vnic_page_dir);
6034         }
6035         if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6036                 ctx_pg = &ctx->stat_mem;
6037                 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6038                 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6039                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6040                                       &req.stat_pg_size_stat_lvl,
6041                                       &req.stat_page_dir);
6042         }
6043         for (i = 0, num_entries = &req.tqm_sp_num_entries,
6044              pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6045              pg_dir = &req.tqm_sp_page_dir,
6046              ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6047              i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6048                 if (!(enables & ena))
6049                         continue;
6050
6051                 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6052                 ctx_pg = ctx->tqm_mem[i];
6053                 *num_entries = cpu_to_le32(ctx_pg->entries);
6054                 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6055         }
6056         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6057         if (rc)
6058                 rc = -EIO;
6059         return rc;
6060 }
6061
6062 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6063                                   struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size)
6064 {
6065         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6066
6067         if (!mem_size)
6068                 return 0;
6069
6070         rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6071         if (rmem->nr_pages > MAX_CTX_PAGES) {
6072                 rmem->nr_pages = 0;
6073                 return -EINVAL;
6074         }
6075         rmem->page_size = BNXT_PAGE_SIZE;
6076         rmem->pg_arr = ctx_pg->ctx_pg_arr;
6077         rmem->dma_arr = ctx_pg->ctx_dma_arr;
6078         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6079         return bnxt_alloc_ring(bp, rmem);
6080 }
6081
6082 static void bnxt_free_ctx_mem(struct bnxt *bp)
6083 {
6084         struct bnxt_ctx_mem_info *ctx = bp->ctx;
6085         int i;
6086
6087         if (!ctx)
6088                 return;
6089
6090         if (ctx->tqm_mem[0]) {
6091                 for (i = 0; i < bp->max_q + 1; i++)
6092                         bnxt_free_ring(bp, &ctx->tqm_mem[i]->ring_mem);
6093                 kfree(ctx->tqm_mem[0]);
6094                 ctx->tqm_mem[0] = NULL;
6095         }
6096
6097         bnxt_free_ring(bp, &ctx->stat_mem.ring_mem);
6098         bnxt_free_ring(bp, &ctx->vnic_mem.ring_mem);
6099         bnxt_free_ring(bp, &ctx->cq_mem.ring_mem);
6100         bnxt_free_ring(bp, &ctx->srq_mem.ring_mem);
6101         bnxt_free_ring(bp, &ctx->qp_mem.ring_mem);
6102         ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6103 }
6104
6105 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6106 {
6107         struct bnxt_ctx_pg_info *ctx_pg;
6108         struct bnxt_ctx_mem_info *ctx;
6109         u32 mem_size, ena, entries;
6110         int i, rc;
6111
6112         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6113         if (rc) {
6114                 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6115                            rc);
6116                 return rc;
6117         }
6118         ctx = bp->ctx;
6119         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6120                 return 0;
6121
6122         ctx_pg = &ctx->qp_mem;
6123         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries;
6124         mem_size = ctx->qp_entry_size * ctx_pg->entries;
6125         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6126         if (rc)
6127                 return rc;
6128
6129         ctx_pg = &ctx->srq_mem;
6130         ctx_pg->entries = ctx->srq_max_l2_entries;
6131         mem_size = ctx->srq_entry_size * ctx_pg->entries;
6132         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6133         if (rc)
6134                 return rc;
6135
6136         ctx_pg = &ctx->cq_mem;
6137         ctx_pg->entries = ctx->cq_max_l2_entries;
6138         mem_size = ctx->cq_entry_size * ctx_pg->entries;
6139         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6140         if (rc)
6141                 return rc;
6142
6143         ctx_pg = &ctx->vnic_mem;
6144         ctx_pg->entries = ctx->vnic_max_vnic_entries +
6145                           ctx->vnic_max_ring_table_entries;
6146         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
6147         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6148         if (rc)
6149                 return rc;
6150
6151         ctx_pg = &ctx->stat_mem;
6152         ctx_pg->entries = ctx->stat_max_entries;
6153         mem_size = ctx->stat_entry_size * ctx_pg->entries;
6154         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6155         if (rc)
6156                 return rc;
6157
6158         entries = ctx->qp_max_l2_entries;
6159         entries = roundup(entries, ctx->tqm_entries_multiple);
6160         entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6161                           ctx->tqm_max_entries_per_ring);
6162         for (i = 0, ena = 0; i < bp->max_q + 1; i++) {
6163                 ctx_pg = ctx->tqm_mem[i];
6164                 ctx_pg->entries = entries;
6165                 mem_size = ctx->tqm_entry_size * entries;
6166                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size);
6167                 if (rc)
6168                         return rc;
6169                 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
6170         }
6171         ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6172         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6173         if (rc)
6174                 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6175                            rc);
6176         else
6177                 ctx->flags |= BNXT_CTX_FLAG_INITED;
6178
6179         return 0;
6180 }
6181
6182 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
6183 {
6184         struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6185         struct hwrm_func_resource_qcaps_input req = {0};
6186         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6187         int rc;
6188
6189         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6190         req.fid = cpu_to_le16(0xffff);
6191
6192         mutex_lock(&bp->hwrm_cmd_lock);
6193         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6194         if (rc) {
6195                 rc = -EIO;
6196                 goto hwrm_func_resc_qcaps_exit;
6197         }
6198
6199         hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6200         if (!all)
6201                 goto hwrm_func_resc_qcaps_exit;
6202
6203         hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6204         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6205         hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6206         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6207         hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6208         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6209         hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6210         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6211         hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6212         hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6213         hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6214         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6215         hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6216         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6217         hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6218         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6219
6220         if (bp->flags & BNXT_FLAG_CHIP_P5) {
6221                 u16 max_msix = le16_to_cpu(resp->max_msix);
6222
6223                 hw_resc->max_irqs = min_t(u16, hw_resc->max_irqs, max_msix);
6224                 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6225         }
6226
6227         if (BNXT_PF(bp)) {
6228                 struct bnxt_pf_info *pf = &bp->pf;
6229
6230                 pf->vf_resv_strategy =
6231                         le16_to_cpu(resp->vf_reservation_strategy);
6232                 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
6233                         pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6234         }
6235 hwrm_func_resc_qcaps_exit:
6236         mutex_unlock(&bp->hwrm_cmd_lock);
6237         return rc;
6238 }
6239
6240 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
6241 {
6242         int rc = 0;
6243         struct hwrm_func_qcaps_input req = {0};
6244         struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6245         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6246         u32 flags;
6247
6248         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6249         req.fid = cpu_to_le16(0xffff);
6250
6251         mutex_lock(&bp->hwrm_cmd_lock);
6252         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6253         if (rc)
6254                 goto hwrm_func_qcaps_exit;
6255
6256         flags = le32_to_cpu(resp->flags);
6257         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
6258                 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6259         if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
6260                 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
6261
6262         bp->tx_push_thresh = 0;
6263         if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
6264                 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6265
6266         hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6267         hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6268         hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6269         hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6270         hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6271         if (!hw_resc->max_hw_ring_grps)
6272                 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6273         hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6274         hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6275         hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6276
6277         if (BNXT_PF(bp)) {
6278                 struct bnxt_pf_info *pf = &bp->pf;
6279
6280                 pf->fw_fid = le16_to_cpu(resp->fid);
6281                 pf->port_id = le16_to_cpu(resp->port_id);
6282                 bp->dev->dev_port = pf->port_id;
6283                 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
6284                 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
6285                 pf->max_vfs = le16_to_cpu(resp->max_vfs);
6286                 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
6287                 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
6288                 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
6289                 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
6290                 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
6291                 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
6292                 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
6293                         bp->flags |= BNXT_FLAG_WOL_CAP;
6294         } else {
6295 #ifdef CONFIG_BNXT_SRIOV
6296                 struct bnxt_vf_info *vf = &bp->vf;
6297
6298                 vf->fw_fid = le16_to_cpu(resp->fid);
6299                 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
6300 #endif
6301         }
6302
6303 hwrm_func_qcaps_exit:
6304         mutex_unlock(&bp->hwrm_cmd_lock);
6305         return rc;
6306 }
6307
6308 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
6309
6310 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
6311 {
6312         int rc;
6313
6314         rc = __bnxt_hwrm_func_qcaps(bp);
6315         if (rc)
6316                 return rc;
6317         rc = bnxt_hwrm_queue_qportcfg(bp);
6318         if (rc) {
6319                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
6320                 return rc;
6321         }
6322         if (bp->hwrm_spec_code >= 0x10803) {
6323                 rc = bnxt_alloc_ctx_mem(bp);
6324                 if (rc)
6325                         return rc;
6326                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
6327                 if (!rc)
6328                         bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
6329         }
6330         return 0;
6331 }
6332
6333 static int bnxt_hwrm_func_reset(struct bnxt *bp)
6334 {
6335         struct hwrm_func_reset_input req = {0};
6336
6337         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
6338         req.enables = 0;
6339
6340         return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
6341 }
6342
6343 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
6344 {
6345         int rc = 0;
6346         struct hwrm_queue_qportcfg_input req = {0};
6347         struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
6348         u8 i, j, *qptr;
6349         bool no_rdma;
6350
6351         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
6352
6353         mutex_lock(&bp->hwrm_cmd_lock);
6354         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6355         if (rc)
6356                 goto qportcfg_exit;
6357
6358         if (!resp->max_configurable_queues) {
6359                 rc = -EINVAL;
6360                 goto qportcfg_exit;
6361         }
6362         bp->max_tc = resp->max_configurable_queues;
6363         bp->max_lltc = resp->max_configurable_lossless_queues;
6364         if (bp->max_tc > BNXT_MAX_QUEUE)
6365                 bp->max_tc = BNXT_MAX_QUEUE;
6366
6367         no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
6368         qptr = &resp->queue_id0;
6369         for (i = 0, j = 0; i < bp->max_tc; i++) {
6370                 bp->q_info[j].queue_id = *qptr;
6371                 bp->q_ids[i] = *qptr++;
6372                 bp->q_info[j].queue_profile = *qptr++;
6373                 bp->tc_to_qidx[j] = j;
6374                 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
6375                     (no_rdma && BNXT_PF(bp)))
6376                         j++;
6377         }
6378         bp->max_q = bp->max_tc;
6379         bp->max_tc = max_t(u8, j, 1);
6380
6381         if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
6382                 bp->max_tc = 1;
6383
6384         if (bp->max_lltc > bp->max_tc)
6385                 bp->max_lltc = bp->max_tc;
6386
6387 qportcfg_exit:
6388         mutex_unlock(&bp->hwrm_cmd_lock);
6389         return rc;
6390 }
6391
6392 static int bnxt_hwrm_ver_get(struct bnxt *bp)
6393 {
6394         int rc;
6395         struct hwrm_ver_get_input req = {0};
6396         struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
6397         u32 dev_caps_cfg;
6398
6399         bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
6400         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
6401         req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
6402         req.hwrm_intf_min = HWRM_VERSION_MINOR;
6403         req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
6404         mutex_lock(&bp->hwrm_cmd_lock);
6405         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6406         if (rc)
6407                 goto hwrm_ver_get_exit;
6408
6409         memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
6410
6411         bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
6412                              resp->hwrm_intf_min_8b << 8 |
6413                              resp->hwrm_intf_upd_8b;
6414         if (resp->hwrm_intf_maj_8b < 1) {
6415                 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
6416                             resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
6417                             resp->hwrm_intf_upd_8b);
6418                 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
6419         }
6420         snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
6421                  resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
6422                  resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
6423
6424         bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
6425         if (!bp->hwrm_cmd_timeout)
6426                 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
6427
6428         if (resp->hwrm_intf_maj_8b >= 1) {
6429                 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
6430                 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
6431         }
6432         if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
6433                 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
6434
6435         bp->chip_num = le16_to_cpu(resp->chip_num);
6436         if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
6437             !resp->chip_metal)
6438                 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
6439
6440         dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
6441         if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
6442             (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
6443                 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
6444
6445 hwrm_ver_get_exit:
6446         mutex_unlock(&bp->hwrm_cmd_lock);
6447         return rc;
6448 }
6449
6450 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
6451 {
6452         struct hwrm_fw_set_time_input req = {0};
6453         struct tm tm;
6454         time64_t now = ktime_get_real_seconds();
6455
6456         if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
6457             bp->hwrm_spec_code < 0x10400)
6458                 return -EOPNOTSUPP;
6459
6460         time64_to_tm(now, 0, &tm);
6461         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
6462         req.year = cpu_to_le16(1900 + tm.tm_year);
6463         req.month = 1 + tm.tm_mon;
6464         req.day = tm.tm_mday;
6465         req.hour = tm.tm_hour;
6466         req.minute = tm.tm_min;
6467         req.second = tm.tm_sec;
6468         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6469 }
6470
6471 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
6472 {
6473         int rc;
6474         struct bnxt_pf_info *pf = &bp->pf;
6475         struct hwrm_port_qstats_input req = {0};
6476
6477         if (!(bp->flags & BNXT_FLAG_PORT_STATS))
6478                 return 0;
6479
6480         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
6481         req.port_id = cpu_to_le16(pf->port_id);
6482         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
6483         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
6484         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6485         return rc;
6486 }
6487
6488 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
6489 {
6490         struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
6491         struct hwrm_port_qstats_ext_input req = {0};
6492         struct bnxt_pf_info *pf = &bp->pf;
6493         int rc;
6494
6495         if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
6496                 return 0;
6497
6498         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
6499         req.port_id = cpu_to_le16(pf->port_id);
6500         req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
6501         req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
6502         req.tx_stat_size = cpu_to_le16(sizeof(struct tx_port_stats_ext));
6503         req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
6504         mutex_lock(&bp->hwrm_cmd_lock);
6505         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6506         if (!rc) {
6507                 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
6508                 bp->fw_tx_stats_ext_size = le16_to_cpu(resp->tx_stat_size) / 8;
6509         } else {
6510                 bp->fw_rx_stats_ext_size = 0;
6511                 bp->fw_tx_stats_ext_size = 0;
6512         }
6513         mutex_unlock(&bp->hwrm_cmd_lock);
6514         return rc;
6515 }
6516
6517 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
6518 {
6519         if (bp->vxlan_port_cnt) {
6520                 bnxt_hwrm_tunnel_dst_port_free(
6521                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
6522         }
6523         bp->vxlan_port_cnt = 0;
6524         if (bp->nge_port_cnt) {
6525                 bnxt_hwrm_tunnel_dst_port_free(
6526                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
6527         }
6528         bp->nge_port_cnt = 0;
6529 }
6530
6531 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
6532 {
6533         int rc, i;
6534         u32 tpa_flags = 0;
6535
6536         if (set_tpa)
6537                 tpa_flags = bp->flags & BNXT_FLAG_TPA;
6538         for (i = 0; i < bp->nr_vnics; i++) {
6539                 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
6540                 if (rc) {
6541                         netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
6542                                    i, rc);
6543                         return rc;
6544                 }
6545         }
6546         return 0;
6547 }
6548
6549 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
6550 {
6551         int i;
6552
6553         for (i = 0; i < bp->nr_vnics; i++)
6554                 bnxt_hwrm_vnic_set_rss(bp, i, false);
6555 }
6556
6557 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
6558                                     bool irq_re_init)
6559 {
6560         if (bp->vnic_info) {
6561                 bnxt_hwrm_clear_vnic_filter(bp);
6562                 /* clear all RSS setting before free vnic ctx */
6563                 bnxt_hwrm_clear_vnic_rss(bp);
6564                 bnxt_hwrm_vnic_ctx_free(bp);
6565                 /* before free the vnic, undo the vnic tpa settings */
6566                 if (bp->flags & BNXT_FLAG_TPA)
6567                         bnxt_set_tpa(bp, false);
6568                 bnxt_hwrm_vnic_free(bp);
6569         }
6570         bnxt_hwrm_ring_free(bp, close_path);
6571         bnxt_hwrm_ring_grp_free(bp);
6572         if (irq_re_init) {
6573                 bnxt_hwrm_stat_ctx_free(bp);
6574                 bnxt_hwrm_free_tunnel_ports(bp);
6575         }
6576 }
6577
6578 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
6579 {
6580         struct hwrm_func_cfg_input req = {0};
6581         int rc;
6582
6583         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
6584         req.fid = cpu_to_le16(0xffff);
6585         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
6586         if (br_mode == BRIDGE_MODE_VEB)
6587                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
6588         else if (br_mode == BRIDGE_MODE_VEPA)
6589                 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
6590         else
6591                 return -EINVAL;
6592         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6593         if (rc)
6594                 rc = -EIO;
6595         return rc;
6596 }
6597
6598 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
6599 {
6600         struct hwrm_func_cfg_input req = {0};
6601         int rc;
6602
6603         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
6604                 return 0;
6605
6606         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
6607         req.fid = cpu_to_le16(0xffff);
6608         req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
6609         req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
6610         if (size == 128)
6611                 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
6612
6613         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6614         if (rc)
6615                 rc = -EIO;
6616         return rc;
6617 }
6618
6619 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
6620 {
6621         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
6622         int rc;
6623
6624         if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
6625                 goto skip_rss_ctx;
6626
6627         /* allocate context for vnic */
6628         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
6629         if (rc) {
6630                 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
6631                            vnic_id, rc);
6632                 goto vnic_setup_err;
6633         }
6634         bp->rsscos_nr_ctxs++;
6635
6636         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
6637                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
6638                 if (rc) {
6639                         netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
6640                                    vnic_id, rc);
6641                         goto vnic_setup_err;
6642                 }
6643                 bp->rsscos_nr_ctxs++;
6644         }
6645
6646 skip_rss_ctx:
6647         /* configure default vnic, ring grp */
6648         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
6649         if (rc) {
6650                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
6651                            vnic_id, rc);
6652                 goto vnic_setup_err;
6653         }
6654
6655         /* Enable RSS hashing on vnic */
6656         rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
6657         if (rc) {
6658                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
6659                            vnic_id, rc);
6660                 goto vnic_setup_err;
6661         }
6662
6663         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6664                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
6665                 if (rc) {
6666                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
6667                                    vnic_id, rc);
6668                 }
6669         }
6670
6671 vnic_setup_err:
6672         return rc;
6673 }
6674
6675 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
6676 {
6677         int rc, i, nr_ctxs;
6678
6679         nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
6680         for (i = 0; i < nr_ctxs; i++) {
6681                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
6682                 if (rc) {
6683                         netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
6684                                    vnic_id, i, rc);
6685                         break;
6686                 }
6687                 bp->rsscos_nr_ctxs++;
6688         }
6689         if (i < nr_ctxs)
6690                 return -ENOMEM;
6691
6692         rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
6693         if (rc) {
6694                 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
6695                            vnic_id, rc);
6696                 return rc;
6697         }
6698         rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
6699         if (rc) {
6700                 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
6701                            vnic_id, rc);
6702                 return rc;
6703         }
6704         if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6705                 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
6706                 if (rc) {
6707                         netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
6708                                    vnic_id, rc);
6709                 }
6710         }
6711         return rc;
6712 }
6713
6714 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
6715 {
6716         if (bp->flags & BNXT_FLAG_CHIP_P5)
6717                 return __bnxt_setup_vnic_p5(bp, vnic_id);
6718         else
6719                 return __bnxt_setup_vnic(bp, vnic_id);
6720 }
6721
6722 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
6723 {
6724 #ifdef CONFIG_RFS_ACCEL
6725         int i, rc = 0;
6726
6727         for (i = 0; i < bp->rx_nr_rings; i++) {
6728                 struct bnxt_vnic_info *vnic;
6729                 u16 vnic_id = i + 1;
6730                 u16 ring_id = i;
6731
6732                 if (vnic_id >= bp->nr_vnics)
6733                         break;
6734
6735                 vnic = &bp->vnic_info[vnic_id];
6736                 vnic->flags |= BNXT_VNIC_RFS_FLAG;
6737                 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
6738                         vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
6739                 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
6740                 if (rc) {
6741                         netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
6742                                    vnic_id, rc);
6743                         break;
6744                 }
6745                 rc = bnxt_setup_vnic(bp, vnic_id);
6746                 if (rc)
6747                         break;
6748         }
6749         return rc;
6750 #else
6751         return 0;
6752 #endif
6753 }
6754
6755 /* Allow PF and VF with default VLAN to be in promiscuous mode */
6756 static bool bnxt_promisc_ok(struct bnxt *bp)
6757 {
6758 #ifdef CONFIG_BNXT_SRIOV
6759         if (BNXT_VF(bp) && !bp->vf.vlan)
6760                 return false;
6761 #endif
6762         return true;
6763 }
6764
6765 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
6766 {
6767         unsigned int rc = 0;
6768
6769         rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
6770         if (rc) {
6771                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
6772                            rc);
6773                 return rc;
6774         }
6775
6776         rc = bnxt_hwrm_vnic_cfg(bp, 1);
6777         if (rc) {
6778                 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
6779                            rc);
6780                 return rc;
6781         }
6782         return rc;
6783 }
6784
6785 static int bnxt_cfg_rx_mode(struct bnxt *);
6786 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
6787
6788 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
6789 {
6790         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
6791         int rc = 0;
6792         unsigned int rx_nr_rings = bp->rx_nr_rings;
6793
6794         if (irq_re_init) {
6795                 rc = bnxt_hwrm_stat_ctx_alloc(bp);
6796                 if (rc) {
6797                         netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
6798                                    rc);
6799                         goto err_out;
6800                 }
6801         }
6802
6803         rc = bnxt_hwrm_ring_alloc(bp);
6804         if (rc) {
6805                 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
6806                 goto err_out;
6807         }
6808
6809         rc = bnxt_hwrm_ring_grp_alloc(bp);
6810         if (rc) {
6811                 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
6812                 goto err_out;
6813         }
6814
6815         if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6816                 rx_nr_rings--;
6817
6818         /* default vnic 0 */
6819         rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
6820         if (rc) {
6821                 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
6822                 goto err_out;
6823         }
6824
6825         rc = bnxt_setup_vnic(bp, 0);
6826         if (rc)
6827                 goto err_out;
6828
6829         if (bp->flags & BNXT_FLAG_RFS) {
6830                 rc = bnxt_alloc_rfs_vnics(bp);
6831                 if (rc)
6832                         goto err_out;
6833         }
6834
6835         if (bp->flags & BNXT_FLAG_TPA) {
6836                 rc = bnxt_set_tpa(bp, true);
6837                 if (rc)
6838                         goto err_out;
6839         }
6840
6841         if (BNXT_VF(bp))
6842                 bnxt_update_vf_mac(bp);
6843
6844         /* Filter for default vnic 0 */
6845         rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
6846         if (rc) {
6847                 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
6848                 goto err_out;
6849         }
6850         vnic->uc_filter_count = 1;
6851
6852         vnic->rx_mask = 0;
6853         if (bp->dev->flags & IFF_BROADCAST)
6854                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
6855
6856         if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
6857                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
6858
6859         if (bp->dev->flags & IFF_ALLMULTI) {
6860                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
6861                 vnic->mc_list_count = 0;
6862         } else {
6863                 u32 mask = 0;
6864
6865                 bnxt_mc_list_updated(bp, &mask);
6866                 vnic->rx_mask |= mask;
6867         }
6868
6869         rc = bnxt_cfg_rx_mode(bp);
6870         if (rc)
6871                 goto err_out;
6872
6873         rc = bnxt_hwrm_set_coal(bp);
6874         if (rc)
6875                 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
6876                                 rc);
6877
6878         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
6879                 rc = bnxt_setup_nitroa0_vnic(bp);
6880                 if (rc)
6881                         netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
6882                                    rc);
6883         }
6884
6885         if (BNXT_VF(bp)) {
6886                 bnxt_hwrm_func_qcfg(bp);
6887                 netdev_update_features(bp->dev);
6888         }
6889
6890         return 0;
6891
6892 err_out:
6893         bnxt_hwrm_resource_free(bp, 0, true);
6894
6895         return rc;
6896 }
6897
6898 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
6899 {
6900         bnxt_hwrm_resource_free(bp, 1, irq_re_init);
6901         return 0;
6902 }
6903
6904 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
6905 {
6906         bnxt_init_cp_rings(bp);
6907         bnxt_init_rx_rings(bp);
6908         bnxt_init_tx_rings(bp);
6909         bnxt_init_ring_grps(bp, irq_re_init);
6910         bnxt_init_vnics(bp);
6911
6912         return bnxt_init_chip(bp, irq_re_init);
6913 }
6914
6915 static int bnxt_set_real_num_queues(struct bnxt *bp)
6916 {
6917         int rc;
6918         struct net_device *dev = bp->dev;
6919
6920         rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
6921                                           bp->tx_nr_rings_xdp);
6922         if (rc)
6923                 return rc;
6924
6925         rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
6926         if (rc)
6927                 return rc;
6928
6929 #ifdef CONFIG_RFS_ACCEL
6930         if (bp->flags & BNXT_FLAG_RFS)
6931                 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
6932 #endif
6933
6934         return rc;
6935 }
6936
6937 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6938                            bool shared)
6939 {
6940         int _rx = *rx, _tx = *tx;
6941
6942         if (shared) {
6943                 *rx = min_t(int, _rx, max);
6944                 *tx = min_t(int, _tx, max);
6945         } else {
6946                 if (max < 2)
6947                         return -ENOMEM;
6948
6949                 while (_rx + _tx > max) {
6950                         if (_rx > _tx && _rx > 1)
6951                                 _rx--;
6952                         else if (_tx > 1)
6953                                 _tx--;
6954                 }
6955                 *rx = _rx;
6956                 *tx = _tx;
6957         }
6958         return 0;
6959 }
6960
6961 static void bnxt_setup_msix(struct bnxt *bp)
6962 {
6963         const int len = sizeof(bp->irq_tbl[0].name);
6964         struct net_device *dev = bp->dev;
6965         int tcs, i;
6966
6967         tcs = netdev_get_num_tc(dev);
6968         if (tcs > 1) {
6969                 int i, off, count;
6970
6971                 for (i = 0; i < tcs; i++) {
6972                         count = bp->tx_nr_rings_per_tc;
6973                         off = i * count;
6974                         netdev_set_tc_queue(dev, i, count, off);
6975                 }
6976         }
6977
6978         for (i = 0; i < bp->cp_nr_rings; i++) {
6979                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
6980                 char *attr;
6981
6982                 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6983                         attr = "TxRx";
6984                 else if (i < bp->rx_nr_rings)
6985                         attr = "rx";
6986                 else
6987                         attr = "tx";
6988
6989                 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
6990                          attr, i);
6991                 bp->irq_tbl[map_idx].handler = bnxt_msix;
6992         }
6993 }
6994
6995 static void bnxt_setup_inta(struct bnxt *bp)
6996 {
6997         const int len = sizeof(bp->irq_tbl[0].name);
6998
6999         if (netdev_get_num_tc(bp->dev))
7000                 netdev_reset_tc(bp->dev);
7001
7002         snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7003                  0);
7004         bp->irq_tbl[0].handler = bnxt_inta;
7005 }
7006
7007 static int bnxt_setup_int_mode(struct bnxt *bp)
7008 {
7009         int rc;
7010
7011         if (bp->flags & BNXT_FLAG_USING_MSIX)
7012                 bnxt_setup_msix(bp);
7013         else
7014                 bnxt_setup_inta(bp);
7015
7016         rc = bnxt_set_real_num_queues(bp);
7017         return rc;
7018 }
7019
7020 #ifdef CONFIG_RFS_ACCEL
7021 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7022 {
7023         return bp->hw_resc.max_rsscos_ctxs;
7024 }
7025
7026 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7027 {
7028         return bp->hw_resc.max_vnics;
7029 }
7030 #endif
7031
7032 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7033 {
7034         return bp->hw_resc.max_stat_ctxs;
7035 }
7036
7037 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
7038 {
7039         bp->hw_resc.max_stat_ctxs = max;
7040 }
7041
7042 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7043 {
7044         return bp->hw_resc.max_cp_rings;
7045 }
7046
7047 unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
7048 {
7049         unsigned int cp = bp->hw_resc.max_cp_rings;
7050
7051         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7052                 cp -= bnxt_get_ulp_msix_num(bp);
7053
7054         return cp;
7055 }
7056
7057 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7058 {
7059         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7060
7061         return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7062 }
7063
7064 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
7065 {
7066         bp->hw_resc.max_irqs = max_irqs;
7067 }
7068
7069 int bnxt_get_avail_msix(struct bnxt *bp, int num)
7070 {
7071         int max_cp = bnxt_get_max_func_cp_rings(bp);
7072         int max_irq = bnxt_get_max_func_irqs(bp);
7073         int total_req = bp->cp_nr_rings + num;
7074         int max_idx, avail_msix;
7075
7076         max_idx = bp->total_irqs;
7077         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7078                 max_idx = min_t(int, bp->total_irqs, max_cp);
7079         avail_msix = max_idx - bp->cp_nr_rings;
7080         if (!BNXT_NEW_RM(bp) || avail_msix >= num)
7081                 return avail_msix;
7082
7083         if (max_irq < total_req) {
7084                 num = max_irq - bp->cp_nr_rings;
7085                 if (num <= 0)
7086                         return 0;
7087         }
7088         return num;
7089 }
7090
7091 static int bnxt_get_num_msix(struct bnxt *bp)
7092 {
7093         if (!BNXT_NEW_RM(bp))
7094                 return bnxt_get_max_func_irqs(bp);
7095
7096         return bnxt_nq_rings_in_use(bp);
7097 }
7098
7099 static int bnxt_init_msix(struct bnxt *bp)
7100 {
7101         int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
7102         struct msix_entry *msix_ent;
7103
7104         total_vecs = bnxt_get_num_msix(bp);
7105         max = bnxt_get_max_func_irqs(bp);
7106         if (total_vecs > max)
7107                 total_vecs = max;
7108
7109         if (!total_vecs)
7110                 return 0;
7111
7112         msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
7113         if (!msix_ent)
7114                 return -ENOMEM;
7115
7116         for (i = 0; i < total_vecs; i++) {
7117                 msix_ent[i].entry = i;
7118                 msix_ent[i].vector = 0;
7119         }
7120
7121         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
7122                 min = 2;
7123
7124         total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
7125         ulp_msix = bnxt_get_ulp_msix_num(bp);
7126         if (total_vecs < 0 || total_vecs < ulp_msix) {
7127                 rc = -ENODEV;
7128                 goto msix_setup_exit;
7129         }
7130
7131         bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
7132         if (bp->irq_tbl) {
7133                 for (i = 0; i < total_vecs; i++)
7134                         bp->irq_tbl[i].vector = msix_ent[i].vector;
7135
7136                 bp->total_irqs = total_vecs;
7137                 /* Trim rings based upon num of vectors allocated */
7138                 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
7139                                      total_vecs - ulp_msix, min == 1);
7140                 if (rc)
7141                         goto msix_setup_exit;
7142
7143                 bp->cp_nr_rings = (min == 1) ?
7144                                   max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
7145                                   bp->tx_nr_rings + bp->rx_nr_rings;
7146
7147         } else {
7148                 rc = -ENOMEM;
7149                 goto msix_setup_exit;
7150         }
7151         bp->flags |= BNXT_FLAG_USING_MSIX;
7152         kfree(msix_ent);
7153         return 0;
7154
7155 msix_setup_exit:
7156         netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
7157         kfree(bp->irq_tbl);
7158         bp->irq_tbl = NULL;
7159         pci_disable_msix(bp->pdev);
7160         kfree(msix_ent);
7161         return rc;
7162 }
7163
7164 static int bnxt_init_inta(struct bnxt *bp)
7165 {
7166         bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
7167         if (!bp->irq_tbl)
7168                 return -ENOMEM;
7169
7170         bp->total_irqs = 1;
7171         bp->rx_nr_rings = 1;
7172         bp->tx_nr_rings = 1;
7173         bp->cp_nr_rings = 1;
7174         bp->flags |= BNXT_FLAG_SHARED_RINGS;
7175         bp->irq_tbl[0].vector = bp->pdev->irq;
7176         return 0;
7177 }
7178
7179 static int bnxt_init_int_mode(struct bnxt *bp)
7180 {
7181         int rc = 0;
7182
7183         if (bp->flags & BNXT_FLAG_MSIX_CAP)
7184                 rc = bnxt_init_msix(bp);
7185
7186         if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
7187                 /* fallback to INTA */
7188                 rc = bnxt_init_inta(bp);
7189         }
7190         return rc;
7191 }
7192
7193 static void bnxt_clear_int_mode(struct bnxt *bp)
7194 {
7195         if (bp->flags & BNXT_FLAG_USING_MSIX)
7196                 pci_disable_msix(bp->pdev);
7197
7198         kfree(bp->irq_tbl);
7199         bp->irq_tbl = NULL;
7200         bp->flags &= ~BNXT_FLAG_USING_MSIX;
7201 }
7202
7203 int bnxt_reserve_rings(struct bnxt *bp)
7204 {
7205         int tcs = netdev_get_num_tc(bp->dev);
7206         int rc;
7207
7208         if (!bnxt_need_reserve_rings(bp))
7209                 return 0;
7210
7211         rc = __bnxt_reserve_rings(bp);
7212         if (rc) {
7213                 netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc);
7214                 return rc;
7215         }
7216         if (BNXT_NEW_RM(bp) && (bnxt_get_num_msix(bp) != bp->total_irqs)) {
7217                 bnxt_ulp_irq_stop(bp);
7218                 bnxt_clear_int_mode(bp);
7219                 rc = bnxt_init_int_mode(bp);
7220                 bnxt_ulp_irq_restart(bp, rc);
7221                 if (rc)
7222                         return rc;
7223         }
7224         if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
7225                 netdev_err(bp->dev, "tx ring reservation failure\n");
7226                 netdev_reset_tc(bp->dev);
7227                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
7228                 return -ENOMEM;
7229         }
7230         bp->num_stat_ctxs = bp->cp_nr_rings;
7231         return 0;
7232 }
7233
7234 static void bnxt_free_irq(struct bnxt *bp)
7235 {
7236         struct bnxt_irq *irq;
7237         int i;
7238
7239 #ifdef CONFIG_RFS_ACCEL
7240         free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
7241         bp->dev->rx_cpu_rmap = NULL;
7242 #endif
7243         if (!bp->irq_tbl || !bp->bnapi)
7244                 return;
7245
7246         for (i = 0; i < bp->cp_nr_rings; i++) {
7247                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7248
7249                 irq = &bp->irq_tbl[map_idx];
7250                 if (irq->requested) {
7251                         if (irq->have_cpumask) {
7252                                 irq_set_affinity_hint(irq->vector, NULL);
7253                                 free_cpumask_var(irq->cpu_mask);
7254                                 irq->have_cpumask = 0;
7255                         }
7256                         free_irq(irq->vector, bp->bnapi[i]);
7257                 }
7258
7259                 irq->requested = 0;
7260         }
7261 }
7262
7263 static int bnxt_request_irq(struct bnxt *bp)
7264 {
7265         int i, j, rc = 0;
7266         unsigned long flags = 0;
7267 #ifdef CONFIG_RFS_ACCEL
7268         struct cpu_rmap *rmap;
7269 #endif
7270
7271         rc = bnxt_setup_int_mode(bp);
7272         if (rc) {
7273                 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
7274                            rc);
7275                 return rc;
7276         }
7277 #ifdef CONFIG_RFS_ACCEL
7278         rmap = bp->dev->rx_cpu_rmap;
7279 #endif
7280         if (!(bp->flags & BNXT_FLAG_USING_MSIX))
7281                 flags = IRQF_SHARED;
7282
7283         for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
7284                 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7285                 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
7286
7287 #ifdef CONFIG_RFS_ACCEL
7288                 if (rmap && bp->bnapi[i]->rx_ring) {
7289                         rc = irq_cpu_rmap_add(rmap, irq->vector);
7290                         if (rc)
7291                                 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
7292                                             j);
7293                         j++;
7294                 }
7295 #endif
7296                 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
7297                                  bp->bnapi[i]);
7298                 if (rc)
7299                         break;
7300
7301                 irq->requested = 1;
7302
7303                 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
7304                         int numa_node = dev_to_node(&bp->pdev->dev);
7305
7306                         irq->have_cpumask = 1;
7307                         cpumask_set_cpu(cpumask_local_spread(i, numa_node),
7308                                         irq->cpu_mask);
7309                         rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
7310                         if (rc) {
7311                                 netdev_warn(bp->dev,
7312                                             "Set affinity failed, IRQ = %d\n",
7313                                             irq->vector);
7314                                 break;
7315                         }
7316                 }
7317         }
7318         return rc;
7319 }
7320
7321 static void bnxt_del_napi(struct bnxt *bp)
7322 {
7323         int i;
7324
7325         if (!bp->bnapi)
7326                 return;
7327
7328         for (i = 0; i < bp->cp_nr_rings; i++) {
7329                 struct bnxt_napi *bnapi = bp->bnapi[i];
7330
7331                 napi_hash_del(&bnapi->napi);
7332                 netif_napi_del(&bnapi->napi);
7333         }
7334         /* We called napi_hash_del() before netif_napi_del(), we need
7335          * to respect an RCU grace period before freeing napi structures.
7336          */
7337         synchronize_net();
7338 }
7339
7340 static void bnxt_init_napi(struct bnxt *bp)
7341 {
7342         int i;
7343         unsigned int cp_nr_rings = bp->cp_nr_rings;
7344         struct bnxt_napi *bnapi;
7345
7346         if (bp->flags & BNXT_FLAG_USING_MSIX) {
7347                 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
7348
7349                 if (bp->flags & BNXT_FLAG_CHIP_P5)
7350                         poll_fn = bnxt_poll_p5;
7351                 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7352                         cp_nr_rings--;
7353                 for (i = 0; i < cp_nr_rings; i++) {
7354                         bnapi = bp->bnapi[i];
7355                         netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
7356                 }
7357                 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7358                         bnapi = bp->bnapi[cp_nr_rings];
7359                         netif_napi_add(bp->dev, &bnapi->napi,
7360                                        bnxt_poll_nitroa0, 64);
7361                 }
7362         } else {
7363                 bnapi = bp->bnapi[0];
7364                 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
7365         }
7366 }
7367
7368 static void bnxt_disable_napi(struct bnxt *bp)
7369 {
7370         int i;
7371
7372         if (!bp->bnapi)
7373                 return;
7374
7375         for (i = 0; i < bp->cp_nr_rings; i++) {
7376                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
7377
7378                 if (bp->bnapi[i]->rx_ring)
7379                         cancel_work_sync(&cpr->dim.work);
7380
7381                 napi_disable(&bp->bnapi[i]->napi);
7382         }
7383 }
7384
7385 static void bnxt_enable_napi(struct bnxt *bp)
7386 {
7387         int i;
7388
7389         for (i = 0; i < bp->cp_nr_rings; i++) {
7390                 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
7391                 bp->bnapi[i]->in_reset = false;
7392
7393                 if (bp->bnapi[i]->rx_ring) {
7394                         INIT_WORK(&cpr->dim.work, bnxt_dim_work);
7395                         cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
7396                 }
7397                 napi_enable(&bp->bnapi[i]->napi);
7398         }
7399 }
7400
7401 void bnxt_tx_disable(struct bnxt *bp)
7402 {
7403         int i;
7404         struct bnxt_tx_ring_info *txr;
7405
7406         if (bp->tx_ring) {
7407                 for (i = 0; i < bp->tx_nr_rings; i++) {
7408                         txr = &bp->tx_ring[i];
7409                         txr->dev_state = BNXT_DEV_STATE_CLOSING;
7410                 }
7411         }
7412         /* Stop all TX queues */
7413         netif_tx_disable(bp->dev);
7414         netif_carrier_off(bp->dev);
7415 }
7416
7417 void bnxt_tx_enable(struct bnxt *bp)
7418 {
7419         int i;
7420         struct bnxt_tx_ring_info *txr;
7421
7422         for (i = 0; i < bp->tx_nr_rings; i++) {
7423                 txr = &bp->tx_ring[i];
7424                 txr->dev_state = 0;
7425         }
7426         netif_tx_wake_all_queues(bp->dev);
7427         if (bp->link_info.link_up)
7428                 netif_carrier_on(bp->dev);
7429 }
7430
7431 static void bnxt_report_link(struct bnxt *bp)
7432 {
7433         if (bp->link_info.link_up) {
7434                 const char *duplex;
7435                 const char *flow_ctrl;
7436                 u32 speed;
7437                 u16 fec;
7438
7439                 netif_carrier_on(bp->dev);
7440                 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
7441                         duplex = "full";
7442                 else
7443                         duplex = "half";
7444                 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
7445                         flow_ctrl = "ON - receive & transmit";
7446                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
7447                         flow_ctrl = "ON - transmit";
7448                 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
7449                         flow_ctrl = "ON - receive";
7450                 else
7451                         flow_ctrl = "none";
7452                 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
7453                 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
7454                             speed, duplex, flow_ctrl);
7455                 if (bp->flags & BNXT_FLAG_EEE_CAP)
7456                         netdev_info(bp->dev, "EEE is %s\n",
7457                                     bp->eee.eee_active ? "active" :
7458                                                          "not active");
7459                 fec = bp->link_info.fec_cfg;
7460                 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
7461                         netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
7462                                     (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
7463                                     (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
7464                                      (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
7465         } else {
7466                 netif_carrier_off(bp->dev);
7467                 netdev_err(bp->dev, "NIC Link is Down\n");
7468         }
7469 }
7470
7471 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
7472 {
7473         int rc = 0;
7474         struct hwrm_port_phy_qcaps_input req = {0};
7475         struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7476         struct bnxt_link_info *link_info = &bp->link_info;
7477
7478         if (bp->hwrm_spec_code < 0x10201)
7479                 return 0;
7480
7481         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
7482
7483         mutex_lock(&bp->hwrm_cmd_lock);
7484         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7485         if (rc)
7486                 goto hwrm_phy_qcaps_exit;
7487
7488         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
7489                 struct ethtool_eee *eee = &bp->eee;
7490                 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
7491
7492                 bp->flags |= BNXT_FLAG_EEE_CAP;
7493                 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7494                 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
7495                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
7496                 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
7497                                  PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
7498         }
7499         if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
7500                 if (bp->test_info)
7501                         bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
7502         }
7503         if (resp->supported_speeds_auto_mode)
7504                 link_info->support_auto_speeds =
7505                         le16_to_cpu(resp->supported_speeds_auto_mode);
7506
7507         bp->port_count = resp->port_cnt;
7508
7509 hwrm_phy_qcaps_exit:
7510         mutex_unlock(&bp->hwrm_cmd_lock);
7511         return rc;
7512 }
7513
7514 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
7515 {
7516         int rc = 0;
7517         struct bnxt_link_info *link_info = &bp->link_info;
7518         struct hwrm_port_phy_qcfg_input req = {0};
7519         struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7520         u8 link_up = link_info->link_up;
7521         u16 diff;
7522
7523         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
7524
7525         mutex_lock(&bp->hwrm_cmd_lock);
7526         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7527         if (rc) {
7528                 mutex_unlock(&bp->hwrm_cmd_lock);
7529                 return rc;
7530         }
7531
7532         memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
7533         link_info->phy_link_status = resp->link;
7534         link_info->duplex = resp->duplex_cfg;
7535         if (bp->hwrm_spec_code >= 0x10800)
7536                 link_info->duplex = resp->duplex_state;
7537         link_info->pause = resp->pause;
7538         link_info->auto_mode = resp->auto_mode;
7539         link_info->auto_pause_setting = resp->auto_pause;
7540         link_info->lp_pause = resp->link_partner_adv_pause;
7541         link_info->force_pause_setting = resp->force_pause;
7542         link_info->duplex_setting = resp->duplex_cfg;
7543         if (link_info->phy_link_status == BNXT_LINK_LINK)
7544                 link_info->link_speed = le16_to_cpu(resp->link_speed);
7545         else
7546                 link_info->link_speed = 0;
7547         link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
7548         link_info->support_speeds = le16_to_cpu(resp->support_speeds);
7549         link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
7550         link_info->lp_auto_link_speeds =
7551                 le16_to_cpu(resp->link_partner_adv_speeds);
7552         link_info->preemphasis = le32_to_cpu(resp->preemphasis);
7553         link_info->phy_ver[0] = resp->phy_maj;
7554         link_info->phy_ver[1] = resp->phy_min;
7555         link_info->phy_ver[2] = resp->phy_bld;
7556         link_info->media_type = resp->media_type;
7557         link_info->phy_type = resp->phy_type;
7558         link_info->transceiver = resp->xcvr_pkg_type;
7559         link_info->phy_addr = resp->eee_config_phy_addr &
7560                               PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
7561         link_info->module_status = resp->module_status;
7562
7563         if (bp->flags & BNXT_FLAG_EEE_CAP) {
7564                 struct ethtool_eee *eee = &bp->eee;
7565                 u16 fw_speeds;
7566
7567                 eee->eee_active = 0;
7568                 if (resp->eee_config_phy_addr &
7569                     PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
7570                         eee->eee_active = 1;
7571                         fw_speeds = le16_to_cpu(
7572                                 resp->link_partner_adv_eee_link_speed_mask);
7573                         eee->lp_advertised =
7574                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7575                 }
7576
7577                 /* Pull initial EEE config */
7578                 if (!chng_link_state) {
7579                         if (resp->eee_config_phy_addr &
7580                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
7581                                 eee->eee_enabled = 1;
7582
7583                         fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
7584                         eee->advertised =
7585                                 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
7586
7587                         if (resp->eee_config_phy_addr &
7588                             PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
7589                                 __le32 tmr;
7590
7591                                 eee->tx_lpi_enabled = 1;
7592                                 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
7593                                 eee->tx_lpi_timer = le32_to_cpu(tmr) &
7594                                         PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
7595                         }
7596                 }
7597         }
7598
7599         link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
7600         if (bp->hwrm_spec_code >= 0x10504)
7601                 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
7602
7603         /* TODO: need to add more logic to report VF link */
7604         if (chng_link_state) {
7605                 if (link_info->phy_link_status == BNXT_LINK_LINK)
7606                         link_info->link_up = 1;
7607                 else
7608                         link_info->link_up = 0;
7609                 if (link_up != link_info->link_up)
7610                         bnxt_report_link(bp);
7611         } else {
7612                 /* alwasy link down if not require to update link state */
7613                 link_info->link_up = 0;
7614         }
7615         mutex_unlock(&bp->hwrm_cmd_lock);
7616
7617         if (!BNXT_SINGLE_PF(bp))
7618                 return 0;
7619
7620         diff = link_info->support_auto_speeds ^ link_info->advertising;
7621         if ((link_info->support_auto_speeds | diff) !=
7622             link_info->support_auto_speeds) {
7623                 /* An advertised speed is no longer supported, so we need to
7624                  * update the advertisement settings.  Caller holds RTNL
7625                  * so we can modify link settings.
7626                  */
7627                 link_info->advertising = link_info->support_auto_speeds;
7628                 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
7629                         bnxt_hwrm_set_link_setting(bp, true, false);
7630         }
7631         return 0;
7632 }
7633
7634 static void bnxt_get_port_module_status(struct bnxt *bp)
7635 {
7636         struct bnxt_link_info *link_info = &bp->link_info;
7637         struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
7638         u8 module_status;
7639
7640         if (bnxt_update_link(bp, true))
7641                 return;
7642
7643         module_status = link_info->module_status;
7644         switch (module_status) {
7645         case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
7646         case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
7647         case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
7648                 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
7649                             bp->pf.port_id);
7650                 if (bp->hwrm_spec_code >= 0x10201) {
7651                         netdev_warn(bp->dev, "Module part number %s\n",
7652                                     resp->phy_vendor_partnumber);
7653                 }
7654                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
7655                         netdev_warn(bp->dev, "TX is disabled\n");
7656                 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
7657                         netdev_warn(bp->dev, "SFP+ module is shutdown\n");
7658         }
7659 }
7660
7661 static void
7662 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
7663 {
7664         if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
7665                 if (bp->hwrm_spec_code >= 0x10201)
7666                         req->auto_pause =
7667                                 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
7668                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
7669                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
7670                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
7671                         req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
7672                 req->enables |=
7673                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
7674         } else {
7675                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
7676                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
7677                 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
7678                         req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
7679                 req->enables |=
7680                         cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
7681                 if (bp->hwrm_spec_code >= 0x10201) {
7682                         req->auto_pause = req->force_pause;
7683                         req->enables |= cpu_to_le32(
7684                                 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
7685                 }
7686         }
7687 }
7688
7689 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
7690                                       struct hwrm_port_phy_cfg_input *req)
7691 {
7692         u8 autoneg = bp->link_info.autoneg;
7693         u16 fw_link_speed = bp->link_info.req_link_speed;
7694         u16 advertising = bp->link_info.advertising;
7695
7696         if (autoneg & BNXT_AUTONEG_SPEED) {
7697                 req->auto_mode |=
7698                         PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
7699
7700                 req->enables |= cpu_to_le32(
7701                         PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
7702                 req->auto_link_speed_mask = cpu_to_le16(advertising);
7703
7704                 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
7705                 req->flags |=
7706                         cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
7707         } else {
7708                 req->force_link_speed = cpu_to_le16(fw_link_speed);
7709                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
7710         }
7711
7712         /* tell chimp that the setting takes effect immediately */
7713         req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
7714 }
7715
7716 int bnxt_hwrm_set_pause(struct bnxt *bp)
7717 {
7718         struct hwrm_port_phy_cfg_input req = {0};
7719         int rc;
7720
7721         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
7722         bnxt_hwrm_set_pause_common(bp, &req);
7723
7724         if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
7725             bp->link_info.force_link_chng)
7726                 bnxt_hwrm_set_link_common(bp, &req);
7727
7728         mutex_lock(&bp->hwrm_cmd_lock);
7729         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7730         if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
7731                 /* since changing of pause setting doesn't trigger any link
7732                  * change event, the driver needs to update the current pause
7733                  * result upon successfully return of the phy_cfg command
7734                  */
7735                 bp->link_info.pause =
7736                 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
7737                 bp->link_info.auto_pause_setting = 0;
7738                 if (!bp->link_info.force_link_chng)
7739                         bnxt_report_link(bp);
7740         }
7741         bp->link_info.force_link_chng = false;
7742         mutex_unlock(&bp->hwrm_cmd_lock);
7743         return rc;
7744 }
7745
7746 static void bnxt_hwrm_set_eee(struct bnxt *bp,
7747                               struct hwrm_port_phy_cfg_input *req)
7748 {
7749         struct ethtool_eee *eee = &bp->eee;
7750
7751         if (eee->eee_enabled) {
7752                 u16 eee_speeds;
7753                 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
7754
7755                 if (eee->tx_lpi_enabled)
7756                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
7757                 else
7758                         flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
7759
7760                 req->flags |= cpu_to_le32(flags);
7761                 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
7762                 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
7763                 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
7764         } else {
7765                 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
7766         }
7767 }
7768
7769 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
7770 {
7771         struct hwrm_port_phy_cfg_input req = {0};
7772
7773         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
7774         if (set_pause)
7775                 bnxt_hwrm_set_pause_common(bp, &req);
7776
7777         bnxt_hwrm_set_link_common(bp, &req);
7778
7779         if (set_eee)
7780                 bnxt_hwrm_set_eee(bp, &req);
7781         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7782 }
7783
7784 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
7785 {
7786         struct hwrm_port_phy_cfg_input req = {0};
7787
7788         if (!BNXT_SINGLE_PF(bp))
7789                 return 0;
7790
7791         if (pci_num_vf(bp->pdev))
7792                 return 0;
7793
7794         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
7795         req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
7796         return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7797 }
7798
7799 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
7800 {
7801         struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
7802         struct hwrm_func_drv_if_change_input req = {0};
7803         bool resc_reinit = false;
7804         int rc;
7805
7806         if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
7807                 return 0;
7808
7809         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
7810         if (up)
7811                 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
7812         mutex_lock(&bp->hwrm_cmd_lock);
7813         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7814         if (!rc && (resp->flags &
7815                     cpu_to_le32(FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)))
7816                 resc_reinit = true;
7817         mutex_unlock(&bp->hwrm_cmd_lock);
7818
7819         if (up && resc_reinit && BNXT_NEW_RM(bp)) {
7820                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7821
7822                 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7823                 hw_resc->resv_cp_rings = 0;
7824                 hw_resc->resv_irqs = 0;
7825                 hw_resc->resv_tx_rings = 0;
7826                 hw_resc->resv_rx_rings = 0;
7827                 hw_resc->resv_hw_ring_grps = 0;
7828                 hw_resc->resv_vnics = 0;
7829                 bp->tx_nr_rings = 0;
7830                 bp->rx_nr_rings = 0;
7831         }
7832         return rc;
7833 }
7834
7835 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
7836 {
7837         struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7838         struct hwrm_port_led_qcaps_input req = {0};
7839         struct bnxt_pf_info *pf = &bp->pf;
7840         int rc;
7841
7842         if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
7843                 return 0;
7844
7845         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
7846         req.port_id = cpu_to_le16(pf->port_id);
7847         mutex_lock(&bp->hwrm_cmd_lock);
7848         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7849         if (rc) {
7850                 mutex_unlock(&bp->hwrm_cmd_lock);
7851                 return rc;
7852         }
7853         if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
7854                 int i;
7855
7856                 bp->num_leds = resp->num_leds;
7857                 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
7858                                                  bp->num_leds);
7859                 for (i = 0; i < bp->num_leds; i++) {
7860                         struct bnxt_led_info *led = &bp->leds[i];
7861                         __le16 caps = led->led_state_caps;
7862
7863                         if (!led->led_group_id ||
7864                             !BNXT_LED_ALT_BLINK_CAP(caps)) {
7865                                 bp->num_leds = 0;
7866                                 break;
7867                         }
7868                 }
7869         }
7870         mutex_unlock(&bp->hwrm_cmd_lock);
7871         return 0;
7872 }
7873
7874 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
7875 {
7876         struct hwrm_wol_filter_alloc_input req = {0};
7877         struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
7878         int rc;
7879
7880         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
7881         req.port_id = cpu_to_le16(bp->pf.port_id);
7882         req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
7883         req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
7884         memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
7885         mutex_lock(&bp->hwrm_cmd_lock);
7886         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7887         if (!rc)
7888                 bp->wol_filter_id = resp->wol_filter_id;
7889         mutex_unlock(&bp->hwrm_cmd_lock);
7890         return rc;
7891 }
7892
7893 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
7894 {
7895         struct hwrm_wol_filter_free_input req = {0};
7896         int rc;
7897
7898         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
7899         req.port_id = cpu_to_le16(bp->pf.port_id);
7900         req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
7901         req.wol_filter_id = bp->wol_filter_id;
7902         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7903         return rc;
7904 }
7905
7906 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
7907 {
7908         struct hwrm_wol_filter_qcfg_input req = {0};
7909         struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7910         u16 next_handle = 0;
7911         int rc;
7912
7913         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
7914         req.port_id = cpu_to_le16(bp->pf.port_id);
7915         req.handle = cpu_to_le16(handle);
7916         mutex_lock(&bp->hwrm_cmd_lock);
7917         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7918         if (!rc) {
7919                 next_handle = le16_to_cpu(resp->next_handle);
7920                 if (next_handle != 0) {
7921                         if (resp->wol_type ==
7922                             WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
7923                                 bp->wol = 1;
7924                                 bp->wol_filter_id = resp->wol_filter_id;
7925                         }
7926                 }
7927         }
7928         mutex_unlock(&bp->hwrm_cmd_lock);
7929         return next_handle;
7930 }
7931
7932 static void bnxt_get_wol_settings(struct bnxt *bp)
7933 {
7934         u16 handle = 0;
7935
7936         if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
7937                 return;
7938
7939         do {
7940                 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
7941         } while (handle && handle != 0xffff);
7942 }
7943
7944 #ifdef CONFIG_BNXT_HWMON
7945 static ssize_t bnxt_show_temp(struct device *dev,
7946                               struct device_attribute *devattr, char *buf)
7947 {
7948         struct hwrm_temp_monitor_query_input req = {0};
7949         struct hwrm_temp_monitor_query_output *resp;
7950         struct bnxt *bp = dev_get_drvdata(dev);
7951         u32 temp = 0;
7952
7953         resp = bp->hwrm_cmd_resp_addr;
7954         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
7955         mutex_lock(&bp->hwrm_cmd_lock);
7956         if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
7957                 temp = resp->temp * 1000; /* display millidegree */
7958         mutex_unlock(&bp->hwrm_cmd_lock);
7959
7960         return sprintf(buf, "%u\n", temp);
7961 }
7962 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
7963
7964 static struct attribute *bnxt_attrs[] = {
7965         &sensor_dev_attr_temp1_input.dev_attr.attr,
7966         NULL
7967 };
7968 ATTRIBUTE_GROUPS(bnxt);
7969
7970 static void bnxt_hwmon_close(struct bnxt *bp)
7971 {
7972         if (bp->hwmon_dev) {
7973                 hwmon_device_unregister(bp->hwmon_dev);
7974                 bp->hwmon_dev = NULL;
7975         }
7976 }
7977
7978 static void bnxt_hwmon_open(struct bnxt *bp)
7979 {
7980         struct pci_dev *pdev = bp->pdev;
7981
7982         bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
7983                                                           DRV_MODULE_NAME, bp,
7984                                                           bnxt_groups);
7985         if (IS_ERR(bp->hwmon_dev)) {
7986                 bp->hwmon_dev = NULL;
7987                 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
7988         }
7989 }
7990 #else
7991 static void bnxt_hwmon_close(struct bnxt *bp)
7992 {
7993 }
7994
7995 static void bnxt_hwmon_open(struct bnxt *bp)
7996 {
7997 }
7998 #endif
7999
8000 static bool bnxt_eee_config_ok(struct bnxt *bp)
8001 {
8002         struct ethtool_eee *eee = &bp->eee;
8003         struct bnxt_link_info *link_info = &bp->link_info;
8004
8005         if (!(bp->flags & BNXT_FLAG_EEE_CAP))
8006                 return true;
8007
8008         if (eee->eee_enabled) {
8009                 u32 advertising =
8010                         _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
8011
8012                 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8013                         eee->eee_enabled = 0;
8014                         return false;
8015                 }
8016                 if (eee->advertised & ~advertising) {
8017                         eee->advertised = advertising & eee->supported;
8018                         return false;
8019                 }
8020         }
8021         return true;
8022 }
8023
8024 static int bnxt_update_phy_setting(struct bnxt *bp)
8025 {
8026         int rc;
8027         bool update_link = false;
8028         bool update_pause = false;
8029         bool update_eee = false;
8030         struct bnxt_link_info *link_info = &bp->link_info;
8031
8032         rc = bnxt_update_link(bp, true);
8033         if (rc) {
8034                 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
8035                            rc);
8036                 return rc;
8037         }
8038         if (!BNXT_SINGLE_PF(bp))
8039                 return 0;
8040
8041         if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8042             (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
8043             link_info->req_flow_ctrl)
8044                 update_pause = true;
8045         if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
8046             link_info->force_pause_setting != link_info->req_flow_ctrl)
8047                 update_pause = true;
8048         if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
8049                 if (BNXT_AUTO_MODE(link_info->auto_mode))
8050                         update_link = true;
8051                 if (link_info->req_link_speed != link_info->force_link_speed)
8052                         update_link = true;
8053                 if (link_info->req_duplex != link_info->duplex_setting)
8054                         update_link = true;
8055         } else {
8056                 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
8057                         update_link = true;
8058                 if (link_info->advertising != link_info->auto_link_speeds)
8059                         update_link = true;
8060         }
8061
8062         /* The last close may have shutdown the link, so need to call
8063          * PHY_CFG to bring it back up.
8064          */
8065         if (!netif_carrier_ok(bp->dev))
8066                 update_link = true;
8067
8068         if (!bnxt_eee_config_ok(bp))
8069                 update_eee = true;
8070
8071         if (update_link)
8072                 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
8073         else if (update_pause)
8074                 rc = bnxt_hwrm_set_pause(bp);
8075         if (rc) {
8076                 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
8077                            rc);
8078                 return rc;
8079         }
8080
8081         return rc;
8082 }
8083
8084 /* Common routine to pre-map certain register block to different GRC window.
8085  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
8086  * in PF and 3 windows in VF that can be customized to map in different
8087  * register blocks.
8088  */
8089 static void bnxt_preset_reg_win(struct bnxt *bp)
8090 {
8091         if (BNXT_PF(bp)) {
8092                 /* CAG registers map to GRC window #4 */
8093                 writel(BNXT_CAG_REG_BASE,
8094                        bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
8095         }
8096 }
8097
8098 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
8099
8100 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8101 {
8102         int rc = 0;
8103
8104         bnxt_preset_reg_win(bp);
8105         netif_carrier_off(bp->dev);
8106         if (irq_re_init) {
8107                 /* Reserve rings now if none were reserved at driver probe. */
8108                 rc = bnxt_init_dflt_ring_mode(bp);
8109                 if (rc) {
8110                         netdev_err(bp->dev, "Failed to reserve default rings at open\n");
8111                         return rc;
8112                 }
8113         }
8114         rc = bnxt_reserve_rings(bp);
8115         if (rc)
8116                 return rc;
8117         if ((bp->flags & BNXT_FLAG_RFS) &&
8118             !(bp->flags & BNXT_FLAG_USING_MSIX)) {
8119                 /* disable RFS if falling back to INTA */
8120                 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
8121                 bp->flags &= ~BNXT_FLAG_RFS;
8122         }
8123
8124         rc = bnxt_alloc_mem(bp, irq_re_init);
8125         if (rc) {
8126                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8127                 goto open_err_free_mem;
8128         }
8129
8130         if (irq_re_init) {
8131                 bnxt_init_napi(bp);
8132                 rc = bnxt_request_irq(bp);
8133                 if (rc) {
8134                         netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
8135                         goto open_err_irq;
8136                 }
8137         }
8138
8139         bnxt_enable_napi(bp);
8140         bnxt_debug_dev_init(bp);
8141
8142         rc = bnxt_init_nic(bp, irq_re_init);
8143         if (rc) {
8144                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8145                 goto open_err;
8146         }
8147
8148         if (link_re_init) {
8149                 mutex_lock(&bp->link_lock);
8150                 rc = bnxt_update_phy_setting(bp);
8151                 mutex_unlock(&bp->link_lock);
8152                 if (rc) {
8153                         netdev_warn(bp->dev, "failed to update phy settings\n");
8154                         if (BNXT_SINGLE_PF(bp)) {
8155                                 bp->link_info.phy_retry = true;
8156                                 bp->link_info.phy_retry_expires =
8157                                         jiffies + 5 * HZ;
8158                         }
8159                 }
8160         }
8161
8162         if (irq_re_init)
8163                 udp_tunnel_get_rx_info(bp->dev);
8164
8165         set_bit(BNXT_STATE_OPEN, &bp->state);
8166         bnxt_enable_int(bp);
8167         /* Enable TX queues */
8168         bnxt_tx_enable(bp);
8169         mod_timer(&bp->timer, jiffies + bp->current_interval);
8170         /* Poll link status and check for SFP+ module status */
8171         bnxt_get_port_module_status(bp);
8172
8173         /* VF-reps may need to be re-opened after the PF is re-opened */
8174         if (BNXT_PF(bp))
8175                 bnxt_vf_reps_open(bp);
8176         return 0;
8177
8178 open_err:
8179         bnxt_debug_dev_exit(bp);
8180         bnxt_disable_napi(bp);
8181
8182 open_err_irq:
8183         bnxt_del_napi(bp);
8184
8185 open_err_free_mem:
8186         bnxt_free_skbs(bp);
8187         bnxt_free_irq(bp);
8188         bnxt_free_mem(bp, true);
8189         return rc;
8190 }
8191
8192 /* rtnl_lock held */
8193 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8194 {
8195         int rc = 0;
8196
8197         rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
8198         if (rc) {
8199                 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
8200                 dev_close(bp->dev);
8201         }
8202         return rc;
8203 }
8204
8205 /* rtnl_lock held, open the NIC half way by allocating all resources, but
8206  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
8207  * self tests.
8208  */
8209 int bnxt_half_open_nic(struct bnxt *bp)
8210 {
8211         int rc = 0;
8212
8213         rc = bnxt_alloc_mem(bp, false);
8214         if (rc) {
8215                 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
8216                 goto half_open_err;
8217         }
8218         rc = bnxt_init_nic(bp, false);
8219         if (rc) {
8220                 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
8221                 goto half_open_err;
8222         }
8223         return 0;
8224
8225 half_open_err:
8226         bnxt_free_skbs(bp);
8227         bnxt_free_mem(bp, false);
8228         dev_close(bp->dev);
8229         return rc;
8230 }
8231
8232 /* rtnl_lock held, this call can only be made after a previous successful
8233  * call to bnxt_half_open_nic().
8234  */
8235 void bnxt_half_close_nic(struct bnxt *bp)
8236 {
8237         bnxt_hwrm_resource_free(bp, false, false);
8238         bnxt_free_skbs(bp);
8239         bnxt_free_mem(bp, false);
8240 }
8241
8242 static int bnxt_open(struct net_device *dev)
8243 {
8244         struct bnxt *bp = netdev_priv(dev);
8245         int rc;
8246
8247         bnxt_hwrm_if_change(bp, true);
8248         rc = __bnxt_open_nic(bp, true, true);
8249         if (rc)
8250                 bnxt_hwrm_if_change(bp, false);
8251
8252         bnxt_hwmon_open(bp);
8253
8254         return rc;
8255 }
8256
8257 static bool bnxt_drv_busy(struct bnxt *bp)
8258 {
8259         return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
8260                 test_bit(BNXT_STATE_READ_STATS, &bp->state));
8261 }
8262
8263 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
8264                              bool link_re_init)
8265 {
8266         /* Close the VF-reps before closing PF */
8267         if (BNXT_PF(bp))
8268                 bnxt_vf_reps_close(bp);
8269
8270         /* Change device state to avoid TX queue wake up's */
8271         bnxt_tx_disable(bp);
8272
8273         clear_bit(BNXT_STATE_OPEN, &bp->state);
8274         smp_mb__after_atomic();
8275         while (bnxt_drv_busy(bp))
8276                 msleep(20);
8277
8278         /* Flush rings and and disable interrupts */
8279         bnxt_shutdown_nic(bp, irq_re_init);
8280
8281         /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
8282
8283         bnxt_debug_dev_exit(bp);
8284         bnxt_disable_napi(bp);
8285         del_timer_sync(&bp->timer);
8286         bnxt_free_skbs(bp);
8287
8288         if (irq_re_init) {
8289                 bnxt_free_irq(bp);
8290                 bnxt_del_napi(bp);
8291         }
8292         bnxt_free_mem(bp, irq_re_init);
8293 }
8294
8295 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
8296 {
8297         int rc = 0;
8298
8299 #ifdef CONFIG_BNXT_SRIOV
8300         if (bp->sriov_cfg) {
8301                 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
8302                                                       !bp->sriov_cfg,
8303                                                       BNXT_SRIOV_CFG_WAIT_TMO);
8304                 if (rc)
8305                         netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
8306         }
8307 #endif
8308         __bnxt_close_nic(bp, irq_re_init, link_re_init);
8309         return rc;
8310 }
8311
8312 static int bnxt_close(struct net_device *dev)
8313 {
8314         struct bnxt *bp = netdev_priv(dev);
8315
8316         bnxt_hwmon_close(bp);
8317         bnxt_close_nic(bp, true, true);
8318         bnxt_hwrm_shutdown_link(bp);
8319         bnxt_hwrm_if_change(bp, false);
8320         return 0;
8321 }
8322
8323 /* rtnl_lock held */
8324 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
8325 {
8326         switch (cmd) {
8327         case SIOCGMIIPHY:
8328                 /* fallthru */
8329         case SIOCGMIIREG: {
8330                 if (!netif_running(dev))
8331                         return -EAGAIN;
8332
8333                 return 0;
8334         }
8335
8336         case SIOCSMIIREG:
8337                 if (!netif_running(dev))
8338                         return -EAGAIN;
8339
8340                 return 0;
8341
8342         default:
8343                 /* do nothing */
8344                 break;
8345         }
8346         return -EOPNOTSUPP;
8347 }
8348
8349 static void
8350 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
8351 {
8352         u32 i;
8353         struct bnxt *bp = netdev_priv(dev);
8354
8355         set_bit(BNXT_STATE_READ_STATS, &bp->state);
8356         /* Make sure bnxt_close_nic() sees that we are reading stats before
8357          * we check the BNXT_STATE_OPEN flag.
8358          */
8359         smp_mb__after_atomic();
8360         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
8361                 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
8362                 return;
8363         }
8364
8365         /* TODO check if we need to synchronize with bnxt_close path */
8366         for (i = 0; i < bp->cp_nr_rings; i++) {
8367                 struct bnxt_napi *bnapi = bp->bnapi[i];
8368                 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8369                 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
8370
8371                 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
8372                 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
8373                 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
8374
8375                 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
8376                 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
8377                 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
8378
8379                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
8380                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
8381                 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
8382
8383                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
8384                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
8385                 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
8386
8387                 stats->rx_missed_errors +=
8388                         le64_to_cpu(hw_stats->rx_discard_pkts);
8389
8390                 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
8391
8392                 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
8393         }
8394
8395         if (bp->flags & BNXT_FLAG_PORT_STATS) {
8396                 struct rx_port_stats *rx = bp->hw_rx_port_stats;
8397                 struct tx_port_stats *tx = bp->hw_tx_port_stats;
8398
8399                 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
8400                 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
8401                 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
8402                                           le64_to_cpu(rx->rx_ovrsz_frames) +
8403                                           le64_to_cpu(rx->rx_runt_frames);
8404                 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
8405                                    le64_to_cpu(rx->rx_jbr_frames);
8406                 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
8407                 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
8408                 stats->tx_errors = le64_to_cpu(tx->tx_err);
8409         }
8410         clear_bit(BNXT_STATE_READ_STATS, &bp->state);
8411 }
8412
8413 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
8414 {
8415         struct net_device *dev = bp->dev;
8416         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8417         struct netdev_hw_addr *ha;
8418         u8 *haddr;
8419         int mc_count = 0;
8420         bool update = false;
8421         int off = 0;
8422
8423         netdev_for_each_mc_addr(ha, dev) {
8424                 if (mc_count >= BNXT_MAX_MC_ADDRS) {
8425                         *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8426                         vnic->mc_list_count = 0;
8427                         return false;
8428                 }
8429                 haddr = ha->addr;
8430                 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
8431                         memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
8432                         update = true;
8433                 }
8434                 off += ETH_ALEN;
8435                 mc_count++;
8436         }
8437         if (mc_count)
8438                 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
8439
8440         if (mc_count != vnic->mc_list_count) {
8441                 vnic->mc_list_count = mc_count;
8442                 update = true;
8443         }
8444         return update;
8445 }
8446
8447 static bool bnxt_uc_list_updated(struct bnxt *bp)
8448 {
8449         struct net_device *dev = bp->dev;
8450         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8451         struct netdev_hw_addr *ha;
8452         int off = 0;
8453
8454         if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
8455                 return true;
8456
8457         netdev_for_each_uc_addr(ha, dev) {
8458                 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
8459                         return true;
8460
8461                 off += ETH_ALEN;
8462         }
8463         return false;
8464 }
8465
8466 static void bnxt_set_rx_mode(struct net_device *dev)
8467 {
8468         struct bnxt *bp = netdev_priv(dev);
8469         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8470         u32 mask = vnic->rx_mask;
8471         bool mc_update = false;
8472         bool uc_update;
8473
8474         if (!netif_running(dev))
8475                 return;
8476
8477         mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
8478                   CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
8479                   CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
8480                   CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
8481
8482         if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8483                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8484
8485         uc_update = bnxt_uc_list_updated(bp);
8486
8487         if (dev->flags & IFF_BROADCAST)
8488                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8489         if (dev->flags & IFF_ALLMULTI) {
8490                 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8491                 vnic->mc_list_count = 0;
8492         } else {
8493                 mc_update = bnxt_mc_list_updated(bp, &mask);
8494         }
8495
8496         if (mask != vnic->rx_mask || uc_update || mc_update) {
8497                 vnic->rx_mask = mask;
8498
8499                 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
8500                 bnxt_queue_sp_work(bp);
8501         }
8502 }
8503
8504 static int bnxt_cfg_rx_mode(struct bnxt *bp)
8505 {
8506         struct net_device *dev = bp->dev;
8507         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8508         struct netdev_hw_addr *ha;
8509         int i, off = 0, rc;
8510         bool uc_update;
8511
8512         netif_addr_lock_bh(dev);
8513         uc_update = bnxt_uc_list_updated(bp);
8514         netif_addr_unlock_bh(dev);
8515
8516         if (!uc_update)
8517                 goto skip_uc;
8518
8519         mutex_lock(&bp->hwrm_cmd_lock);
8520         for (i = 1; i < vnic->uc_filter_count; i++) {
8521                 struct hwrm_cfa_l2_filter_free_input req = {0};
8522
8523                 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
8524                                        -1);
8525
8526                 req.l2_filter_id = vnic->fw_l2_filter_id[i];
8527
8528                 rc = _hwrm_send_message(bp, &req, sizeof(req),
8529                                         HWRM_CMD_TIMEOUT);
8530         }
8531         mutex_unlock(&bp->hwrm_cmd_lock);
8532
8533         vnic->uc_filter_count = 1;
8534
8535         netif_addr_lock_bh(dev);
8536         if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
8537                 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8538         } else {
8539                 netdev_for_each_uc_addr(ha, dev) {
8540                         memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
8541                         off += ETH_ALEN;
8542                         vnic->uc_filter_count++;
8543                 }
8544         }
8545         netif_addr_unlock_bh(dev);
8546
8547         for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
8548                 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
8549                 if (rc) {
8550                         netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
8551                                    rc);
8552                         vnic->uc_filter_count = i;
8553                         return rc;
8554                 }
8555         }
8556
8557 skip_uc:
8558         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
8559         if (rc)
8560                 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
8561                            rc);
8562
8563         return rc;
8564 }
8565
8566 static bool bnxt_can_reserve_rings(struct bnxt *bp)
8567 {
8568 #ifdef CONFIG_BNXT_SRIOV
8569         if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
8570                 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8571
8572                 /* No minimum rings were provisioned by the PF.  Don't
8573                  * reserve rings by default when device is down.
8574                  */
8575                 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
8576                         return true;
8577
8578                 if (!netif_running(bp->dev))
8579                         return false;
8580         }
8581 #endif
8582         return true;
8583 }
8584
8585 /* If the chip and firmware supports RFS */
8586 static bool bnxt_rfs_supported(struct bnxt *bp)
8587 {
8588         if (bp->flags & BNXT_FLAG_CHIP_P5)
8589                 return false;
8590         if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
8591                 return true;
8592         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8593                 return true;
8594         return false;
8595 }
8596
8597 /* If runtime conditions support RFS */
8598 static bool bnxt_rfs_capable(struct bnxt *bp)
8599 {
8600 #ifdef CONFIG_RFS_ACCEL
8601         int vnics, max_vnics, max_rss_ctxs;
8602
8603         if (bp->flags & BNXT_FLAG_CHIP_P5)
8604                 return false;
8605         if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
8606                 return false;
8607
8608         vnics = 1 + bp->rx_nr_rings;
8609         max_vnics = bnxt_get_max_func_vnics(bp);
8610         max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
8611
8612         /* RSS contexts not a limiting factor */
8613         if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8614                 max_rss_ctxs = max_vnics;
8615         if (vnics > max_vnics || vnics > max_rss_ctxs) {
8616                 if (bp->rx_nr_rings > 1)
8617                         netdev_warn(bp->dev,
8618                                     "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
8619                                     min(max_rss_ctxs - 1, max_vnics - 1));
8620                 return false;
8621         }
8622
8623         if (!BNXT_NEW_RM(bp))
8624                 return true;
8625
8626         if (vnics == bp->hw_resc.resv_vnics)
8627                 return true;
8628
8629         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics);
8630         if (vnics <= bp->hw_resc.resv_vnics)
8631                 return true;
8632
8633         netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
8634         bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1);
8635         return false;
8636 #else
8637         return false;
8638 #endif
8639 }
8640
8641 static netdev_features_t bnxt_fix_features(struct net_device *dev,
8642                                            netdev_features_t features)
8643 {
8644         struct bnxt *bp = netdev_priv(dev);
8645
8646         if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
8647                 features &= ~NETIF_F_NTUPLE;
8648
8649         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
8650                 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
8651
8652         if (!(features & NETIF_F_GRO))
8653                 features &= ~NETIF_F_GRO_HW;
8654
8655         if (features & NETIF_F_GRO_HW)
8656                 features &= ~NETIF_F_LRO;
8657
8658         /* Both CTAG and STAG VLAN accelaration on the RX side have to be
8659          * turned on or off together.
8660          */
8661         if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
8662             (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
8663                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
8664                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
8665                                       NETIF_F_HW_VLAN_STAG_RX);
8666                 else
8667                         features |= NETIF_F_HW_VLAN_CTAG_RX |
8668                                     NETIF_F_HW_VLAN_STAG_RX;
8669         }
8670 #ifdef CONFIG_BNXT_SRIOV
8671         if (BNXT_VF(bp)) {
8672                 if (bp->vf.vlan) {
8673                         features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
8674                                       NETIF_F_HW_VLAN_STAG_RX);
8675                 }
8676         }
8677 #endif
8678         return features;
8679 }
8680
8681 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
8682 {
8683         struct bnxt *bp = netdev_priv(dev);
8684         u32 flags = bp->flags;
8685         u32 changes;
8686         int rc = 0;
8687         bool re_init = false;
8688         bool update_tpa = false;
8689
8690         flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
8691         if (features & NETIF_F_GRO_HW)
8692                 flags |= BNXT_FLAG_GRO;
8693         else if (features & NETIF_F_LRO)
8694                 flags |= BNXT_FLAG_LRO;
8695
8696         if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
8697                 flags &= ~BNXT_FLAG_TPA;
8698
8699         if (features & NETIF_F_HW_VLAN_CTAG_RX)
8700                 flags |= BNXT_FLAG_STRIP_VLAN;
8701
8702         if (features & NETIF_F_NTUPLE)
8703                 flags |= BNXT_FLAG_RFS;
8704
8705         changes = flags ^ bp->flags;
8706         if (changes & BNXT_FLAG_TPA) {
8707                 update_tpa = true;
8708                 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
8709                     (flags & BNXT_FLAG_TPA) == 0)
8710                         re_init = true;
8711         }
8712
8713         if (changes & ~BNXT_FLAG_TPA)
8714                 re_init = true;
8715
8716         if (flags != bp->flags) {
8717                 u32 old_flags = bp->flags;
8718
8719                 bp->flags = flags;
8720
8721                 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
8722                         if (update_tpa)
8723                                 bnxt_set_ring_params(bp);
8724                         return rc;
8725                 }
8726
8727                 if (re_init) {
8728                         bnxt_close_nic(bp, false, false);
8729                         if (update_tpa)
8730                                 bnxt_set_ring_params(bp);
8731
8732                         return bnxt_open_nic(bp, false, false);
8733                 }
8734                 if (update_tpa) {
8735                         rc = bnxt_set_tpa(bp,
8736                                           (flags & BNXT_FLAG_TPA) ?
8737                                           true : false);
8738                         if (rc)
8739                                 bp->flags = old_flags;
8740                 }
8741         }
8742         return rc;
8743 }
8744
8745 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
8746                                        u32 ring_id, u32 *prod, u32 *cons)
8747 {
8748         struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
8749         struct hwrm_dbg_ring_info_get_input req = {0};
8750         int rc;
8751
8752         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
8753         req.ring_type = ring_type;
8754         req.fw_ring_id = cpu_to_le32(ring_id);
8755         mutex_lock(&bp->hwrm_cmd_lock);
8756         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8757         if (!rc) {
8758                 *prod = le32_to_cpu(resp->producer_index);
8759                 *cons = le32_to_cpu(resp->consumer_index);
8760         }
8761         mutex_unlock(&bp->hwrm_cmd_lock);
8762         return rc;
8763 }
8764
8765 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
8766 {
8767         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
8768         int i = bnapi->index;
8769
8770         if (!txr)
8771                 return;
8772
8773         netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
8774                     i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
8775                     txr->tx_cons);
8776 }
8777
8778 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
8779 {
8780         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
8781         int i = bnapi->index;
8782
8783         if (!rxr)
8784                 return;
8785
8786         netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
8787                     i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
8788                     rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
8789                     rxr->rx_sw_agg_prod);
8790 }
8791
8792 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
8793 {
8794         struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8795         int i = bnapi->index;
8796
8797         netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
8798                     i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
8799 }
8800
8801 static void bnxt_dbg_dump_states(struct bnxt *bp)
8802 {
8803         int i;
8804         struct bnxt_napi *bnapi;
8805
8806         for (i = 0; i < bp->cp_nr_rings; i++) {
8807                 bnapi = bp->bnapi[i];
8808                 if (netif_msg_drv(bp)) {
8809                         bnxt_dump_tx_sw_state(bnapi);
8810                         bnxt_dump_rx_sw_state(bnapi);
8811                         bnxt_dump_cp_sw_state(bnapi);
8812                 }
8813         }
8814 }
8815
8816 static void bnxt_reset_task(struct bnxt *bp, bool silent)
8817 {
8818         if (!silent)
8819                 bnxt_dbg_dump_states(bp);
8820         if (netif_running(bp->dev)) {
8821                 int rc;
8822
8823                 if (!silent)
8824                         bnxt_ulp_stop(bp);
8825                 bnxt_close_nic(bp, false, false);
8826                 rc = bnxt_open_nic(bp, false, false);
8827                 if (!silent && !rc)
8828                         bnxt_ulp_start(bp);
8829         }
8830 }
8831
8832 static void bnxt_tx_timeout(struct net_device *dev)
8833 {
8834         struct bnxt *bp = netdev_priv(dev);
8835
8836         netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
8837         set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
8838         bnxt_queue_sp_work(bp);
8839 }
8840
8841 static void bnxt_timer(struct timer_list *t)
8842 {
8843         struct bnxt *bp = from_timer(bp, t, timer);
8844         struct net_device *dev = bp->dev;
8845
8846         if (!netif_running(dev))
8847                 return;
8848
8849         if (atomic_read(&bp->intr_sem) != 0)
8850                 goto bnxt_restart_timer;
8851
8852         if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
8853             bp->stats_coal_ticks) {
8854                 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
8855                 bnxt_queue_sp_work(bp);
8856         }
8857
8858         if (bnxt_tc_flower_enabled(bp)) {
8859                 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
8860                 bnxt_queue_sp_work(bp);
8861         }
8862
8863         if (bp->link_info.phy_retry) {
8864                 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
8865                         bp->link_info.phy_retry = 0;
8866                         netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
8867                 } else {
8868                         set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
8869                         bnxt_queue_sp_work(bp);
8870                 }
8871         }
8872
8873         if ((bp->flags & BNXT_FLAG_CHIP_P5) && netif_carrier_ok(dev)) {
8874                 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
8875                 bnxt_queue_sp_work(bp);
8876         }
8877 bnxt_restart_timer:
8878         mod_timer(&bp->timer, jiffies + bp->current_interval);
8879 }
8880
8881 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
8882 {
8883         /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
8884          * set.  If the device is being closed, bnxt_close() may be holding
8885          * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
8886          * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
8887          */
8888         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
8889         rtnl_lock();
8890 }
8891
8892 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
8893 {
8894         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
8895         rtnl_unlock();
8896 }
8897
8898 /* Only called from bnxt_sp_task() */
8899 static void bnxt_reset(struct bnxt *bp, bool silent)
8900 {
8901         bnxt_rtnl_lock_sp(bp);
8902         if (test_bit(BNXT_STATE_OPEN, &bp->state))
8903                 bnxt_reset_task(bp, silent);
8904         bnxt_rtnl_unlock_sp(bp);
8905 }
8906
8907 static void bnxt_chk_missed_irq(struct bnxt *bp)
8908 {
8909         int i;
8910
8911         if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8912                 return;
8913
8914         for (i = 0; i < bp->cp_nr_rings; i++) {
8915                 struct bnxt_napi *bnapi = bp->bnapi[i];
8916                 struct bnxt_cp_ring_info *cpr;
8917                 u32 fw_ring_id;
8918                 int j;
8919
8920                 if (!bnapi)
8921                         continue;
8922
8923                 cpr = &bnapi->cp_ring;
8924                 for (j = 0; j < 2; j++) {
8925                         struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
8926                         u32 val[2];
8927
8928                         if (!cpr2 || cpr2->has_more_work ||
8929                             !bnxt_has_work(bp, cpr2))
8930                                 continue;
8931
8932                         if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
8933                                 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
8934                                 continue;
8935                         }
8936                         fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
8937                         bnxt_dbg_hwrm_ring_info_get(bp,
8938                                 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
8939                                 fw_ring_id, &val[0], &val[1]);
8940                         cpr->missed_irqs++;
8941                 }
8942         }
8943 }
8944
8945 static void bnxt_cfg_ntp_filters(struct bnxt *);
8946
8947 static void bnxt_sp_task(struct work_struct *work)
8948 {
8949         struct bnxt *bp = container_of(work, struct bnxt, sp_task);
8950
8951         set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
8952         smp_mb__after_atomic();
8953         if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
8954                 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
8955                 return;
8956         }
8957
8958         if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
8959                 bnxt_cfg_rx_mode(bp);
8960
8961         if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
8962                 bnxt_cfg_ntp_filters(bp);
8963         if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
8964                 bnxt_hwrm_exec_fwd_req(bp);
8965         if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
8966                 bnxt_hwrm_tunnel_dst_port_alloc(
8967                         bp, bp->vxlan_port,
8968                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8969         }
8970         if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
8971                 bnxt_hwrm_tunnel_dst_port_free(
8972                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8973         }
8974         if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
8975                 bnxt_hwrm_tunnel_dst_port_alloc(
8976                         bp, bp->nge_port,
8977                         TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8978         }
8979         if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
8980                 bnxt_hwrm_tunnel_dst_port_free(
8981                         bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8982         }
8983         if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
8984                 bnxt_hwrm_port_qstats(bp);
8985                 bnxt_hwrm_port_qstats_ext(bp);
8986         }
8987
8988         if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
8989                 int rc;
8990
8991                 mutex_lock(&bp->link_lock);
8992                 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
8993                                        &bp->sp_event))
8994                         bnxt_hwrm_phy_qcaps(bp);
8995
8996                 rc = bnxt_update_link(bp, true);
8997                 mutex_unlock(&bp->link_lock);
8998                 if (rc)
8999                         netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
9000                                    rc);
9001         }
9002         if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
9003                 int rc;
9004
9005                 mutex_lock(&bp->link_lock);
9006                 rc = bnxt_update_phy_setting(bp);
9007                 mutex_unlock(&bp->link_lock);
9008                 if (rc) {
9009                         netdev_warn(bp->dev, "update phy settings retry failed\n");
9010                 } else {
9011                         bp->link_info.phy_retry = false;
9012                         netdev_info(bp->dev, "update phy settings retry succeeded\n");
9013                 }
9014         }
9015         if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
9016                 mutex_lock(&bp->link_lock);
9017                 bnxt_get_port_module_status(bp);
9018                 mutex_unlock(&bp->link_lock);
9019         }
9020
9021         if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
9022                 bnxt_tc_flow_stats_work(bp);
9023
9024         if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
9025                 bnxt_chk_missed_irq(bp);
9026
9027         /* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
9028          * must be the last functions to be called before exiting.
9029          */
9030         if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
9031                 bnxt_reset(bp, false);
9032
9033         if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
9034                 bnxt_reset(bp, true);
9035
9036         smp_mb__before_atomic();
9037         clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
9038 }
9039
9040 /* Under rtnl_lock */
9041 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
9042                      int tx_xdp)
9043 {
9044         int max_rx, max_tx, tx_sets = 1;
9045         int tx_rings_needed;
9046         int rx_rings = rx;
9047         int cp, vnics, rc;
9048
9049         if (tcs)
9050                 tx_sets = tcs;
9051
9052         rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
9053         if (rc)
9054                 return rc;
9055
9056         if (max_rx < rx)
9057                 return -ENOMEM;
9058
9059         tx_rings_needed = tx * tx_sets + tx_xdp;
9060         if (max_tx < tx_rings_needed)
9061                 return -ENOMEM;
9062
9063         vnics = 1;
9064         if (bp->flags & BNXT_FLAG_RFS)
9065                 vnics += rx_rings;
9066
9067         if (bp->flags & BNXT_FLAG_AGG_RINGS)
9068                 rx_rings <<= 1;
9069         cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
9070         if (BNXT_NEW_RM(bp))
9071                 cp += bnxt_get_ulp_msix_num(bp);
9072         return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
9073                                      vnics);
9074 }
9075
9076 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
9077 {
9078         if (bp->bar2) {
9079                 pci_iounmap(pdev, bp->bar2);
9080                 bp->bar2 = NULL;
9081         }
9082
9083         if (bp->bar1) {
9084                 pci_iounmap(pdev, bp->bar1);
9085                 bp->bar1 = NULL;
9086         }
9087
9088         if (bp->bar0) {
9089                 pci_iounmap(pdev, bp->bar0);
9090                 bp->bar0 = NULL;
9091         }
9092 }
9093
9094 static void bnxt_cleanup_pci(struct bnxt *bp)
9095 {
9096         bnxt_unmap_bars(bp, bp->pdev);
9097         pci_release_regions(bp->pdev);
9098         pci_disable_device(bp->pdev);
9099 }
9100
9101 static void bnxt_init_dflt_coal(struct bnxt *bp)
9102 {
9103         struct bnxt_coal *coal;
9104
9105         /* Tick values in micro seconds.
9106          * 1 coal_buf x bufs_per_record = 1 completion record.
9107          */
9108         coal = &bp->rx_coal;
9109         coal->coal_ticks = 14;
9110         coal->coal_bufs = 30;
9111         coal->coal_ticks_irq = 1;
9112         coal->coal_bufs_irq = 2;
9113         coal->idle_thresh = 50;
9114         coal->bufs_per_record = 2;
9115         coal->budget = 64;              /* NAPI budget */
9116
9117         coal = &bp->tx_coal;
9118         coal->coal_ticks = 28;
9119         coal->coal_bufs = 30;
9120         coal->coal_ticks_irq = 2;
9121         coal->coal_bufs_irq = 2;
9122         coal->bufs_per_record = 1;
9123
9124         bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
9125 }
9126
9127 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
9128 {
9129         int rc;
9130         struct bnxt *bp = netdev_priv(dev);
9131
9132         SET_NETDEV_DEV(dev, &pdev->dev);
9133
9134         /* enable device (incl. PCI PM wakeup), and bus-mastering */
9135         rc = pci_enable_device(pdev);
9136         if (rc) {
9137                 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
9138                 goto init_err;
9139         }
9140
9141         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
9142                 dev_err(&pdev->dev,
9143                         "Cannot find PCI device base address, aborting\n");
9144                 rc = -ENODEV;
9145                 goto init_err_disable;
9146         }
9147
9148         rc = pci_request_regions(pdev, DRV_MODULE_NAME);
9149         if (rc) {
9150                 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
9151                 goto init_err_disable;
9152         }
9153
9154         if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
9155             dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
9156                 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
9157                 goto init_err_disable;
9158         }
9159
9160         pci_set_master(pdev);
9161
9162         bp->dev = dev;
9163         bp->pdev = pdev;
9164
9165         bp->bar0 = pci_ioremap_bar(pdev, 0);
9166         if (!bp->bar0) {
9167                 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
9168                 rc = -ENOMEM;
9169                 goto init_err_release;
9170         }
9171
9172         bp->bar1 = pci_ioremap_bar(pdev, 2);
9173         if (!bp->bar1) {
9174                 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
9175                 rc = -ENOMEM;
9176                 goto init_err_release;
9177         }
9178
9179         bp->bar2 = pci_ioremap_bar(pdev, 4);
9180         if (!bp->bar2) {
9181                 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
9182                 rc = -ENOMEM;
9183                 goto init_err_release;
9184         }
9185
9186         pci_enable_pcie_error_reporting(pdev);
9187
9188         INIT_WORK(&bp->sp_task, bnxt_sp_task);
9189
9190         spin_lock_init(&bp->ntp_fltr_lock);
9191 #if BITS_PER_LONG == 32
9192         spin_lock_init(&bp->db_lock);
9193 #endif
9194
9195         bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
9196         bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
9197
9198         bnxt_init_dflt_coal(bp);
9199
9200         timer_setup(&bp->timer, bnxt_timer, 0);
9201         bp->current_interval = BNXT_TIMER_INTERVAL;
9202
9203         clear_bit(BNXT_STATE_OPEN, &bp->state);
9204         return 0;
9205
9206 init_err_release:
9207         bnxt_unmap_bars(bp, pdev);
9208         pci_release_regions(pdev);
9209
9210 init_err_disable:
9211         pci_disable_device(pdev);
9212
9213 init_err:
9214         return rc;
9215 }
9216
9217 /* rtnl_lock held */
9218 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
9219 {
9220         struct sockaddr *addr = p;
9221         struct bnxt *bp = netdev_priv(dev);
9222         int rc = 0;
9223
9224         if (!is_valid_ether_addr(addr->sa_data))
9225                 return -EADDRNOTAVAIL;
9226
9227         if (ether_addr_equal(addr->sa_data, dev->dev_addr))
9228                 return 0;
9229
9230         rc = bnxt_approve_mac(bp, addr->sa_data, true);
9231         if (rc)
9232                 return rc;
9233
9234         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
9235         if (netif_running(dev)) {
9236                 bnxt_close_nic(bp, false, false);
9237                 rc = bnxt_open_nic(bp, false, false);
9238         }
9239
9240         return rc;
9241 }
9242
9243 /* rtnl_lock held */
9244 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
9245 {
9246         struct bnxt *bp = netdev_priv(dev);
9247
9248         if (netif_running(dev))
9249                 bnxt_close_nic(bp, false, false);
9250
9251         dev->mtu = new_mtu;
9252         bnxt_set_ring_params(bp);
9253
9254         if (netif_running(dev))
9255                 return bnxt_open_nic(bp, false, false);
9256
9257         return 0;
9258 }
9259
9260 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
9261 {
9262         struct bnxt *bp = netdev_priv(dev);
9263         bool sh = false;
9264         int rc;
9265
9266         if (tc > bp->max_tc) {
9267                 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
9268                            tc, bp->max_tc);
9269                 return -EINVAL;
9270         }
9271
9272         if (netdev_get_num_tc(dev) == tc)
9273                 return 0;
9274
9275         if (bp->flags & BNXT_FLAG_SHARED_RINGS)
9276                 sh = true;
9277
9278         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
9279                               sh, tc, bp->tx_nr_rings_xdp);
9280         if (rc)
9281                 return rc;
9282
9283         /* Needs to close the device and do hw resource re-allocations */
9284         if (netif_running(bp->dev))
9285                 bnxt_close_nic(bp, true, false);
9286
9287         if (tc) {
9288                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
9289                 netdev_set_num_tc(dev, tc);
9290         } else {
9291                 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
9292                 netdev_reset_tc(dev);
9293         }
9294         bp->tx_nr_rings += bp->tx_nr_rings_xdp;
9295         bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9296                                bp->tx_nr_rings + bp->rx_nr_rings;
9297         bp->num_stat_ctxs = bp->cp_nr_rings;
9298
9299         if (netif_running(bp->dev))
9300                 return bnxt_open_nic(bp, true, false);
9301
9302         return 0;
9303 }
9304
9305 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
9306                                   void *cb_priv)
9307 {
9308         struct bnxt *bp = cb_priv;
9309
9310         if (!bnxt_tc_flower_enabled(bp) ||
9311             !tc_cls_can_offload_and_chain0(bp->dev, type_data))
9312                 return -EOPNOTSUPP;
9313
9314         switch (type) {
9315         case TC_SETUP_CLSFLOWER:
9316                 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
9317         default:
9318                 return -EOPNOTSUPP;
9319         }
9320 }
9321
9322 static int bnxt_setup_tc_block(struct net_device *dev,
9323                                struct tc_block_offload *f)
9324 {
9325         struct bnxt *bp = netdev_priv(dev);
9326
9327         if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
9328                 return -EOPNOTSUPP;
9329
9330         switch (f->command) {
9331         case TC_BLOCK_BIND:
9332                 return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb,
9333                                              bp, bp, f->extack);
9334         case TC_BLOCK_UNBIND:
9335                 tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp);
9336                 return 0;
9337         default:
9338                 return -EOPNOTSUPP;
9339         }
9340 }
9341
9342 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
9343                          void *type_data)
9344 {
9345         switch (type) {
9346         case TC_SETUP_BLOCK:
9347                 return bnxt_setup_tc_block(dev, type_data);
9348         case TC_SETUP_QDISC_MQPRIO: {
9349                 struct tc_mqprio_qopt *mqprio = type_data;
9350
9351                 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
9352
9353                 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
9354         }
9355         default:
9356                 return -EOPNOTSUPP;
9357         }
9358 }
9359
9360 #ifdef CONFIG_RFS_ACCEL
9361 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
9362                             struct bnxt_ntuple_filter *f2)
9363 {
9364         struct flow_keys *keys1 = &f1->fkeys;
9365         struct flow_keys *keys2 = &f2->fkeys;
9366
9367         if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
9368             keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
9369             keys1->ports.ports == keys2->ports.ports &&
9370             keys1->basic.ip_proto == keys2->basic.ip_proto &&
9371             keys1->basic.n_proto == keys2->basic.n_proto &&
9372             keys1->control.flags == keys2->control.flags &&
9373             ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
9374             ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
9375                 return true;
9376
9377         return false;
9378 }
9379
9380 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
9381                               u16 rxq_index, u32 flow_id)
9382 {
9383         struct bnxt *bp = netdev_priv(dev);
9384         struct bnxt_ntuple_filter *fltr, *new_fltr;
9385         struct flow_keys *fkeys;
9386         struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
9387         int rc = 0, idx, bit_id, l2_idx = 0;
9388         struct hlist_head *head;
9389
9390         if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
9391                 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9392                 int off = 0, j;
9393
9394                 netif_addr_lock_bh(dev);
9395                 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
9396                         if (ether_addr_equal(eth->h_dest,
9397                                              vnic->uc_list + off)) {
9398                                 l2_idx = j + 1;
9399                                 break;
9400                         }
9401                 }
9402                 netif_addr_unlock_bh(dev);
9403                 if (!l2_idx)
9404                         return -EINVAL;
9405         }
9406         new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
9407         if (!new_fltr)
9408                 return -ENOMEM;
9409
9410         fkeys = &new_fltr->fkeys;
9411         if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
9412                 rc = -EPROTONOSUPPORT;
9413                 goto err_free;
9414         }
9415
9416         if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
9417              fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
9418             ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
9419              (fkeys->basic.ip_proto != IPPROTO_UDP))) {
9420                 rc = -EPROTONOSUPPORT;
9421                 goto err_free;
9422         }
9423         if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
9424             bp->hwrm_spec_code < 0x10601) {
9425                 rc = -EPROTONOSUPPORT;
9426                 goto err_free;
9427         }
9428         if ((fkeys->control.flags & FLOW_DIS_ENCAPSULATION) &&
9429             bp->hwrm_spec_code < 0x10601) {
9430                 rc = -EPROTONOSUPPORT;
9431                 goto err_free;
9432         }
9433
9434         memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
9435         memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
9436
9437         idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
9438         head = &bp->ntp_fltr_hash_tbl[idx];
9439         rcu_read_lock();
9440         hlist_for_each_entry_rcu(fltr, head, hash) {
9441                 if (bnxt_fltr_match(fltr, new_fltr)) {
9442                         rcu_read_unlock();
9443                         rc = 0;
9444                         goto err_free;
9445                 }
9446         }
9447         rcu_read_unlock();
9448
9449         spin_lock_bh(&bp->ntp_fltr_lock);
9450         bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
9451                                          BNXT_NTP_FLTR_MAX_FLTR, 0);
9452         if (bit_id < 0) {
9453                 spin_unlock_bh(&bp->ntp_fltr_lock);
9454                 rc = -ENOMEM;
9455                 goto err_free;
9456         }
9457
9458         new_fltr->sw_id = (u16)bit_id;
9459         new_fltr->flow_id = flow_id;
9460         new_fltr->l2_fltr_idx = l2_idx;
9461         new_fltr->rxq = rxq_index;
9462         hlist_add_head_rcu(&new_fltr->hash, head);
9463         bp->ntp_fltr_count++;
9464         spin_unlock_bh(&bp->ntp_fltr_lock);
9465
9466         set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
9467         bnxt_queue_sp_work(bp);
9468
9469         return new_fltr->sw_id;
9470
9471 err_free:
9472         kfree(new_fltr);
9473         return rc;
9474 }
9475
9476 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
9477 {
9478         int i;
9479
9480         for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
9481                 struct hlist_head *head;
9482                 struct hlist_node *tmp;
9483                 struct bnxt_ntuple_filter *fltr;
9484                 int rc;
9485
9486                 head = &bp->ntp_fltr_hash_tbl[i];
9487                 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
9488                         bool del = false;
9489
9490                         if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
9491                                 if (rps_may_expire_flow(bp->dev, fltr->rxq,
9492                                                         fltr->flow_id,
9493                                                         fltr->sw_id)) {
9494                                         bnxt_hwrm_cfa_ntuple_filter_free(bp,
9495                                                                          fltr);
9496                                         del = true;
9497                                 }
9498                         } else {
9499                                 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
9500                                                                        fltr);
9501                                 if (rc)
9502                                         del = true;
9503                                 else
9504                                         set_bit(BNXT_FLTR_VALID, &fltr->state);
9505                         }
9506
9507                         if (del) {
9508                                 spin_lock_bh(&bp->ntp_fltr_lock);
9509                                 hlist_del_rcu(&fltr->hash);
9510                                 bp->ntp_fltr_count--;
9511                                 spin_unlock_bh(&bp->ntp_fltr_lock);
9512                                 synchronize_rcu();
9513                                 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
9514                                 kfree(fltr);
9515                         }
9516                 }
9517         }
9518         if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
9519                 netdev_info(bp->dev, "Receive PF driver unload event!");
9520 }
9521
9522 #else
9523
9524 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
9525 {
9526 }
9527
9528 #endif /* CONFIG_RFS_ACCEL */
9529
9530 static void bnxt_udp_tunnel_add(struct net_device *dev,
9531                                 struct udp_tunnel_info *ti)
9532 {
9533         struct bnxt *bp = netdev_priv(dev);
9534
9535         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
9536                 return;
9537
9538         if (!netif_running(dev))
9539                 return;
9540
9541         switch (ti->type) {
9542         case UDP_TUNNEL_TYPE_VXLAN:
9543                 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
9544                         return;
9545
9546                 bp->vxlan_port_cnt++;
9547                 if (bp->vxlan_port_cnt == 1) {
9548                         bp->vxlan_port = ti->port;
9549                         set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
9550                         bnxt_queue_sp_work(bp);
9551                 }
9552                 break;
9553         case UDP_TUNNEL_TYPE_GENEVE:
9554                 if (bp->nge_port_cnt && bp->nge_port != ti->port)
9555                         return;
9556
9557                 bp->nge_port_cnt++;
9558                 if (bp->nge_port_cnt == 1) {
9559                         bp->nge_port = ti->port;
9560                         set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
9561                 }
9562                 break;
9563         default:
9564                 return;
9565         }
9566
9567         bnxt_queue_sp_work(bp);
9568 }
9569
9570 static void bnxt_udp_tunnel_del(struct net_device *dev,
9571                                 struct udp_tunnel_info *ti)
9572 {
9573         struct bnxt *bp = netdev_priv(dev);
9574
9575         if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
9576                 return;
9577
9578         if (!netif_running(dev))
9579                 return;
9580
9581         switch (ti->type) {
9582         case UDP_TUNNEL_TYPE_VXLAN:
9583                 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
9584                         return;
9585                 bp->vxlan_port_cnt--;
9586
9587                 if (bp->vxlan_port_cnt != 0)
9588                         return;
9589
9590                 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
9591                 break;
9592         case UDP_TUNNEL_TYPE_GENEVE:
9593                 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
9594                         return;
9595                 bp->nge_port_cnt--;
9596
9597                 if (bp->nge_port_cnt != 0)
9598                         return;
9599
9600                 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
9601                 break;
9602         default:
9603                 return;
9604         }
9605
9606         bnxt_queue_sp_work(bp);
9607 }
9608
9609 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9610                                struct net_device *dev, u32 filter_mask,
9611                                int nlflags)
9612 {
9613         struct bnxt *bp = netdev_priv(dev);
9614
9615         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
9616                                        nlflags, filter_mask, NULL);
9617 }
9618
9619 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
9620                                u16 flags)
9621 {
9622         struct bnxt *bp = netdev_priv(dev);
9623         struct nlattr *attr, *br_spec;
9624         int rem, rc = 0;
9625
9626         if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
9627                 return -EOPNOTSUPP;
9628
9629         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9630         if (!br_spec)
9631                 return -EINVAL;
9632
9633         nla_for_each_nested(attr, br_spec, rem) {
9634                 u16 mode;
9635
9636                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9637                         continue;
9638
9639                 if (nla_len(attr) < sizeof(mode))
9640                         return -EINVAL;
9641
9642                 mode = nla_get_u16(attr);
9643                 if (mode == bp->br_mode)
9644                         break;
9645
9646                 rc = bnxt_hwrm_set_br_mode(bp, mode);
9647                 if (!rc)
9648                         bp->br_mode = mode;
9649                 break;
9650         }
9651         return rc;
9652 }
9653
9654 static int bnxt_get_phys_port_name(struct net_device *dev, char *buf,
9655                                    size_t len)
9656 {
9657         struct bnxt *bp = netdev_priv(dev);
9658         int rc;
9659
9660         /* The PF and it's VF-reps only support the switchdev framework */
9661         if (!BNXT_PF(bp))
9662                 return -EOPNOTSUPP;
9663
9664         rc = snprintf(buf, len, "p%d", bp->pf.port_id);
9665
9666         if (rc >= len)
9667                 return -EOPNOTSUPP;
9668         return 0;
9669 }
9670
9671 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
9672 {
9673         if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
9674                 return -EOPNOTSUPP;
9675
9676         /* The PF and it's VF-reps only support the switchdev framework */
9677         if (!BNXT_PF(bp))
9678                 return -EOPNOTSUPP;
9679
9680         switch (attr->id) {
9681         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
9682                 attr->u.ppid.id_len = sizeof(bp->switch_id);
9683                 memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len);
9684                 break;
9685         default:
9686                 return -EOPNOTSUPP;
9687         }
9688         return 0;
9689 }
9690
9691 static int bnxt_swdev_port_attr_get(struct net_device *dev,
9692                                     struct switchdev_attr *attr)
9693 {
9694         return bnxt_port_attr_get(netdev_priv(dev), attr);
9695 }
9696
9697 static const struct switchdev_ops bnxt_switchdev_ops = {
9698         .switchdev_port_attr_get        = bnxt_swdev_port_attr_get
9699 };
9700
9701 static const struct net_device_ops bnxt_netdev_ops = {
9702         .ndo_open               = bnxt_open,
9703         .ndo_start_xmit         = bnxt_start_xmit,
9704         .ndo_stop               = bnxt_close,
9705         .ndo_get_stats64        = bnxt_get_stats64,
9706         .ndo_set_rx_mode        = bnxt_set_rx_mode,
9707         .ndo_do_ioctl           = bnxt_ioctl,
9708         .ndo_validate_addr      = eth_validate_addr,
9709         .ndo_set_mac_address    = bnxt_change_mac_addr,
9710         .ndo_change_mtu         = bnxt_change_mtu,
9711         .ndo_fix_features       = bnxt_fix_features,
9712         .ndo_set_features       = bnxt_set_features,
9713         .ndo_tx_timeout         = bnxt_tx_timeout,
9714 #ifdef CONFIG_BNXT_SRIOV
9715         .ndo_get_vf_config      = bnxt_get_vf_config,
9716         .ndo_set_vf_mac         = bnxt_set_vf_mac,
9717         .ndo_set_vf_vlan        = bnxt_set_vf_vlan,
9718         .ndo_set_vf_rate        = bnxt_set_vf_bw,
9719         .ndo_set_vf_link_state  = bnxt_set_vf_link_state,
9720         .ndo_set_vf_spoofchk    = bnxt_set_vf_spoofchk,
9721         .ndo_set_vf_trust       = bnxt_set_vf_trust,
9722 #endif
9723         .ndo_setup_tc           = bnxt_setup_tc,
9724 #ifdef CONFIG_RFS_ACCEL
9725         .ndo_rx_flow_steer      = bnxt_rx_flow_steer,
9726 #endif
9727         .ndo_udp_tunnel_add     = bnxt_udp_tunnel_add,
9728         .ndo_udp_tunnel_del     = bnxt_udp_tunnel_del,
9729         .ndo_bpf                = bnxt_xdp,
9730         .ndo_bridge_getlink     = bnxt_bridge_getlink,
9731         .ndo_bridge_setlink     = bnxt_bridge_setlink,
9732         .ndo_get_phys_port_name = bnxt_get_phys_port_name
9733 };
9734
9735 static void bnxt_remove_one(struct pci_dev *pdev)
9736 {
9737         struct net_device *dev = pci_get_drvdata(pdev);
9738         struct bnxt *bp = netdev_priv(dev);
9739
9740         if (BNXT_PF(bp)) {
9741                 bnxt_sriov_disable(bp);
9742                 bnxt_dl_unregister(bp);
9743         }
9744
9745         pci_disable_pcie_error_reporting(pdev);
9746         unregister_netdev(dev);
9747         bnxt_shutdown_tc(bp);
9748         bnxt_cancel_sp_work(bp);
9749         bp->sp_event = 0;
9750
9751         bnxt_clear_int_mode(bp);
9752         bnxt_hwrm_func_drv_unrgtr(bp);
9753         bnxt_free_hwrm_resources(bp);
9754         bnxt_free_hwrm_short_cmd_req(bp);
9755         bnxt_ethtool_free(bp);
9756         bnxt_dcb_free(bp);
9757         kfree(bp->edev);
9758         bp->edev = NULL;
9759         bnxt_free_ctx_mem(bp);
9760         kfree(bp->ctx);
9761         bp->ctx = NULL;
9762         bnxt_cleanup_pci(bp);
9763         free_netdev(dev);
9764 }
9765
9766 static int bnxt_probe_phy(struct bnxt *bp)
9767 {
9768         int rc = 0;
9769         struct bnxt_link_info *link_info = &bp->link_info;
9770
9771         rc = bnxt_hwrm_phy_qcaps(bp);
9772         if (rc) {
9773                 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
9774                            rc);
9775                 return rc;
9776         }
9777         mutex_init(&bp->link_lock);
9778
9779         rc = bnxt_update_link(bp, false);
9780         if (rc) {
9781                 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
9782                            rc);
9783                 return rc;
9784         }
9785
9786         /* Older firmware does not have supported_auto_speeds, so assume
9787          * that all supported speeds can be autonegotiated.
9788          */
9789         if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
9790                 link_info->support_auto_speeds = link_info->support_speeds;
9791
9792         /*initialize the ethool setting copy with NVM settings */
9793         if (BNXT_AUTO_MODE(link_info->auto_mode)) {
9794                 link_info->autoneg = BNXT_AUTONEG_SPEED;
9795                 if (bp->hwrm_spec_code >= 0x10201) {
9796                         if (link_info->auto_pause_setting &
9797                             PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
9798                                 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
9799                 } else {
9800                         link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
9801                 }
9802                 link_info->advertising = link_info->auto_link_speeds;
9803         } else {
9804                 link_info->req_link_speed = link_info->force_link_speed;
9805                 link_info->req_duplex = link_info->duplex_setting;
9806         }
9807         if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
9808                 link_info->req_flow_ctrl =
9809                         link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
9810         else
9811                 link_info->req_flow_ctrl = link_info->force_pause_setting;
9812         return rc;
9813 }
9814
9815 static int bnxt_get_max_irq(struct pci_dev *pdev)
9816 {
9817         u16 ctrl;
9818
9819         if (!pdev->msix_cap)
9820                 return 1;
9821
9822         pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
9823         return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
9824 }
9825
9826 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
9827                                 int *max_cp)
9828 {
9829         struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9830         int max_ring_grps = 0;
9831
9832         *max_tx = hw_resc->max_tx_rings;
9833         *max_rx = hw_resc->max_rx_rings;
9834         *max_cp = min_t(int, bnxt_get_max_func_cp_rings_for_en(bp),
9835                         hw_resc->max_irqs - bnxt_get_ulp_msix_num(bp));
9836         *max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs);
9837         max_ring_grps = hw_resc->max_hw_ring_grps;
9838         if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
9839                 *max_cp -= 1;
9840                 *max_rx -= 2;
9841         }
9842         if (bp->flags & BNXT_FLAG_AGG_RINGS)
9843                 *max_rx >>= 1;
9844         *max_rx = min_t(int, *max_rx, max_ring_grps);
9845 }
9846
9847 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
9848 {
9849         int rx, tx, cp;
9850
9851         _bnxt_get_max_rings(bp, &rx, &tx, &cp);
9852         *max_rx = rx;
9853         *max_tx = tx;
9854         if (!rx || !tx || !cp)
9855                 return -ENOMEM;
9856
9857         return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
9858 }
9859
9860 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
9861                                bool shared)
9862 {
9863         int rc;
9864
9865         rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
9866         if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
9867                 /* Not enough rings, try disabling agg rings. */
9868                 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
9869                 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
9870                 if (rc) {
9871                         /* set BNXT_FLAG_AGG_RINGS back for consistency */
9872                         bp->flags |= BNXT_FLAG_AGG_RINGS;
9873                         return rc;
9874                 }
9875                 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
9876                 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9877                 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9878                 bnxt_set_ring_params(bp);
9879         }
9880
9881         if (bp->flags & BNXT_FLAG_ROCE_CAP) {
9882                 int max_cp, max_stat, max_irq;
9883
9884                 /* Reserve minimum resources for RoCE */
9885                 max_cp = bnxt_get_max_func_cp_rings(bp);
9886                 max_stat = bnxt_get_max_func_stat_ctxs(bp);
9887                 max_irq = bnxt_get_max_func_irqs(bp);
9888                 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
9889                     max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
9890                     max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
9891                         return 0;
9892
9893                 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
9894                 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
9895                 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
9896                 max_cp = min_t(int, max_cp, max_irq);
9897                 max_cp = min_t(int, max_cp, max_stat);
9898                 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
9899                 if (rc)
9900                         rc = 0;
9901         }
9902         return rc;
9903 }
9904
9905 /* In initial default shared ring setting, each shared ring must have a
9906  * RX/TX ring pair.
9907  */
9908 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
9909 {
9910         bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
9911         bp->rx_nr_rings = bp->cp_nr_rings;
9912         bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
9913         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
9914 }
9915
9916 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
9917 {
9918         int dflt_rings, max_rx_rings, max_tx_rings, rc;
9919
9920         if (!bnxt_can_reserve_rings(bp))
9921                 return 0;
9922
9923         if (sh)
9924                 bp->flags |= BNXT_FLAG_SHARED_RINGS;
9925         dflt_rings = netif_get_num_default_rss_queues();
9926         /* Reduce default rings on multi-port cards so that total default
9927          * rings do not exceed CPU count.
9928          */
9929         if (bp->port_count > 1) {
9930                 int max_rings =
9931                         max_t(int, num_online_cpus() / bp->port_count, 1);
9932
9933                 dflt_rings = min_t(int, dflt_rings, max_rings);
9934         }
9935         rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
9936         if (rc)
9937                 return rc;
9938         bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
9939         bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
9940         if (sh)
9941                 bnxt_trim_dflt_sh_rings(bp);
9942         else
9943                 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
9944         bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
9945
9946         rc = __bnxt_reserve_rings(bp);
9947         if (rc)
9948                 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
9949         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9950         if (sh)
9951                 bnxt_trim_dflt_sh_rings(bp);
9952
9953         /* Rings may have been trimmed, re-reserve the trimmed rings. */
9954         if (bnxt_need_reserve_rings(bp)) {
9955                 rc = __bnxt_reserve_rings(bp);
9956                 if (rc)
9957                         netdev_warn(bp->dev, "2nd rings reservation failed.\n");
9958                 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9959         }
9960         bp->num_stat_ctxs = bp->cp_nr_rings;
9961         if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9962                 bp->rx_nr_rings++;
9963                 bp->cp_nr_rings++;
9964         }
9965         return rc;
9966 }
9967
9968 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
9969 {
9970         int rc;
9971
9972         if (bp->tx_nr_rings)
9973                 return 0;
9974
9975         bnxt_ulp_irq_stop(bp);
9976         bnxt_clear_int_mode(bp);
9977         rc = bnxt_set_dflt_rings(bp, true);
9978         if (rc) {
9979                 netdev_err(bp->dev, "Not enough rings available.\n");
9980                 goto init_dflt_ring_err;
9981         }
9982         rc = bnxt_init_int_mode(bp);
9983         if (rc)
9984                 goto init_dflt_ring_err;
9985
9986         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9987         if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
9988                 bp->flags |= BNXT_FLAG_RFS;
9989                 bp->dev->features |= NETIF_F_NTUPLE;
9990         }
9991 init_dflt_ring_err:
9992         bnxt_ulp_irq_restart(bp, rc);
9993         return rc;
9994 }
9995
9996 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
9997 {
9998         int rc;
9999
10000         ASSERT_RTNL();
10001         bnxt_hwrm_func_qcaps(bp);
10002
10003         if (netif_running(bp->dev))
10004                 __bnxt_close_nic(bp, true, false);
10005
10006         bnxt_ulp_irq_stop(bp);
10007         bnxt_clear_int_mode(bp);
10008         rc = bnxt_init_int_mode(bp);
10009         bnxt_ulp_irq_restart(bp, rc);
10010
10011         if (netif_running(bp->dev)) {
10012                 if (rc)
10013                         dev_close(bp->dev);
10014                 else
10015                         rc = bnxt_open_nic(bp, true, false);
10016         }
10017
10018         return rc;
10019 }
10020
10021 static int bnxt_init_mac_addr(struct bnxt *bp)
10022 {
10023         int rc = 0;
10024
10025         if (BNXT_PF(bp)) {
10026                 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
10027         } else {
10028 #ifdef CONFIG_BNXT_SRIOV
10029                 struct bnxt_vf_info *vf = &bp->vf;
10030                 bool strict_approval = true;
10031
10032                 if (is_valid_ether_addr(vf->mac_addr)) {
10033                         /* overwrite netdev dev_addr with admin VF MAC */
10034                         memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
10035                         /* Older PF driver or firmware may not approve this
10036                          * correctly.
10037                          */
10038                         strict_approval = false;
10039                 } else {
10040                         eth_hw_addr_random(bp->dev);
10041                 }
10042                 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
10043 #endif
10044         }
10045         return rc;
10046 }
10047
10048 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
10049 {
10050         static int version_printed;
10051         struct net_device *dev;
10052         struct bnxt *bp;
10053         int rc, max_irqs;
10054
10055         if (pci_is_bridge(pdev))
10056                 return -ENODEV;
10057
10058         if (version_printed++ == 0)
10059                 pr_info("%s", version);
10060
10061         max_irqs = bnxt_get_max_irq(pdev);
10062         dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
10063         if (!dev)
10064                 return -ENOMEM;
10065
10066         bp = netdev_priv(dev);
10067         bnxt_set_max_func_irqs(bp, max_irqs);
10068
10069         if (bnxt_vf_pciid(ent->driver_data))
10070                 bp->flags |= BNXT_FLAG_VF;
10071
10072         if (pdev->msix_cap)
10073                 bp->flags |= BNXT_FLAG_MSIX_CAP;
10074
10075         rc = bnxt_init_board(pdev, dev);
10076         if (rc < 0)
10077                 goto init_err_free;
10078
10079         dev->netdev_ops = &bnxt_netdev_ops;
10080         dev->watchdog_timeo = BNXT_TX_TIMEOUT;
10081         dev->ethtool_ops = &bnxt_ethtool_ops;
10082         SWITCHDEV_SET_OPS(dev, &bnxt_switchdev_ops);
10083         pci_set_drvdata(pdev, dev);
10084
10085         rc = bnxt_alloc_hwrm_resources(bp);
10086         if (rc)
10087                 goto init_err_pci_clean;
10088
10089         mutex_init(&bp->hwrm_cmd_lock);
10090         rc = bnxt_hwrm_ver_get(bp);
10091         if (rc)
10092                 goto init_err_pci_clean;
10093
10094         if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10095             bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10096                 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10097                 if (rc)
10098                         goto init_err_pci_clean;
10099         }
10100
10101         if (BNXT_CHIP_P5(bp))
10102                 bp->flags |= BNXT_FLAG_CHIP_P5;
10103
10104         rc = bnxt_hwrm_func_reset(bp);
10105         if (rc)
10106                 goto init_err_pci_clean;
10107
10108         bnxt_hwrm_fw_set_time(bp);
10109
10110         dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10111                            NETIF_F_TSO | NETIF_F_TSO6 |
10112                            NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10113                            NETIF_F_GSO_IPXIP4 |
10114                            NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10115                            NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
10116                            NETIF_F_RXCSUM | NETIF_F_GRO;
10117
10118         if (BNXT_SUPPORTS_TPA(bp))
10119                 dev->hw_features |= NETIF_F_LRO;
10120
10121         dev->hw_enc_features =
10122                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
10123                         NETIF_F_TSO | NETIF_F_TSO6 |
10124                         NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
10125                         NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
10126                         NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
10127         dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
10128                                     NETIF_F_GSO_GRE_CSUM;
10129         dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
10130         dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
10131                             NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
10132         if (BNXT_SUPPORTS_TPA(bp))
10133                 dev->hw_features |= NETIF_F_GRO_HW;
10134         dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
10135         if (dev->features & NETIF_F_GRO_HW)
10136                 dev->features &= ~NETIF_F_LRO;
10137         dev->priv_flags |= IFF_UNICAST_FLT;
10138
10139 #ifdef CONFIG_BNXT_SRIOV
10140         init_waitqueue_head(&bp->sriov_cfg_wait);
10141         mutex_init(&bp->sriov_lock);
10142 #endif
10143         if (BNXT_SUPPORTS_TPA(bp)) {
10144                 bp->gro_func = bnxt_gro_func_5730x;
10145                 if (BNXT_CHIP_P4(bp))
10146                         bp->gro_func = bnxt_gro_func_5731x;
10147         }
10148         if (!BNXT_CHIP_P4_PLUS(bp))
10149                 bp->flags |= BNXT_FLAG_DOUBLE_DB;
10150
10151         rc = bnxt_hwrm_func_drv_rgtr(bp);
10152         if (rc)
10153                 goto init_err_pci_clean;
10154
10155         rc = bnxt_hwrm_func_rgtr_async_events(bp, NULL, 0);
10156         if (rc)
10157                 goto init_err_pci_clean;
10158
10159         bp->ulp_probe = bnxt_ulp_probe;
10160
10161         rc = bnxt_hwrm_queue_qportcfg(bp);
10162         if (rc) {
10163                 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
10164                            rc);
10165                 rc = -1;
10166                 goto init_err_pci_clean;
10167         }
10168         /* Get the MAX capabilities for this function */
10169         rc = bnxt_hwrm_func_qcaps(bp);
10170         if (rc) {
10171                 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10172                            rc);
10173                 rc = -1;
10174                 goto init_err_pci_clean;
10175         }
10176         rc = bnxt_init_mac_addr(bp);
10177         if (rc) {
10178                 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
10179                 rc = -EADDRNOTAVAIL;
10180                 goto init_err_pci_clean;
10181         }
10182
10183         bnxt_hwrm_func_qcfg(bp);
10184         bnxt_hwrm_vnic_qcaps(bp);
10185         bnxt_hwrm_port_led_qcaps(bp);
10186         bnxt_ethtool_init(bp);
10187         bnxt_dcb_init(bp);
10188
10189         /* MTU range: 60 - FW defined max */
10190         dev->min_mtu = ETH_ZLEN;
10191         dev->max_mtu = bp->max_mtu;
10192
10193         rc = bnxt_probe_phy(bp);
10194         if (rc)
10195                 goto init_err_pci_clean;
10196
10197         bnxt_set_rx_skb_mode(bp, false);
10198         bnxt_set_tpa_flags(bp);
10199         bnxt_set_ring_params(bp);
10200         rc = bnxt_set_dflt_rings(bp, true);
10201         if (rc) {
10202                 netdev_err(bp->dev, "Not enough rings available.\n");
10203                 rc = -ENOMEM;
10204                 goto init_err_pci_clean;
10205         }
10206
10207         /* Default RSS hash cfg. */
10208         bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10209                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10210                            VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10211                            VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10212         if (BNXT_CHIP_P4(bp) && bp->hwrm_spec_code >= 0x10501) {
10213                 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10214                 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10215                                     VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10216         }
10217
10218         if (bnxt_rfs_supported(bp)) {
10219                 dev->hw_features |= NETIF_F_NTUPLE;
10220                 if (bnxt_rfs_capable(bp)) {
10221                         bp->flags |= BNXT_FLAG_RFS;
10222                         dev->features |= NETIF_F_NTUPLE;
10223                 }
10224         }
10225
10226         if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
10227                 bp->flags |= BNXT_FLAG_STRIP_VLAN;
10228
10229         rc = bnxt_init_int_mode(bp);
10230         if (rc)
10231                 goto init_err_pci_clean;
10232
10233         /* No TC has been set yet and rings may have been trimmed due to
10234          * limited MSIX, so we re-initialize the TX rings per TC.
10235          */
10236         bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
10237
10238         bnxt_get_wol_settings(bp);
10239         if (bp->flags & BNXT_FLAG_WOL_CAP)
10240                 device_set_wakeup_enable(&pdev->dev, bp->wol);
10241         else
10242                 device_set_wakeup_capable(&pdev->dev, false);
10243
10244         bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10245
10246         bnxt_hwrm_coal_params_qcaps(bp);
10247
10248         if (BNXT_PF(bp)) {
10249                 if (!bnxt_pf_wq) {
10250                         bnxt_pf_wq =
10251                                 create_singlethread_workqueue("bnxt_pf_wq");
10252                         if (!bnxt_pf_wq) {
10253                                 dev_err(&pdev->dev, "Unable to create workqueue.\n");
10254                                 goto init_err_pci_clean;
10255                         }
10256                 }
10257                 bnxt_init_tc(bp);
10258         }
10259
10260         rc = register_netdev(dev);
10261         if (rc)
10262                 goto init_err_cleanup_tc;
10263
10264         if (BNXT_PF(bp))
10265                 bnxt_dl_register(bp);
10266
10267         netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
10268                     board_info[ent->driver_data].name,
10269                     (long)pci_resource_start(pdev, 0), dev->dev_addr);
10270         pcie_print_link_status(pdev);
10271
10272         return 0;
10273
10274 init_err_cleanup_tc:
10275         bnxt_shutdown_tc(bp);
10276         bnxt_clear_int_mode(bp);
10277
10278 init_err_pci_clean:
10279         bnxt_free_hwrm_resources(bp);
10280         bnxt_free_ctx_mem(bp);
10281         kfree(bp->ctx);
10282         bp->ctx = NULL;
10283         bnxt_cleanup_pci(bp);
10284
10285 init_err_free:
10286         free_netdev(dev);
10287         return rc;
10288 }
10289
10290 static void bnxt_shutdown(struct pci_dev *pdev)
10291 {
10292         struct net_device *dev = pci_get_drvdata(pdev);
10293         struct bnxt *bp;
10294
10295         if (!dev)
10296                 return;
10297
10298         rtnl_lock();
10299         bp = netdev_priv(dev);
10300         if (!bp)
10301                 goto shutdown_exit;
10302
10303         if (netif_running(dev))
10304                 dev_close(dev);
10305
10306         bnxt_ulp_shutdown(bp);
10307
10308         if (system_state == SYSTEM_POWER_OFF) {
10309                 bnxt_clear_int_mode(bp);
10310                 pci_wake_from_d3(pdev, bp->wol);
10311                 pci_set_power_state(pdev, PCI_D3hot);
10312         }
10313
10314 shutdown_exit:
10315         rtnl_unlock();
10316 }
10317
10318 #ifdef CONFIG_PM_SLEEP
10319 static int bnxt_suspend(struct device *device)
10320 {
10321         struct pci_dev *pdev = to_pci_dev(device);
10322         struct net_device *dev = pci_get_drvdata(pdev);
10323         struct bnxt *bp = netdev_priv(dev);
10324         int rc = 0;
10325
10326         rtnl_lock();
10327         if (netif_running(dev)) {
10328                 netif_device_detach(dev);
10329                 rc = bnxt_close(dev);
10330         }
10331         bnxt_hwrm_func_drv_unrgtr(bp);
10332         rtnl_unlock();
10333         return rc;
10334 }
10335
10336 static int bnxt_resume(struct device *device)
10337 {
10338         struct pci_dev *pdev = to_pci_dev(device);
10339         struct net_device *dev = pci_get_drvdata(pdev);
10340         struct bnxt *bp = netdev_priv(dev);
10341         int rc = 0;
10342
10343         rtnl_lock();
10344         if (bnxt_hwrm_ver_get(bp) || bnxt_hwrm_func_drv_rgtr(bp)) {
10345                 rc = -ENODEV;
10346                 goto resume_exit;
10347         }
10348         rc = bnxt_hwrm_func_reset(bp);
10349         if (rc) {
10350                 rc = -EBUSY;
10351                 goto resume_exit;
10352         }
10353         bnxt_get_wol_settings(bp);
10354         if (netif_running(dev)) {
10355                 rc = bnxt_open(dev);
10356                 if (!rc)
10357                         netif_device_attach(dev);
10358         }
10359
10360 resume_exit:
10361         rtnl_unlock();
10362         return rc;
10363 }
10364
10365 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
10366 #define BNXT_PM_OPS (&bnxt_pm_ops)
10367
10368 #else
10369
10370 #define BNXT_PM_OPS NULL
10371
10372 #endif /* CONFIG_PM_SLEEP */
10373
10374 /**
10375  * bnxt_io_error_detected - called when PCI error is detected
10376  * @pdev: Pointer to PCI device
10377  * @state: The current pci connection state
10378  *
10379  * This function is called after a PCI bus error affecting
10380  * this device has been detected.
10381  */
10382 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
10383                                                pci_channel_state_t state)
10384 {
10385         struct net_device *netdev = pci_get_drvdata(pdev);
10386         struct bnxt *bp = netdev_priv(netdev);
10387
10388         netdev_info(netdev, "PCI I/O error detected\n");
10389
10390         rtnl_lock();
10391         netif_device_detach(netdev);
10392
10393         bnxt_ulp_stop(bp);
10394
10395         if (state == pci_channel_io_perm_failure) {
10396                 rtnl_unlock();
10397                 return PCI_ERS_RESULT_DISCONNECT;
10398         }
10399
10400         if (netif_running(netdev))
10401                 bnxt_close(netdev);
10402
10403         pci_disable_device(pdev);
10404         rtnl_unlock();
10405
10406         /* Request a slot slot reset. */
10407         return PCI_ERS_RESULT_NEED_RESET;
10408 }
10409
10410 /**
10411  * bnxt_io_slot_reset - called after the pci bus has been reset.
10412  * @pdev: Pointer to PCI device
10413  *
10414  * Restart the card from scratch, as if from a cold-boot.
10415  * At this point, the card has exprienced a hard reset,
10416  * followed by fixups by BIOS, and has its config space
10417  * set up identically to what it was at cold boot.
10418  */
10419 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
10420 {
10421         struct net_device *netdev = pci_get_drvdata(pdev);
10422         struct bnxt *bp = netdev_priv(netdev);
10423         int err = 0;
10424         pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
10425
10426         netdev_info(bp->dev, "PCI Slot Reset\n");
10427
10428         rtnl_lock();
10429
10430         if (pci_enable_device(pdev)) {
10431                 dev_err(&pdev->dev,
10432                         "Cannot re-enable PCI device after reset.\n");
10433         } else {
10434                 pci_set_master(pdev);
10435
10436                 err = bnxt_hwrm_func_reset(bp);
10437                 if (!err && netif_running(netdev))
10438                         err = bnxt_open(netdev);
10439
10440                 if (!err) {
10441                         result = PCI_ERS_RESULT_RECOVERED;
10442                         bnxt_ulp_start(bp);
10443                 }
10444         }
10445
10446         if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
10447                 dev_close(netdev);
10448
10449         rtnl_unlock();
10450
10451         return PCI_ERS_RESULT_RECOVERED;
10452 }
10453
10454 /**
10455  * bnxt_io_resume - called when traffic can start flowing again.
10456  * @pdev: Pointer to PCI device
10457  *
10458  * This callback is called when the error recovery driver tells
10459  * us that its OK to resume normal operation.
10460  */
10461 static void bnxt_io_resume(struct pci_dev *pdev)
10462 {
10463         struct net_device *netdev = pci_get_drvdata(pdev);
10464
10465         rtnl_lock();
10466
10467         netif_device_attach(netdev);
10468
10469         rtnl_unlock();
10470 }
10471
10472 static const struct pci_error_handlers bnxt_err_handler = {
10473         .error_detected = bnxt_io_error_detected,
10474         .slot_reset     = bnxt_io_slot_reset,
10475         .resume         = bnxt_io_resume
10476 };
10477
10478 static struct pci_driver bnxt_pci_driver = {
10479         .name           = DRV_MODULE_NAME,
10480         .id_table       = bnxt_pci_tbl,
10481         .probe          = bnxt_init_one,
10482         .remove         = bnxt_remove_one,
10483         .shutdown       = bnxt_shutdown,
10484         .driver.pm      = BNXT_PM_OPS,
10485         .err_handler    = &bnxt_err_handler,
10486 #if defined(CONFIG_BNXT_SRIOV)
10487         .sriov_configure = bnxt_sriov_configure,
10488 #endif
10489 };
10490
10491 static int __init bnxt_init(void)
10492 {
10493         bnxt_debug_init();
10494         return pci_register_driver(&bnxt_pci_driver);
10495 }
10496
10497 static void __exit bnxt_exit(void)
10498 {
10499         pci_unregister_driver(&bnxt_pci_driver);
10500         if (bnxt_pf_wq)
10501                 destroy_workqueue(bnxt_pf_wq);
10502         bnxt_debug_exit();
10503 }
10504
10505 module_init(bnxt_init);
10506 module_exit(bnxt_exit);